smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adapt kiwix-lib to the new libzim api.

This commit is contained in:
Matthieu Gautier 2020-07-23 12:03:45 +02:00
parent d87079ec13
commit 1a5a2e7a8e
17 changed files with 186 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
include/entry.h
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr> * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -21,7 +21,8 @@
#define KIWIX_ENTRY_H #define KIWIX_ENTRY_H
#include <stdio.h> #include <stdio.h>
#include <zim/article.h> #include <zim/entry.h>
#include <zim/item.h>
#include <exception> #include <exception>
#include <string> #include <string>
@ -41,19 +42,12 @@ class NoEntry : public std::exception {};
class Entry class Entry
{ {
public: public:
/**
* Default constructor.
*
* Construct an invalid entry.
*/
Entry() = default;
/** /**
* Construct an entry making reference to an zim article. * Construct an entry making reference to an zim article.
* *
* @param article a zim::Article object * @param article a zim::Article object
*/ */
Entry(zim::Article article); Entry(zim::Entry entry);
virtual ~Entry() = default; virtual ~Entry() = default;
/** /**
@ -63,14 +57,14 @@ class Entry
* *
* @return the path of the entry. * @return the path of the entry.
*/ */
std::string getPath() const; std::string getPath() const { return entry.getPath(); }
/** /**
* Get the title of the entry. * Get the title of the entry.
* *
* @return the title of the entry. * @return the title of the entry.
*/ */
std::string getTitle() const; std::string getTitle() const { return entry.getTitle(); }
/** /**
* Get the content of the entry. * Get the content of the entry.
@ -80,7 +74,7 @@ class Entry
* *
* @return the content of the entry. * @return the content of the entry.
*/ */
std::string getContent() const; std::string getContent() const { return entry.getItem().getData(); }
/** /**
* Get the blob of the entry. * Get the blob of the entry.
@ -90,7 +84,7 @@ class Entry
* @param offset The starting offset of the blob. * @param offset The starting offset of the blob.
* @return the blob of the entry. * @return the blob of the entry.
*/ */
zim::Blob getBlob(offset_type offset = 0) const; zim::Blob getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
/** /**
* Get the blob of the entry. * Get the blob of the entry.
@ -101,7 +95,7 @@ class Entry
* @param size The size of the blob. * @param size The size of the blob.
* @return the blob of the entry. * @return the blob of the entry.
*/ */
zim::Blob getBlob(offset_type offset, size_type size) const; zim::Blob getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
/** /**
* Get the info for direct access to the content of the entry. * Get the info for direct access to the content of the entry.
@ -117,7 +111,7 @@ class Entry
* The offset is the offset to read in the file. * The offset is the offset to read in the file.
* Return <"",0> if is not possible to read directly. * Return <"",0> if is not possible to read directly.
*/ */
std::pair<std::string, offset_type> getDirectAccessInfo() const; std::pair<std::string, offset_type> getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
/** /**
* Get the size of the entry. * Get the size of the entry.
@ -174,17 +168,14 @@ class Entry
Entry getFinalEntry() const; Entry getFinalEntry() const;
/** /**
* Convert the entry to a boolean value. * Get the zim entry wrapped by this (kiwix) entry
* *
* @return True if the entry is valid. * @return the zim entry
*/ */
explicit operator bool() const { return good(); } const zim::Entry& getZimEntry() const { return entry; }
private: private:
zim::Article article; zim::Entry entry;
mutable zim::Article final_article;
bool good() const { return article.good(); }
}; };
} }

21
include/reader.h
View File

@ -21,10 +21,8 @@
#define KIWIX_READER_H #define KIWIX_READER_H
#include <stdio.h> #include <stdio.h>
#include <zim/article.h>
#include <zim/file.h>
#include <zim/fileiterator.h>
#include <zim/zim.h> #include <zim/zim.h>
#include <zim/archive.h>
#include <exception> #include <exception>
#include <map> #include <map>
#include <sstream> #include <sstream>
@ -58,7 +56,7 @@ class Reader
* (.zim extesion). * (.zim extesion).
*/ */
Reader(const string zimFilePath); Reader(const string zimFilePath);
~Reader(); ~Reader() = default;
/** /**
* Get the number of "displayable" entries in the zim file. * Get the number of "displayable" entries in the zim file.
@ -110,13 +108,6 @@ class Reader
*/ */
Entry getRandomPage() const; Entry getRandomPage() const;
/**
* Get the entry of the first page.
*
* @return The first entry in the 'A' namespace.
*/
Entry getFirstPage() const;
/** /**
* Get the entry of the main page. * Get the entry of the main page.
* *
@ -455,14 +446,10 @@ class Reader
* *
* @return The libzim file handler. * @return The libzim file handler.
*/ */
zim::File* getZimFileHandler() const; zim::Archive* getZimArchive() const;
protected: protected:
zim::File* zimFileHandler; std::unique_ptr<zim::Archive> zimArchive;
zim::size_type firstArticleOffset;
zim::size_type lastArticleOffset;
zim::size_type nsACount;
zim::size_type nsICount;
std::string zimFilePath; std::string zimFilePath;
SuggestionsList_t suggestions; SuggestionsList_t suggestions;

2
include/tools/otherTools.h
View File

@ -43,7 +43,7 @@ namespace kiwix
const std::string& tagName); const std::string& tagName);
bool convertStrToBool(const std::string& value); bool convertStrToBool(const std::string& value);
using MimeCounterType = std::map<const std::string, zim::article_index_type>; using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
MimeCounterType parseMimetypeCounter(const std::string& counterData); MimeCounterType parseMimetypeCounter(const std::string& counterData);
} }

99
src/entry.cpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org> * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -25,116 +25,51 @@
namespace kiwix namespace kiwix
{ {
Entry::Entry(zim::Article article) Entry::Entry(zim::Entry entry)
: article(article) : entry(entry)
{ {
} }
#define RETURN_IF_INVALID(WHAT) if(!good()) { return (WHAT); }
std::string Entry::getPath() const
{
RETURN_IF_INVALID("");
return article.getLongUrl();
}
std::string Entry::getTitle() const
{
RETURN_IF_INVALID("");
return article.getTitle();
}
std::string Entry::getContent() const
{
RETURN_IF_INVALID("");
return article.getData();
}
zim::Blob Entry::getBlob(offset_type offset) const
{
RETURN_IF_INVALID(zim::Blob());
return article.getData(offset);
}
zim::Blob Entry::getBlob(offset_type offset, size_type size) const
{
RETURN_IF_INVALID(zim::Blob());
return article.getData(offset, size);
}
std::pair<std::string, offset_type> Entry::getDirectAccessInfo() const
{
RETURN_IF_INVALID(std::make_pair("", 0));
return article.getDirectAccessInformation();
}
size_type Entry::getSize() const size_type Entry::getSize() const
{ {
RETURN_IF_INVALID(0); if (entry.isRedirect()) {
return article.getArticleSize(); return 0;
} else {
return entry.getItem().getSize();
}
} }
std::string Entry::getMimetype() const std::string Entry::getMimetype() const
{ {
RETURN_IF_INVALID(""); return entry.getItem(true).getMimetype();
try {
return article.getMimeType();
} catch (exception& e) {
return "application/octet-stream";
}
} }
bool Entry::isRedirect() const bool Entry::isRedirect() const
{ {
RETURN_IF_INVALID(false); return entry.isRedirect();
return article.isRedirect();
}
bool Entry::isLinkTarget() const
{
RETURN_IF_INVALID(false);
return article.isLinktarget();
}
bool Entry::isDeleted() const
{
RETURN_IF_INVALID(false);
return article.isDeleted();
} }
Entry Entry::getRedirectEntry() const Entry Entry::getRedirectEntry() const
{ {
RETURN_IF_INVALID(Entry()); if ( !entry.isRedirect() ) {
if ( !article.isRedirect() ) {
throw NoEntry(); throw NoEntry();
} }
auto targeted_article = article.getRedirectArticle(); return entry.getRedirectEntry();
if ( !targeted_article.good()) {
throw NoEntry();
}
return targeted_article;
} }
Entry Entry::getFinalEntry() const Entry Entry::getFinalEntry() const
{ {
RETURN_IF_INVALID(Entry());
if (final_article.good()) {
return final_article;
}
int loopCounter = 42; int loopCounter = 42;
final_article = article; auto final_entry = entry;
while (final_article.isRedirect() && loopCounter--) { while (final_entry.isRedirect() && loopCounter--) {
final_article = final_article.getRedirectArticle(); final_entry = final_entry.getRedirectEntry();
if ( !final_article.good()) {
throw NoEntry();
}
} }
// Prevent infinite loops. // Prevent infinite loops.
if (final_article.isRedirect()) { if (final_entry.isRedirect()) {
throw NoEntry(); throw NoEntry();
} }
return final_article; return final_entry;
} }
} }

264
src/reader.cpp
View File

@ -21,6 +21,8 @@
#include <time.h> #include <time.h>
#include <zim/search.h> #include <zim/search.h>
#include <zim/item.h>
#include <zim/error.h>
#include "tools/otherTools.h" #include "tools/otherTools.h"
@ -65,7 +67,9 @@ std::string hexUUID(std::string in)
namespace kiwix namespace kiwix
{ {
/* Constructor */ /* Constructor */
Reader::Reader(const string zimFilePath) : zimFileHandler(NULL) Reader::Reader(const string zimFilePath)
: zimArchive(nullptr),
zimFilePath(zimFilePath)
{ {
string tmpZimFilePath = zimFilePath; string tmpZimFilePath = zimFilePath;
@ -76,43 +80,21 @@ Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
tmpZimFilePath.resize(tmpZimFilePath.size() - 2); tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
} }
this->zimFileHandler = new zim::File(tmpZimFilePath); zimArchive.reset(new zim::Archive(tmpZimFilePath));
if (this->zimFileHandler != NULL) {
this->firstArticleOffset
= this->zimFileHandler->getNamespaceBeginOffset('A');
this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
this->nsACount = this->zimFileHandler->getNamespaceCount('A');
this->nsICount = this->zimFileHandler->getNamespaceCount('I');
this->zimFilePath = zimFilePath;
}
/* initialize random seed: */ /* initialize random seed: */
srand(time(NULL)); srand(time(nullptr));
} }
/* Destructor */ zim::Archive* Reader::getZimArchive() const
Reader::~Reader()
{ {
if (this->zimFileHandler != NULL) { return zimArchive.get();
delete this->zimFileHandler;
}
}
zim::File* Reader::getZimFileHandler() const
{
return this->zimFileHandler;
} }
MimeCounterType Reader::parseCounterMetadata() const MimeCounterType Reader::parseCounterMetadata() const
{ {
zim::Article article = this->zimFileHandler->getArticle('M', "Counter"); auto counterContent = zimArchive->getMetadata("Counter");
return parseMimetypeCounter(counterContent);
if (article.good()) {
return parseMimetypeCounter(article.getData());
}
return MimeCounterType();
} }
/* Get the count of articles which can be indexed/displayed */ /* Get the count of articles which can be indexed/displayed */
@ -122,15 +104,11 @@ unsigned int Reader::getArticleCount() const
= this->parseCounterMetadata(); = this->parseCounterMetadata();
unsigned int counter = 0; unsigned int counter = 0;
if (counterMap.empty()) {
counter = this->nsACount;
} else {
for(auto &pair:counterMap) { for(auto &pair:counterMap) {
if (startsWith(pair.first, "text/html")) { if (startsWith(pair.first, "text/html")) {
counter += pair.second; counter += pair.second;
} }
} }
}
return counter; return counter;
} }
@ -142,9 +120,6 @@ unsigned int Reader::getMediaCount() const
= this->parseCounterMetadata(); = this->parseCounterMetadata();
unsigned int counter = 0; unsigned int counter = 0;
if (counterMap.empty()) {
counter = this->nsICount;
} else {
auto it = counterMap.find("image/jpeg"); auto it = counterMap.find("image/jpeg");
if (it != counterMap.end()) { if (it != counterMap.end()) {
counter += it->second; counter += it->second;
@ -159,72 +134,47 @@ unsigned int Reader::getMediaCount() const
if (it != counterMap.end()) { if (it != counterMap.end()) {
counter += it->second; counter += it->second;
} }
}
return counter; return counter;
} }
/* Get the total of all items of a ZIM file, redirects included */ /* Get the total of all items of a ZIM file, redirects included */
unsigned int Reader::getGlobalCount() const unsigned int Reader::getGlobalCount() const
{ {
return this->zimFileHandler->getCountArticles(); return zimArchive->getEntryCount();
} }
/* Return the UID of the ZIM file */ /* Return the UID of the ZIM file */
string Reader::getId() const string Reader::getId() const
{ {
std::ostringstream s; std::ostringstream s;
s << this->zimFileHandler->getFileheader().getUuid(); s << zimArchive->getUuid();
return s.str(); return s.str();
} }
Entry Reader::getRandomPage() const Entry Reader::getRandomPage() const
{ {
if (!this->zimFileHandler) { auto mainPagePath = zimArchive->getMainEntry().getPath();
throw NoEntry();
}
zim::Article article;
std::string mainPagePath = this->getMainPage().getPath();
int watchdog = 42; int watchdog = 42;
do { while (--watchdog){
auto idx = this->firstArticleOffset auto idx = (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
+ (zim::size_type)((double)rand() / ((double)RAND_MAX + 1) * zimArchive->getEntryCount());
* this->nsACount); auto entry = zimArchive->getEntryByPath(idx);
article = zimFileHandler->getArticle(idx);
if (!watchdog--) { if (entry.getPath()==mainPagePath) {
continue;
}
auto item = entry.getItem(true);
if (item.getMimetype() == "text/html") {
return entry;
}
}
throw NoEntry(); throw NoEntry();
} }
} while (!article.good() && article.getLongUrl() == mainPagePath);
return article;
}
/* Return the welcome page URL */
string Reader::getMainPageUrl() const
{
return getMainPage().getPath();
}
Entry Reader::getMainPage() const Entry Reader::getMainPage() const
{ {
if (!this->zimFileHandler) { return zimArchive->getMainEntry();
throw NoEntry();
}
zim::Article article;
if (this->zimFileHandler->getFileheader().hasMainPage())
{
article = zimFileHandler->getArticle(
this->zimFileHandler->getFileheader().getMainPage());
}
if (!article.good())
{
return getFirstPage();
}
return article;
} }
bool Reader::getFavicon(string& content, string& mimeType) const bool Reader::getFavicon(string& content, string& mimeType) const
@ -233,12 +183,12 @@ bool Reader::getFavicon(string& content, string& mimeType) const
for (auto &path: paths) { for (auto &path: paths) {
try { try {
auto entry = getEntryFromPath(path); auto entry = zimArchive->getEntryByPath(path);
entry = entry.getFinalEntry(); auto item = entry.getItem(true);
content = entry.getContent(); content = item.getData();
mimeType = entry.getMimetype(); mimeType = item.getMimetype();
return true; return true;
} catch(NoEntry& e) {}; } catch(zim::EntryNotFound& e) {};
} }
return false; return false;
@ -246,16 +196,15 @@ bool Reader::getFavicon(string& content, string& mimeType) const
string Reader::getZimFilePath() const string Reader::getZimFilePath() const
{ {
return this->zimFilePath; return zimFilePath;
} }
/* Return a metatag value */ /* Return a metatag value */
bool Reader::getMetadata(const string& name, string& value) const bool Reader::getMetadata(const string& name, string& value) const
{ {
try { try {
auto entry = getEntryFromPath("M/"+name); value = zimArchive->getMetadata(name);
value = entry.getContent();
return true; return true;
} catch(NoEntry& e) { } catch(zim::EntryNotFound& e) {
return false; return false;
} }
} }
@ -269,10 +218,9 @@ string Reader::getName() const
string Reader::getTitle() const string Reader::getTitle() const
{ {
string value; string value = zimArchive->getMetadata("Title");
this->getMetadata("Title", value);
if (value.empty()) { if (value.empty()) {
value = getLastPathElement(zimFileHandler->getFilename()); value = getLastPathElement(zimFilePath);
std::replace(value.begin(), value.end(), '_', ' '); std::replace(value.begin(), value.end(), '_', ' ');
size_t pos = value.find(".zim"); size_t pos = value.find(".zim");
value = value.substr(0, pos); value = value.substr(0, pos);
@ -393,42 +341,21 @@ string Reader::getOrigId() const
return origID; return origID;
} }
Entry Reader::getFirstPage() const
{
if (!this->zimFileHandler) {
throw NoEntry();
}
auto firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
auto article = zimFileHandler->getArticle(firstPageOffset);
if (! article.good()) {
throw NoEntry();
}
return article;
}
Entry Reader::getEntryFromPath(const std::string& path) const Entry Reader::getEntryFromPath(const std::string& path) const
{ {
char ns = 0; if (!this->zimArchive) {
std::string short_url;
if (!this->zimFileHandler) {
throw NoEntry(); throw NoEntry();
} }
_parseUrl(path, &ns, short_url);
if (short_url.empty() && ns == 0) { if (path.empty() || path == "/") {
return getMainPage(); return getMainPage();
} }
auto article = zimFileHandler->getArticle(ns, short_url); try {
if (!article.good()) { return zimArchive->getEntryByPath(path);
} catch (zim::EntryNotFound& e) {
throw NoEntry(); throw NoEntry();
} }
return article;
} }
Entry Reader::getEntryFromEncodedPath(const std::string& path) const Entry Reader::getEntryFromEncodedPath(const std::string& path) const
@ -438,56 +365,47 @@ Entry Reader::getEntryFromEncodedPath(const std::string& path) const
Entry Reader::getEntryFromTitle(const std::string& title) const Entry Reader::getEntryFromTitle(const std::string& title) const
{ {
if (!this->zimFileHandler) { if (!this->zimArchive) {
throw NoEntry(); throw NoEntry();
} }
auto article = this->zimFileHandler->getArticleByTitle('A', title);
if (!article.good()) {
throw NoEntry();
}
return article;
}
/* Return the mimeType without the content */
bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
{
try { try {
auto entry = getEntryFromPath(url); return zimArchive->getEntryByTitle(title);
mimeType = entry.getMimetype(); } catch(zim::EntryNotFound& e) {
return true; throw NoEntry();
} catch (NoEntry& e) {
mimeType = "";
return false;
} }
} }
bool Reader::pathExists(const string& path) const bool Reader::pathExists(const string& path) const
{ {
if (!zimFileHandler) if (!zimArchive)
{ {
return false; return false;
} }
char ns = 0; return zimArchive->hasEntryByPath(path);
string titleStr;
_parseUrl(path, &ns, titleStr);
zim::File::const_iterator findItr = zimFileHandler->find(ns, titleStr);
return findItr != zimFileHandler->end() && findItr->getUrl() == titleStr;
} }
/* Does the ZIM file has a fulltext index */ /* Does the ZIM file has a fulltext index */
bool Reader::hasFulltextIndex() const bool Reader::hasFulltextIndex() const
{ {
if (!zimFileHandler || zimFileHandler->is_multiPart() ) if (!zimArchive)
{ {
return false; return false;
} }
return ( pathExists("Z//fulltextIndex/xapian") for(auto path: {"Z//fulltextIndex/xapian", "X/fulltext/xapian"}) {
|| pathExists("X/fulltext/xapian")); try {
auto entry = zimArchive->getEntryByPath(path);
auto item = entry.getItem(true);
auto accessInfo = item.getDirectAccessInformation();
if (accessInfo.second) {
return true;
}
} catch(...) {}
}
return false;
} }
/* Search titles by prefix */ /* Search titles by prefix */
@ -527,23 +445,17 @@ bool Reader::searchSuggestions(const string& prefix,
return false; return false;
} }
for (auto articleItr = zimFileHandler->findByTitle('A', prefix); for (auto& entry: zimArchive->findByTitle(prefix)) {
articleItr != zimFileHandler->end() if (results.size() >= suggestionsCount) {
&& articleItr->getTitle().compare(0, prefix.size(), prefix) == 0 break;
&& results.size() < suggestionsCount; }
++articleItr) {
/* Extract the interesting part of article title & url */ /* Extract the interesting part of article title & url */
std::string normalizedArticleTitle std::string normalizedArticleTitle
= kiwix::normalize(articleItr->getTitle()); = kiwix::normalize(entry.getTitle());
std::string articleFinalUrl = "/A/" + articleItr->getUrl();
if (articleItr->isRedirect()) { // Get the final path.
zim::Article article = *articleItr; auto item = entry.getItem(true);
unsigned int loopCounter = 0; std::string articleFinalUrl = item.getPath();
while (article.isRedirect() && loopCounter++ < 42) {
article = article.getRedirectArticle();
}
articleFinalUrl = "/A/" + article.getUrl();
}
/* Go through all already found suggestions and skip if this /* Go through all already found suggestions and skip if this
article is already in the suggestions list (with an other article is already in the suggestions list (with an other
@ -565,7 +477,7 @@ bool Reader::searchSuggestions(const string& prefix,
/* Insert if possible */ /* Insert if possible */
if (insert) { if (insert) {
std::vector<std::string> suggestion; std::vector<std::string> suggestion;
suggestion.push_back(articleItr->getTitle()); suggestion.push_back(entry.getTitle());
suggestion.push_back(articleFinalUrl); suggestion.push_back(articleFinalUrl);
suggestion.push_back(normalizedArticleTitle); suggestion.push_back(normalizedArticleTitle);
results.insert(suggestionItr, suggestion); results.insert(suggestionItr, suggestion);
@ -612,19 +524,18 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
bool retVal = false; bool retVal = false;
/* Try to search in the title using fulltext search database */ /* Try to search in the title using fulltext search database */
const auto suggestionSearch auto suggestionSearch = zim::Search(*zimArchive);
= this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount); suggestionSearch.set_query(prefix);
suggestionSearch.set_range(0, suggestionsCount);
suggestionSearch.set_suggestion_mode(true);
if (suggestionSearch->get_matches_estimated()) { if (suggestionSearch.get_matches_estimated()) {
for (auto current = suggestionSearch->begin(); for (auto current = suggestionSearch.begin();
current != suggestionSearch->end(); current != suggestionSearch.end();
current++) { current++) {
if (!current->good()) {
continue;
}
std::vector<std::string> suggestion; std::vector<std::string> suggestion;
suggestion.push_back(current->getTitle()); suggestion.push_back(current->getTitle());
suggestion.push_back("/A/" + current->getUrl()); suggestion.push_back(current->getPath());
suggestion.push_back(kiwix::normalize(current->getTitle())); suggestion.push_back(kiwix::normalize(current->getTitle()));
results.push_back(suggestion); results.push_back(suggestion);
} }
@ -676,14 +587,14 @@ bool Reader::getNextSuggestion(string& title, string& url)
/* Check if the file has as checksum */ /* Check if the file has as checksum */
bool Reader::canCheckIntegrity() const bool Reader::canCheckIntegrity() const
{ {
return this->zimFileHandler->getChecksum() != ""; return zimArchive->hasChecksum();
} }
/* Return true if corrupted, false otherwise */ /* Return true if corrupted, false otherwise */
bool Reader::isCorrupted() const bool Reader::isCorrupted() const
{ {
try { try {
if (this->zimFileHandler->verify() == true) { if (zimArchive->check() == true) {
return false; return false;
} }
} catch (exception& e) { } catch (exception& e) {
@ -697,13 +608,10 @@ bool Reader::isCorrupted() const
/* Return the file size, works also for splitted files */ /* Return the file size, works also for splitted files */
unsigned int Reader::getFileSize() const unsigned int Reader::getFileSize() const
{ {
zim::File* file = this->getZimFileHandler(); if (!zimArchive) {
zim::size_type size = 0; return 0;
}
if (file != NULL) { return zimArchive->getFilesize() / 1024;
size = file->getFilesize();
} }
return (size / 1024);
}
} }

23
src/searcher.cpp
View File

@ -115,14 +115,14 @@ void Searcher::search(const std::string& search,
if (resultStart != resultEnd) { if (resultStart != resultEnd) {
/* Perform the search */ /* Perform the search */
string unaccentedSearch = removeAccents(search); string unaccentedSearch = removeAccents(search);
std::vector<const zim::File*> zims; std::vector<zim::Archive> archives;
for (auto current = this->readers.begin(); current != this->readers.end(); for (auto current = this->readers.begin(); current != this->readers.end();
current++) { current++) {
if ( (*current)->hasFulltextIndex() ) { if ( (*current)->hasFulltextIndex() ) {
zims.push_back((*current)->getZimFileHandler()); archives.push_back(*(*current)->getZimArchive());
} }
} }
zim::Search* search = new zim::Search(zims); zim::Search* search = new zim::Search(archives);
search->set_verbose(verbose); search->set_verbose(verbose);
search->set_query(unaccentedSearch); search->set_query(unaccentedSearch);
search->set_range(resultStart, resultEnd); search->set_range(resultStart, resultEnd);
@ -158,12 +158,12 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
return; return;
} }
std::vector<const zim::File*> zims; std::vector<zim::Archive> archives;
for (auto current = this->readers.begin(); current != this->readers.end(); for (auto current = this->readers.begin(); current != this->readers.end();
current++) { current++) {
zims.push_back((*current)->getZimFileHandler()); archives.push_back(*(*current)->getZimArchive());
} }
zim::Search* search = new zim::Search(zims); zim::Search* search = new zim::Search(archives);
search->set_verbose(verbose); search->set_verbose(verbose);
search->set_query(""); search->set_query("");
search->set_georange(latitude, longitude, distance); search->set_georange(latitude, longitude, distance);
@ -213,12 +213,12 @@ void Searcher::suggestions(std::string& searchPattern, const bool verbose)
this->resultEnd = 10; this->resultEnd = 10;
string unaccentedSearch = removeAccents(searchPattern); string unaccentedSearch = removeAccents(searchPattern);
std::vector<const zim::File*> zims; std::vector<zim::Archive> archives;
for (auto current = this->readers.begin(); current != this->readers.end(); for (auto current = this->readers.begin(); current != this->readers.end();
current++) { current++) {
zims.push_back((*current)->getZimFileHandler()); archives.push_back(*(*current)->getZimArchive());
} }
zim::Search* search = new zim::Search(zims); zim::Search* search = new zim::Search(archives);
search->set_verbose(verbose); search->set_verbose(verbose);
search->set_query(unaccentedSearch); search->set_query(unaccentedSearch);
search->set_range(resultStart, resultEnd); search->set_range(resultStart, resultEnd);
@ -257,10 +257,7 @@ std::string _Result::get_snippet()
} }
std::string _Result::get_content() std::string _Result::get_content()
{ {
if (iterator->good()) { return iterator->getItem(true).getData();
return iterator->getData();
}
return "";
} }
int _Result::get_size() int _Result::get_size()
{ {

19
src/server/internalServer.cpp
View File

@ -766,23 +766,14 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
urlStr = urlStr.substr(1); urlStr = urlStr.substr(1);
} }
kiwix::Entry entry;
try { try {
entry = reader->getEntryFromPath(urlStr); auto entry = reader->getEntryFromPath(urlStr);
if (entry.isRedirect() || urlStr.empty()) { if (entry.isRedirect() || urlStr.empty()) {
// If urlStr is empty, we want to mainPage. // If urlStr is empty, we want to mainPage.
// We must do a redirection to the real page. // We must do a redirection to the real page.
return build_redirect(bookName, entry.getFinalEntry()); return build_redirect(bookName, entry.getFinalEntry());
} }
} catch(kiwix::NoEntry& e) { auto response = ItemResponse::build(*this, request, entry.getZimEntry().getItem());
if (m_verbose.load())
printf("Failed to find %s\n", urlStr.c_str());
return Response::build_404(*this, request, bookName);
}
auto response = EntryResponse::build(*this, request, entry);
try { try {
dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle()); dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
} catch (std::bad_cast& e) {} } catch (std::bad_cast& e) {}
@ -793,6 +784,12 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
} }
return response; return response;
} catch(kiwix::NoEntry& e) {
if (m_verbose.load())
printf("Failed to find %s\n", urlStr.c_str());
return Response::build_404(*this, request, bookName);
}
} }
} }

2
src/server/internalServer.h
View File

@ -104,7 +104,7 @@ class InternalServer {
friend std::unique_ptr<Response> Response::build(const InternalServer& server); friend std::unique_ptr<Response> Response::build(const InternalServer& server);
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
friend std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry); friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg); friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
}; };

46
src/server/response.cpp
View File

@ -33,10 +33,10 @@ std::string render_template(const std::string& template_str, kainjow::mustache::
return ss.str(); return ss.str();
} }
std::string get_mime_type(const kiwix::Entry& entry) std::string get_mime_type(const zim::Item& item)
{ {
try { try {
return entry.getMimetype(); return item.getMimetype();
} catch (exception& e) { } catch (exception& e) {
return "application/octet-stream"; return "application/octet-stream";
} }
@ -131,17 +131,17 @@ static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
struct RunningResponse { struct RunningResponse {
kiwix::Entry entry; zim::Item item;
int range_start; int range_start;
RunningResponse(kiwix::Entry entry, RunningResponse(zim::Item item,
int range_start) : int range_start) :
entry(entry), item(item),
range_start(range_start) range_start(range_start)
{} {}
}; };
static ssize_t callback_reader_from_entry(void* cls, static ssize_t callback_reader_from_item(void* cls,
uint64_t pos, uint64_t pos,
char* buf, char* buf,
size_t max) size_t max)
@ -150,13 +150,13 @@ static ssize_t callback_reader_from_entry(void* cls,
size_t max_size_to_set = min<size_t>( size_t max_size_to_set = min<size_t>(
max, max,
response->entry.getSize() - pos - response->range_start); response->item.getSize() - pos - response->range_start);
if (max_size_to_set <= 0) { if (max_size_to_set <= 0) {
return MHD_CONTENT_READER_END_WITH_ERROR; return MHD_CONTENT_READER_END_WITH_ERROR;
} }
zim::Blob blob = response->entry.getBlob(response->range_start+pos, max_size_to_set); zim::Blob blob = response->item.getData(response->range_start+pos, max_size_to_set);
memcpy(buf, blob.data(), max_size_to_set); memcpy(buf, blob.data(), max_size_to_set);
return max_size_to_set; return max_size_to_set;
} }
@ -178,8 +178,6 @@ void print_response_info(int retCode, MHD_Response* response)
} }
void ContentResponse::introduce_taskbar() void ContentResponse::introduce_taskbar()
{ {
kainjow::mustache::data data; kainjow::mustache::data data;
@ -342,9 +340,9 @@ std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& se
return ContentResponse::build(server, content, mimetype); return ContentResponse::build(server, content, mimetype);
} }
EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
Response(verbose), Response(verbose),
m_entry(entry), m_item(item),
m_mimeType(mimetype) m_mimeType(mimetype)
{ {
m_byteRange = byterange; m_byteRange = byterange;
@ -352,48 +350,46 @@ EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType); add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
} }
std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry) std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
{ {
const std::string mimetype = get_mime_type(entry); const std::string mimetype = get_mime_type(item);
auto byteRange = request.get_range().resolve(entry.getSize()); auto byteRange = request.get_range().resolve(item.getSize());
const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT; const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
if (noRange && is_compressible_mime_type(mimetype)) { if (noRange && is_compressible_mime_type(mimetype)) {
// Return a contentResponse // Return a contentResponse
zim::Blob raw_content = entry.getBlob(); auto response = ContentResponse::build(server, item.getData(), mimetype);
const std::string content = string(raw_content.data(), raw_content.size());
auto response = ContentResponse::build(server, content, mimetype);
response->set_cacheable(); response->set_cacheable();
response->m_byteRange = byteRange; response->m_byteRange = byteRange;
return std::move(response); return std::move(response);
} }
if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
auto response = Response::build_416(server, entry.getSize()); auto response = Response::build_416(server, item.getSize());
response->set_cacheable(); response->set_cacheable();
return response; return response;
} }
return std::unique_ptr<Response>(new EntryResponse( return std::unique_ptr<Response>(new ItemResponse(
server.m_verbose.load(), server.m_verbose.load(),
entry, item,
mimetype, mimetype,
byteRange)); byteRange));
} }
MHD_Response* MHD_Response*
EntryResponse::create_mhd_response(const RequestContext& request) ItemResponse::create_mhd_response(const RequestContext& request)
{ {
const auto content_length = m_byteRange.length(); const auto content_length = m_byteRange.length();
MHD_Response* response = MHD_create_response_from_callback(content_length, MHD_Response* response = MHD_create_response_from_callback(content_length,
16384, 16384,
callback_reader_from_entry, callback_reader_from_item,
new RunningResponse(m_entry, m_byteRange.first()), new RunningResponse(m_item, m_byteRange.first()),
callback_free_response); callback_free_response);
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) { if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
std::ostringstream oss; std::ostringstream oss;
oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last() oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
<< "/" << m_entry.getSize(); << "/" << m_item.getSize();
MHD_add_response_header(response, MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());

10
src/server/response.h
View File

@ -72,7 +72,7 @@ class Response {
ETag m_etag; ETag m_etag;
std::map<std::string, std::string> m_customHeaders; std::map<std::string, std::string> m_customHeaders;
friend class EntryResponse; // temporary to allow the builder to change m_mode friend class ItemResponse;
}; };
@ -104,15 +104,15 @@ class ContentResponse : public Response {
std::string m_bookTitle; std::string m_bookTitle;
}; };
class EntryResponse : public Response { class ItemResponse : public Response {
public: public:
EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange); ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const Entry& entry); static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
private: private:
MHD_Response* create_mhd_response(const RequestContext& request); MHD_Response* create_mhd_response(const RequestContext& request);
Entry m_entry; zim::Item m_item;
std::string m_mimeType; std::string m_mimeType;
}; };

1
src/wrapper/java/kiwixreader.cpp
View File

@ -20,7 +20,6 @@
#include <jni.h> #include <jni.h>
#include <zim/file.h>
#include <exception> #include <exception>
#include "org_kiwix_kiwixlib_JNIKiwixReader.h" #include "org_kiwix_kiwixlib_JNIKiwixReader.h"

1
src/wrapper/java/kiwixsearcher.cpp
View File

@ -19,7 +19,6 @@
*/ */
#include <zim/file.h>
#include "org_kiwix_kiwixlib_JNIKiwixSearcher.h" #include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
#include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h" #include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"

1
src/wrapper/java/kiwixserver.cpp
View File

@ -20,7 +20,6 @@
#include <jni.h> #include <jni.h>
#include <zim/file.h>
#include "org_kiwix_kiwixlib_JNIKiwixServer.h" #include "org_kiwix_kiwixlib_JNIKiwixServer.h"
#include "tools/base64.h" #include "tools/base64.h"

1
src/wrapper/java/manager.cpp
View File

@ -19,7 +19,6 @@
#include <jni.h> #include <jni.h>
#include <zim/file.h>
#include "org_kiwix_kiwixlib_Manager.h" #include "org_kiwix_kiwixlib_Manager.h"
#include "manager.h" #include "manager.h"

2
test/counterParsing.cpp
View File

@ -24,7 +24,7 @@
#include <zim/zim.h> #include <zim/zim.h>
namespace kiwix { namespace kiwix {
using CounterType = std::map<const std::string, zim::article_index_type>; using CounterType = std::map<const std::string, zim::entry_index_type>;
CounterType parseMimetypeCounter(const std::string& counterData); CounterType parseMimetypeCounter(const std::string& counterData);
}; };

1
test/meson.build
View File

@ -1,5 +1,4 @@
tests = [ tests = [
'parseUrl',
'library', 'library',
'regex', 'regex',
'tagParsing', 'tagParsing',

67
test/parseUrl.cpp
View File

@ -1,67 +0,0 @@
/*
* Copyright (C) 2013 Tommi Maekitalo
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
* NON-INFRINGEMENT. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#include "gtest/gtest.h"
#include <string>
namespace kiwix {
bool _parseUrl(const std::string& url, char* ns, std::string& title);
};
using namespace kiwix;
namespace
{
TEST(ParseUrlTest, invalid)
{
char ns;
std::string title;
ASSERT_FALSE(_parseUrl("", &ns, title));
ASSERT_FALSE(_parseUrl("A", &ns, title));
ASSERT_FALSE(_parseUrl("/", &ns, title));
ASSERT_FALSE(_parseUrl("//", &ns, title));
ASSERT_FALSE(_parseUrl("/A", &ns, title));
ASSERT_FALSE(_parseUrl("/A/", &ns, title));
ASSERT_FALSE(_parseUrl("/AB", &ns, title));
ASSERT_FALSE(_parseUrl("//A/title", &ns, title));
}
TEST(ParseUrlTest, valid)
{
char ns;
std::string title;
ASSERT_TRUE(_parseUrl("A/title", &ns, title));
ASSERT_EQ(ns, 'A');
ASSERT_EQ(title, "title");
ASSERT_TRUE(_parseUrl("/A/title", &ns, title));
ASSERT_EQ(ns, 'A');
ASSERT_EQ(title, "title");
ASSERT_TRUE(_parseUrl("A//title", &ns, title));
ASSERT_EQ(ns, 'A');
ASSERT_EQ(title, "/title");
ASSERT_TRUE(_parseUrl("/A//title", &ns, title));
ASSERT_EQ(ns, 'A');
ASSERT_EQ(title, "/title");
}
};