From 7275f9b8e366acb60c238ec73bd38eb90e73a954 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jan 2020 14:28:38 +0100 Subject: [PATCH 1/4] Move function to convert and use tags inside otherTools. --- include/tools/otherTools.h | 16 ++++++-- src/reader.cpp | 70 ++--------------------------------- src/tools/otherTools.cpp | 76 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 70 deletions(-) diff --git a/include/tools/otherTools.h b/include/tools/otherTools.h index a6dc47234..2f95f3fc2 100644 --- a/include/tools/otherTools.h +++ b/include/tools/otherTools.h @@ -21,6 +21,7 @@ #define KIWIX_OTHERTOOLS_H #include +#include namespace pugi { class xml_node; @@ -28,9 +29,18 @@ namespace pugi { namespace kiwix { -void sleep(unsigned int milliseconds); -std::string nodeToString(const pugi::xml_node& node); -std::string converta2toa3(const std::string& a2code); + void sleep(unsigned int milliseconds); + std::string nodeToString(const pugi::xml_node& node); + std::string converta2toa3(const std::string& a2code); + + /* + * Convert all format tag string to new format + */ + std::vector convertTags(const std::string& tags_str); + std::string getTagValueFromTagList(const std::vector& tagList, + const std::string& tagName); + bool convertStrToBool(const std::string& value); + } #endif diff --git a/src/reader.cpp b/src/reader.cpp index 87cedc550..d6874724a 100644 --- a/src/reader.cpp +++ b/src/reader.cpp @@ -22,6 +22,8 @@ #include +#include "tools/otherTools.h" + inline char hi(char v) { char hex[] = "0123456789abcdef"; @@ -352,43 +354,6 @@ string Reader::getLicense() const METADATA("License") } -std::vector convertTags(const std::string& tags_str) -{ - auto tags = split(tags_str, ";"); - std::vector tagsList; - bool picSeen(false), vidSeen(false), detSeen(false), indexSeen(false); - for (auto tag: tags) { - picSeen |= (tag == "nopic" || startsWith(tag, "_pictures:")); - vidSeen |= (tag == "novid" || startsWith(tag, "_videos:")); - detSeen |= (tag == "nodet" || startsWith(tag, "_details:")); - indexSeen |= startsWith(tag, "_ftindex"); - if (tag == "nopic") { - tagsList.push_back("_pictures:no"); - } else if (tag == "novid") { - tagsList.push_back("_videos:no"); - } else if (tag == "nodet") { - tagsList.push_back("_details:no"); - } else if (tag == "_ftindex") { - tagsList.push_back("_ftindex:yes"); - } else { - tagsList.push_back(tag); - } - } - if (!indexSeen) { - tagsList.push_back("_ftindex:no"); - } - if (!picSeen) { - tagsList.push_back("_pictures:yes"); - } - if (!vidSeen) { - tagsList.push_back("_videos:yes"); - } - if (!detSeen) { - tagsList.push_back("_details:yes"); - } - return tagsList; -} - string Reader::getTags(bool original) const { string tags_str; @@ -400,26 +365,6 @@ string Reader::getTags(bool original) const return join(tags, ";"); } -string getTagValueFromTagList(const std::vector& tagList, const std::string& tagName) -{ - for (auto tag: tagList) { - if (tag[0] == '_') { - auto delimPos = tag.find(':'); - if (delimPos == string::npos) { - // No delimiter... what to do ? - continue; - } - auto cTagName = tag.substr(1, delimPos-1); - auto cTagValue = tag.substr(delimPos+1); - if (cTagName == tagName) { - return cTagValue; - } - } - } - std::stringstream ss; - ss << tagName << " cannot be found"; - throw std::out_of_range(ss.str()); -} string Reader::getTagStr(const std::string& tagName) const { @@ -430,16 +375,7 @@ string Reader::getTagStr(const std::string& tagName) const bool Reader::getTagBool(const std::string& tagName) const { - auto tagValue = getTagStr(tagName); - if (tagValue == "yes") { - return true; - } else if (tagValue == "no") { - return false; - } else { - std::stringstream ss; - ss << "Tag value '" << tagValue << "' for " << tagName << " cannot be converted to bool."; - throw std::domain_error(ss.str()); - } + return convertStrToBool(getTagStr(tagName)); } string Reader::getRelation() const diff --git a/src/tools/otherTools.cpp b/src/tools/otherTools.cpp index 538d87104..e4ae850be 100644 --- a/src/tools/otherTools.cpp +++ b/src/tools/otherTools.cpp @@ -25,7 +25,10 @@ #include #endif +#include "tools/stringTools.h" + #include +#include #include @@ -204,3 +207,76 @@ std::string kiwix::nodeToString(const pugi::xml_node& node) std::string kiwix::converta2toa3(const std::string& a2code){ return codeisomapping.at(a2code); } + +std::vector kiwix::convertTags(const std::string& tags_str) +{ + auto tags = kiwix::split(tags_str, ";"); + std::vector tagsList; + bool picSeen(false), vidSeen(false), detSeen(false), indexSeen(false); + for (auto tag: tags) { + picSeen |= (tag == "nopic" || startsWith(tag, "_pictures:")); + vidSeen |= (tag == "novid" || startsWith(tag, "_videos:")); + detSeen |= (tag == "nodet" || startsWith(tag, "_details:")); + indexSeen |= kiwix::startsWith(tag, "_ftindex"); + if (tag == "nopic") { + tagsList.push_back("_pictures:no"); + } else if (tag == "novid") { + tagsList.push_back("_videos:no"); + } else if (tag == "nodet") { + tagsList.push_back("_details:no"); + } else if (tag == "_ftindex") { + tagsList.push_back("_ftindex:yes"); + } else { + tagsList.push_back(tag); + } + } + if (!indexSeen) { + tagsList.push_back("_ftindex:no"); + } + if (!picSeen) { + tagsList.push_back("_pictures:yes"); + } + if (!vidSeen) { + tagsList.push_back("_videos:yes"); + } + if (!detSeen) { + tagsList.push_back("_details:yes"); + } + return tagsList; +} + +std::string kiwix::getTagValueFromTagList( + const std::vector& tagList, const std::string& tagName) +{ + for (auto tag: tagList) { + if (tag[0] == '_') { + auto delimPos = tag.find(':'); + if (delimPos == std::string::npos) { + // No delimiter... what to do ? + continue; + } + auto cTagName = tag.substr(1, delimPos-1); + auto cTagValue = tag.substr(delimPos+1); + if (cTagName == tagName) { + return cTagValue; + } + } + } + std::stringstream ss; + ss << tagName << " cannot be found"; + throw std::out_of_range(ss.str()); +} + +bool kiwix::convertStrToBool(const std::string& value) +{ + if (value == "yes") { + return true; + } else if (value == "no") { + return false; + } + + std::stringstream ss; + ss << "Tag value '" << value << "' cannot be converted to bool."; + throw std::domain_error(ss.str()); +} + From bc257d2d6d21014935103fed9543c828ebbb29f5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jan 2020 14:46:01 +0100 Subject: [PATCH 2/4] Add method to get value of tag from a book. Only the `getTagStr` method is available on android because we need a proper exception handling on wrapping side. Fix #298 --- include/book.h | 2 ++ src/book.cpp | 9 +++++++++ src/wrapper/java/book.cpp | 7 +++++++ src/wrapper/java/org/kiwix/kiwixlib/Book.java | 7 +++++++ 4 files changed, 25 insertions(+) diff --git a/include/book.h b/include/book.h index e3bf185cd..7e3bb0cf9 100644 --- a/include/book.h +++ b/include/book.h @@ -60,6 +60,8 @@ class Book const std::string& getUrl() const { return m_url; } const std::string& getName() const { return m_name; } const std::string& getTags() const { return m_tags; } + std::string getTagStr(const std::string& tagName) const; + bool getTagBool(const std::string& tagName) const; const std::string& getOrigId() const { return m_origId; } const uint64_t& getArticleCount() const { return m_articleCount; } const uint64_t& getMediaCount() const { return m_mediaCount; } diff --git a/src/book.cpp b/src/book.cpp index 15bd31743..103535e60 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -23,6 +23,7 @@ #include "tools/base64.h" #include "tools/regexTools.h" #include "tools/networkTools.h" +#include "tools/otherTools.h" #include @@ -203,4 +204,12 @@ const std::string& Book::getFavicon() const { return m_favicon; } +std::string Book::getTagStr(const std::string& tagName) const { + return getTagValueFromTagList(convertTags(m_tags), tagName); +} + +bool Book::getTagBool(const std::string& tagName) const { + return convertStrToBool(getTagStr(tagName)); +} + } diff --git a/src/wrapper/java/book.cpp b/src/wrapper/java/book.cpp index b76492c31..77cf55dcd 100644 --- a/src/wrapper/java/book.cpp +++ b/src/wrapper/java/book.cpp @@ -66,4 +66,11 @@ GETTER(jstring, getFavicon) GETTER(jstring, getFaviconUrl) GETTER(jstring, getFaviconMimeType) +METHOD(jstring, Book, getTagStr, jstring tagName) try { + auto cRet = Book->getTagStr(jni2c(tagName, env)); + return c2jni(cRet, env); +} catch(...) { + return c2jni("", env); +} + #undef GETTER diff --git a/src/wrapper/java/org/kiwix/kiwixlib/Book.java b/src/wrapper/java/org/kiwix/kiwixlib/Book.java index 06b72e11d..f4f45d68e 100644 --- a/src/wrapper/java/org/kiwix/kiwixlib/Book.java +++ b/src/wrapper/java/org/kiwix/kiwixlib/Book.java @@ -20,6 +20,13 @@ public class Book public native String getUrl(); public native String getName(); public native String getTags(); + /** + * Return the value associated to the tag tagName + * + * @param tagName the tag name to search for. + * @return The value of the tag. If the tag is not found, return empty string. + */ + public native String getTagStr(String tagName); public native long getArticleCount(); public native long getMediaCount(); From a756e7f8f32f9c509b410b4791915833af808b93 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jan 2020 14:51:37 +0100 Subject: [PATCH 3/4] Add flavour attribute to book. Fix #259 Fix kiwix/kiwix-tools#316 --- include/book.h | 3 +++ src/book.cpp | 5 +++++ src/libxml_dumper.cpp | 1 + src/opds_dumper.cpp | 1 + src/wrapper/java/book.cpp | 1 + src/wrapper/java/org/kiwix/kiwixlib/Book.java | 1 + 6 files changed, 12 insertions(+) diff --git a/include/book.h b/include/book.h index 7e3bb0cf9..8d137f302 100644 --- a/include/book.h +++ b/include/book.h @@ -62,6 +62,7 @@ class Book const std::string& getTags() const { return m_tags; } std::string getTagStr(const std::string& tagName) const; bool getTagBool(const std::string& tagName) const; + const std::string& getFlavour() const { return m_flavour; } const std::string& getOrigId() const { return m_origId; } const uint64_t& getArticleCount() const { return m_articleCount; } const uint64_t& getMediaCount() const { return m_mediaCount; } @@ -83,6 +84,7 @@ class Book void setDate(const std::string& date) { m_date = date; } void setUrl(const std::string& url) { m_url = url; } void setName(const std::string& name) { m_name = name; } + void setFlavour(const std::string& flavour) { m_flavour = flavour; } void setTags(const std::string& tags) { m_tags = tags; } void setOrigId(const std::string& origId) { m_origId = origId; } void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; } @@ -105,6 +107,7 @@ class Book std::string m_date; std::string m_url; std::string m_name; + std::string m_flavour; std::string m_tags; std::string m_origId; uint64_t m_articleCount; diff --git a/src/book.cpp b/src/book.cpp index 103535e60..8b2f30db7 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -59,6 +59,7 @@ bool Book::update(const kiwix::Book& other) m_date = other.m_date; m_url = other.m_url; m_name = other.m_name; + m_flavour = other.m_flavour; m_tags = other.m_tags; m_origId = other.m_origId; m_articleCount = other.m_articleCount; @@ -85,6 +86,7 @@ void Book::update(const kiwix::Reader& reader) m_publisher = reader.getPublisher(); m_date = reader.getDate(); m_name = reader.getName(); + m_flavour = reader.getFlavour(); m_tags = reader.getTags(); m_origId = reader.getOrigId(); m_articleCount = reader.getArticleCount(); @@ -112,6 +114,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir) m_date = ATTR("date"); m_url = ATTR("url"); m_name = ATTR("name"); + m_flavour = ATTR("flavour"); m_tags = ATTR("tags"); m_origId = ATTR("origId"); m_articleCount = strtoull(ATTR("articleCount"), 0, 0); @@ -142,6 +145,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost if (!m_id.compare(0, 9, "urn:uuid:")) { m_id.erase(0, 9); } + // No path on opds. m_title = VALUE("title"); m_description = VALUE("summary"); m_language = VALUE("language"); @@ -149,6 +153,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost m_publisher = node.child("publisher").child("name").child_value(); m_date = fromOpdsDate(VALUE("updated")); m_name = VALUE("name"); + m_flavour = VALUE("flavour"); m_tags = VALUE("tags"); for(auto linkNode = node.child("link"); linkNode; linkNode = linkNode.next_sibling("link")) { diff --git a/src/libxml_dumper.cpp b/src/libxml_dumper.cpp index 56ec4b44d..3c0cefab9 100644 --- a/src/libxml_dumper.cpp +++ b/src/libxml_dumper.cpp @@ -58,6 +58,7 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) { ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator()); ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher()); ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName()); + ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour()); ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags()); ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType()); ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl()); diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 04cdb5436..dd584e29f 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -76,6 +76,7 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) { ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage()); ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate())); ADD_TEXT_ENTRY(entry_node, "name", book.getName()); + ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour()); ADD_TEXT_ENTRY(entry_node, "tags", book.getTags()); ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()); diff --git a/src/wrapper/java/book.cpp b/src/wrapper/java/book.cpp index 77cf55dcd..9f7df5b73 100644 --- a/src/wrapper/java/book.cpp +++ b/src/wrapper/java/book.cpp @@ -58,6 +58,7 @@ GETTER(jstring, getPublisher) GETTER(jstring, getDate) GETTER(jstring, getUrl) GETTER(jstring, getName) +GETTER(jstring, getFlavour) GETTER(jstring, getTags) GETTER(jlong, getArticleCount) GETTER(jlong, getMediaCount) diff --git a/src/wrapper/java/org/kiwix/kiwixlib/Book.java b/src/wrapper/java/org/kiwix/kiwixlib/Book.java index f4f45d68e..3f00956a9 100644 --- a/src/wrapper/java/org/kiwix/kiwixlib/Book.java +++ b/src/wrapper/java/org/kiwix/kiwixlib/Book.java @@ -19,6 +19,7 @@ public class Book public native String getDate(); public native String getUrl(); public native String getName(); + public native String getFlavour(); public native String getTags(); /** * Return the value associated to the tag tagName From fff2524ee23c051446cf1c7b1f2f65fd26811b39 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jan 2020 14:53:11 +0100 Subject: [PATCH 4/4] Fix opensearch description. --- static/opensearchdescription.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/opensearchdescription.xml b/static/opensearchdescription.xml index 709beaa7f..47fd790b5 100644 --- a/static/opensearchdescription.xml +++ b/static/opensearchdescription.xml @@ -4,6 +4,7 @@ Search zim files in the catalog. + template="/{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/>