Merge pull request #171 from kiwix/remoteContentManager

Remote content manager

include/book.h

@@ -45,7 +45,7 @@ class Book
   bool update(const Book& other);
   void update(const Reader& reader);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
-  void updateFromOpds(const pugi::xml_node& node);
+  void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
   std::string getHumanReadableIdFromPath();
 
   bool readOnly() const { return m_readOnly; }
@@ -67,7 +67,7 @@ class Book
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const { return m_favicon; }
+  const std::string& getFavicon() const;
   const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
   const std::string& getDownloadId() const { return m_downloadId; }
 
@@ -115,7 +115,8 @@ class Book
   uint64_t m_mediaCount;
   bool m_readOnly;
   uint64_t m_size;
-  std::string m_favicon;
+  mutable std::string m_favicon;
+  std::string m_faviconUrl;
   std::string m_faviconMimeType;
 };
 

include/library.h

@@ -33,7 +33,15 @@ class Book;
 class OPDSDumper;
 
 enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
-enum supportedListMode { ALL, REMOTE, LOCAL };
+enum supportedListMode {
+  ALL = 0,
+  LOCAL = 1,
+  REMOTE = 1 << 1,
+  NOLOCAL = 1 << 2,
+  NOREMOTE = 1 << 3,
+  VALID = 1 << 4,
+  NOVALID = 1 << 5
+};
 /**
  * A Library store several books.
  */
@@ -127,10 +135,15 @@ class Library
    * List books in the library.
    *
    * @param mode The mode of listing :
-   *             - ALL list all books.
+   *             - LOCAL    : list only local books (with a path).
-   *               (LOCAL and REMOTE. Other filters are applied).
+   *             - REMOTE   : list only remote books (with an url).
-   *             - LOCAL list only local books.
+   *             - VALID    : list only valid books (without a path or with a
-   *             - REMOTE list only remote books.
+   *                          path pointing to a valid zim file).
+   *             - NOLOCAL  : list only books without valid path.
+   *             - NOREMOTE : list only books without url.
+   *             - NOVALID  : list only books not valid.
+   *             - ALL : Do not do any filter (LOCAL or REMOTE)
+   *             - Flags can be combined.
    * @param sortBy Attribute to sort by the book list.
    * @param search List only books with search in the title, description.
    * @param language List only books in this language.
@@ -141,7 +154,7 @@ class Library
    * @return The list of bookIds corresponding to the query.
    */
   std::vector<std::string> listBooksIds(
-    supportedListMode = ALL,
+    int supportedListMode = ALL,
     supportedListSortBy sortBy = UNSORTED,
     const std::string& search = "",
     const std::string& language = "",

include/opds_dumper.h

@@ -84,6 +84,15 @@ class OPDSDumper
    */
   void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; }
 
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+
   /**
    * Set the library to dump.
    *
@@ -98,6 +107,10 @@ class OPDSDumper
   std::string date;
   std::string rootLocation;
   std::string searchDescriptionUrl;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+  bool m_isSearchResult = false;
 
  private:
   pugi::xml_node handleBook(Book book, pugi::xml_node root_node);

src/book.cpp

@@ -22,6 +22,7 @@
 
 #include "common/base64.h"
 #include "common/regexTools.h"
+#include "common/networkTools.h"
 
 #include <pugixml.hpp>
 
@@ -131,7 +132,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
 
 
 #define VALUE(name) node.child(name).child_value()
-void Book::updateFromOpds(const pugi::xml_node& node)
+void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
 {
   m_id = VALUE("id");
   if (!m_id.compare(0, 9, "urn:uuid:")) {
@@ -149,7 +150,10 @@ void Book::updateFromOpds(const pugi::xml_node& node)
     if (rel == "http://opds-spec.org/acquisition/open-access") {
       m_url = linkNode.attribute("href").value();
       m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
-      break;
+    }
+    if (rel == "http://opds-spec.org/image/thumbnail") {
+      m_faviconUrl = urlHost + linkNode.attribute("href").value();
+      m_faviconMimeType = linkNode.attribute("type").value();
     }
  }
 
@@ -189,4 +193,15 @@ void Book::setIndexPath(const std::string& indexPath)
     : indexPath;
 }
 
+const std::string& Book::getFavicon() const {
+  if (m_favicon.empty() && !m_faviconUrl.empty()) {
+    try {
+      m_favicon = download(m_faviconUrl);
+    } catch(...) {
+      std::cerr << "Cannot download favicon from " << m_faviconUrl;
+    }
+  }
+  return m_favicon;
+}
+
 }

src/common/networkTools.cpp

@@ -191,7 +191,6 @@ std::string kiwix::download(const std::string& url) {
   auto curl = curl_easy_init();
   std::stringstream ss;
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-  curl_easy_setopt(curl, CURLOPT_PORT, 80);
   curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);

src/library.cpp

@@ -303,7 +303,7 @@ std::string Comparator<PUBLISHER>::get_keys(const std::string& id)
 
 
 std::vector<std::string> Library::listBooksIds(
-    supportedListMode mode,
+    int mode,
     supportedListSortBy sortBy,
     const std::string& search,
     const std::string& language,
@@ -314,9 +314,20 @@ std::vector<std::string> Library::listBooksIds(
   std::vector<std::string> bookIds;
   for(auto& pair:books) {
     auto& book = pair.second;
-    if (mode == LOCAL && book.getPath().empty())
+    auto local = !book.getPath().empty();
+    if (mode & LOCAL && !local)
       continue;
-    if (mode == REMOTE && (!book.getPath().empty() || book.getUrl().empty()))
+    if (mode & NOLOCAL && local)
+      continue;
+    auto valid = book.isPathValid();
+    if (mode & VALID && !valid)
+      continue;
+    if (mode & NOVALID && valid)
+      continue;
+    auto remote = !book.getUrl().empty();
+    if (mode & REMOTE && !remote)
+      continue;
+    if (mode & NOREMOTE && remote)
       continue;
     if (maxSize != 0 && book.getSize() > maxSize)
       continue;

src/manager.cpp

@@ -18,7 +18,6 @@
  */
 
 #include "manager.h"
-#include "common/networkTools.h"
 
 #include <pugixml.hpp>
 
@@ -101,22 +100,7 @@ bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& url
     kiwix::Book book;
 
     book.setReadOnly(false);
-    book.updateFromOpds(entryNode);
+    book.updateFromOpds(entryNode, urlHost);
-    for(pugi::xml_node linkNode = entryNode.child("link"); linkNode;
-        linkNode = linkNode.next_sibling("link")) {
-       std::string rel = linkNode.attribute("rel").value();
-
-       if (rel == "http://opds-spec.org/image/thumbnail") {
-         auto faviconUrl = urlHost + linkNode.attribute("href").value();
-         try {
-           book.setFavicon(download(faviconUrl));
-           book.setFaviconMimeType(linkNode.attribute("type").value());
-         } catch (...) {
-           std::cerr << "Cannot get favicon content from " << faviconUrl << std::endl;
-         }
-         break;
-       }
-    }
 
     /* Update the book properties with the new importer */
     manipulator->addBookToLibrary(book);

src/opds_dumper.cpp

@@ -50,6 +50,14 @@ std::string gen_date_str()
   return is.str();
 }
 
+void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+  m_isSearchResult = true;
+}
+
 #define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
 
 pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
@@ -98,6 +106,12 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds)
   ADD_TEXT_ENTRY(root_node, "title", title);
   ADD_TEXT_ENTRY(root_node, "updated", date);
 
+  if (m_isSearchResult) {
+    ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults));
+    ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex));
+    ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count));
+  }
+
   auto self_link_node = root_node.append_child("link");
   self_link_node.append_attribute("rel") = "self";
   self_link_node.append_attribute("href") = "";