Merge pull request #170 from kiwix/content_manager

Content manager

include/book.h

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOK_H
+#define KIWIX_BOOK_H
+
+#include <string>
+
+namespace pugi {
+class xml_node;
+}
+
+namespace kiwix
+{
+enum supportedIndexType { UNKNOWN, XAPIAN };
+
+class OPDSDumper;
+class Reader;
+
+/**
+ * A class to store information about a book (a zim file)
+ */
+class Book
+{
+ public:
+  Book();
+  ~Book();
+
+  bool update(const Book& other);
+  void update(const Reader& reader);
+  void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
+  void updateFromOpds(const pugi::xml_node& node);
+  std::string getHumanReadableIdFromPath();
+
+  bool readOnly() const { return m_readOnly; }
+  const std::string& getId() const { return m_id; }
+  const std::string& getPath() const { return m_path; }
+  bool isPathValid() const { return m_pathValid; }
+  const std::string& getIndexPath() const { return m_indexPath; }
+  const supportedIndexType& getIndexType() const { return m_indexType; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getDescription() const { return m_description; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getCreator() const { return m_creator; }
+  const std::string& getPublisher() const { return m_publisher; }
+  const std::string& getDate() const { return m_date; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getName() const { return m_name; }
+  const std::string& getTags() const { return m_tags; }
+  const std::string& getOrigId() const { return m_origId; }
+  const uint64_t& getArticleCount() const { return m_articleCount; }
+  const uint64_t& getMediaCount() const { return m_mediaCount; }
+  const uint64_t& getSize() const { return m_size; }
+  const std::string& getFavicon() const { return m_favicon; }
+  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
+  const std::string& getDownloadId() const { return m_downloadId; }
+
+  void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
+  void setId(const std::string& id) { m_id = id; }
+  void setPath(const std::string& path);
+  void setPathValid(bool valid) { m_pathValid = valid; }
+  void setIndexPath(const std::string& indexPath);
+  void setIndexType(supportedIndexType indexType) { m_indexType = indexType;}
+  void setTitle(const std::string& title) { m_title = title; }
+  void setDescription(const std::string& description) { m_description = description; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setCreator(const std::string& creator) { m_creator = creator; }
+  void setPublisher(const std::string& publisher) { m_publisher = publisher; }
+  void setDate(const std::string& date) { m_date = date; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setName(const std::string& name) { m_name = name; }
+  void setTags(const std::string& tags) { m_tags = tags; }
+  void setOrigId(const std::string& origId) { m_origId = origId; }
+  void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
+  void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
+  void setSize(uint64_t size) { m_size = size; }
+  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
+  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
+  void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
+
+ protected:
+  std::string m_id;
+  std::string m_downloadId;
+  std::string m_path;
+  bool m_pathValid;
+  std::string m_indexPath;
+  supportedIndexType m_indexType;
+  std::string m_title;
+  std::string m_description;
+  std::string m_language;
+  std::string m_creator;
+  std::string m_publisher;
+  std::string m_date;
+  std::string m_url;
+  std::string m_name;
+  std::string m_tags;
+  std::string m_origId;
+  uint64_t m_articleCount;
+  uint64_t m_mediaCount;
+  bool m_readOnly;
+  uint64_t m_size;
+  std::string m_favicon;
+  std::string m_faviconMimeType;
+};
+
+}
+
+#endif

include/common/base64.h

@@ -1,4 +1,4 @@
 #include <string>
 
-std::string base64_encode(unsigned char const* , unsigned int len);
+std::string base64_encode(const std::string& inString);
-std::string base64_decode(std::string const& s);
+std::string base64_decode(const std::string& s);

include/common/networkTools.h

@@ -20,30 +20,14 @@
 #ifndef KIWIX_NETWORKTOOLS_H
 #define KIWIX_NETWORKTOOLS_H
 
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#else
-#include <net/if.h>
-#include <netdb.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-
-#include <iostream>
 #include <map>
 #include <string>
-#include <vector>
 
 namespace kiwix
 {
 std::map<std::string, std::string> getNetworkInterfaces();
 std::string getBestPublicIp();
+std::string download(const std::string& url);
 }
 
 #endif

include/common/otherTools.h

@@ -26,9 +26,12 @@
 #include <unistd.h>
 #endif
 
+#include <pugixml.hpp>
+
 namespace kiwix
 {
 void sleep(unsigned int milliseconds);
+std::string nodeToString(pugi::xml_node node);
 }
 
 #endif

include/common/pathTools.h

@@ -59,5 +59,6 @@ bool copyFile(const string& sourcePath, const string& destPath);
 string getLastPathElement(const string& path);
 string getExecutablePath();
 string getCurrentDirectory();
+string getDataDirectory();
 bool writeTextFile(const string& path, const string& content);
 #endif

include/common/stringTools.h

@@ -35,8 +35,8 @@ namespace kiwix
 {
 #ifndef __ANDROID__
 
-std::string beautifyInteger(const unsigned int number);
+std::string beautifyInteger(uint64_t number);
-std::string beautifyFileSize(const unsigned int number);
+std::string beautifyFileSize(uint64_t number);
 void printStringInHexadecimal(const char* s);
 void printStringInHexadecimal(icu::UnicodeString s);
 void stringReplacement(std::string& str,
@@ -64,6 +64,12 @@ std::string lcFirst(const std::string& word);
 std::string toTitle(const std::string& word);
 
 std::string normalize(const std::string& word);
+template<typename T>
+std::string to_string(T value)
+{
+    std::ostringstream oss;
+    oss << value;
+    return oss.str();
 }
-
+} //namespace kiwix
 #endif

include/downloader.h

@@ -21,15 +21,15 @@
 #define KIWIX_DOWNLOADER_H
 
 #include <string>
-#ifdef ENABLE_LIBARIA2
+#include <vector>
-# include <aria2/aria2.h>
+#include <map>
-#endif
 #include <pthread.h>
+#include <memory>
 
 namespace kiwix
 {
 
-
+class Aria2;
 struct DownloadedFile {
   DownloadedFile()
    : success(false) {}
@@ -37,6 +37,46 @@ struct DownloadedFile {
   std::string path;
 };
 
+class AriaError : public std::runtime_error {
+ public:
+  AriaError(const std::string& message) : std::runtime_error(message) {}
+};
+
+
+class Download {
+ public:
+  typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
+
+  Download() :
+    m_status(K_UNKNOWN) {}
+  Download(std::shared_ptr<Aria2> p_aria, std::string did)
+    : mp_aria(p_aria),
+      m_status(K_UNKNOWN),
+      m_did(did) {};
+  void updateStatus(bool follow=false);
+  StatusResult getStatus()          { return m_status; }
+  std::string  getDid()             { return m_did; }
+  std::string  getFollowedBy()      { return m_followedBy; }
+  uint64_t     getTotalLength()     { return m_totalLength; }
+  uint64_t     getCompletedLength() { return m_completedLength; }
+  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
+  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
+  std::string  getPath()            { return m_path; }
+  std::vector<std::string>&  getUris()             { return m_uris; }
+
+ protected:
+  std::shared_ptr<Aria2> mp_aria;
+  StatusResult m_status;
+  std::string m_did = "";
+  std::string m_followedBy = "";
+  uint64_t m_totalLength;
+  uint64_t m_completedLength;
+  uint64_t m_downloadSpeed;
+  uint64_t m_verifiedLength;
+  std::vector<std::string> m_uris;
+  std::string m_path;
+};
+
 /**
  * A tool to download things.
  *
@@ -45,28 +85,19 @@ class Downloader
 {
  public:
   Downloader();
-  ~Downloader();
+  virtual ~Downloader();
 
-  /**
+  void close();
-   * Download a content.
+
-   *
+  Download* startDownload(const std::string& uri);
-   * @param url the url to download
+  Download* getDownload(const std::string& did);
-   * @return the content downloaded.
+
-   */
+  size_t getNbDownload() { return m_knownDownloads.size(); }
-  DownloadedFile download(const std::string& url);
+  std::vector<std::string> getDownloadIds();
 
  private:
-  static pthread_mutex_t globalLock;
+  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
-
+  std::shared_ptr<Aria2> mp_aria;
-  std::string tmpDir;
-#ifdef ENABLE_LIBARIA2
-  DownloadedFile* fileHandle;
-  aria2::Session* session;
-  static int downloadEventCallback(aria2::Session* session,
-                                   aria2::DownloadEvent event,
-                                   aria2::A2Gid gid,
-                                   void* userData);
-#endif
 };
 }
 

include/library.h

@@ -20,78 +20,31 @@
 #ifndef KIWIX_LIBRARY_H
 #define KIWIX_LIBRARY_H
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stack>
 #include <string>
 #include <vector>
-
+#include <map>
-#include "common/regexTools.h"
-#include "common/stringTools.h"
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
-using namespace std;
-
 namespace kiwix
 {
-enum supportedIndexType { UNKNOWN, XAPIAN };
 
+class Book;
+class OPDSDumper;
 
-/**
+enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
- * A class to store information about a book (a zim file)
+enum supportedListMode { ALL, REMOTE, LOCAL };
- */
-class Book
-{
- public:
-  Book();
-  ~Book();
-
-  static bool sortByLastOpen(const Book& a, const Book& b);
-  static bool sortByTitle(const Book& a, const Book& b);
-  static bool sortBySize(const Book& a, const Book& b);
-  static bool sortByDate(const Book& a, const Book& b);
-  static bool sortByCreator(const Book& a, const Book& b);
-  static bool sortByPublisher(const Book& a, const Book& b);
-  static bool sortByLanguage(const Book& a, const Book& b);
-  string getHumanReadableIdFromPath();
-
-  string id;
-  string path;
-  string pathAbsolute;
-  string last;
-  string indexPath;
-  string indexPathAbsolute;
-  supportedIndexType indexType;
-  string title;
-  string description;
-  string language;
-  string creator;
-  string publisher;
-  string date;
-  string url;
-  string name;
-  string tags;
-  string origId;
-  string articleCount;
-  string mediaCount;
-  bool readOnly;
-  string size;
-  string favicon;
-  string faviconMimeType;
-};
-
 /**
  * A Library store several books.
  */
 class Library
 {
+  std::map<std::string, kiwix::Book> books;
  public:
   Library();
   ~Library();
 
-  string version;
+  std::string version;
   /**
    * Add a book to the library.
    *
@@ -104,26 +57,99 @@ class Library
    */
   bool addBook(const Book& book);
 
+  Book& getBookById(const std::string& id);
+
   /**
    * Remove a book from the library.
    *
-   * @param bookIndex the index of the book to remove.
+   * @param id the id of the book to remove.
-   * @return True
+   * @return True if the book were in the lirbrary and has been removed.
    */
-  bool removeBookByIndex(const unsigned int bookIndex);
+  bool removeBookById(const std::string& id);
-  vector<kiwix::Book> books;
 
-  /*
+  /**
-   * 'current' is the variable storing the current content/book id
+   * Write the library to a file.
-   * in the library. This is used to be able to load per default a
+   *
-   * content. As Kiwix may work with many library XML files, you may
+   * @param path the path of the file to write to.
-   * have "current" defined many time with different values. The
+   * @return True if the library has been correctly save.
-   * last XML file read has the priority, Although we do not have an
-   * library object for each file, we want to be able to fallback to
-   * an 'old' current book if the one which should be load
-   * failed. That is the reason why we need a stack here
    */
-  stack<string> current;
+  bool writeToFile(const std::string& path);
+
+  /**
+   * Get the number of book in the library.
+   *
+   * @param localBooks If we must count local books (books with a path).
+   * @param remoteBooks If we must count remote books (books with an url)
+   * @return The number of books.
+   */
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
+
+  /**
+   * Get all langagues of the books in the library.
+   *
+   * @return A list of languages.
+   */
+  std::vector<std::string> getBooksLanguages();
+
+  /**
+   * Get all book creators of the books in the library.
+   *
+   * @return A list of book creators.
+   */
+  std::vector<std::string> getBooksCreators();
+
+  /**
+   * Get all book publishers of the books in the library.
+   *
+   * @return A list of book publishers.
+   */
+  std::vector<std::string> getBooksPublishers();
+
+  /**
+   * Get all book ids of the books in the library.
+   *
+   * @return A list of book ids.
+   */
+  std::vector<std::string> getBooksIds();
+
+  /**
+   * Filter the library and generate a new one with the keep elements.
+   *
+   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
+   *
+   * @param search List only books with search in the title or description.
+   * @return The list of bookIds corresponding to the query.
+   */
+  std::vector<std::string> filter(const std::string& search);
+
+
+  /**
+   * List books in the library.
+   *
+   * @param mode The mode of listing :
+   *             - ALL list all books.
+   *               (LOCAL and REMOTE. Other filters are applied).
+   *             - LOCAL list only local books.
+   *             - REMOTE list only remote books.
+   * @param sortBy Attribute to sort by the book list.
+   * @param search List only books with search in the title, description.
+   * @param language List only books in this language.
+   * @param creator List only books of this creator.
+   * @param publisher List only books of this publisher.
+   * @param maxSize Do not list book bigger than maxSize.
+   *                Set to 0 to cancel this filter.
+   * @return The list of bookIds corresponding to the query.
+   */
+  std::vector<std::string> listBooksIds(
+    supportedListMode = ALL,
+    supportedListSortBy sortBy = UNSORTED,
+    const std::string& search = "",
+    const std::string& language = "",
+    const std::string& creator = "",
+    const std::string& publisher = "",
+    size_t maxSize = 0);
+
+  friend class OPDSDumper;
 };
 }
 

include/manager.h

@@ -20,24 +20,37 @@
 #ifndef KIWIX_MANAGER_H
 #define KIWIX_MANAGER_H
 
-#include <time.h>
+#include "book.h"
-#include <sstream>
-#include <string>
-
-#include <pugixml.hpp>
-
-#include "common/base64.h"
-#include "common/pathTools.h"
-#include "common/regexTools.h"
 #include "library.h"
 #include "reader.h"
 
-using namespace std;
+#include <string>
+#include <vector>
+
+namespace pugi {
+class xml_document;
+}
 
 namespace kiwix
 {
-enum supportedListMode { LASTOPEN, REMOTE, LOCAL };
+
-enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
+class LibraryManipulator {
+ public:
+  virtual ~LibraryManipulator() {}
+  virtual bool addBookToLibrary(Book book) = 0;
+};
+
+class DefaultLibraryManipulator : public LibraryManipulator {
+ public:
+  DefaultLibraryManipulator(Library* library) :
+    library(library) {}
+  virtual ~DefaultLibraryManipulator() {}
+  bool addBookToLibrary(Book book) {
+    return library->addBook(book);
+  }
+ private:
+   kiwix::Library* library;
+};
 
 /**
  * A tool to manage a `Library`.
@@ -48,7 +61,8 @@ enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
 class Manager
 {
  public:
-  Manager();
+  Manager(LibraryManipulator* manipulator);
+  Manager(Library* library);
   ~Manager();
 
   /**
@@ -59,7 +73,7 @@ class Manager
    *                 updated content.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const string path, const bool readOnly = true);
+  bool readFile(const std::string& path, const bool readOnly = true);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
@@ -71,8 +85,8 @@ class Manager
    *                 updated content.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const string nativePath,
+  bool readFile(const std::string& nativePath,
-                const string UTF8Path,
+                const std::string& UTF8Path,
                 const bool readOnly = true);
 
   /**
@@ -84,9 +98,9 @@ class Manager
    * @param libraryPath The library path (used to resolve relative path)
    * @return True if the content has been properly parsed.
    */
-  bool readXml(const string& xml,
+  bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const string libraryPath = "");
+               const std::string& libraryPath = "");
 
   /**
    * Load a library content stored in a OPDS stream.
@@ -97,69 +111,7 @@ class Manager
    * @param libraryPath The library path (used to resolve relative path)
    * @return True if the content has been properly parsed.
    */
-  bool readOpds(const string& content, const std::string& urlHost);
+  bool readOpds(const std::string& content, const std::string& urlHost);
-
-  /**
-   * Write the library to a file.
-   *
-   * @param path the path of the file to write.
-   * @return True.
-   */
-  bool writeFile(const string path);
-
-
-  /**
-   * Remove a book from the library.
-   *
-   * @param bookIndex the index of the book to remove
-   * @return True
-   */
-  bool removeBookByIndex(const unsigned int bookIndex);
-
-  /**
-   * Remove a book from the library.
-   *
-   * @param id the id of the book to remove.
-   * @return True if the book were in the library.
-   */
-  bool removeBookById(const string id);
-
-  /**
-   * Set the current book.
-   *
-   * @param id The id to add to the stack of current books.
-   *           If id is empty, remove the current book from the stack.
-   * @return True
-   */
-  bool setCurrentBookId(const string id);
-
-  /**
-   * Get the current book id.
-   *
-   * @return The id of the current book (or empty string if no current book).
-   */
-  string getCurrentBookId() const;
-
-  /**
-   * Set the path of the external fulltext index associated to a book.
-   *
-   * @param id The id of the book to set.
-   * @param path The path of the external fullext index.
-   * @param supportedIndexType The type of the fulltext index.
-   * @return True if the book is in the library.
-   */
-  bool setBookIndex(const string id,
-                    const string path,
-                    const supportedIndexType type = XAPIAN);
-
-  /**
-   * Set the path of the zim file associated to a book.
-   *
-   * @param id The id of the book to set.
-   * @param path The path of the zim file.
-   * @return True if the book is in the library.
-   */
-  bool setBookPath(const string id, const string path);
 
   /**
    * Add a book to the library.
@@ -172,9 +124,9 @@ class Manager
    * @return The id of the book if the book has been added to the library.
    *         Else, an empty string.
    */
-  string addBookFromPathAndGetId(const string pathToOpen,
+  std::string addBookFromPathAndGetId(const std::string& pathToOpen,
-                                 const string pathToSave = "",
+                                 const std::string& pathToSave = "",
-                                 const string url = "",
+                                 const std::string& url = "",
                                  const bool checkMetaData = false);
 
   /**
@@ -188,18 +140,11 @@ class Manager
    * @return True if the book has been added to the library.
    */
 
-  bool addBookFromPath(const string pathToOpen,
+  bool addBookFromPath(const std::string& pathToOpen,
-                       const string pathToSave = "",
+                       const std::string& pathToSave = "",
-                       const string url = "",
+                       const std::string& url = "",
                        const bool checkMetaData = false);
 
-  /**
-   * Clone and return the internal library.
-   *
-   * @return A clone of the library.
-   */
-  Library cloneLibrary();
-
   /**
    * Get the book corresponding to an id.
    *
@@ -207,24 +152,7 @@ class Manager
    * @param[out] book The book corresponding to the id.
    * @return True if the book has been found.
    */
-  bool getBookById(const string id, Book& book);
+  bool getBookById(const std::string& id, Book& book);
-
-  /**
-   * Get the current book.
-   *
-   * @param[out] The current book.
-   * @return True if there is a current book.
-   */
-  bool getCurrentBook(Book& book);
-
-  /**
-   * Get the number of book in the library.
-   *
-   * @param localBooks If we must count local books (books with a path).
-   * @param remoteBooks If we must count remote books (books with an url)
-   * @return The number of books.
-   */
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
 
   /**
    * Update the "last open date" of a book
@@ -232,7 +160,7 @@ class Manager
    * @param id the id of the book.
    * @return True if the book is in the library.
    */
-  bool updateBookLastOpenDateById(const string id);
+  bool updateBookLastOpenDateById(const std::string& id);
 
   /**
    * Remove (set to empty) paths of all books in the library.
@@ -261,64 +189,28 @@ class Manager
   bool listBooks(const supportedListMode mode,
                  const supportedListSortBy sortBy,
                  const unsigned int maxSize,
-                 const string language,
+                 const std::string& language,
-                 const string creator,
+                 const std::string& creator,
-                 const string publisher,
+                 const std::string& publisher,
-                 const string search);
+                 const std::string& search);
 
-  /**
+  std::string writableLibraryPath;
-   * Filter the library and generate a new one with the keep elements.
-   *
-   * @param search List only books with search in the title or description.
-   * @return A `Library`.
-   */
-  Library filter(const string& search);
 
-
+  std::vector<std::string> bookIdList;
-  /**
-   * Get all langagues of the books in the library.
-   *
-   * @return A list of languages.
-   */
-  vector<string> getBooksLanguages();
-
-  /**
-   * Get all book creators of the books in the library.
-   *
-   * @return A list of book creators.
-   */
-  vector<string> getBooksCreators();
-
-  /**
-   * Get all book publishers of the books in the library.
-   *
-   * @return A list of book publishers.
-   */
-  vector<string> getBooksPublishers();
-
-  /**
-   * Get all book ids of the books in the library.
-   *
-   * @return A list of book ids.
-   */
-  vector<string> getBooksIds();
-
-  string writableLibraryPath;
-
-  vector<std::string> bookIdList;
 
  protected:
-  kiwix::Library library;
+  kiwix::LibraryManipulator* manipulator;
+  bool mustDeleteManipulator;
 
-  bool readBookFromPath(const string path, Book* book = NULL);
+  bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
                    const bool readOnly,
-                   const string libraryPath);
+                   const std::string& libraryPath);
   bool parseOpdsDom(const pugi::xml_document& doc,
                     const std::string& urlHost);
 
  private:
-  void checkAndCleanBookPaths(Book& book, const string& libraryPath);
+  void checkAndCleanBookPaths(Book& book, const std::string& libraryPath);
 };
 }
 

include/meson.build

@@ -1,4 +1,5 @@
 headers = [
+  'book.h',
   'common.h',
   'library.h',
   'manager.h',

include/opds_dumper.h

@@ -45,7 +45,7 @@ class OPDSDumper
 {
  public:
   OPDSDumper() = default;
-  OPDSDumper(Library library);
+  OPDSDumper(Library* library);
   ~OPDSDumper();
 
   /**
@@ -54,7 +54,7 @@ class OPDSDumper
    * @param id The id of the library.
    * @return The OPDS feed.
    */
-  std::string dumpOPDSFeed();
+  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds);
 
   /**
    * Set the id of the opds stream.
@@ -89,10 +89,10 @@ class OPDSDumper
    *
    * @param library The library to dump.
    */
-  void setLibrary(Library library) { this->library = library; }
+  void setLibrary(Library* library) { this->library = library; }
 
  protected:
-  kiwix::Library library;
+  kiwix::Library* library;
   std::string id;
   std::string title;
   std::string date;

meson.build

@@ -17,7 +17,13 @@ thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
 libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
-libaria2_dep = dependency('libaria2', static:static_deps, required:false)
+libcurl_dep = dependency('libcurl', static:static_deps)
+
+if target_machine.system() == 'windows' and static_deps
+  add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
+endif
+
+
 
 ctpp2_include_path = ''
 has_ctpp2_dep = false
@@ -78,7 +84,7 @@ endif
 
 xapian_dep = dependency('xapian-core', required:false, static:static_deps)
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libaria2_dep]
+all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libcurl_dep]
 if has_ctpp2_dep
   all_deps += [ctpp2_dep]
 endif
@@ -88,7 +94,6 @@ inc = include_directories('include')
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
 conf.set('ENABLE_CTPP2', has_ctpp2_dep)
-conf.set('ENABLE_LIBARIA2', libaria2_dep.found())
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -102,10 +107,7 @@ subdir('static')
 subdir('src')
 subdir('test')
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml']
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
-if libaria2_dep.found()
-    pkg_requires += ['libaria2']
-endif
 if xapian_dep.found()
     pkg_requires += ['xapian-core']
 endif

src/android/kiwixreader.cpp

@@ -163,8 +163,7 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
     std::string cMime;
     READER->getFavicon(cContent, cMime);
     favicon = c2jni(
-        base64_encode(reinterpret_cast<const unsigned char*>(cContent.c_str()),
+        base64_encode(cContent),
-                      cContent.length()),
         env);
   } catch (std::exception& e) {
     __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM favicon");

src/aria2.cpp

@@ -0,0 +1,187 @@
+
+
+#include "aria2.h"
+#include "xmlrpc.h"
+#include <sstream>
+#include <thread>
+#include <chrono>
+#include <common/otherTools.h>
+#include <common/pathTools.h>
+#include <downloader.h> // For AriaError
+
+namespace kiwix {
+
+Aria2::Aria2():
+  mp_aria(nullptr),
+  m_port(42042),
+  m_secret("kiwixariarpc"),
+  mp_curl(nullptr),
+  m_lock(PTHREAD_MUTEX_INITIALIZER)
+{
+  m_downloadDir = getDataDirectory();
+  std::vector<const char*> callCmd;
+
+  std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
+  std::string download_dir = "--dir=" + getDataDirectory();
+  std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
+  std::string session = "--save-session=" + session_file;
+  std::string inputFile = "--input-file=" + session_file;
+//  std::string log_dir = "--log=\"" + logDir + "\"";
+#ifdef _WIN32
+  int pid = GetCurrentProcessId();
+#else
+  pid_t pid = getpid();
+#endif
+  std::string stop_with_pid = "--stop-with-process=" + to_string(pid);
+  std::string rpc_secret = "--rpc-secret=" + m_secret;
+  m_secret = "token:"+m_secret;
+
+  callCmd.push_back("aria2c");
+  callCmd.push_back("--enable-rpc");
+  callCmd.push_back(rpc_secret.c_str());
+  callCmd.push_back(rpc_port.c_str());
+  callCmd.push_back(download_dir.c_str());
+  if (fileExists(session_file)) {
+    callCmd.push_back(inputFile.c_str());
+  }
+  callCmd.push_back(session.c_str());
+//  callCmd.push_back(log_dir.c_str());
+  callCmd.push_back("--auto-save-interval=10");
+  callCmd.push_back(stop_with_pid.c_str());
+  callCmd.push_back("--allow-overwrite=true");
+  callCmd.push_back("--dht-entry-point=router.bittorrent.com:6881");
+  callCmd.push_back("--dht-entry-point6=router.bittorrent.com:6881");
+  callCmd.push_back("--quiet=true");
+  callCmd.push_back("--bt-enable-lpd=true");
+  callCmd.push_back("--always-resume=true");
+  callCmd.push_back("--max-concurrent-downloads=42");
+  callCmd.push_back("--rpc-max-request-size=6M");
+  callCmd.push_back("--file-allocation=none");
+  callCmd.push_back(NULL);
+  mp_aria = Subprocess::run(callCmd);
+  mp_curl = curl_easy_init();
+  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+
+  int watchdog = 50;
+  while(--watchdog) {
+    std::this_thread::sleep_for(std::chrono::microseconds(100));
+    auto res = curl_easy_perform(mp_curl);
+    if (res == CURLE_OK) {
+      break;
+    }
+  }
+  if (!watchdog) {
+    curl_easy_cleanup(mp_curl);
+    throw std::runtime_error("Cannot connect to aria2c rpc");
+  }
+}
+
+Aria2::~Aria2()
+{
+  curl_easy_cleanup(mp_curl);
+}
+
+void Aria2::close()
+{
+  saveSession();
+  shutdown();
+}
+
+size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+  auto str = static_cast<std::stringstream*>(userdata);
+  str->write(ptr, nmemb);
+  return nmemb;
+}
+
+std::string Aria2::doRequest(const MethodCall& methodCall)
+{
+  pthread_mutex_lock(&m_lock);
+  auto requestContent = methodCall.toString();
+  std::stringstream stringstream;
+  CURLcode res;
+  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
+  res = curl_easy_perform(mp_curl);
+  if (res == CURLE_OK) {
+    long response_code;
+    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+    pthread_mutex_unlock(&m_lock);
+    if (response_code != 200) {
+      throw std::runtime_error("Invalid return code from aria");
+    }
+    auto responseContent = stringstream.str();
+    MethodResponse response(responseContent);
+    if (response.isFault()) {
+      throw AriaError(response.getFault().getFaultString());
+    }
+    return responseContent;
+  }
+  pthread_mutex_unlock(&m_lock);
+  throw std::runtime_error("Cannot perform request");
+}
+
+std::string Aria2::addUri(const std::vector<std::string>& uris)
+{
+  MethodCall methodCall("aria2.addUri", m_secret);
+  auto uriParams = methodCall.newParamValue().getArray();
+  for (auto& uri : uris) {
+    uriParams.addValue().set(uri);
+  }
+  auto ret = doRequest(methodCall);
+  MethodResponse response(ret);
+  return response.getParamValue(0).getAsS();
+}
+
+std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::string>& statusKey)
+{
+  MethodCall methodCall("aria2.tellStatus", m_secret);
+  methodCall.newParamValue().set(gid);
+  if (!statusKey.empty()) {
+    auto statusArray = methodCall.newParamValue().getArray();
+    for (auto& key : statusKey) {
+      statusArray.addValue().set(key);
+    }
+  }
+  return doRequest(methodCall);
+}
+
+std::vector<std::string> Aria2::tellActive()
+{
+  MethodCall methodCall("aria2.tellActive", m_secret);
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  statusArray.addValue().set(std::string("following"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> activeGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      activeGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return activeGID;
+}
+
+void Aria2::saveSession()
+{
+  MethodCall methodCall("aria2.saveSession", m_secret);
+  doRequest(methodCall);
+  std::cout << "session saved" << std::endl;
+}
+
+void Aria2::shutdown()
+{
+  MethodCall methodCall("aria2.shutdown", m_secret);
+  doRequest(methodCall);
+}
+
+
+} // end namespace kiwix

src/aria2.h

@@ -0,0 +1,45 @@
+
+
+#ifndef KIWIXLIB_ARIA2_H_
+#define KIWIXLIB_ARIA2_H_
+
+#ifdef _WIN32
+// winsock2.h need to be included before windows.h (included by curl.h)
+# include <winsock2.h>
+#endif
+
+#include "subprocess.h"
+#include "xmlrpc.h"
+
+#include <memory>
+#include <curl/curl.h>
+
+namespace kiwix {
+
+class Aria2
+{
+  private:
+    std::unique_ptr<Subprocess> mp_aria;
+    int m_port;
+    std::string m_secret;
+    std::string m_downloadDir;
+    CURL* mp_curl;
+    pthread_mutex_t m_lock;
+
+    std::string doRequest(const MethodCall& methodCall);
+
+  public:
+    Aria2();
+    virtual ~Aria2();
+    void close();
+
+    std::string addUri(const std::vector<std::string>& uri);
+    std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    std::vector<std::string> tellActive();
+    void saveSession();
+    void shutdown();
+};
+
+}; //end namespace kiwix
+
+#endif // KIWIXLIB_ARIA2_H_

src/book.cpp

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "book.h"
+#include "reader.h"
+
+#include "common/base64.h"
+#include "common/regexTools.h"
+
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Book::Book() : m_readOnly(false)
+{
+}
+/* Destructor */
+Book::~Book()
+{
+}
+
+bool Book::update(const kiwix::Book& other)
+{
+  if (m_readOnly)
+    return false;
+
+  m_readOnly = other.m_readOnly;
+
+  if (m_path.empty()) {
+    m_path = other.m_path;
+  }
+
+  if (m_url.empty()) {
+    m_url = other.m_url;
+  }
+
+  if (m_tags.empty()) {
+    m_tags = other.m_tags;
+  }
+
+  if (m_name.empty()) {
+    m_name = other.m_name;
+  }
+
+  if (m_indexPath.empty()) {
+    m_indexPath = other.m_indexPath;
+    m_indexType = other.m_indexType;
+  }
+
+  if (m_faviconMimeType.empty()) {
+    m_favicon = other.m_favicon;
+    m_faviconMimeType = other.m_faviconMimeType;
+  }
+  return true;
+}
+
+void Book::update(const kiwix::Reader& reader)
+{
+  m_path = reader.getZimFilePath();
+  m_id = reader.getId();
+  m_description = reader.getDescription();
+  m_language = reader.getLanguage();
+  m_date = reader.getDate();
+  m_creator = reader.getCreator();
+  m_publisher = reader.getPublisher();
+  m_title = reader.getTitle();
+  m_name = reader.getName();
+  m_tags = reader.getTags();
+  m_origId = reader.getOrigId();
+  m_articleCount = reader.getArticleCount();
+  m_mediaCount = reader.getMediaCount();
+  m_size = reader.getFileSize();
+
+  reader.getFavicon(m_favicon, m_faviconMimeType);
+}
+
+#define ATTR(name) node.attribute(name).value()
+void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
+{
+  m_id = ATTR("id");
+  std::string path = ATTR("path");
+  if (isRelativePath(path)) {
+    path = computeAbsolutePath(baseDir, path);
+  }
+  m_path = path;
+  path = ATTR("indexPath");
+  if (!path.empty()) {
+    if (isRelativePath(path)) {
+      path = computeAbsolutePath(baseDir, path);
+    }
+    m_indexPath = path;
+    m_indexType = XAPIAN;
+  }
+  m_title = ATTR("title");
+  m_name = ATTR("name");
+  m_tags = ATTR("tags");
+  m_description = ATTR("description");
+  m_language = ATTR("language");
+  m_date = ATTR("date");
+  m_creator = ATTR("creator");
+  m_publisher = ATTR("publisher");
+  m_url = ATTR("url");
+  m_origId = ATTR("origId");
+  m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
+  m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
+  m_size = strtoull(ATTR("size"), 0, 0) << 10;
+  m_favicon = base64_decode(ATTR("favicon"));
+  m_faviconMimeType = ATTR("faviconMimeType");
+  try {
+    m_downloadId = ATTR("downloadId");
+  } catch(...) {}
+}
+#undef ATTR
+
+
+#define VALUE(name) node.child(name).child_value()
+void Book::updateFromOpds(const pugi::xml_node& node)
+{
+  m_id = VALUE("id");
+  if (!m_id.compare(0, 9, "urn:uuid:")) {
+    m_id.erase(0, 9);
+  }
+  m_title = VALUE("title");
+  m_description = VALUE("description");
+  m_language = VALUE("language");
+  m_date = VALUE("updated");
+  m_creator = node.child("author").child("name").child_value();
+  for(auto linkNode = node.child("link"); linkNode;
+           linkNode = linkNode.next_sibling("link")) {
+    std::string rel = linkNode.attribute("rel").value();
+
+    if (rel == "http://opds-spec.org/acquisition/open-access") {
+      m_url = linkNode.attribute("href").value();
+      m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
+      break;
+    }
+ }
+
+}
+#undef VALUE
+
+std::string Book::getHumanReadableIdFromPath()
+{
+  std::string id = m_path;
+  if (!id.empty()) {
+    kiwix::removeAccents(id);
+
+#ifdef _WIN32
+    id = replaceRegex(id, "", "^.*\\\\");
+#else
+    id = replaceRegex(id, "", "^.*/");
+#endif
+
+    id = replaceRegex(id, "", "\\.zim[a-z]*$");
+    id = replaceRegex(id, "_", " ");
+    id = replaceRegex(id, "plus", "\\+");
+  }
+  return id;
+}
+
+void Book::setPath(const std::string& path)
+{
+ m_path = isRelativePath(path)
+   ? computeAbsolutePath(getCurrentDirectory(), path)
+   : path;
+}
+
+void Book::setIndexPath(const std::string& indexPath)
+{
+  m_indexPath = isRelativePath(indexPath)
+    ? computeAbsolutePath(getCurrentDirectory(), indexPath)
+    : indexPath;
+}
+
+}

src/common/base64.cpp

@@ -37,8 +37,10 @@ static inline bool is_base64(unsigned char c) {
   return (isalnum(c) || (c == '+') || (c == '/'));
 }
 
-std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) {
+std::string base64_encode(const std::string& inString) {
   std::string ret;
+  auto in_len = inString.size();
+  const unsigned char* bytes_to_encode = reinterpret_cast<const unsigned char*>(inString.data());
   int i = 0;
   int j = 0;
   unsigned char char_array_3[3];

src/common/networkTools.cpp

@@ -19,6 +19,25 @@
 
 #include <common/networkTools.h>
 
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <net/if.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
+
+#include <curl/curl.h>
+
+#include <sstream>
+#include <iostream>
 
 
 std::map<std::string, std::string> kiwix::getNetworkInterfaces()
@@ -160,3 +179,32 @@ std::string kiwix::getBestPublicIp()
 
   return "127.0.0.1";
 }
+
+size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+  auto str = static_cast<std::stringstream*>(userdata);
+  str->write(ptr, nmemb);
+  return nmemb;
+}
+
+std::string kiwix::download(const std::string& url) {
+  auto curl = curl_easy_init();
+  std::stringstream ss;
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(curl, CURLOPT_PORT, 80);
+  curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);
+  auto res = curl_easy_perform(curl);
+  if (res != CURLE_OK) {
+    curl_easy_cleanup(curl);
+    throw std::runtime_error("Cannot perform request");
+  }
+  long response_code;
+  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(curl);
+  if (response_code != 200) {
+    throw std::runtime_error("Invalid return code from server");
+  }
+  return ss.str();
+}

src/common/otherTools.cpp

@@ -27,3 +27,19 @@ void kiwix::sleep(unsigned int milliseconds)
   usleep(1000 * milliseconds);
 #endif
 }
+
+
+struct XmlStringWriter: pugi::xml_writer
+{
+  std::string result;
+  virtual void write(const void* data, size_t size){
+    result.append(static_cast<const char*>(data), size);
+  }
+};
+
+std::string kiwix::nodeToString(pugi::xml_node node)
+{
+  XmlStringWriter writer;
+  node.print(writer, "  ");
+  return writer.result;
+}

src/common/pathTools.cpp

@@ -310,3 +310,29 @@ string getCurrentDirectory()
   free(a_cwd);
   return s_cwd;
 }
+
+string getDataDirectory()
+{
+#ifdef _WIN32
+  char* cDataDir = ::getenv("APPDATA");
+#else
+  char* cDataDir = ::getenv("KIWIX_DATA_DIR");
+#endif
+  std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
+  if (!dataDir.empty())
+    return dataDir;
+#ifdef _WIN32
+  cDataDir = ::getenv("USERPROFILE");
+  dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
+#else
+  cDataDir = ::getenv("XDG_DATA_HOME");
+  dataDir = cDataDir==nullptr ? "" : cDataDir;
+  if (dataDir.empty()) {
+    cDataDir = ::getenv("HOME");
+    dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
+    dataDir = appendToDirectory(dataDir, ".local");
+    dataDir = appendToDirectory(dataDir, "share");
+  }
+#endif
+  return appendToDirectory(dataDir, "kiwix");
+}

src/common/stringTools.cpp

@@ -60,7 +60,7 @@ std::string kiwix::removeAccents(const std::string& text)
 #ifndef __ANDROID__
 
 /* Prepare integer for display */
-std::string kiwix::beautifyInteger(const unsigned int number)
+std::string kiwix::beautifyInteger(uint64_t number)
 {
   std::stringstream numberStream;
   numberStream << number;
@@ -75,14 +75,19 @@ std::string kiwix::beautifyInteger(const unsigned int number)
   return numberString;
 }
 
-std::string kiwix::beautifyFileSize(const unsigned int number)
+std::string kiwix::beautifyFileSize(uint64_t number)
 {
-  if (number > 1024 * 1024) {
+  std::stringstream ss;
-    return kiwix::beautifyInteger(number / (1024 * 1024)) + " GB";
+  ss << std::fixed << std::setprecision(2);
-  } else {
+  if (number>>30)
-    return kiwix::beautifyInteger(number / 1024 != 0 ? number / 1024 : 1)
+    ss << (number/(1024.0*1024*1024)) << " GB";
-           + " MB";
+  else if (number>>20)
-  }
+    ss << (number/(1024.0*1024)) << " MB";
+  else if (number>>10)
+    ss << (number/1024.0) << " KB";
+  else
+    ss << number << " B";
+  return ss.str();
 }
 
 void kiwix::printStringInHexadecimal(icu::UnicodeString s)

src/config.h.in

@@ -2,5 +2,3 @@
 #mesondefine VERSION
 
 #mesondefine ENABLE_CTPP2
-
-#mesondefine ENABLE_LIBARIA2

src/downloader.cpp

@@ -20,102 +20,135 @@
 #include "downloader.h"
 #include "common/pathTools.h"
 
-#ifndef _WIN32
+#include <algorithm>
-# include <unistd.h>
+#include <thread>
-#endif
+#include <chrono>
+
 #include <iostream>
 
+#include "aria2.h"
+#include "xmlrpc.h"
+#include "common/otherTools.h"
+#include <pugixml.hpp>
+
 namespace kiwix
 {
 
-pthread_mutex_t Downloader::globalLock = PTHREAD_MUTEX_INITIALIZER;
+void Download::updateStatus(bool follow)
-
+{
+  static std::vector<std::string> statusKey = {"status", "files", "totalLength",
+                                               "completedLength", "followedBy",
+                                               "downloadSpeed", "verifiedLength"};
+  std::string strStatus;
+  if(follow && !m_followedBy.empty()) {
+    strStatus = mp_aria->tellStatus(m_followedBy, statusKey);
+  } else {
+    strStatus = mp_aria->tellStatus(m_did, statusKey);
+  }
+//  std::cout << strStatus << std::endl;
+  MethodResponse response(strStatus);
+  if (response.isFault()) {
+    m_status = Download::K_UNKNOWN;
+    return;
+  }
+  auto structNode = response.getParams().getParam(0).getValue().getStruct();
+  auto _status = structNode.getMember("status").getValue().getAsS();
+  auto status = _status == "active" ? Download::K_ACTIVE
+              : _status == "waiting" ? Download::K_WAITING
+              : _status == "paused" ? Download::K_PAUSED
+              : _status == "error" ? Download::K_ERROR
+              : _status == "complete" ? Download::K_COMPLETE
+              : _status == "removed" ? Download::K_REMOVED
+              : Download::K_UNKNOWN;
+  if (status == K_COMPLETE) {
+    try {
+      auto followedByMember = structNode.getMember("followedBy");
+      m_followedBy = followedByMember.getValue().getArray().getValue(0).getAsS();
+      if (follow) {
+        status = K_ACTIVE;
+        updateStatus(true);
+        return;
+      }
+    } catch (InvalidRPCNode& e) { }
+  }
+  m_status = status;
+  m_totalLength = structNode.getMember("totalLength").getValue().getAsI();
+  m_completedLength = structNode.getMember("completedLength").getValue().getAsI();
+  m_downloadSpeed = structNode.getMember("downloadSpeed").getValue().getAsI();
+  try {
+    auto verifiedLengthValue = structNode.getMember("verifiedLength").getValue();
+    m_verifiedLength = verifiedLengthValue.getAsI();
+  } catch (InvalidRPCNode& e) { m_verifiedLength = 0; }
+  auto filesMember = structNode.getMember("files");
+  auto fileStruct = filesMember.getValue().getArray().getValue(0).getStruct();
+  m_path = fileStruct.getMember("path").getValue().getAsS();
+  auto urisArray = fileStruct.getMember("uris").getValue().getArray();
+  int index = 0;
+  m_uris.clear();
+  while(true) {
+    try {
+      auto uriNode = urisArray.getValue(index++).getStruct().getMember("uri");
+      m_uris.push_back(uriNode.getValue().getAsS());
+    } catch(InvalidRPCNode& e) { break; }
+  }
+}
 
 /* Constructor */
-Downloader::Downloader()
+Downloader::Downloader() :
+  mp_aria(new Aria2())
 {
-#ifdef ENABLE_LIBARIA2
+  for (auto gid : mp_aria->tellActive()) {
-  aria2::SessionConfig config;
+    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-  config.downloadEventCallback = Downloader::downloadEventCallback;
+    m_knownDownloads[gid]->updateStatus();
-  config.userData = this;
+  }
-  tmpDir = makeTmpDirectory();
-  aria2::KeyVals options;
-  options.push_back(std::pair<std::string, std::string>("dir", tmpDir));
-  session = aria2::sessionNew(options, config);
-#endif
 }
 
 
 /* Destructor */
 Downloader::~Downloader()
 {
-#ifdef ENABLE_LIBARIA2
-  aria2::sessionFinal(session);
-#endif
-  rmdir(tmpDir.c_str());
 }
 
-#ifdef ENABLE_LIBARIA2
+void Downloader::close()
-int Downloader::downloadEventCallback(aria2::Session* session,
-                                      aria2::DownloadEvent event,
-                                      aria2::A2Gid gid,
-                                      void* userData)
 {
-  Downloader* downloader = static_cast<Downloader*>(userData);
+  mp_aria->close();
-
-  auto fileHandle = downloader->fileHandle;
-  auto dh = aria2::getDownloadHandle(session, gid);
-
-  if (!dh) {
-    return 0;
-  }
-
-  switch (event) {
-    case aria2::EVENT_ON_DOWNLOAD_COMPLETE:
-      {
-        if (dh->getNumFiles() > 0) {
-          auto f = dh->getFile(1);
-          fileHandle->path = f.path;
-          fileHandle->success = true;
-        }
-      }
-      break;
-    case aria2::EVENT_ON_DOWNLOAD_ERROR:
-      {
-        fileHandle->success = false;
-      }
-      break;
-    default:
-      break;
-  }
-  aria2::deleteDownloadHandle(dh);
-  return 0;
 }
-#endif
 
-DownloadedFile Downloader::download(const std::string& url) {
+std::vector<std::string> Downloader::getDownloadIds() {
-  pthread_mutex_lock(&globalLock);
+  std::vector<std::string> ret;
-  DownloadedFile fileHandle;
+  for(auto& p:m_knownDownloads) {
-#ifdef ENABLE_LIBARIA2
+    ret.push_back(p.first);
+  }
+  return ret;
+}
+
+Download* Downloader::startDownload(const std::string& uri)
+{
+  for (auto& p: m_knownDownloads) {
+    auto& d = p.second;
+    auto& uris = d->getUris();
+    if (std::find(uris.begin(), uris.end(), uri) != uris.end())
+      return d.get();
+  }
+  std::vector<std::string> uris = {uri};
+  auto gid = mp_aria->addUri(uris);
+  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+  return m_knownDownloads[gid].get();
+}
+
+Download* Downloader::getDownload(const std::string& did)
+{
   try {
-    std::vector<std::string> uris = {url};
+    return m_knownDownloads.at(did).get();
-    aria2::KeyVals options;
+  } catch(exception& e) {
-    aria2::A2Gid gid;
+    for (auto gid : mp_aria->tellActive()) {
-    int ret;
+      if (gid == did) {
-    DownloadedFile fileHandle;
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    
+        return m_knownDownloads[gid].get();
-    ret = aria2::addUri(session, &gid, uris, options);
+      }
-    if (ret < 0) {
-      std::cerr << "Failed to download" << std::endl;
-    } else {
-      this->fileHandle = &fileHandle;
-      aria2::run(session, aria2::RUN_DEFAULT);
     }
-  } catch (...) {};
+    throw e;
-  this->fileHandle = nullptr;
+  }
-  pthread_mutex_unlock(&globalLock);
-#endif
-  return fileHandle;
 }
 
 }

src/library.cpp

@@ -18,72 +18,17 @@
  */
 
 #include "library.h"
+#include "book.h"
+
+#include "common/base64.h"
+#include "common/regexTools.h"
+#include "common/pathTools.h"
+
+#include <pugixml.hpp>
+#include <algorithm>
 
 namespace kiwix
 {
-/* Constructor */
-Book::Book() : readOnly(false)
-{
-}
-/* Destructor */
-Book::~Book()
-{
-}
-/* Sort functions */
-bool Book::sortByLastOpen(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return atoi(a.last.c_str()) > atoi(b.last.c_str());
-}
-
-bool Book::sortByTitle(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.title.c_str(), b.title.c_str()) < 0;
-}
-
-bool Book::sortByDate(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.date.c_str(), b.date.c_str()) > 0;
-}
-
-bool Book::sortBySize(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return atoi(a.size.c_str()) < atoi(b.size.c_str());
-}
-
-bool Book::sortByPublisher(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.publisher.c_str(), b.publisher.c_str()) < 0;
-}
-
-bool Book::sortByCreator(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.creator.c_str(), b.creator.c_str()) < 0;
-}
-
-bool Book::sortByLanguage(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.language.c_str(), b.language.c_str()) < 0;
-}
-
-std::string Book::getHumanReadableIdFromPath()
-{
-  std::string id = pathAbsolute;
-  if (!id.empty()) {
-    kiwix::removeAccents(id);
-
-#ifdef _WIN32
-    id = replaceRegex(id, "", "^.*\\\\");
-#else
-    id = replaceRegex(id, "", "^.*/");
-#endif
-
-    id = replaceRegex(id, "", "\\.zim[a-z]*$");
-    id = replaceRegex(id, "_", " ");
-    id = replaceRegex(id, "plus", "\\+");
-  }
-  return id;
-}
-
 /* Constructor */
 Library::Library() : version(KIWIX_LIBRARY_VERSION)
 {
@@ -92,63 +37,320 @@ Library::Library() : version(KIWIX_LIBRARY_VERSION)
 Library::~Library()
 {
 }
+
+
 bool Library::addBook(const Book& book)
 {
   /* Try to find it */
   std::vector<kiwix::Book>::iterator itr;
-  for (itr = this->books.begin(); itr != this->books.end(); ++itr) {
+  try {
-    if (itr->id == book.id) {
+    auto& oldbook = books.at(book.getId());
-      if (!itr->readOnly) {
+    oldbook.update(book);
-        itr->readOnly = book.readOnly;
+    return false;
+  } catch (std::out_of_range&) {
+    books[book.getId()] = book;
+    return true;
+  }
+}
 
-        if (itr->path.empty()) {
-          itr->path = book.path;
-        }
 
-        if (itr->pathAbsolute.empty()) {
+bool Library::removeBookById(const std::string& id)
-          itr->pathAbsolute = book.pathAbsolute;
+{
-        }
+  return books.erase(id) == 1;
+}
 
-        if (itr->url.empty()) {
+Book& Library::getBookById(const std::string& id)
-          itr->url = book.url;
+{
-        }
+  return books.at(id);
+}
 
-        if (itr->tags.empty()) {
+unsigned int Library::getBookCount(const bool localBooks,
-          itr->tags = book.tags;
+                                   const bool remoteBooks)
-        }
+{
+  unsigned int result = 0;
+  for (auto& pair: books) {
+    auto& book = pair.second;
+    if ((!book.getPath().empty() && localBooks)
+        || (book.getPath().empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
 
-        if (itr->name.empty()) {
+bool Library::writeToFile(const std::string& path) {
-          itr->name = book.name;
+  pugi::xml_document doc;
-        }
+  auto baseDir = removeLastPathElement(path, true, false);
 
-        if (itr->indexPath.empty()) {
+  /* Add the library node */
-          itr->indexPath = book.indexPath;
+  pugi::xml_node libraryNode = doc.append_child("library");
-          itr->indexType = book.indexType;
-        }
 
-        if (itr->indexPathAbsolute.empty()) {
+  if (!version.empty())
-          itr->indexPathAbsolute = book.indexPathAbsolute;
+    libraryNode.append_attribute("version") = version.c_str();
-          itr->indexType = book.indexType;
-        }
 
-        if (itr->faviconMimeType.empty()) {
+  /* Add each book */
-          itr->favicon = book.favicon;
+  for (auto& pair: books) {
-          itr->faviconMimeType = book.faviconMimeType;
+    auto& book = pair.second;
-        }
+    if (!book.readOnly()) {
+      pugi::xml_node bookNode = libraryNode.append_child("book");
+      bookNode.append_attribute("id") = book.getId().c_str();
+
+      if (!book.getPath().empty()) {
+        bookNode.append_attribute("path") = computeRelativePath(
+            baseDir, book.getPath()).c_str();
       }
 
-      return false;
+      if (!book.getIndexPath().empty()) {
+        bookNode.append_attribute("indexPath") = computeRelativePath(
+            baseDir, book.getIndexPath()).c_str();
+        bookNode.append_attribute("indexType") = "xapian";
+      }
+
+      if (book.getOrigId().empty()) {
+        if (!book.getTitle().empty())
+          bookNode.append_attribute("title") = book.getTitle().c_str();
+
+        if (!book.getName().empty())
+          bookNode.append_attribute("name") = book.getName().c_str();
+
+        if (!book.getTags().empty())
+          bookNode.append_attribute("tags") = book.getTags().c_str();
+
+        if (!book.getDescription().empty())
+          bookNode.append_attribute("description") = book.getDescription().c_str();
+
+        if (!book.getLanguage().empty())
+          bookNode.append_attribute("language") = book.getLanguage().c_str();
+
+        if (!book.getCreator().empty())
+          bookNode.append_attribute("creator") = book.getCreator().c_str();
+
+        if (!book.getPublisher().empty())
+          bookNode.append_attribute("publisher") = book.getPublisher().c_str();
+
+        if (!book.getFavicon().empty())
+          bookNode.append_attribute("favicon") = base64_encode(book.getFavicon()).c_str();
+
+        if (!book.getFaviconMimeType().empty())
+          bookNode.append_attribute("faviconMimeType")
+              = book.getFaviconMimeType().c_str();
+      } else {
+        bookNode.append_attribute("origId") = book.getOrigId().c_str();
+      }
+
+      if (!book.getDate().empty()) {
+        bookNode.append_attribute("date") = book.getDate().c_str();
+      }
+
+      if (!book.getUrl().empty()) {
+        bookNode.append_attribute("url") = book.getUrl().c_str();
+      }
+
+      if (!book.getArticleCount())
+        bookNode.append_attribute("articleCount") = to_string(book.getArticleCount()).c_str();
+
+      if (!book.getMediaCount())
+        bookNode.append_attribute("mediaCount") = to_string(book.getMediaCount()).c_str();
+
+      if (book.getSize()) {
+        bookNode.append_attribute("size") = to_string(book.getSize()>>10).c_str();
+      }
+
+      if (!book.getDownloadId().empty()) {
+        bookNode.append_attribute("downloadId") = book.getDownloadId().c_str();
+      }
     }
   }
 
-  /* otherwise */
+  /* saving file */
-  this->books.push_back(book);
+  return doc.save_file(path.c_str());
-  return true;
 }
 
-bool Library::removeBookByIndex(const unsigned int bookIndex)
+std::vector<std::string> Library::getBooksLanguages()
 {
-  books.erase(books.begin() + bookIndex);
+  std::vector<std::string> booksLanguages;
-  return true;
+  std::map<std::string, bool> booksLanguagesMap;
+
+  for (auto& pair: books) {
+    auto& book = pair.second;
+    auto& language = book.getLanguage();
+    if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
+      if (book.getOrigId().empty()) {
+        booksLanguagesMap[language] = true;
+        booksLanguages.push_back(language);
+      }
+    }
+  }
+
+  return booksLanguages;
+}
+
+std::vector<std::string> Library::getBooksCreators()
+{
+  std::vector<std::string> booksCreators;
+  std::map<std::string, bool> booksCreatorsMap;
+
+  for (auto& pair: books) {
+    auto& book = pair.second;
+    auto& creator = book.getCreator();
+    if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
+      if (book.getOrigId().empty()) {
+        booksCreatorsMap[creator] = true;
+        booksCreators.push_back(creator);
+      }
+    }
+  }
+
+  return booksCreators;
+}
+
+std::vector<std::string> Library::getBooksPublishers()
+{
+  std::vector<std::string> booksPublishers;
+  std::map<std::string, bool> booksPublishersMap;
+
+  for (auto& pair:books) {
+    auto& book = pair.second;
+    auto& publisher = book.getPublisher();
+    if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
+      if (book.getOrigId().empty()) {
+        booksPublishersMap[publisher] = true;
+        booksPublishers.push_back(publisher);
+      }
+    }
+  }
+
+  return booksPublishers;
+}
+
+std::vector<std::string> Library::getBooksIds()
+{
+  std::vector<std::string> bookIds;
+
+  for (auto& pair: books) {
+    bookIds.push_back(pair.first);
+  }
+
+  return bookIds;
+}
+
+std::vector<std::string> Library::filter(const std::string& search)
+{
+  if (search.empty()) {
+    return getBooksIds();
+  }
+
+  std::vector<std::string> bookIds;
+  for(auto& pair:books) {
+     auto& book = pair.second;
+     if (matchRegex(book.getTitle(), "\\Q" + search + "\\E")
+         || matchRegex(book.getDescription(), "\\Q" + search + "\\E")) {
+       bookIds.push_back(pair.first);
+     }
+  }
+
+  return bookIds;
+}
+
+template<supportedListSortBy sort>
+struct Comparator {
+  Library* lib;
+  Comparator(Library* lib) : lib(lib) {}
+
+  bool operator() (const std::string& id1, const std::string& id2) {
+    return get_keys(id1) < get_keys(id2);
+  }
+
+  std::string get_keys(const std::string& id);
+  unsigned int get_keyi(const std::string& id);
+};
+
+template<>
+std::string Comparator<TITLE>::get_keys(const std::string& id)
+{
+  return lib->getBookById(id).getTitle();
+}
+
+template<>
+unsigned int Comparator<SIZE>::get_keyi(const std::string& id)
+{
+  return lib->getBookById(id).getSize();
+}
+
+template<>
+bool Comparator<SIZE>::operator() (const std::string& id1, const std::string& id2)
+{
+  return get_keyi(id1) < get_keyi(id2);
+}
+
+template<>
+std::string Comparator<DATE>::get_keys(const std::string& id)
+{
+  return lib->getBookById(id).getDate();
+}
+
+template<>
+std::string Comparator<CREATOR>::get_keys(const std::string& id)
+{
+  return lib->getBookById(id).getCreator();
+}
+
+template<>
+std::string Comparator<PUBLISHER>::get_keys(const std::string& id)
+{
+  return lib->getBookById(id).getPublisher();
+}
+
+
+std::vector<std::string> Library::listBooksIds(
+    supportedListMode mode,
+    supportedListSortBy sortBy,
+    const std::string& search,
+    const std::string& language,
+    const std::string& creator,
+    const std::string& publisher,
+    size_t maxSize) {
+
+  std::vector<std::string> bookIds;
+  for(auto& pair:books) {
+    auto& book = pair.second;
+    if (mode == LOCAL && book.getPath().empty())
+      continue;
+    if (mode == REMOTE && (!book.getPath().empty() || book.getUrl().empty()))
+      continue;
+    if (maxSize != 0 && book.getSize() > maxSize)
+      continue;
+    if (!language.empty() && book.getLanguage() != language)
+      continue;
+    if (!publisher.empty() && book.getPublisher() != publisher)
+      continue;
+    if (!creator.empty() && book.getCreator() != creator)
+      continue;
+    if (!search.empty() && !(matchRegex(book.getTitle(), "\\Q" + search + "\\E")
+                          || matchRegex(book.getDescription(), "\\Q" + search + "\\E")))
+      continue;
+    bookIds.push_back(pair.first);
+  }
+
+  switch(sortBy) {
+    case TITLE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this));
+      break;
+    case SIZE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<SIZE>(this));
+      break;
+    case DATE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<DATE>(this));
+      break;
+    case CREATOR:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<CREATOR>(this));
+      break;
+    case PUBLISHER:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<PUBLISHER>(this));
+      break;
+    default:
+      break;
+  }
+  return bookIds;
 }
 }

src/manager.cpp

@@ -18,80 +18,66 @@
  */
 
 #include "manager.h"
-#include "downloader.h"
+#include "common/networkTools.h"
+
+#include <pugixml.hpp>
 
 namespace kiwix
 {
 /* Constructor */
-Manager::Manager() : writableLibraryPath("")
+Manager::Manager(LibraryManipulator* manipulator):
+  writableLibraryPath(""),
+  manipulator(manipulator),
+  mustDeleteManipulator(false)
 {
 }
+
+Manager::Manager(Library* library) :
+  writableLibraryPath(""),
+  manipulator(new DefaultLibraryManipulator(library)),
+  mustDeleteManipulator(true)
+{
+}
+
 /* Destructor */
 Manager::~Manager()
 {
+  if (mustDeleteManipulator) {
+    delete manipulator;
+  }
 }
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
                           const bool readOnly,
-                          const string libraryPath)
+                          const std::string& libraryPath)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
-  if (strlen(libraryNode.attribute("current").value()))
+  std::string libraryVersion = libraryNode.attribute("version").value();
-    this->setCurrentBookId(libraryNode.attribute("current").value());
-
-  string libraryVersion = libraryNode.attribute("version").value();
 
   for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode;
        bookNode = bookNode.next_sibling("book")) {
-    bool ok = true;
     kiwix::Book book;
 
-    book.readOnly = readOnly;
+    book.setReadOnly(readOnly);
-    book.id = bookNode.attribute("id").value();
+    book.updateFromXml(bookNode,
-    book.path = bookNode.attribute("path").value();
+                       removeLastPathElement(libraryPath, true, false));
-    book.last = (std::string(bookNode.attribute("last").value()) != "undefined"
-                     ? bookNode.attribute("last").value()
-                     : "");
-    book.indexPath = bookNode.attribute("indexPath").value();
-    book.indexType = XAPIAN;
-    book.title = bookNode.attribute("title").value();
-    book.name = bookNode.attribute("name").value();
-    book.tags = bookNode.attribute("tags").value();
-    book.description = bookNode.attribute("description").value();
-    book.language = bookNode.attribute("language").value();
-    book.date = bookNode.attribute("date").value();
-    book.creator = bookNode.attribute("creator").value();
-    book.publisher = bookNode.attribute("publisher").value();
-    book.url = bookNode.attribute("url").value();
-    book.origId = bookNode.attribute("origId").value();
-    book.articleCount = bookNode.attribute("articleCount").value();
-    book.mediaCount = bookNode.attribute("mediaCount").value();
-    book.size = bookNode.attribute("size").value();
-    book.favicon = bookNode.attribute("favicon").value();
-    book.faviconMimeType = bookNode.attribute("faviconMimeType").value();
-
-    /* Check absolute and relative paths */
-    this->checkAndCleanBookPaths(book, libraryPath);
 
     /* Update the book properties with the new importer */
     if (libraryVersion.empty()
         || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.path.empty()) {
+      if (!book.getPath().empty()) {
-        ok = this->readBookFromPath(book.pathAbsolute);
+        this->readBookFromPath(book.getPath(), &book);
       }
     }
-
+    manipulator->addBookToLibrary(book);
-    if (ok) {
-      library.addBook(book);
-    }
   }
 
   return true;
 }
 
-bool Manager::readXml(const string& xml,
+bool Manager::readXml(const std::string& xml,
                       const bool readOnly,
-                      const string libraryPath)
+                      const std::string& libraryPath)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
@@ -114,36 +100,26 @@ bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& url
        entryNode = entryNode.next_sibling("entry")) {
     kiwix::Book book;
 
-    book.readOnly = false;
+    book.setReadOnly(false);
-    book.id = entryNode.child("id").child_value();
+    book.updateFromOpds(entryNode);
-    book.title = entryNode.child("title").child_value();
-    book.description = entryNode.child("summary").child_value();
-    book.language = entryNode.child("language").child_value();
-    book.date = entryNode.child("updated").child_value();
-    book.creator = entryNode.child("author").child("name").child_value();
     for(pugi::xml_node linkNode = entryNode.child("link"); linkNode;
         linkNode = linkNode.next_sibling("link")) {
        std::string rel = linkNode.attribute("rel").value();
 
        if (rel == "http://opds-spec.org/image/thumbnail") {
          auto faviconUrl = urlHost + linkNode.attribute("href").value();
-         auto downloader = Downloader();
+         try {
-         auto fileHandle = downloader.download(faviconUrl);
+           book.setFavicon(download(faviconUrl));
-         if (fileHandle.success) {
+           book.setFaviconMimeType(linkNode.attribute("type").value());
-           auto content = getFileContent(fileHandle.path);
+         } catch (...) {
-           book.favicon = base64_encode((const unsigned char*)content.data(), content.size());
-           book.faviconMimeType = linkNode.attribute("type").value();
-         } else {
            std::cerr << "Cannot get favicon content from " << faviconUrl << std::endl;
          }
-
+         break;
-       } else if (rel == "http://opds-spec.org/acquisition/open-access") {
-         book.url = linkNode.attribute("href").value();
        }
     }
 
     /* Update the book properties with the new importer */
-    library.addBook(book);
+    manipulator->addBookToLibrary(book);
   }
 
   return true;
@@ -151,7 +127,7 @@ bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& url
 
 
 
-bool Manager::readOpds(const string& content, const std::string& urlHost)
+bool Manager::readOpds(const std::string& content, const std::string& urlHost)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
@@ -165,13 +141,13 @@ bool Manager::readOpds(const string& content, const std::string& urlHost)
   return false;
 }
 
-bool Manager::readFile(const string path, const bool readOnly)
+bool Manager::readFile(const std::string& path, const bool readOnly)
 {
   return this->readFile(path, path, readOnly);
 }
 
-bool Manager::readFile(const string nativePath,
+bool Manager::readFile(const std::string& nativePath,
-                       const string UTF8Path,
+                       const std::string& UTF8Path,
                        const bool readOnly)
 {
   bool retVal = true;
@@ -194,149 +170,31 @@ bool Manager::readFile(const string nativePath,
   return retVal;
 }
 
-bool Manager::writeFile(const string path)
-{
-  pugi::xml_document doc;
-
-  /* Add the library node */
-  pugi::xml_node libraryNode = doc.append_child("library");
-
-  if (!getCurrentBookId().empty()) {
-    libraryNode.append_attribute("current") = getCurrentBookId().c_str();
-  }
-
-  if (!library.version.empty())
-    libraryNode.append_attribute("version") = library.version.c_str();
-
-  /* Add each book */
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (!itr->readOnly) {
-      this->checkAndCleanBookPaths(*itr, path);
-
-      pugi::xml_node bookNode = libraryNode.append_child("book");
-      bookNode.append_attribute("id") = itr->id.c_str();
-
-      if (!itr->path.empty()) {
-        bookNode.append_attribute("path") = itr->path.c_str();
-      }
-
-      if (!itr->last.empty() && itr->last != "undefined") {
-        bookNode.append_attribute("last") = itr->last.c_str();
-      }
-
-      if (!itr->indexPath.empty())
-        bookNode.append_attribute("indexPath") = itr->indexPath.c_str();
-
-      if (!itr->indexPath.empty() || !itr->indexPathAbsolute.empty()) {
-        if (itr->indexType == XAPIAN) {
-          bookNode.append_attribute("indexType") = "xapian";
-        }
-      }
-
-      if (itr->origId.empty()) {
-        if (!itr->title.empty())
-          bookNode.append_attribute("title") = itr->title.c_str();
-
-        if (!itr->name.empty())
-          bookNode.append_attribute("name") = itr->name.c_str();
-
-        if (!itr->tags.empty())
-          bookNode.append_attribute("tags") = itr->tags.c_str();
-
-        if (!itr->description.empty())
-          bookNode.append_attribute("description") = itr->description.c_str();
-
-        if (!itr->language.empty())
-          bookNode.append_attribute("language") = itr->language.c_str();
-
-        if (!itr->creator.empty())
-          bookNode.append_attribute("creator") = itr->creator.c_str();
-
-        if (!itr->publisher.empty())
-          bookNode.append_attribute("publisher") = itr->publisher.c_str();
-
-        if (!itr->favicon.empty())
-          bookNode.append_attribute("favicon") = itr->favicon.c_str();
-
-        if (!itr->faviconMimeType.empty())
-          bookNode.append_attribute("faviconMimeType")
-              = itr->faviconMimeType.c_str();
-      }
-
-      if (!itr->date.empty()) {
-        bookNode.append_attribute("date") = itr->date.c_str();
-      }
-
-      if (!itr->url.empty()) {
-        bookNode.append_attribute("url") = itr->url.c_str();
-      }
-
-      if (!itr->origId.empty())
-        bookNode.append_attribute("origId") = itr->origId.c_str();
-
-      if (!itr->articleCount.empty())
-        bookNode.append_attribute("articleCount") = itr->articleCount.c_str();
-
-      if (!itr->mediaCount.empty())
-        bookNode.append_attribute("mediaCount") = itr->mediaCount.c_str();
-
-      if (!itr->size.empty()) {
-        bookNode.append_attribute("size") = itr->size.c_str();
-      }
-    }
-  }
-
-  /* saving file */
-  doc.save_file(path.c_str());
-
-  return true;
-}
-
-
-bool Manager::setCurrentBookId(const string id)
-{
-  if (library.current.empty() || library.current.top() != id) {
-    if (id.empty() && !library.current.empty()) {
-      library.current.pop();
-    } else {
-      library.current.push(id);
-    }
-  }
-  return true;
-}
-
-string Manager::getCurrentBookId() const
-{
-  return library.current.empty() ? "" : library.current.top();
-}
 
 /* Add a book to the library. Return empty string if failed, book id otherwise
  */
-string Manager::addBookFromPathAndGetId(const string pathToOpen,
+std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
-                                        const string pathToSave,
+                                             const std::string& pathToSave,
-                                        const string url,
+                                             const std::string& url,
-                                        const bool checkMetaData)
+                                             const bool checkMetaData)
 {
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
     if (pathToSave != pathToOpen) {
-      book.path = pathToSave;
+      book.setPath(isRelativePath(pathToSave)
-      book.pathAbsolute
-          = isRelativePath(pathToSave)
                 ? computeAbsolutePath(
                       removeLastPathElement(writableLibraryPath, true, false),
                       pathToSave)
-                : pathToSave;
+                : pathToSave);
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.title.empty() && !book.language.empty()
+        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
-            && !book.date.empty())) {
+            && !book.getDate().empty())) {
-      book.url = url;
+      book.setUrl(url);
-      library.addBook(book);
+      manipulator->addBookToLibrary(book);
-      return book.id;
+      return book.getId();
     }
   }
 
@@ -345,9 +203,9 @@ string Manager::addBookFromPathAndGetId(const string pathToOpen,
 
 /* Wrapper over Manager::addBookFromPath which return a bool instead of a string
  */
-bool Manager::addBookFromPath(const string pathToOpen,
+bool Manager::addBookFromPath(const std::string& pathToOpen,
-                              const string pathToSave,
+                              const std::string& pathToSave,
-                              const string url,
+                              const std::string& url,
                               const bool checkMetaData)
 {
   return !(
@@ -355,380 +213,19 @@ bool Manager::addBookFromPath(const string pathToOpen,
           .empty());
 }
 
-bool Manager::readBookFromPath(const string path, kiwix::Book* book)
+bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
   try {
-    kiwix::Reader* reader = new kiwix::Reader(path);
+    kiwix::Reader reader(path);
-
+    book->update(reader);
-    if (book != NULL) {
+    book->setPathValid(true);
-      book->path = path;
-      book->pathAbsolute = path;
-      book->id = reader->getId();
-      book->description = reader->getDescription();
-      book->language = reader->getLanguage();
-      book->date = reader->getDate();
-      book->creator = reader->getCreator();
-      book->publisher = reader->getPublisher();
-      book->title = reader->getTitle();
-      book->name = reader->getName();
-      book->tags = reader->getTags();
-      book->origId = reader->getOrigId();
-      std::ostringstream articleCountStream;
-      articleCountStream << reader->getArticleCount();
-      book->articleCount = articleCountStream.str();
-
-      std::ostringstream mediaCountStream;
-      mediaCountStream << reader->getMediaCount();
-      book->mediaCount = mediaCountStream.str();
-
-      ostringstream convert;
-      convert << reader->getFileSize();
-      book->size = convert.str();
-
-      string favicon;
-      string faviconMimeType;
-      if (reader->getFavicon(favicon, faviconMimeType)) {
-        book->favicon = base64_encode(
-            reinterpret_cast<const unsigned char*>(favicon.c_str()),
-            favicon.length());
-        book->faviconMimeType = faviconMimeType;
-      }
-    }
-
-    delete reader;
   } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
+    std::cerr << "Invalid " << path << " : " << e.what() << std::endl;
+    book->setPathValid(false);
     return false;
   }
 
   return true;
 }
 
-bool Manager::removeBookByIndex(const unsigned int bookIndex)
-{
-  return this->library.removeBookByIndex(bookIndex);
-}
-
-bool Manager::removeBookById(const string id)
-{
-  unsigned int bookIndex = 0;
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      return this->library.removeBookByIndex(bookIndex);
-    }
-    bookIndex++;
-  }
-  return false;
-}
-
-vector<string> Manager::getBooksLanguages()
-{
-  std::vector<string> booksLanguages;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksLanguagesMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksLanguagesMap.find(itr->language) == booksLanguagesMap.end()) {
-      if (itr->origId.empty()) {
-        booksLanguagesMap[itr->language] = true;
-        booksLanguages.push_back(itr->language);
-      }
-    }
-  }
-
-  return booksLanguages;
-}
-
-vector<string> Manager::getBooksCreators()
-{
-  std::vector<string> booksCreators;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksCreatorsMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksCreatorsMap.find(itr->creator) == booksCreatorsMap.end()) {
-      if (itr->origId.empty()) {
-        booksCreatorsMap[itr->creator] = true;
-        booksCreators.push_back(itr->creator);
-      }
-    }
-  }
-
-  return booksCreators;
-}
-
-vector<string> Manager::getBooksIds()
-{
-  std::vector<string> booksIds;
-  std::vector<kiwix::Book>::iterator itr;
-
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    booksIds.push_back(itr->id);
-  }
-
-  return booksIds;
-}
-
-vector<string> Manager::getBooksPublishers()
-{
-  std::vector<string> booksPublishers;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksPublishersMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksPublishersMap.find(itr->publisher) == booksPublishersMap.end()) {
-      if (itr->origId.empty()) {
-        booksPublishersMap[itr->publisher] = true;
-        booksPublishers.push_back(itr->publisher);
-      }
-    }
-  }
-
-  return booksPublishers;
-}
-
-kiwix::Library Manager::cloneLibrary()
-{
-  return this->library;
-}
-bool Manager::getCurrentBook(Book& book)
-{
-  string currentBookId = getCurrentBookId();
-  if (currentBookId.empty()) {
-    return false;
-  } else {
-    getBookById(currentBookId, book);
-    return true;
-  }
-}
-
-bool Manager::getBookById(const string id, Book& book)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      book = *itr;
-      return true;
-    }
-  }
-  return false;
-}
-
-bool Manager::updateBookLastOpenDateById(const string id)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      char unixdate[12];
-      sprintf(unixdate, "%d", (int)time(NULL));
-      itr->last = unixdate;
-      return true;
-    }
-  }
-
-  return false;
-}
-
-bool Manager::setBookIndex(const string id,
-                           const string path,
-                           const supportedIndexType type)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      itr->indexPath = path;
-      itr->indexPathAbsolute
-          = isRelativePath(path)
-                ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
-                      path)
-                : path;
-      itr->indexType = type;
-      return true;
-    }
-  }
-
-  return false;
-}
-
-bool Manager::setBookPath(const string id, const string path)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      itr->path = path;
-      itr->pathAbsolute
-          = isRelativePath(path)
-                ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
-                      path)
-                : path;
-      return true;
-    }
-  }
-
-  return false;
-}
-
-void Manager::removeBookPaths()
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    itr->path = "";
-    itr->pathAbsolute = "";
-  }
-}
-
-unsigned int Manager::getBookCount(const bool localBooks,
-                                   const bool remoteBooks)
-{
-  unsigned int result = 0;
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if ((!itr->path.empty() && localBooks)
-        || (itr->path.empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
-}
-
-bool Manager::listBooks(const supportedListMode mode,
-                        const supportedListSortBy sortBy,
-                        const unsigned int maxSize,
-                        const string language,
-                        const string creator,
-                        const string publisher,
-                        const string search)
-{
-  this->bookIdList.clear();
-  std::vector<kiwix::Book>::iterator itr;
-
-  /* Sort */
-  if (sortBy == TITLE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
-  } else if (sortBy == SIZE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
-  } else if (sortBy == DATE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
-  } else if (sortBy == CREATOR) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
-  } else if (sortBy == PUBLISHER) {
-    std::sort(library.books.begin(),
-              library.books.end(),
-              kiwix::Book::sortByPublisher);
-  }
-
-  /* Special sort for LASTOPEN */
-  if (mode == LASTOPEN) {
-    std::sort(library.books.begin(),
-              library.books.end(),
-              kiwix::Book::sortByLastOpen);
-    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-      if (!itr->last.empty()) {
-        this->bookIdList.push_back(itr->id);
-      }
-    }
-  } else {
-    /* Generate the list of book id */
-    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-      bool ok = true;
-
-      if (mode == LOCAL && itr->path.empty()) {
-        ok = false;
-      }
-
-      if (ok == true && mode == REMOTE
-          && (!itr->path.empty() || itr->url.empty())) {
-        ok = false;
-      }
-
-      if (ok == true && maxSize != 0
-          && (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024) {
-        ok = false;
-      }
-
-      if (ok == true && !language.empty()
-          && !matchRegex(itr->language, language)) {
-        ok = false;
-      }
-
-      if (ok == true && !creator.empty() && itr->creator != creator) {
-        ok = false;
-      }
-
-      if (ok == true && !publisher.empty() && itr->publisher != publisher) {
-        ok = false;
-      }
-
-      if ((ok == true && !search.empty())
-          && !(matchRegex(itr->title, "\\Q" + search + "\\E")
-               || matchRegex(itr->description, "\\Q" + search + "\\E")
-               || matchRegex(itr->language, "\\Q" + search + "\\E"))) {
-        ok = false;
-      }
-
-      if (ok == true) {
-        this->bookIdList.push_back(itr->id);
-      }
-    }
-  }
-
-  return true;
-}
-
-
-Library Manager::filter(const std::string& search) {
-  Library library;
-
-  if (search.empty()) {
-    return library;
-  }
-
-  for(auto book:this->library.books) {
-     if (matchRegex(book.title, "\\Q" + search + "\\E")
-         || matchRegex(book.description, "\\Q" + search + "\\E")) {
-       library.addBook(book);
-     }
-  }
-
-  return library;
-}
-
-void Manager::checkAndCleanBookPaths(Book& book, const string& libraryPath)
-{
-  if (!book.path.empty()) {
-    if (isRelativePath(book.path)) {
-      book.pathAbsolute = computeAbsolutePath(
-          removeLastPathElement(libraryPath, true, false), book.path);
-    } else {
-      book.pathAbsolute = book.path;
-      book.path = computeRelativePath(
-          removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
-    }
-  }
-
-  if (!book.indexPath.empty()) {
-    if (isRelativePath(book.indexPath)) {
-      book.indexPathAbsolute = computeAbsolutePath(
-          removeLastPathElement(libraryPath, true, false), book.indexPath);
-    } else {
-      book.indexPathAbsolute = book.indexPath;
-      book.indexPath
-          = computeRelativePath(removeLastPathElement(libraryPath, true, false),
-                                book.indexPathAbsolute);
-    }
-  }
-}
 }

src/meson.build

@@ -1,4 +1,5 @@
 kiwix_sources = [
+  'book.cpp',
   'library.cpp',
   'manager.cpp',
   'opds_dumper.cpp',
@@ -6,6 +7,8 @@ kiwix_sources = [
   'reader.cpp',
   'entry.cpp',
   'searcher.cpp',
+  'subprocess.cpp',
+  'aria2.cpp',
   'common/base64.cpp',
   'common/pathTools.cpp',
   'common/regexTools.cpp',
@@ -17,6 +20,12 @@ kiwix_sources = [
 ]
 kiwix_sources += lib_resources
 
+if host_machine.system() == 'windows'
+  kiwix_sources += 'subprocess_windows.cpp'
+else
+  kiwix_sources += 'subprocess_unix.cpp'
+endif
+
 if xapian_dep.found()
   kiwix_sources += ['xapianSearcher.cpp']
 endif

src/opds_dumper.cpp

@@ -18,11 +18,14 @@
  */
 
 #include "opds_dumper.h"
+#include "book.h"
+
+#include <common/otherTools.h>
 
 namespace kiwix
 {
 /* Constructor */
-OPDSDumper::OPDSDumper(Library library)
+OPDSDumper::OPDSDumper(Library* library)
   : library(library)
 {
 }
@@ -31,24 +34,6 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-struct xml_string_writer: pugi::xml_writer
-{
-    std::string result;
-
-    virtual void write(const void* data, size_t size)
-    {
-        result.append(static_cast<const char*>(data), size);
-    }
-};
-
-std::string node_to_string(pugi::xml_node node)
-{
-    xml_string_writer writer;
-    node.print(writer, "  ");
-
-    return writer.result;
-}
-
 std::string gen_date_str()
 {
   auto now = time(0);
@@ -69,36 +54,37 @@ std::string gen_date_str()
 
 pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "title", book.title);
+  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
-  ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.id);
+  ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
   ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
   ADD_TEXT_ENTRY(entry_node, "updated", date);
-  ADD_TEXT_ENTRY(entry_node, "summary", book.description);
+  ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
 
   auto content_node = entry_node.append_child("link");
   content_node.append_attribute("type") = "text/html";
   content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str();
 
   auto author_node = entry_node.append_child("author");
-  ADD_TEXT_ENTRY(author_node, "name", book.creator);
+  ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
 
-  if (! book.url.empty()) {
+  if (! book.getUrl().empty()) {
     auto acquisition_link = entry_node.append_child("link");
     acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";
     acquisition_link.append_attribute("type") = "application/x-zim";
-    acquisition_link.append_attribute("href") = book.url.c_str();
+    acquisition_link.append_attribute("href") = book.getUrl().c_str();
+    acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str();
   }
 
-  if (! book.faviconMimeType.empty() ) {
+  if (! book.getFaviconMimeType().empty() ) {
     auto image_link = entry_node.append_child("link");
     image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail";
-    image_link.append_attribute("type") = book.faviconMimeType.c_str();
+    image_link.append_attribute("type") = book.getFaviconMimeType().c_str();
     image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str();
   }
   return entry_node;
 }
 
-string OPDSDumper::dumpOPDSFeed()
+string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds)
 {
   date = gen_date_str();
   pugi::xml_document doc;
@@ -125,11 +111,13 @@ string OPDSDumper::dumpOPDSFeed()
     search_link.append_attribute("href") = searchDescriptionUrl.c_str();
   }
 
-  for (auto book: library.books) {
+  if (library) {
-    handleBook(book, root_node);
+    for (auto& bookId: bookIds) {
+      handleBook(library->getBookById(bookId), root_node);
+    }
   }
 
-  return node_to_string(root_node);
+  return nodeToString(root_node);
 }
 
 }

src/subprocess.cpp

@@ -0,0 +1,40 @@
+
+
+#include "subprocess.h"
+
+#ifdef _WIN32
+# include "subprocess_windows.h"
+#else
+# include "subprocess_unix.h"
+#endif
+
+Subprocess::Subprocess(std::unique_ptr<SubprocessImpl> impl, const commandLine_t& commandLine) :
+  mp_impl(std::move(impl))
+{
+  mp_impl->run(commandLine);
+}
+
+Subprocess::~Subprocess()
+{
+  mp_impl->kill();
+}
+
+std::unique_ptr<Subprocess> Subprocess::run(const commandLine_t& commandLine)
+{
+#ifdef _WIN32
+  auto impl = std::unique_ptr<SubprocessImpl>(new WinImpl);
+#else
+  auto impl = std::unique_ptr<UnixImpl>(new UnixImpl);
+#endif
+  return std::unique_ptr<Subprocess>(new Subprocess(std::move(impl), commandLine));
+}
+
+bool Subprocess::isRunning()
+{
+  return mp_impl->isRunning();
+}
+
+bool Subprocess::kill()
+{
+  return mp_impl->kill();
+}

src/subprocess.h

@@ -0,0 +1,36 @@
+
+#ifndef KIWIX_SUBPROCESS_H_
+#define KIWIX_SUBPROCESS_H_
+
+#include <string>
+#include <memory>
+#include <vector>
+
+typedef std::vector<const char *> commandLine_t;
+
+class SubprocessImpl
+{
+  public:
+    virtual void run(const commandLine_t& commandLine) = 0;
+    virtual bool kill() = 0;
+    virtual bool isRunning() = 0;
+    virtual ~SubprocessImpl() = default;
+};
+
+class Subprocess
+{
+  private:
+    // Impl depends of the system (window, unix, ...)
+    std::unique_ptr<SubprocessImpl> mp_impl;
+    Subprocess(std::unique_ptr<SubprocessImpl> impl, const commandLine_t& commandLine);
+
+  public:
+    static std::unique_ptr<Subprocess> run(const commandLine_t& commandLine);
+    ~Subprocess();
+
+    bool isRunning();
+    bool kill();
+};
+
+
+#endif // KIWIX_SUBPROCESS_H_

src/subprocess_unix.cpp

@@ -0,0 +1,93 @@
+
+
+#include "subprocess_unix.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <iostream>
+#include <signal.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+
+UnixImpl::UnixImpl():
+  m_pid(0),
+  m_running(false),
+  m_mutex(PTHREAD_MUTEX_INITIALIZER),
+  m_waitingThread()
+{
+}
+
+UnixImpl::~UnixImpl()
+{
+  kill();
+// Android has no pthread_cancel :(
+#ifdef __ANDROID__
+  pthread_kill(m_waitingThread, SIGUSR1);
+#else
+  pthread_cancel(m_waitingThread);
+#endif
+}
+
+#ifdef __ANDROID__
+void thread_exit_handler(int sig) {
+  pthread_exit(0);
+}
+#endif
+
+void* UnixImpl::waitForPID(void* _self)
+{
+#ifdef __ANDROID__
+  struct sigaction actions;
+  memset(&actions, 0, sizeof(actions));
+  sigemptyset(&actions.sa_mask);
+  actions.sa_flags = 0;
+  actions.sa_handler = thread_exit_handler;
+  sigaction(SIGUSR1, &actions, NULL);
+#endif
+
+  UnixImpl* self = static_cast<UnixImpl*>(_self);
+  waitpid(self->m_pid, NULL, WEXITED);
+
+  pthread_mutex_lock(&self->m_mutex);
+  self->m_running = false;
+  pthread_mutex_unlock(&self->m_mutex);
+
+  return self;
+}
+
+void UnixImpl::run(const commandLine_t& commandLine)
+{
+  const char* binary = commandLine[0];
+  std::cerr << "running " << binary << std::endl;
+  int pid = fork();
+  switch(pid) {
+    case -1:
+      std::cerr << "cannot fork" << std::endl;
+      break;
+    case 0:
+      if (execvp(binary, const_cast<char* const*>(commandLine.data()))) {
+        perror("Cannot launch\n");
+        exit(-1);
+      }
+
+      break;
+    default:
+      m_pid = pid;
+      m_running = true;
+      pthread_create(&m_waitingThread, NULL, waitForPID, this);
+      break;
+  }
+}
+
+bool UnixImpl::kill()
+{
+  return (::kill(m_pid, SIGKILL) == 0);
+}
+
+bool UnixImpl::isRunning()
+{
+  pthread_mutex_lock(&m_mutex);
+  bool ret = m_running;
+  pthread_mutex_unlock(&m_mutex);
+  return ret;
+}

src/subprocess_unix.h

@@ -0,0 +1,28 @@
+#ifndef KIWIX_SUBPROCESS_UNIX_H_
+#define KIWIX_SUBPROCESS_UNIX_H_
+
+#include "subprocess.h"
+
+#include <pthread.h>
+
+
+class UnixImpl : public SubprocessImpl
+{
+  private:
+    int m_pid;
+    bool m_running;
+    pthread_mutex_t m_mutex;
+    pthread_t m_waitingThread;
+
+  public:
+    UnixImpl();
+    virtual ~UnixImpl();
+
+    void run(const commandLine_t& commandLine);
+    bool kill();
+    bool isRunning();
+
+    static void* waitForPID(void* self);
+};
+
+#endif //KIWIX_SUBPROCESS_UNIX_H_

src/subprocess_windows.cpp

@@ -0,0 +1,94 @@
+
+
+#include "subprocess_windows.h"
+
+#include <windows.h>
+#include <winbase.h>
+#include <iostream>
+#include <sstream>
+
+WinImpl::WinImpl():
+  m_pid(0),
+  m_running(false),
+  m_handle(INVALID_HANDLE_VALUE)
+{
+  InitializeCriticalSection(&m_criticalSection);
+}
+
+WinImpl::~WinImpl()
+{
+  kill();
+  CloseHandle(m_handle);
+  DeleteCriticalSection(&m_criticalSection);
+}
+
+DWORD WINAPI WinImpl::waitForPID(void* _self)
+{
+  WinImpl* self = static_cast<WinImpl*>(_self);
+  WaitForSingleObject(self->m_handle, INFINITE);
+
+  EnterCriticalSection(&self->m_criticalSection);
+  self->m_running = false;
+  LeaveCriticalSection(&self->m_criticalSection);
+
+  return 0;
+}
+
+std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
+{
+  auto size = MultiByteToWideChar(CP_UTF8, 0,
+                value.c_str(), -1, nullptr, 0);
+  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size]);
+  auto ret = MultiByteToWideChar(CP_UTF8, 0,
+                value.c_str(), -1, wdata.get(), size);
+  if (0 == ret) {
+    std::ostringstream oss;
+    oss << "Cannot convert to wchar : " << GetLastError();
+    throw std::runtime_error(oss.str());
+  }
+  return wdata;
+}
+
+
+void WinImpl::run(const commandLine_t& commandLine)
+{
+  STARTUPINFOW startInfo = {0};
+  PROCESS_INFORMATION procInfo;
+  startInfo.cb = sizeof(startInfo);
+  const char* binary = commandLine[0];
+  std::cerr << "running " << binary << std::endl;
+  std::ostringstream oss;
+  for(auto& item: commandLine) {
+    oss << item << " ";
+  }
+  if (CreateProcessW(
+    toWideChar(binary).get(),
+    toWideChar(oss.str()).get(),
+    NULL,
+    NULL,
+    false,
+    CREATE_NO_WINDOW,
+    NULL,
+    NULL,
+    &startInfo,
+    &procInfo)) {
+    m_pid = procInfo.dwProcessId;
+    m_handle = procInfo.hProcess;
+    CloseHandle(procInfo.hThread);
+    m_running = true;
+    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
+  }
+}
+
+bool WinImpl::kill()
+{
+  return TerminateProcess(m_handle, 0);
+}
+
+bool WinImpl::isRunning()
+{
+  EnterCriticalSection(&m_criticalSection);
+  bool ret = m_running;
+  LeaveCriticalSection(&m_criticalSection);
+  return ret;
+}

src/subprocess_windows.h

@@ -0,0 +1,27 @@
+#ifndef KIWIX_SUBPROCESS_WINDOWS_H_
+#define KIWIX_SUBPROCESS_WINDOWS_H_
+
+#include "subprocess.h"
+
+#include <synchapi.h>
+
+class WinImpl : public SubprocessImpl
+{
+  private:
+    int m_pid;
+    bool m_running;
+    HANDLE m_handle;
+    CRITICAL_SECTION m_criticalSection;
+
+  public:
+    WinImpl();
+    virtual ~WinImpl();
+
+    void run(const commandLine_t& commandLine);
+    bool kill();
+    bool isRunning();
+
+    static DWORD WINAPI waitForPID(void* self);
+};
+
+#endif //KIWIX_SUBPROCESS_WINDOWS_H_

src/xmlrpc.h

@@ -0,0 +1,264 @@
+
+
+#ifndef KIWIX_XMLRPC_H_
+#define KIWIX_XMLRPC_H_
+
+#include <common/otherTools.h>
+
+namespace kiwix {
+
+class InvalidRPCNode : public std::runtime_error {
+  public:
+    InvalidRPCNode(const std::string& msg) : std::runtime_error(msg) {};
+};
+
+class Struct;
+class Array;
+
+class Value {
+    pugi::xml_node m_value;
+
+  public:
+    Value(pugi::xml_node value) : m_value(value) { }
+
+    void set(int value) {
+      if (!m_value.child("int"))
+        m_value.append_child("int");
+      m_value.child("int").text().set(value);
+    };
+
+    int getAsI() const {
+      if (!m_value.child("int"))
+        throw InvalidRPCNode("Type Error");
+      return m_value.child("int").text().as_int();
+    }
+
+    void set(bool value) {
+      if (!m_value.child("boolean"))
+        m_value.append_child("boolean");
+      m_value.child("boolean").text().set(value);
+    };
+
+    int getAsB() const {
+      if (!m_value.child("boolean"))
+        throw InvalidRPCNode("Type Error");
+      return m_value.child("boolean").text().as_bool();
+    }
+
+    void set(const std::string& value) {
+      if (!m_value.child("string"))
+        m_value.append_child("string");
+      m_value.child("string").text().set(value.c_str());
+    };
+
+    std::string getAsS() const {
+      if (!m_value.child("string"))
+        throw InvalidRPCNode("Type Error");
+      return m_value.child("string").text().as_string();
+    }
+
+    void set(double value) {
+      if (!m_value.child("double"))
+        m_value.append_child("double");
+      m_value.child("double").text().set(value);
+    };
+
+    double getAsD() const {
+      if (!m_value.child("double"))
+        throw InvalidRPCNode("Type Error");
+      return m_value.child("double").text().as_double();
+    }
+
+    inline Struct getStruct();
+    inline Array getArray();
+};
+
+class Array {
+    pugi::xml_node m_array;
+  public:
+    Array(pugi::xml_node array) : m_array(array) {
+      if (!m_array.child("data"))
+        m_array.append_child("data");
+    }
+
+    Value addValue() {
+      auto value = m_array.child("data").append_child("value");
+      return Value(value);
+    }
+
+    Value getValue(int index) const {
+      auto value = m_array.child("data").child("value");
+      while(index && value) {
+        value = value.next_sibling();
+        index--;
+      }
+      if (0==index) {
+        return Value(value);
+      } else {
+        throw InvalidRPCNode("Index error");
+      }
+    }
+
+};
+
+class Member {
+    pugi::xml_node m_member;
+
+  public:
+    Member(pugi::xml_node member) : m_member(member) { }
+
+    Value getValue() const {
+      return Value(m_member.child("value"));
+    };
+};
+
+class Struct {
+    pugi::xml_node m_struct;
+
+  public:
+    Struct(pugi::xml_node _struct) : m_struct(_struct) { }
+
+    Member getMember(const std::string& name) const {
+      for(auto member=m_struct.first_child(); member; member=member.next_sibling()) {
+        std::string member_name = member.child("name").text().get();
+        if (member_name == name) {
+          return Member(member);
+        }
+      }
+      throw InvalidRPCNode("Key Error");
+   }
+
+   Member addMember(const std::string& name) {
+     auto member = m_struct.append_child("member");
+     member.append_child("name").text().set(name.c_str());
+     member.append_child("value");
+     return Member(member);
+   }
+};
+
+class Fault : public Struct {
+  public:
+    Fault(pugi::xml_node fault) : Struct(fault) {};
+
+    int getFaultCode() const {
+      return getMember("faultCode").getValue().getAsI();
+    }
+
+    std::string getFaultString() const {
+      return getMember("faultString").getValue().getAsS();
+    }
+};
+
+Struct Value::getStruct() {
+  if (!m_value.child("struct"))
+    m_value.append_child("struct");
+  return Struct(m_value.child("struct"));
+}
+
+Array Value::getArray() {
+  if (!m_value.child("array"))
+    m_value.append_child("array");
+  return Array(m_value.child("array"));
+}
+
+class Param {
+    pugi::xml_node m_param;
+  public:
+    Param(pugi::xml_node param) : m_param(param) {
+      if (!m_param.child("value"))
+        m_param.append_child("value");
+    };
+
+    Value getValue() const {
+      return Value(m_param.child("value"));
+    };
+};
+
+class Params {
+    pugi::xml_node m_params;
+
+  public:
+    Params(pugi::xml_node params) : m_params(params) {};
+
+    Param addParam() {
+      auto param = m_params.append_child("param");
+      return Param(param);
+    }
+    Param getParam(int index) const {
+      auto param = m_params.child("param");
+      while(index && param) {
+        param = param.next_sibling();
+        index--;
+      }
+      if (0==index) {
+        return Param(param);
+      } else {
+        throw InvalidRPCNode("Index Error");
+      }
+    }
+};
+
+class MethodCall {
+    pugi::xml_document m_doc;
+
+  public:
+    MethodCall(const std::string& methodName, const std::string& secret) {
+      auto mCall = m_doc.append_child("methodCall");
+      mCall.append_child("methodName").text().set(methodName.c_str());
+      mCall.append_child("params");
+      if (!secret.empty()) {
+        getParams().addParam().getValue().set(secret);
+      }
+    }
+
+    Params getParams() const {
+      return Params(m_doc.child("methodCall").child("params"));
+    }
+
+    Value newParamValue() {
+      return getParams().addParam().getValue();
+    }
+
+    std::string toString() const {
+      return nodeToString(m_doc);
+    }
+};
+
+
+class MethodResponse {
+    pugi::xml_document m_doc;
+
+  public:
+    MethodResponse(const std::string& content) {
+      m_doc.load_buffer(content.c_str(), content.size());
+    }
+
+    Params getParams() const {
+      auto params = m_doc.child("methodResponse").child("params");
+      if (!params)
+        throw InvalidRPCNode("No params");
+      return Params(params);
+    }
+
+    Value getParamValue(int index) const {
+      return getParams().getParam(index).getValue();
+    }
+
+    bool isFault() const {
+      return (!!m_doc.child("methodResponse").child("fault"));
+    }
+
+    Fault getFault() const {
+      auto fault = m_doc.child("methodResponse").child("fault");
+      if (!fault)
+        throw InvalidRPCNode("No fault");
+      return Fault(fault.child("value").child("struct"));
+    }
+};
+
+
+
+};
+
+
+#endif // KIWIX_XMLRPC_H_

travis/install_deps.sh

@@ -9,8 +9,6 @@ ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz
 cd $HOME
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]
 then
-  brew update
-  brew upgrade python3
   pip3 install meson==0.43.0
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip