Merge pull request #215 from kiwix/new_version

New version 5.0.0

.travis.yml

@@ -18,6 +18,7 @@ addons:
   apt:
     packages:
       - cmake
+      - python3.5
       - python3-pip
       - libbz2-dev
       - ccache

ChangeLog

@@ -1,3 +1,60 @@
+kiwix-lib 5.0.0
+===============
+
+ * Remove error message when trying to open an wrong zim file.
+ * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
+ * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
+ * Remove rpath
+ * Detect infinite (and too long) redirect loops.
+
+kiwix-lib 4.1.0
+===============
+
+ * Allow the library to be filtered by tags.
+ * Fix language mapping.
+ * Update README about mustache dependency.
+
+kiwix-lib 4.0.1
+===============
+
+ * Fix "maybe uninitialize variable" issue.
+ * Ensure path are stored correctly (absolute path) in the library.
+ * [CI] Use the new deps archive xz
+
+kiwix-lib 4.0.0
+===============
+
+ * [API break] Remove support for external index.
+ * Move to the mustache templating system instead of ctpp2.
+ * Make meson.build works for meson>=0.43.0
+ * [API break] Move the basic tools from the `common` directory to `tools`.
+
+kiwix-lib 3.1.1
+===============
+
+ * The OPDS feed book's date must be the date of the book, not the date of the
+ feed generation.
+ * Convert the standard opds date to our format (YYYY-MM-DD)
+ * Remove duplicate language attribute in the libxml dumper.
+ * Create the datadirectory to not fail to write a file in a non-existent
+   directory
+
+kiwix-lib 3.1.0
+===============
+
+ * Add a method to get the favicon url of book (if available).
+ * Move dump code of library.xml in a specific class.
+ * Add a first support to bookmarks
+
+kiwix-lib 3.0.3
+===============
+
+ * Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
+ ('eng').
+ * Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
+ * Correctly fill the book size for the zim file size.
+ * Fix launch of aria2c.
+
 kiwix-lib 3.0.2
 ===============
 

README.md

@@ -33,12 +33,11 @@ libraries need to be available:
 (package libzim-dev on Ubuntu)
 * Pugixml ........................................ http://pugixml.org/
 (package libpugixml-dev on Ubuntu)
-* ctpp2 ........................................ http://ctpp.havoc.ru/
-(package libctpp2-dev on Ubuntu)
-* Xapian ......................................... https://xapian.org/
-(package libxapian-dev on Ubuntu)
 * libaria2 .................................. https://aria2.github.io/
 (no package on Ubuntu)
+* Mustache ....................... https://github.com/kainjow/Mustache
+(Just copy the header mustache.hpp somewhere it can be found by the
+compiler and/or set CPPFLAGS with correct '-I' option)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -49,10 +48,6 @@ version by hand.
 If you want to install these dependencies locally, then use the
 kiwix-lib directory as install prefix.
 
-If you compile ctpp2 from source and want to compile the Kiwix library
-statically then you will probably need to rename ctpp2 static library
-from ctpp2-st.a to ctpp2.a.
-
 Environment
 -------------
 

include/book.h

@@ -28,7 +28,6 @@ class xml_node;
 
 namespace kiwix
 {
-enum supportedIndexType { UNKNOWN, XAPIAN };
 
 class OPDSDumper;
 class Reader;
@@ -52,8 +51,6 @@ class Book
   const std::string& getId() const { return m_id; }
   const std::string& getPath() const { return m_path; }
   bool isPathValid() const { return m_pathValid; }
-  const std::string& getIndexPath() const { return m_indexPath; }
-  const supportedIndexType& getIndexType() const { return m_indexType; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
   const std::string& getLanguage() const { return m_language; }
@@ -68,6 +65,7 @@ class Book
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
   const std::string& getFavicon() const;
+  const std::string& getFaviconUrl() const { return m_faviconUrl; }
   const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
   const std::string& getDownloadId() const { return m_downloadId; }
 
@@ -75,8 +73,6 @@ class Book
   void setId(const std::string& id) { m_id = id; }
   void setPath(const std::string& path);
   void setPathValid(bool valid) { m_pathValid = valid; }
-  void setIndexPath(const std::string& indexPath);
-  void setIndexType(supportedIndexType indexType) { m_indexType = indexType;}
   void setTitle(const std::string& title) { m_title = title; }
   void setDescription(const std::string& description) { m_description = description; }
   void setLanguage(const std::string& language) { m_language = language; }
@@ -99,8 +95,6 @@ class Book
   std::string m_downloadId;
   std::string m_path;
   bool m_pathValid;
-  std::string m_indexPath;
-  supportedIndexType m_indexType;
   std::string m_title;
   std::string m_description;
   std::string m_language;

include/bookmark.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOKMARK_H
+#define KIWIX_BOOKMARK_H
+
+#include <string>
+
+namespace pugi {
+class xml_node;
+}
+
+namespace kiwix
+{
+
+/**
+ * A class to store information about a bookmark (an article in a book)
+ */
+class Bookmark
+{
+ public:
+  Bookmark();
+  ~Bookmark();
+
+  void updateFromXml(const pugi::xml_node& node);
+
+  const std::string& getBookId() const { return m_bookId; }
+  const std::string& getBookTitle()  const { return m_bookTitle; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getDate() const { return m_date; }
+
+  void setBookId(const std::string& bookId) { m_bookId = bookId; }
+  void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setTitle(const std::string& title) { m_title = title; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setDate(const std::string& date) { m_date = date; }
+
+ protected:
+  std::string m_bookId;
+  std::string m_bookTitle;
+  std::string m_url;
+  std::string m_title;
+  std::string m_language;
+  std::string m_date;
+};
+
+}
+
+#endif

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -1,79 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef _CTPP2_VM_STRING_LOADER_HPP__
-#define _CTPP2_VM_STRING_LOADER_HPP__ 1
-
-#include <ctpp2/CTPP2VMLoader.hpp>
-#include <ctpp2/CTPP2Util.hpp>
-#include <ctpp2/CTPP2Exception.hpp>
-#include <ctpp2/CTPP2VMExecutable.hpp>
-#include <ctpp2/CTPP2VMInstruction.hpp>
-#include <ctpp2/CTPP2VMMemoryCore.hpp>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <iostream>
-#include <string>
-
-/**
-  @file VMStringLoader.hpp
-  @brief Load program core from file
-*/
-
-namespace CTPP // C++ Template Engine
-{
-// FWD
-struct VMExecutable;
-
-/**
-  @class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
-  @brief Load program core from file
-*/
-class CTPP2DECL VMStringLoader:
-  public VMLoader
-{
-public:
-	/**
-	*/
-	VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
-	/**
-	  @brief Get ready-to-run program
-	*/
-	const VMMemoryCore * GetCore() const;
-
-	/**
-	  @brief A destructor
-	*/
-	~VMStringLoader() throw();
-private:
-	/** Program core             */
-	VMExecutable  * oCore;
-	/** Ready-to-run program     */
-	VMMemoryCore  * pVMMemoryCore;
-};
-
-} // namespace CTPP
-#endif // _CTPP2_VM_STRING_LOADER_HPP__
-// End.

include/entry.h

@@ -24,7 +24,6 @@
 #include <zim/article.h>
 #include <exception>
 #include <string>
-#include "common.h"
 
 using namespace std;
 

include/library.h

@@ -24,12 +24,14 @@
 #include <vector>
 #include <map>
 
+#include "book.h"
+#include "bookmark.h"
+
 #define KIWIX_LIBRARY_VERSION "20110515"
 
 namespace kiwix
 {
 
-class Book;
 class OPDSDumper;
 
 enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
@@ -47,12 +49,13 @@ enum supportedListMode {
  */
 class Library
 {
-  std::map<std::string, kiwix::Book> books;
+  std::map<std::string, kiwix::Book> m_books;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+
  public:
   Library();
   ~Library();
 
-  std::string version;
   /**
    * Add a book to the library.
    *
@@ -65,6 +68,22 @@ class Library
    */
   bool addBook(const Book& book);
 
+  /**
+   * Add a bookmark to the library.
+   *
+   * @param bookmark the book to add.
+   */
+  void addBookmark(const Bookmark& bookmark);
+
+  /**
+   * Remove a bookmarkk
+   *
+   * @param zimId The zimId of the bookmark.
+   * @param url The url of the bookmark.
+   * @return True if the bookmark has been removed.
+   */
+  bool removeBookmark(const std::string& zimId, const std::string& url);
+
   Book& getBookById(const std::string& id);
 
   /**
@@ -79,10 +98,18 @@ class Library
    * Write the library to a file.
    *
    * @param path the path of the file to write to.
-   * @return True if the library has been correctly save.
+   * @return True if the library has been correctly saved.
    */
   bool writeToFile(const std::string& path);
 
+  /**
+   * Write the library bookmarks to a file.
+   *
+   * @param path the path of the file to write to.
+   * @return True if the library has been correctly saved.
+   */
+  bool writeBookmarksToFile(const std::string& path);
+
   /**
    * Get the number of book in the library.
    *
@@ -113,6 +140,13 @@ class Library
    */
   std::vector<std::string> getBooksPublishers();
 
+  /**
+   * Get all bookmarks.
+   *
+   * @return A list of bookmarks
+   */
+  const std::vector<kiwix::Bookmark>& getBookmarks() { return m_bookmarks; }
+
   /**
    * Get all book ids of the books in the library.
    *
@@ -160,9 +194,11 @@ class Library
     const std::string& language = "",
     const std::string& creator = "",
     const std::string& publisher = "",
+    const std::vector<std::string>& tags = {},
     size_t maxSize = 0);
 
   friend class OPDSDumper;
+  friend class libXMLDumper;
 };
 }
 

include/libxml_dumper.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBXML_DUMPER_H
+#define KIWIX_LIBXML_DUMPER_H
+
+#include <string>
+#include <vector>
+
+#include <pugixml.hpp>
+
+#include "library.h"
+
+namespace kiwix
+{
+
+/**
+ * A tool to dump a `Library` into a basic library.xml
+ *
+ */
+class LibXMLDumper
+{
+ public:
+  LibXMLDumper() = default;
+  LibXMLDumper(Library* library);
+  ~LibXMLDumper();
+
+  /**
+   * Dump the library.xml
+   *
+   * @param id The id of the library.
+   * @return The library.xml content.
+   */
+  std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
+
+
+  /**
+   * Dump the bookmark of the library.
+   *
+   * @return The bookmark.xml content.
+   */
+  std::string dumpLibXMLBookmark();
+
+  /**
+   * Set the base directory used.
+   *
+   * @param baseDir the base directory to use.
+   */
+  void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
+
+  /**
+   * Set the library to dump.
+   *
+   * @param library The library to dump.
+   */
+  void setLibrary(Library* library) { this->library = library; }
+
+ protected:
+  kiwix::Library* library;
+  std::string baseDir;
+ private:
+  void handleBook(Book book, pugi::xml_node root_node);
+  void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
+};
+}
+
+#endif // KIWIX_OPDS_DUMPER_H

include/manager.h

@@ -38,6 +38,7 @@ class LibraryManipulator {
  public:
   virtual ~LibraryManipulator() {}
   virtual bool addBookToLibrary(Book book) = 0;
+  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
 };
 
 class DefaultLibraryManipulator : public LibraryManipulator {
@@ -48,6 +49,9 @@ class DefaultLibraryManipulator : public LibraryManipulator {
   bool addBookToLibrary(Book book) {
     return library->addBook(book);
   }
+  void addBookmarkToLibrary(Bookmark bookmark) {
+    library->addBookmark(bookmark);
+  }
  private:
    kiwix::Library* library;
 };
@@ -113,6 +117,15 @@ class Manager
    */
   bool readOpds(const std::string& content, const std::string& urlHost);
 
+
+  /**
+   * Load a bookmark file.
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed.
+   */
+  bool readBookmarkFile(const std::string& path);
+
   /**
    * Add a book to the library.
    *

include/meson.build

@@ -1,8 +1,10 @@
 headers = [
   'book.h',
+  'bookmark.h',
   'common.h',
   'library.h',
   'manager.h',
+  'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
   'reader.h',
@@ -10,26 +12,15 @@ headers = [
   'searcher.h'
 ]
 
-if xapian_dep.found()
-  headers += ['xapianSearcher.h']
-endif
-
 install_headers(headers, subdir:'kiwix')
 
 install_headers(
-  'common/base64.h',
-  'common/networkTools.h',
-  'common/otherTools.h',
-  'common/pathTools.h',
-  'common/regexTools.h',
-  'common/stringTools.h',
-  subdir:'kiwix/common'
+  'tools/base64.h',
+  'tools/networkTools.h',
+  'tools/otherTools.h',
+  'tools/pathTools.h',
+  'tools/regexTools.h',
+  'tools/stringTools.h',
+  subdir:'kiwix/tools'
 )
 
-if has_ctpp2_dep
-  install_headers(
-    'ctpp2/CTPP2VMStringLoader.hpp',
-    subdir:'kiwix/ctpp2'
-  )
-endif
-

include/opds_dumper.h

@@ -26,9 +26,9 @@
 
 #include <pugixml.hpp>
 
-#include "common/base64.h"
-#include "common/pathTools.h"
-#include "common/regexTools.h"
+#include "tools/base64.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
 #include "library.h"
 #include "reader.h"
 

include/reader.h

@@ -31,8 +31,8 @@
 #include <string>
 #include "common.h"
 #include "entry.h"
-#include "common/pathTools.h"
-#include "common/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 
 using namespace std;
 

include/searcher.h

@@ -29,8 +29,8 @@
 #include <string>
 #include <vector>
 #include <vector>
-#include "common/pathTools.h"
-#include "common/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 #include "kiwix_config.h"
 
 using namespace std;
@@ -57,24 +57,7 @@ struct SearcherInternal;
  * The Searcher class is reponsible to do different kind of search using the
  * fulltext index.
  *
- * Historically, there are two kind of fulltext index :
- *  - The legacy one, is the external fulltext index. A directory stored outside
- *    of the zim file.
- *  - The new one, a embedded fulltext index in the zim file.
- *
- * Legacy external fulltext index has to be considered as obsolet format with
- * less functionnalities:
- *  - No multi zim search ;
- *  - No geo_search ;
- *  - No suggestions search ;
- *
- * To reflect this, there is two Search creation "API":
- *  - One for the external fulltext index, using the constructor taking a
- *    xapianDirectoryPath) ;
- *  - One for the embedded fulltext index, using a "empty" constructor and the
- *  `add_reader` method".
- *
- *  On top of that, the Searcher may (if compiled with ctpp2) be used to
+ *  Searcher may (if compiled with ctpp2) be used to
  *  generate a html page for the search result. This use a template that need a
  *  humanReaderName. This feature is only used by kiwix-serve and this should be
  *  move outside of Searcher (and with a better API). If you don't use the html
@@ -92,18 +75,6 @@ class Searcher
    */
   Searcher(const string& humanReadableName = "");
 
-  /**
-   * The constructor for legacy external fulltext index.
-   *
-   * @param xapianDirectoryPath The path to the external index directory.
-   * @param reader The reader associated to the external index.
-   *               It will be used retrive the article content or generate
-   *               the snippet.
-   * @param humanReadableName The humanReadableName for the zim.
-   */
-  Searcher(const string& xapianDirectoryPath,
-           Reader* reader,
-           const string& humanReadableName);
   ~Searcher();
 
   /**
@@ -192,12 +163,10 @@ class Searcher
    */
   bool setSearchProtocolPrefix(const std::string prefix);
 
-#ifdef ENABLE_CTPP2
   /**
    * Generate the html page with the resutls of the search.
    */
   string getHtml();
-#endif
 
  protected:
   std::string beautifyInteger(const unsigned int number);

include/tools/networkTools.h

@@ -20,13 +20,10 @@
 #ifndef KIWIX_NETWORKTOOLS_H
 #define KIWIX_NETWORKTOOLS_H
 
-#include <map>
 #include <string>
 
 namespace kiwix
 {
-std::map<std::string, std::string> getNetworkInterfaces();
-std::string getBestPublicIp();
 std::string download(const std::string& url);
 }
 

include/tools/stringTools.h

@@ -33,8 +33,6 @@
 
 namespace kiwix
 {
-#ifndef __ANDROID__
-
 std::string beautifyInteger(uint64_t number);
 std::string beautifyFileSize(uint64_t number);
 void printStringInHexadecimal(const char* s);
@@ -44,8 +42,6 @@ void stringReplacement(std::string& str,
                        const std::string& newStr);
 std::string encodeDiples(const std::string& str);
 
-#endif
-
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
 

include/xapianSearcher.h

@@ -1,98 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_XAPIAN_SEARCHER_H
-#define KIWIX_XAPIAN_SEARCHER_H
-
-#include <xapian.h>
-#include "reader.h"
-#include "searcher.h"
-
-#include <map>
-#include <string>
-
-using namespace std;
-
-namespace kiwix
-{
-class XapianSearcher;
-
-class XapianResult : public Result
-{
- public:
-  XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
-  virtual ~XapianResult(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual int get_readerIndex() { return 0; };
-
- private:
-  XapianSearcher* searcher;
-  Xapian::MSetIterator iterator;
-  Xapian::Document document;
-};
-
-class NoXapianIndexInZim : public exception
-{
-  virtual const char* what() const throw()
-  {
-    return "There is no fulltext index in the zim file";
-  }
-};
-
-class XapianSearcher
-{
-  friend class XapianResult;
-
- public:
-  XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
-  virtual ~XapianSearcher(){};
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-  virtual Result* getNextResult();
-  void restart_search();
-
-  Xapian::MSet results;
-
- protected:
-  void closeIndex();
-  void openIndex(const string& xapianDirectoryPath);
-  void setup_queryParser();
-
-  Reader* reader;
-  Xapian::Database readableDatabase;
-  std::string language;
-  std::string stopwords;
-  Xapian::QueryParser queryParser;
-  Xapian::Stem stemmer;
-  Xapian::SimpleStopper stopper;
-  Xapian::MSetIterator current_result;
-  std::map<std::string, int> valuesmap;
-};
-}
-
-#endif

meson.build

@@ -1,10 +1,9 @@
 project('kiwix-lib', 'cpp',
-  version : '3.0.2',
+  version : '5.0.0',
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
-find_library_in_compiler = meson.version().version_compare('>=0.31.0')
 
 static_deps = get_option('android') or get_option('default_library') == 'static'
 if get_option('android')
@@ -19,83 +18,22 @@ libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 
+if not compiler.has_header('mustache.hpp')
+  error('Cannot found header mustache.hpp')
+endif
+
 extra_cflags = ''
 if target_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-
-
-ctpp2_include_path = ''
-has_ctpp2_dep = false
-ctpp2_prefix_install = get_option('ctpp2-install-prefix')
-ctpp2_link_args = []
-if ctpp2_prefix_install == ''
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp')
-    if find_library_in_compiler
-      ctpp2_lib = compiler.find_library('ctpp2')
-    else
-      ctpp2_lib = find_library('ctpp2')
-    endif
-    ctpp2_link_args = ['-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      if find_library_in_compiler
-        iconv_lib = compiler.find_library('iconv', required:false)
-      else
-        iconv_lib = find_library('iconv', required:false)
-      endif
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
-else
-  if not find_library_in_compiler
-    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
-  endif
-  ctpp2_include_path = ctpp2_prefix_install + '/include'
-  ctpp2_include_args = ['-I'+ctpp2_include_path]
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
-    ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
-    ctpp2_lib_path = join_paths(ctpp2_prefix_install, get_option('libdir'))
-    message(ctpp2_lib_path)
-    ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path, required:false)
-    if not ctpp2_lib.found()
-      ctpp2_lib_path = join_paths(ctpp2_prefix_install, 'lib')
-      message(ctpp2_lib_path)
-      ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
-    endif
-    ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      iconv_lib = compiler.find_library('iconv', required:false)
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
-endif
-
-xapian_dep = dependency('xapian-core', required:false, static:static_deps)
-
-all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libcurl_dep]
-if has_ctpp2_dep
-  all_deps += [ctpp2_dep]
-endif
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep]
 
 inc = include_directories('include')
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
-conf.set('ENABLE_CTPP2', has_ctpp2_dep)
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -110,16 +48,6 @@ subdir('src')
 subdir('test')
 
 pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
-if xapian_dep.found()
-    pkg_requires += ['xapian-core']
-endif
-
-if has_ctpp2_dep
-  extra_libs += ctpp2_link_args
-  if ctpp2_include_path != ''
-    extra_cflags = ' -I'+ctpp2_include_path
-  endif
-endif
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,4 +1,2 @@
-option('ctpp2-install-prefix', type : 'string', value : '',
-  description : 'Prefix where ctpp libs has been installed')
 option('android', type : 'boolean', value : false,
   description : 'Do we make a kiwix-lib for android')

scripts/ctpp2c.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-
-ctpp2c=$1
-SOURCE=$(pwd)/$2
-DEST=$3
-
-$ctpp2c $SOURCE $DEST

scripts/meson.build

@@ -1,5 +1,4 @@
 
 res_compiler = find_program('kiwix-compile-resources')
-intermediate_ctpp2c = find_program('ctpp2c.sh')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))

src/android/kiwixreader.cpp

@@ -24,7 +24,7 @@
 #include <android/log.h>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
-#include "common/base64.h"
+#include "tools/base64.h"
 #include "reader.h"
 #include "utils.h"
 
@@ -91,7 +91,7 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
 JNIEXPORT jint JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
 {
-  jint size;
+  jint size = 0;
 
   try {
     int cSize = READER->getFileSize();

src/aria2.cpp

@@ -5,10 +5,17 @@
 #include <sstream>
 #include <thread>
 #include <chrono>
-#include <common/otherTools.h>
-#include <common/pathTools.h>
+#include <tools/otherTools.h>
+#include <tools/pathTools.h>
 #include <downloader.h> // For AriaError
 
+#ifdef _WIN32
+# define ARIA2_CMD "aria2c.exe"
+#else
+# define ARIA2_CMD "aria2c"
+#endif
+
+
 namespace kiwix {
 
 Aria2::Aria2():
@@ -19,6 +26,7 @@ Aria2::Aria2():
   m_lock(PTHREAD_MUTEX_INITIALIZER)
 {
   m_downloadDir = getDataDirectory();
+  makeDirectory(m_downloadDir);
   std::vector<const char*> callCmd;
 
   std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
@@ -36,11 +44,16 @@ Aria2::Aria2():
   std::string rpc_secret = "--rpc-secret=" + m_secret;
   m_secret = "token:"+m_secret;
 
-#ifdef _WIN32
-  callCmd.push_back("aria2c.exe");
-#else
-  callCmd.push_back("aria2c");
-#endif
+  std::string aria2cmd = appendToDirectory(
+    removeLastPathElement(getExecutablePath(), true, true),
+    ARIA2_CMD);
+  if (fileExists(aria2cmd)) {
+    // A local aria2c exe exists (packaged with kiwix-desktop), use it.
+    callCmd.push_back(aria2cmd.c_str());
+  } else {
+    // Try to use a potential installed aria2c.
+    callCmd.push_back(ARIA2_CMD);
+  }
   callCmd.push_back("--enable-rpc");
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
@@ -69,7 +82,7 @@ Aria2::Aria2():
 
   int watchdog = 50;
   while(--watchdog) {
-    std::this_thread::sleep_for(std::chrono::microseconds(100));
+    std::this_thread::sleep_for(std::chrono::microseconds(10000));
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;

src/book.cpp

@@ -20,9 +20,9 @@
 #include "book.h"
 #include "reader.h"
 
-#include "common/base64.h"
-#include "common/regexTools.h"
-#include "common/networkTools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/networkTools.h"
 
 #include <pugixml.hpp>
 
@@ -60,11 +60,6 @@ bool Book::update(const kiwix::Book& other)
     m_name = other.m_name;
   }
 
-  if (m_indexPath.empty()) {
-    m_indexPath = other.m_indexPath;
-    m_indexType = other.m_indexType;
-  }
-
   if (m_faviconMimeType.empty()) {
     m_favicon = other.m_favicon;
     m_faviconMimeType = other.m_faviconMimeType;
@@ -87,7 +82,7 @@ void Book::update(const kiwix::Reader& reader)
   m_origId = reader.getOrigId();
   m_articleCount = reader.getArticleCount();
   m_mediaCount = reader.getMediaCount();
-  m_size = reader.getFileSize() << 10;
+  m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
 
   reader.getFavicon(m_favicon, m_faviconMimeType);
 }
@@ -101,14 +96,6 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  path = ATTR("indexPath");
-  if (!path.empty()) {
-    if (isRelativePath(path)) {
-      path = computeAbsolutePath(baseDir, path);
-    }
-    m_indexPath = path;
-    m_indexType = XAPIAN;
-  }
   m_title = ATTR("title");
   m_name = ATTR("name");
   m_tags = ATTR("tags");
@@ -131,6 +118,14 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
 #undef ATTR
 
 
+static std::string fromOpdsDate(const std::string& date)
+{
+  //The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
+  //and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
+  return date.substr(0, 10);
+}
+
+
 #define VALUE(name) node.child(name).child_value()
 void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
 {
@@ -141,7 +136,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_title = VALUE("title");
   m_description = VALUE("description");
   m_language = VALUE("language");
-  m_date = VALUE("updated");
+  m_date = fromOpdsDate(VALUE("updated"));
   m_creator = node.child("author").child("name").child_value();
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
@@ -186,13 +181,6 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-void Book::setIndexPath(const std::string& indexPath)
-{
-  m_indexPath = isRelativePath(indexPath)
-    ? computeAbsolutePath(getCurrentDirectory(), indexPath)
-    : indexPath;
-}
-
 const std::string& Book::getFavicon() const {
   if (m_favicon.empty() && !m_faviconUrl.empty()) {
     try {

src/bookmark.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "bookmark.h"
+
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Bookmark::Bookmark()
+{
+}
+
+/* Destructor */
+Bookmark::~Bookmark()
+{
+}
+
+void Bookmark::updateFromXml(const pugi::xml_node& node)
+{
+  auto bookNode = node.child("book");
+  m_bookId = bookNode.child("id").child_value();
+  m_bookTitle = bookNode.child("title").child_value();
+  m_language = bookNode.child("language").child_value();
+  m_date = bookNode.child("date").child_value();
+  m_title = node.child("title").child_value();
+  m_url = node.child("url").child_value();
+}
+
+}

src/common/networkTools.cpp

@@ -1,209 +0,0 @@
-/*
- * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/networkTools.h>
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#else
-#include <net/if.h>
-#include <netdb.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-
-#include <curl/curl.h>
-
-#include <sstream>
-#include <iostream>
-
-
-std::map<std::string, std::string> kiwix::getNetworkInterfaces()
-{
-  std::map<std::string, std::string> interfaces;
-
-#ifdef _WIN32
-  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
-  if (sd == (SOCKET)SOCKET_ERROR) {
-    std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  INTERFACE_INFO InterfaceList[20];
-  unsigned long nBytesReturned;
-  if (WSAIoctl(sd,
-               SIO_GET_INTERFACE_LIST,
-               0,
-               0,
-               &InterfaceList,
-               sizeof(InterfaceList),
-               &nBytesReturned,
-               0,
-               0)
-      == SOCKET_ERROR) {
-    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
-  for (int i = 0; i < nNumInterfaces; ++i) {
-    sockaddr_in* pAddress;
-    pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
-
-    /* Add to the map */
-    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
-    std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
-    interfaces.insert(
-        std::pair<std::string, std::string>(interfaceName, interfaceIp));
-  }
-#else
-  /* Get Network interfaces information */
-  char buf[16384];
-  struct ifconf ifconf;
-  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
-  ifconf.ifc_len = sizeof buf;
-  ifconf.ifc_buf = buf;
-  if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
-    perror("ioctl(SIOCGIFCONF)");
-    exit(EXIT_FAILURE);
-  }
-
-  /* Go through each interface */
-  int i;
-  size_t len;
-  struct ifreq* ifreq;
-  ifreq = ifconf.ifc_req;
-  for (i = 0; i < ifconf.ifc_len;) {
-    if (ifreq->ifr_addr.sa_family == AF_INET) {
-      /* Get the network interface ip */
-      char host[128] = {0};
-      const int error = getnameinfo(&(ifreq->ifr_addr),
-                                    sizeof ifreq->ifr_addr,
-                                    host,
-                                    sizeof host,
-                                    0,
-                                    0,
-                                    NI_NUMERICHOST);
-      if (!error) {
-        std::string interfaceName = std::string(ifreq->ifr_name);
-        std::string interfaceIp = std::string(host);
-        /* Add to the map */
-        interfaces.insert(
-            std::pair<std::string, std::string>(interfaceName, interfaceIp));
-      } else {
-        perror("getnameinfo()");
-      }
-    }
-
-/* some systems have ifr_addr.sa_len and adjust the length that
- * way, but not mine. weird */
-#ifndef __linux__
-    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
-#else
-    len = sizeof *ifreq;
-#endif
-    ifreq = (struct ifreq*)((char*)ifreq + len);
-    i += len;
-  }
-#endif
-  return interfaces;
-}
-
-std::string kiwix::getBestPublicIp()
-{
-  std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
-
-#ifndef _WIN32
-  const char* const prioritizedNames[]
-      = {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
-  const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
-  for (int i = 0; i < count; ++i) {
-    std::map<std::string, std::string>::const_iterator it
-        = interfaces.find(prioritizedNames[i]);
-    if (it != interfaces.end()) {
-      return it->second;
-    }
-  }
-#endif
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
-      return interfaceIp;
-    }
-  }
-
-  return "127.0.0.1";
-}
-
-size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
-{
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
-  return nmemb;
-}
-
-std::string kiwix::download(const std::string& url) {
-  auto curl = curl_easy_init();
-  std::stringstream ss;
-  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-  curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
-  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);
-  auto res = curl_easy_perform(curl);
-  if (res != CURLE_OK) {
-    curl_easy_cleanup(curl);
-    throw std::runtime_error("Cannot perform request");
-  }
-  long response_code;
-  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-  curl_easy_cleanup(curl);
-  if (response_code != 200) {
-    throw std::runtime_error("Invalid return code from server");
-  }
-  return ss.str();
-}

src/common/otherTools.cpp

@@ -1,300 +0,0 @@
-/*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/otherTools.h>
-#include <map>
-
-std::map<std::string, std::string> codeisomapping {
-  { "af", "afg" },
-  { "ax", "ala" },
-  { "al", "alb" },
-  { "dz", "dza" },
-  { "as", "asm" },
-  { "ad", "and" },
-  { "ao", "ago" },
-  { "ai", "aia" },
-  { "aq", "ata" },
-  { "ag", "atg" },
-  { "ar", "arg" },
-  { "am", "arm" },
-  { "aw", "abw" },
-  { "au", "aus" },
-  { "at", "aut" },
-  { "az", "aze" },
-  { "bs", "bhs" },
-  { "bh", "bhr" },
-  { "bd", "bgd" },
-  { "bb", "brb" },
-  { "by", "blr" },
-  { "be", "bel" },
-  { "bz", "blz" },
-  { "bj", "ben" },
-  { "bm", "bmu" },
-  { "bt", "btn" },
-  { "bo", "bol" },
-  { "ba", "bih" },
-  { "bw", "bwa" },
-  { "bv", "bvt" },
-  { "br", "bra" },
-  { "vg", "vgb" },
-  { "io", "iot" },
-  { "bn", "brn" },
-  { "bg", "bgr" },
-  { "bf", "bfa" },
-  { "bi", "bdi" },
-  { "kh", "khm" },
-  { "cm", "cmr" },
-  { "ca", "can" },
-  { "cv", "cpv" },
-  { "ky", "cym" },
-  { "cf", "caf" },
-  { "td", "tcd" },
-  { "cl", "chl" },
-  { "cn", "chn" },
-  { "hk", "hkg" },
-  { "mo", "mac" },
-  { "cx", "cxr" },
-  { "cc", "cck" },
-  { "co", "col" },
-  { "km", "com" },
-  { "cg", "cog" },
-  { "cd", "cod" },
-  { "ck", "cok" },
-  { "cr", "cri" },
-  { "ci", "civ" },
-  { "hr", "hrv" },
-  { "cu", "cub" },
-  { "cy", "cyp" },
-  { "cz", "cze" },
-  { "dk", "dnk" },
-  { "dj", "dji" },
-  { "dm", "dma" },
-  { "do", "dom" },
-  { "ec", "ecu" },
-  { "eg", "egy" },
-  { "sv", "slv" },
-  { "gq", "gnq" },
-  { "er", "eri" },
-  { "ee", "est" },
-  { "et", "eth" },
-  { "fk", "flk" },
-  { "fo", "fro" },
-  { "fj", "fji" },
-  { "fi", "fin" },
-  { "fr", "fra" },
-  { "gf", "guf" },
-  { "pf", "pyf" },
-  { "tf", "atf" },
-  { "ga", "gab" },
-  { "gm", "gmb" },
-  { "ge", "geo" },
-  { "de", "deu" },
-  { "gh", "gha" },
-  { "gi", "gib" },
-  { "gr", "grc" },
-  { "gl", "grl" },
-  { "gd", "grd" },
-  { "gp", "glp" },
-  { "gu", "gum" },
-  { "gt", "gtm" },
-  { "gg", "ggy" },
-  { "gn", "gin" },
-  { "gw", "gnb" },
-  { "gy", "guy" },
-  { "ht", "hti" },
-  { "hm", "hmd" },
-  { "va", "vat" },
-  { "hn", "hnd" },
-  { "hu", "hun" },
-  { "is", "isl" },
-  { "in", "ind" },
-  { "id", "idn" },
-  { "ir", "irn" },
-  { "iq", "irq" },
-  { "ie", "irl" },
-  { "im", "imn" },
-  { "il", "isr" },
-  { "it", "ita" },
-  { "jm", "jam" },
-  { "jp", "jpn" },
-  { "je", "jey" },
-  { "jo", "jor" },
-  { "kz", "kaz" },
-  { "ke", "ken" },
-  { "ki", "kir" },
-  { "kp", "prk" },
-  { "kr", "kor" },
-  { "kw", "kwt" },
-  { "kg", "kgz" },
-  { "la", "lao" },
-  { "lv", "lva" },
-  { "lb", "lbn" },
-  { "ls", "lso" },
-  { "lr", "lbr" },
-  { "ly", "lby" },
-  { "li", "lie" },
-  { "lt", "ltu" },
-  { "lu", "lux" },
-  { "mk", "mkd" },
-  { "mg", "mdg" },
-  { "mw", "mwi" },
-  { "my", "mys" },
-  { "mv", "mdv" },
-  { "ml", "mli" },
-  { "mt", "mlt" },
-  { "mh", "mhl" },
-  { "mq", "mtq" },
-  { "mr", "mrt" },
-  { "mu", "mus" },
-  { "yt", "myt" },
-  { "mx", "mex" },
-  { "fm", "fsm" },
-  { "md", "mda" },
-  { "mc", "mco" },
-  { "mn", "mng" },
-  { "me", "mne" },
-  { "ms", "msr" },
-  { "ma", "mar" },
-  { "mz", "moz" },
-  { "mm", "mmr" },
-  { "na", "nam" },
-  { "nr", "nru" },
-  { "np", "npl" },
-  { "nl", "nld" },
-  { "an", "ant" },
-  { "nc", "ncl" },
-  { "nz", "nzl" },
-  { "ni", "nic" },
-  { "ne", "ner" },
-  { "ng", "nga" },
-  { "nu", "niu" },
-  { "nf", "nfk" },
-  { "mp", "mnp" },
-  { "no", "nor" },
-  { "om", "omn" },
-  { "pk", "pak" },
-  { "pw", "plw" },
-  { "ps", "pse" },
-  { "pa", "pan" },
-  { "pg", "png" },
-  { "py", "pry" },
-  { "pe", "per" },
-  { "ph", "phl" },
-  { "pn", "pcn" },
-  { "pl", "pol" },
-  { "pt", "prt" },
-  { "pr", "pri" },
-  { "qa", "qat" },
-  { "re", "reu" },
-  { "ro", "rou" },
-  { "ru", "rus" },
-  { "rw", "rwa" },
-  { "bl", "blm" },
-  { "sh", "shn" },
-  { "kn", "kna" },
-  { "lc", "lca" },
-  { "mf", "maf" },
-  { "pm", "spm" },
-  { "vc", "vct" },
-  { "ws", "wsm" },
-  { "sm", "smr" },
-  { "st", "stp" },
-  { "sa", "sau" },
-  { "sn", "sen" },
-  { "rs", "srb" },
-  { "sc", "syc" },
-  { "sl", "sle" },
-  { "sg", "sgp" },
-  { "sk", "svk" },
-  { "si", "svn" },
-  { "sb", "slb" },
-  { "so", "som" },
-  { "za", "zaf" },
-  { "gs", "sgs" },
-  { "ss", "ssd" },
-  { "es", "esp" },
-  { "lk", "lka" },
-  { "sd", "sdn" },
-  { "sr", "sur" },
-  { "sj", "sjm" },
-  { "sz", "swz" },
-  { "se", "swe" },
-  { "ch", "che" },
-  { "sy", "syr" },
-  { "tw", "twn" },
-  { "tj", "tjk" },
-  { "tz", "tza" },
-  { "th", "tha" },
-  { "tl", "tls" },
-  { "tg", "tgo" },
-  { "tk", "tkl" },
-  { "to", "ton" },
-  { "tt", "tto" },
-  { "tn", "tun" },
-  { "tr", "tur" },
-  { "tm", "tkm" },
-  { "tc", "tca" },
-  { "tv", "tuv" },
-  { "ug", "uga" },
-  { "ua", "ukr" },
-  { "ae", "are" },
-  { "gb", "gbr" },
-  { "us", "usa" },
-  { "um", "umi" },
-  { "uy", "ury" },
-  { "uz", "uzb" },
-  { "vu", "vut" },
-  { "ve", "ven" },
-  { "vn", "vnm" },
-  { "vi", "vir" },
-  { "wf", "wlf" },
-  { "eh", "esh" },
-  { "ye", "yem" },
-  { "zm", "zmb" },
-  { "zw", "zwe" }
-};
-
-void kiwix::sleep(unsigned int milliseconds)
-{
-#ifdef _WIN32
-  Sleep(milliseconds);
-#else
-  usleep(1000 * milliseconds);
-#endif
-}
-
-
-struct XmlStringWriter: pugi::xml_writer
-{
-  std::string result;
-  virtual void write(const void* data, size_t size){
-    result.append(static_cast<const char*>(data), size);
-  }
-};
-
-std::string kiwix::nodeToString(pugi::xml_node node)
-{
-  XmlStringWriter writer;
-  node.print(writer, "  ");
-  return writer.result;
-}
-
-std::string kiwix::converta2toa3(const std::string& a2code){
-  return codeisomapping.at(a2code);
-}

src/config.h.in

@@ -1,4 +1,3 @@
 
 #mesondefine VERSION
 
-#mesondefine ENABLE_CTPP2

src/ctpp2/CTPP2VMStringLoader.cpp

@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <ctpp2/CTPP2VMStringLoader.hpp>
-
-namespace CTPP // C++ Template Engine
-{
-
-//
-// Convert byte order
-//
-static void ConvertExecutable(VMExecutable * oCore)
-{
-	// Code entry point
-	oCore -> entry_point = Swap32(oCore -> entry_point);
-	// Offset of code segment
-	oCore -> code_offset = Swap32(oCore -> code_offset);
-
-	// Code segment size
-	oCore -> code_size   = Swap32(oCore -> code_size);
-
-	// Offset of static text segment
-	oCore -> syscalls_offset    = Swap32(oCore -> syscalls_offset);
-	// Static text segment size
-	oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
-
-	// Offset of static text index segment
-	oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
-	// Static text index segment size
-	oCore -> syscalls_index_size   = Swap32(oCore -> syscalls_index_size);
-
-	// Offset of static data segment
-	oCore -> static_data_offset    = Swap32(oCore -> static_data_offset);
-
-	// Static data segment size
-	oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
-
-	// Offset of static text segment
-	oCore -> static_text_offset    = Swap32(oCore -> static_text_offset);
-	// Static text segment size
-	oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
-
-	// Offset of static text index segment
-	oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
-	// Static text index segment size
-	oCore -> static_text_index_size   = Swap32(oCore -> static_text_index_size);
-
-	// Version 2.2+
-	// Offset of static data bit index
-	oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
-	/// Offset of static data bit index
-	oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
-
-	// Platform
-	oCore -> platform      = Swap64(oCore -> platform);
-
-	// Ugly-jolly hack!
-	// ... dereferencing type-punned pointer will break strict-aliasing rules ...
-	UINT_64 iTMP;
-	memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
-	iTMP = Swap64(iTMP);
-	memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
-
-	// Cyclic Redundancy Check
-	oCore -> crc = 0;
-
-	// Convert data structures
-
-	// Convert code segment
-	VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
-	UINT_32 iI = 0;
-	UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pInstructions -> instruction = Swap32(pInstructions -> instruction);
-		pInstructions -> argument    = Swap32(pInstructions -> argument);
-		pInstructions -> reserved    = Swap64(pInstructions -> reserved);
-		++pInstructions;
-	}
-
-	// Convert syscalls index
-	TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
-	iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static text index
-	pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
-	iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static data
-	StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
-	iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
-		++pStaticDataVar;
-	}
-}
-
-//
-// Constructor
-//
-VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
-{
-    oCore = (VMExecutable *)malloc(rawContentSize + 1);
-    memcpy(oCore, rawContent, rawContentSize);
-
-	if (oCore -> magic[0] == 'C' &&
-	    oCore -> magic[1] == 'T' &&
-	    oCore -> magic[2] == 'P' &&
-	    oCore -> magic[3] == 'P')
-	{
-		// Check version
-		if (oCore -> version[0] >= 1)
-		{
-			// Platform-dependent data (byte order)
-			if (oCore -> platform == 0x4142434445464748ull)
-			{
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
-#endif
-
-				// Nothing to do, only check crc
-				UINT_32 iCRC = oCore -> crc;
-				oCore -> crc = 0;
-
-				// Calculate CRC of file
-                // KELSON: next line used to refer to oStat.st_size
-                // changed it to rawContentSize
-				if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
-				{
-					free(oCore);
-					throw CTPPLogicError("CRC checksum invalid");
-				}
-			}
-			// Platform-dependent data (byte order)
-			else if (oCore -> platform == 0x4847464544434241ull)
-			{
-				// Need to reconvert data
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
-#endif
-				ConvertExecutable(oCore);
-			}
-			else
-			{
-				free(oCore);
-				throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
-			}
-
-			// Check IEEE 754 format
-			if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
-			{
-				free(oCore);
-				throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
-			}
-		}
-
-		pVMMemoryCore = new VMMemoryCore(oCore);
-	}
-	else
-	{
-		free(oCore);
-		throw CTPPLogicError("Not an CTPP bytecode file.");
-	}
-}
-
-//
-// Get ready-to-run program
-//
-const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
-
-//
-// A destructor
-//
-VMStringLoader::~VMStringLoader() throw()
-{
-	delete pVMMemoryCore;
-	free(oCore);
-}
-
-} // namespace CTPP
-// End.

src/downloader.cpp

@@ -18,7 +18,7 @@
  */
 
 #include "downloader.h"
-#include "common/pathTools.h"
+#include "tools/pathTools.h"
 
 #include <algorithm>
 #include <thread>
@@ -28,7 +28,7 @@
 
 #include "aria2.h"
 #include "xmlrpc.h"
-#include "common/otherTools.h"
+#include "tools/otherTools.h"
 #include <pugixml.hpp>
 
 namespace kiwix

src/entry.cpp

@@ -122,7 +122,6 @@ Entry Entry::getFinalEntry() const
   if (final_article.good()) {
     return final_article;
   }
-
   int loopCounter = 42;
   final_article = article;
   while (final_article.isRedirect() && loopCounter--) {
@@ -131,7 +130,10 @@ Entry Entry::getFinalEntry() const
       throw NoEntry();
     }
   }
-
+  // Prevent infinite loops.
+  if (final_article.isRedirect()) {
+    throw NoEntry();
+  }
   return final_article;
 }
 

src/library.cpp

@@ -19,18 +19,20 @@
 
 #include "library.h"
 #include "book.h"
+#include "libxml_dumper.h"
 
-#include "common/base64.h"
-#include "common/regexTools.h"
-#include "common/pathTools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/pathTools.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
+#include <set>
 
 namespace kiwix
 {
 /* Constructor */
-Library::Library() : version(KIWIX_LIBRARY_VERSION)
+Library::Library()
 {
 }
 /* Destructor */
@@ -43,31 +45,48 @@ bool Library::addBook(const Book& book)
 {
   /* Try to find it */
   try {
-    auto& oldbook = books.at(book.getId());
+    auto& oldbook = m_books.at(book.getId());
     oldbook.update(book);
     return false;
   } catch (std::out_of_range&) {
-    books[book.getId()] = book;
+    m_books[book.getId()] = book;
     return true;
   }
 }
 
+void Library::addBookmark(const Bookmark& bookmark)
+{
+  m_bookmarks.push_back(bookmark);
+}
+
+bool Library::removeBookmark(const std::string& zimId, const std::string& url)
+{
+  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
+    if (it->getBookId() == zimId && it->getUrl() == url) {
+      m_bookmarks.erase(it);
+      return true;
+    }
+  }
+  return false;
+}
+
+
 
 bool Library::removeBookById(const std::string& id)
 {
-  return books.erase(id) == 1;
+  return m_books.erase(id) == 1;
 }
 
 Book& Library::getBookById(const std::string& id)
 {
-  return books.at(id);
+  return m_books.at(id);
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks)
 {
   unsigned int result = 0;
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     if ((!book.getPath().empty() && localBooks)
         || (book.getPath().empty() && remoteBooks)) {
@@ -78,91 +97,15 @@ unsigned int Library::getBookCount(const bool localBooks,
 }
 
 bool Library::writeToFile(const std::string& path) {
-  pugi::xml_document doc;
   auto baseDir = removeLastPathElement(path, true, false);
+  LibXMLDumper dumper(this);
+  dumper.setBaseDir(baseDir);
+  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
+}
 
-  /* Add the library node */
-  pugi::xml_node libraryNode = doc.append_child("library");
-
-  if (!version.empty())
-    libraryNode.append_attribute("version") = version.c_str();
-
-  /* Add each book */
-  for (auto& pair: books) {
-    auto& book = pair.second;
-    if (!book.readOnly()) {
-      pugi::xml_node bookNode = libraryNode.append_child("book");
-      bookNode.append_attribute("id") = book.getId().c_str();
-
-      if (!book.getPath().empty()) {
-        bookNode.append_attribute("path") = computeRelativePath(
-            baseDir, book.getPath()).c_str();
-      }
-
-      if (!book.getIndexPath().empty()) {
-        bookNode.append_attribute("indexPath") = computeRelativePath(
-            baseDir, book.getIndexPath()).c_str();
-        bookNode.append_attribute("indexType") = "xapian";
-      }
-
-      if (book.getOrigId().empty()) {
-        if (!book.getTitle().empty())
-          bookNode.append_attribute("title") = book.getTitle().c_str();
-
-        if (!book.getName().empty())
-          bookNode.append_attribute("name") = book.getName().c_str();
-
-        if (!book.getTags().empty())
-          bookNode.append_attribute("tags") = book.getTags().c_str();
-
-        if (!book.getDescription().empty())
-          bookNode.append_attribute("description") = book.getDescription().c_str();
-
-        if (!book.getLanguage().empty())
-          bookNode.append_attribute("language") = book.getLanguage().c_str();
-
-        if (!book.getCreator().empty())
-          bookNode.append_attribute("creator") = book.getCreator().c_str();
-
-        if (!book.getPublisher().empty())
-          bookNode.append_attribute("publisher") = book.getPublisher().c_str();
-
-        if (!book.getFavicon().empty())
-          bookNode.append_attribute("favicon") = base64_encode(book.getFavicon()).c_str();
-
-        if (!book.getFaviconMimeType().empty())
-          bookNode.append_attribute("faviconMimeType")
-              = book.getFaviconMimeType().c_str();
-      } else {
-        bookNode.append_attribute("origId") = book.getOrigId().c_str();
-      }
-
-      if (!book.getDate().empty()) {
-        bookNode.append_attribute("date") = book.getDate().c_str();
-      }
-
-      if (!book.getUrl().empty()) {
-        bookNode.append_attribute("url") = book.getUrl().c_str();
-      }
-
-      if (!book.getArticleCount())
-        bookNode.append_attribute("articleCount") = to_string(book.getArticleCount()).c_str();
-
-      if (!book.getMediaCount())
-        bookNode.append_attribute("mediaCount") = to_string(book.getMediaCount()).c_str();
-
-      if (book.getSize()) {
-        bookNode.append_attribute("size") = to_string(book.getSize()>>10).c_str();
-      }
-
-      if (!book.getDownloadId().empty()) {
-        bookNode.append_attribute("downloadId") = book.getDownloadId().c_str();
-      }
-    }
-  }
-
-  /* saving file */
-  return doc.save_file(path.c_str());
+bool Library::writeBookmarksToFile(const std::string& path) {
+  LibXMLDumper dumper(this);
+  return writeTextFile(path, dumper.dumpLibXMLBookmark());
 }
 
 std::vector<std::string> Library::getBooksLanguages()
@@ -170,7 +113,7 @@ std::vector<std::string> Library::getBooksLanguages()
   std::vector<std::string> booksLanguages;
   std::map<std::string, bool> booksLanguagesMap;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     auto& language = book.getLanguage();
     if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
@@ -189,7 +132,7 @@ std::vector<std::string> Library::getBooksCreators()
   std::vector<std::string> booksCreators;
   std::map<std::string, bool> booksCreatorsMap;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     auto& creator = book.getCreator();
     if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
@@ -208,7 +151,7 @@ std::vector<std::string> Library::getBooksPublishers()
   std::vector<std::string> booksPublishers;
   std::map<std::string, bool> booksPublishersMap;
 
-  for (auto& pair:books) {
+  for (auto& pair:m_books) {
     auto& book = pair.second;
     auto& publisher = book.getPublisher();
     if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
@@ -226,7 +169,7 @@ std::vector<std::string> Library::getBooksIds()
 {
   std::vector<std::string> bookIds;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     bookIds.push_back(pair.first);
   }
 
@@ -240,7 +183,7 @@ std::vector<std::string> Library::filter(const std::string& search)
   }
 
   std::vector<std::string> bookIds;
-  for(auto& pair:books) {
+  for(auto& pair:m_books) {
      auto& book = pair.second;
      if (matchRegex(book.getTitle(), "\\Q" + search + "\\E")
          || matchRegex(book.getDescription(), "\\Q" + search + "\\E")) {
@@ -308,10 +251,11 @@ std::vector<std::string> Library::listBooksIds(
     const std::string& language,
     const std::string& creator,
     const std::string& publisher,
+    const std::vector<std::string>& tags,
     size_t maxSize) {
 
   std::vector<std::string> bookIds;
-  for(auto& pair:books) {
+  for(auto& pair:m_books) {
     auto& book = pair.second;
     auto local = !book.getPath().empty();
     if (mode & LOCAL && !local)
@@ -328,6 +272,23 @@ std::vector<std::string> Library::listBooksIds(
       continue;
     if (mode & NOREMOTE && remote)
       continue;
+    if (!tags.empty()) {
+      auto vBookTags = split(book.getTags(), ";");
+      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
+      bool ok = true;
+      for (auto& t: tags) {
+        if (sBookTags.find(t) == sBookTags.end()) {
+          // A "filter" tag is not in the book tag.
+          // No need to loop for all "filter" tags.
+          ok = false;
+          break;
+        }
+      }
+      if (! ok ) {
+        // Skip the book
+        continue;
+      }
+    }
     if (maxSize != 0 && book.getSize() > maxSize)
       continue;
     if (!language.empty() && book.getLanguage() != language)

src/libxml_dumper.cpp

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "libxml_dumper.h"
+#include "book.h"
+
+#include <tools/base64.h>
+#include <tools/stringTools.h>
+#include <tools/otherTools.h>
+
+namespace kiwix
+{
+/* Constructor */
+LibXMLDumper::LibXMLDumper(Library* library)
+  : library(library)
+{
+}
+/* Destructor */
+LibXMLDumper::~LibXMLDumper()
+{
+}
+
+#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
+#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
+
+void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
+  if (book.readOnly())
+    return;
+
+  auto entry_node = root_node.append_child("book");
+  ADD_ATTRIBUTE(entry_node, "id", book.getId());
+
+  if (!book.getPath().empty()) {
+    ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
+  }
+
+  if (book.getOrigId().empty()) {
+    ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
+    ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
+    ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
+    if (!book.getFavicon().empty())
+      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
+  } else {
+    ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
+  }
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
+  ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
+
+  if (book.getArticleCount())
+    ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
+
+  if (book.getMediaCount())
+    ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
+
+  if (book.getSize())
+    ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
+}
+
+#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
+
+void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
+
+  auto entry_node = root_node.append_child("bookmark");
+  auto book_node = entry_node.append_child("book");
+
+  try {
+    auto book = library->getBookById(bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "id", book.getId());
+    ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
+    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", book.getDate());
+  } catch (...) {
+    ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
+    ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
+  }
+  ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
+  ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
+}
+
+
+std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node libraryNode = doc.append_child("library");
+
+  libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
+
+  if (library) {
+    for (auto& bookId: bookIds) {
+      handleBook(library->getBookById(bookId), libraryNode);
+    }
+  }
+  return nodeToString(libraryNode);
+}
+
+std::string LibXMLDumper::dumpLibXMLBookmark()
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
+
+  if (library) {
+    for (auto& bookmark: library->getBookmarks()) {
+      handleBookmark(bookmark, bookmarksNode);
+    }
+  }
+  return nodeToString(bookmarksNode);
+}
+
+}

src/manager.cpp

@@ -208,12 +208,15 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
 
 bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
+  std::string tmp_path = path;
+  if (isRelativePath(path)) {
+    tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
+  }
   try {
-    kiwix::Reader reader(path);
+    kiwix::Reader reader(tmp_path);
     book->update(reader);
     book->setPathValid(true);
   } catch (const std::exception& e) {
-    std::cerr << "Invalid " << path << " : " << e.what() << std::endl;
     book->setPathValid(false);
     return false;
   }
@@ -221,4 +224,27 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
   return true;
 }
 
+bool Manager::readBookmarkFile(const std::string& path)
+{
+  pugi::xml_document doc;
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+
+  if (!result) {
+    return false;
+  }
+
+  pugi::xml_node libraryNode = doc.child("bookmarks");
+
+  for (pugi::xml_node node = libraryNode.child("bookmark"); node;
+       node = node.next_sibling("bookmark")) {
+    kiwix::Bookmark bookmark;
+
+    bookmark.updateFromXml(node);
+
+    manipulator->addBookmarkToLibrary(bookmark);
+  }
+
+  return true;
+}
+
 }

src/meson.build

@@ -1,7 +1,9 @@
 kiwix_sources = [
   'book.cpp',
+  'bookmark.cpp',
   'library.cpp',
   'manager.cpp',
+  'libxml_dumper.cpp',
   'opds_dumper.cpp',
   'downloader.cpp',
   'reader.cpp',
@@ -9,14 +11,12 @@ kiwix_sources = [
   'searcher.cpp',
   'subprocess.cpp',
   'aria2.cpp',
-  'common/base64.cpp',
-  'common/pathTools.cpp',
-  'common/regexTools.cpp',
-  'common/stringTools.cpp',
-  'common/networkTools.cpp',
-  'common/otherTools.cpp',
-  'xapian/htmlparse.cc',
-  'xapian/myhtmlparse.cc'
+  'tools/base64.cpp',
+  'tools/pathTools.cpp',
+  'tools/regexTools.cpp',
+  'tools/stringTools.cpp',
+  'tools/networkTools.cpp',
+  'tools/otherTools.cpp',
 ]
 kiwix_sources += lib_resources
 
@@ -26,10 +26,6 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if xapian_dep.found()
-  kiwix_sources += ['xapianSearcher.cpp']
-endif
-
 if get_option('android')
   subdir('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
@@ -37,11 +33,6 @@ else
   install_dir = get_option('libdir')
 endif
 
-
-if has_ctpp2_dep
-  kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
-endif
-
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
                input : 'config.h.in')
@@ -53,5 +44,4 @@ kiwixlib = library('kiwix',
                    dependencies : all_deps,
                    version: meson.project_version(),
                    install: true,
-                   install_dir: install_dir,
-                   install_rpath: '$ORIGIN')
+                   install_dir: install_dir)

src/opds_dumper.cpp

@@ -20,7 +20,7 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include <common/otherTools.h>
+#include <tools/otherTools.h>
 
 namespace kiwix
 {
@@ -50,6 +50,13 @@ std::string gen_date_str()
   return is.str();
 }
 
+static std::string gen_date_from_yyyy_mm_dd(const std::string& date)
+{
+  std::stringstream is;
+  is << date << "T00:00::00:Z";
+  return is.str();
+}
+
 void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 {
   m_totalResults = totalResults;
@@ -65,7 +72,7 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
   ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-  ADD_TEXT_ENTRY(entry_node, "updated", date);
+  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
   ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
 
   auto content_node = entry_node.append_child("link");

src/reader.cpp

@@ -241,8 +241,6 @@ Entry Reader::getMainPage() const
     throw NoEntry();
   }
 
-  string url = "";
-
   zim::Article article;
   if (this->zimFileHandler->getFileheader().hasMainPage())
   {
@@ -762,7 +760,7 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal;
+  bool retVal = false;
 
   this->suggestions.clear();
   this->suggestionsOffset = this->suggestions.begin();
@@ -851,7 +849,7 @@ bool Reader::isCorrupted() const
 unsigned int Reader::getFileSize() const
 {
   zim::File* file = this->getZimFileHandler();
-  zim::offset_type size = 0;
+  zim::size_type size = 0;
 
   if (file != NULL) {
     size = file->getFilesize();

src/searcher.cpp

@@ -22,20 +22,12 @@
 
 #include "searcher.h"
 #include "reader.h"
-#include "xapianSearcher.h"
 
 #include <zim/search.h>
 
-#ifdef ENABLE_CTPP2
-#include <ctpp2/CDT.hpp>
-#include <ctpp2/CTPP2FileLogger.hpp>
-#include <ctpp2/CTPP2SimpleVM.hpp>
-#include "ctpp2/CTPP2VMStringLoader.hpp"
+#include <mustache.hpp>
 #include "kiwixlib-resources.h"
 
-using namespace CTPP;
-#endif
-
 #define MAX_SEARCH_LEN 140
 
 namespace kiwix
@@ -61,42 +53,18 @@ class _Result : public Result
 
 struct SearcherInternal {
   const zim::Search* _search;
-  XapianSearcher* _xapianSearcher;
   zim::Search::iterator current_iterator;
 
-  SearcherInternal() : _search(NULL), _xapianSearcher(NULL) {}
+  SearcherInternal() : _search(NULL) {}
   ~SearcherInternal()
   {
     if (_search != NULL) {
       delete _search;
     }
-    if (_xapianSearcher != NULL) {
-      delete _xapianSearcher;
-    }
   }
 };
 
 /* Constructor */
-Searcher::Searcher(const string& xapianDirectoryPath,
-                   Reader* reader,
-                   const string& humanReadableName)
-    : internal(new SearcherInternal()),
-      searchPattern(""),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      resultCountPerPage(0),
-      estimatedResultCount(0),
-      resultStart(0),
-      resultEnd(0),
-      contentHumanReadableId(humanReadableName)
-{
-  loadICUExternalTables();
-  if (!reader || !reader->hasFulltextIndex()) {
-    internal->_xapianSearcher = new XapianSearcher(xapianDirectoryPath, reader);
-  }
-  this->humanReaderNames.push_back(humanReadableName);
-}
-
 Searcher::Searcher(const std::string& humanReadableName)
     : internal(new SearcherInternal()),
       searchPattern(""),
@@ -160,26 +128,19 @@ void Searcher::search(std::string& search,
     this->resultStart = resultStart;
     this->resultEnd = resultEnd;
     string unaccentedSearch = removeAccents(search);
-    if (internal->_xapianSearcher) {
-      internal->_xapianSearcher->searchInIndex(
-          unaccentedSearch, resultStart, resultEnd, verbose);
-      this->estimatedResultCount
-          = internal->_xapianSearcher->results.get_matches_estimated();
-    } else {
-      std::vector<const zim::File*> zims;
-      for (auto current = this->readers.begin(); current != this->readers.end();
-           current++) {
-        if ( (*current)->hasFulltextIndex() ) {
-            zims.push_back((*current)->getZimFileHandler());
-        }
+    std::vector<const zim::File*> zims;
+    for (auto current = this->readers.begin(); current != this->readers.end();
+         current++) {
+      if ( (*current)->hasFulltextIndex() ) {
+          zims.push_back((*current)->getZimFileHandler());
       }
-      zim::Search* search = new zim::Search(zims);
-      search->set_query(unaccentedSearch);
-      search->set_range(resultStart, resultEnd);
-      internal->_search = search;
-      internal->current_iterator = internal->_search->begin();
-      this->estimatedResultCount = internal->_search->get_matches_estimated();
     }
+    zim::Search* search = new zim::Search(zims);
+    search->set_query(unaccentedSearch);
+    search->set_range(resultStart, resultEnd);
+    internal->_search = search;
+    internal->current_iterator = internal->_search->begin();
+    this->estimatedResultCount = internal->_search->get_matches_estimated();
   }
 
   return;
@@ -209,10 +170,6 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     return;
   }
 
-  if (internal->_xapianSearcher) {
-    return;
-  }
-
   /* Avoid big researches */
   this->resultCountPerPage = resultEnd - resultStart;
   if (this->resultCountPerPage > MAX_SEARCH_LEN) {
@@ -244,18 +201,14 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
 
 void Searcher::restart_search()
 {
-  if (internal->_xapianSearcher) {
-    internal->_xapianSearcher->restart_search();
-  } else if (internal->_search) {
+  if (internal->_search) {
     internal->current_iterator = internal->_search->begin();
   }
 }
 
 Result* Searcher::getNextResult()
 {
-  if (internal->_xapianSearcher) {
-    return internal->_xapianSearcher->getNextResult();
-  } else if (internal->_search &&
+  if (internal->_search &&
              internal->current_iterator != internal->_search->end()) {
     Result* result = new _Result(internal->current_iterator);
     internal->current_iterator++;
@@ -272,37 +225,31 @@ void Searcher::reset()
   return;
 }
 
-void Searcher::suggestions(std::string& search, const bool verbose)
+void Searcher::suggestions(std::string& searchPattern, const bool verbose)
 {
   this->reset();
 
   if (verbose == true) {
-    cout << "Performing suggestion query `" << search << "`" << endl;
+    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
   }
 
-  this->searchPattern = search;
+  this->searchPattern = searchPattern;
   this->resultStart = 0;
   this->resultEnd = 10;
-  string unaccentedSearch = removeAccents(search);
+  string unaccentedSearch = removeAccents(searchPattern);
 
-  if (internal->_xapianSearcher) {
-    /* [TODO] Suggestion on a external database ?
-     * We do not support that. */
-    this->estimatedResultCount = 0;
-  } else {
-    std::vector<const zim::File*> zims;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      zims.push_back((*current)->getZimFileHandler());
-    }
-    zim::Search* search = new zim::Search(zims);
-    search->set_query(unaccentedSearch);
-    search->set_range(resultStart, resultEnd);
-    search->set_suggestion_mode(true);
-    internal->_search = search;
-    internal->current_iterator = internal->_search->begin();
-    this->estimatedResultCount = internal->_search->get_matches_estimated();
+  std::vector<const zim::File*> zims;
+  for (auto current = this->readers.begin(); current != this->readers.end();
+       current++) {
+    zims.push_back((*current)->getZimFileHandler());
   }
+  zim::Search* search = new zim::Search(zims);
+  search->set_query(unaccentedSearch);
+  search->set_range(resultStart, resultEnd);
+  search->set_suggestion_mode(true);
+  internal->_search = search;
+  internal->current_iterator = internal->_search->begin();
+  this->estimatedResultCount = internal->_search->get_matches_estimated();
 }
 
 /* Return the result count estimation */
@@ -363,46 +310,30 @@ int _Result::get_readerIndex()
 {
   return iterator.get_fileIndex();
 }
-#ifdef ENABLE_CTPP2
 
 string Searcher::getHtml()
 {
-  SimpleVM oSimpleVM(
-      1024, //iIMaxFunctions (default value)
-      4096, //iIMaxArgStackSize (default value)
-      4096, //iIMaxCodeStackSize (default value)
-      10240 * 2 //iIMaxSteps (default*2)
-  );
-
-  // Fill data
-  CDT oData;
-  CDT resultsCDT(CDT::ARRAY_VAL);
+  kainjow::mustache::data results{kainjow::mustache::data::type::list};
 
   this->restart_search();
   Result* p_result = NULL;
   while ((p_result = this->getNextResult())) {
-    CDT result;
-    result["title"] = p_result->get_title();
-    result["url"] = p_result->get_url();
-    result["snippet"] = p_result->get_snippet();
-    result["contentId"] = humanReaderNames[p_result->get_readerIndex()];
-
-    if (p_result->get_size() >= 0) {
-      result["size"] = kiwix::beautifyInteger(p_result->get_size());
-    }
+    kainjow::mustache::data result;
+    result.set("title", p_result->get_title());
+    result.set("url", p_result->get_url());
+    result.set("snippet", p_result->get_snippet());
+    result.set("resultContentId", humanReaderNames[p_result->get_readerIndex()]);
 
     if (p_result->get_wordCount() >= 0) {
-      result["wordCount"] = kiwix::beautifyInteger(p_result->get_wordCount());
+      result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
     }
 
-    resultsCDT.PushBack(result);
+    results.push_back(result);
     delete p_result;
   }
-  this->restart_search();
-  oData["results"] = resultsCDT;
 
   // pages
-  CDT pagesCDT(CDT::ARRAY_VAL);
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
   unsigned int pageStart
       = this->resultStart / this->resultCountPerPage >= 5
@@ -418,48 +349,41 @@ string Searcher::getHtml()
   }
 
   for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
-    CDT page;
-    page["label"] = i + 1;
-    page["start"] = i * this->resultCountPerPage;
-    page["end"] = (i + 1) * this->resultCountPerPage;
+    kainjow::mustache::data page;
+    page.set("label", to_string(i + 1));
+    page.set("start", to_string(i * this->resultCountPerPage));
+    page.set("end", to_string((i + 1) * this->resultCountPerPage));
 
     if (i * this->resultCountPerPage == this->resultStart) {
-      page["selected"] = true;
+      page.set("selected", true);
     }
-
-    pagesCDT.PushBack(page);
+    pages.push_back(page);
   }
-  oData["pages"] = pagesCDT;
 
-  oData["count"] = kiwix::beautifyInteger(this->estimatedResultCount);
-  oData["searchPattern"] = kiwix::encodeDiples(this->searchPattern);
-  oData["searchPatternEncoded"] = urlEncode(this->searchPattern);
-  oData["resultStart"] = this->resultStart + 1;
-  oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount
-                            ? this->estimatedResultCount
-                            : this->resultEnd);
-  oData["resultRange"] = this->resultCountPerPage;
-  oData["resultLastPageStart"]
-      = this->estimatedResultCount > this->resultCountPerPage
-            ? std::round(this->estimatedResultCount / this->resultCountPerPage) * this->resultCountPerPage
-            : 0;
-  oData["protocolPrefix"] = this->protocolPrefix;
-  oData["searchProtocolPrefix"] = this->searchProtocolPrefix;
-  oData["contentId"] = this->contentHumanReadableId;
+  std::string template_str = RESOURCE::search_result_tmpl;
+  kainjow::mustache::mustache tmpl(template_str);
 
-  std::string template_ct2 = RESOURCE::results_ct2;
-  VMStringLoader oLoader(template_ct2.c_str(), template_ct2.size());
+  kainjow::mustache::data allData;
+  allData.set("results", results);
+  allData.set("pages", pages);
+  allData.set("hasResult", this->estimatedResultCount != 0);
+  allData.set("count", kiwix::beautifyInteger(this->estimatedResultCount));
+  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
+  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
+  allData.set("resultStart", to_string(this->resultStart + 1));
+  allData.set("resultEnd", to_string(min(this->resultEnd, this->estimatedResultCount)));
+  allData.set("resultRange", to_string(this->resultCountPerPage));
+  allData.set("resultLastPageStart", to_string(this->estimatedResultCount > this->resultCountPerPage
+             ? round(this->estimatedResultCount / this->resultCountPerPage) * this->resultCountPerPage
+             : 0));
+  allData.set("lastResult", to_string(this->estimatedResultCount));
+  allData.set("protocolPrefix", this->protocolPrefix);
+  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
+  allData.set("contentId", this->contentHumanReadableId);
 
-  FileLogger oLogger(stderr);
-
-  // DEBUG only (write output to stdout)
-  // oSimpleVM.Run(oData, oLoader, stdout, oLogger);
-
-  std::string sResult;
-  oSimpleVM.Run(oData, oLoader, sResult, oLogger);
-
-  return sResult;
+  std::stringstream ss;
+  tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
 }
 
-#endif
 }

src/tools/base64.cpp

@@ -24,7 +24,7 @@
    René Nyffenegger rene.nyffenegger@adp-gmbh.ch
 */
 
-#include <common/base64.h>
+#include <tools/base64.h>
 #include <iostream>
 
 static const std::string base64_chars = 

src/tools/networkTools.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <tools/networkTools.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <curl/curl.h>
+
+#include <sstream>
+#include <iostream>
+#include <stdexcept>
+
+
+size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+  auto str = static_cast<std::stringstream*>(userdata);
+  str->write(ptr, nmemb);
+  return nmemb;
+}
+
+std::string kiwix::download(const std::string& url) {
+  auto curl = curl_easy_init();
+  std::stringstream ss;
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);
+  auto res = curl_easy_perform(curl);
+  if (res != CURLE_OK) {
+    curl_easy_cleanup(curl);
+    throw std::runtime_error("Cannot perform request");
+  }
+  long response_code;
+  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(curl);
+  if (response_code != 200) {
+    throw std::runtime_error("Invalid return code from server");
+  }
+  return ss.str();
+}

src/tools/otherTools.cpp

@@ -0,0 +1,197 @@
+/*
+ * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <tools/otherTools.h>
+#include <map>
+
+static std::map<std::string, std::string> codeisomapping {
+{ "aa", "aar" },
+{ "af", "afr" },
+{ "ak", "aka" },
+{ "am", "amh" },
+{ "ar", "ara" },
+{ "as", "asm" },
+{ "az", "aze" },
+{ "ba", "bak" },
+{ "be", "bel" },
+{ "bg", "bul" },
+{ "bm", "bam" },
+{ "bn", "ben" },
+{ "bo", "bod" },
+{ "br", "bre" },
+{ "bs", "bos" },
+{ "ca", "cat" },
+{ "ce", "che" },
+{ "co", "cos" },
+{ "cs", "ces" },
+{ "cu", "chu" },
+{ "cv", "chv" },
+{ "cy", "cym" },
+{ "da", "dan" },
+{ "de", "deu" },
+{ "dv", "div" },
+{ "dz", "dzo" },
+{ "ee", "ewe" },
+{ "el", "ell" },
+{ "en", "eng" },
+{ "es", "spa" },
+{ "et", "est" },
+{ "eu", "eus" },
+{ "fa", "fas" },
+{ "ff", "ful" },
+{ "fi", "fin" },
+{ "fo", "fao" },
+{ "fr", "fra" },
+{ "fy", "fry" },
+{ "ga", "gle" },
+{ "gd", "gla" },
+{ "gl", "glg" },
+{ "gn", "grn" },
+{ "gu", "guj" },
+{ "gv", "glv" },
+{ "ha", "hau" },
+{ "he", "heb" },
+{ "hi", "hin" },
+{ "hr", "hrv" },
+{ "hu", "hun" },
+{ "hy", "hye" },
+{ "ia", "ina" },
+{ "id", "ind" },
+{ "ig", "ibo" },
+{ "is", "isl" },
+{ "it", "ita" },
+{ "iu", "iku" },
+{ "ja", "jpn" },
+{ "jv", "jav" },
+{ "ka", "kat" },
+{ "ki", "kik" },
+{ "kk", "kaz" },
+{ "kl", "kal" },
+{ "km", "khm" },
+{ "kn", "kan" },
+{ "ko", "kor" },
+{ "ks", "kas" },
+{ "ku", "kur" },
+{ "kw", "cor" },
+{ "ky", "kir" },
+{ "lb", "ltz" },
+{ "lg", "lug" },
+{ "ln", "lin" },
+{ "lo", "lao" },
+{ "lt", "lit" },
+{ "lv", "lav" },
+{ "mg", "mlg" },
+{ "mi", "mri" },
+{ "mi", "mri" },
+{ "mk", "mkd" },
+{ "ml", "mal" },
+{ "mn", "mon" },
+{ "mr", "mar" },
+{ "ms", "msa" },
+{ "mt", "mlt" },
+{ "my", "mya" },
+{ "nb", "nob" },
+{ "ne", "nep" },
+{ "nl", "nld" },
+{ "nn", "nno" },
+{ "no", "nor" },
+{ "ny", "nya" },
+{ "oc", "oci" },
+{ "om", "orm" },
+{ "or", "ori" },
+{ "os", "oss" },
+{ "pa", "pan" },
+{ "pl", "pol" },
+{ "ps", "pus" },
+{ "pt", "por" },
+{ "qu", "que" },
+{ "rm", "roh" },
+{ "rn", "run" },
+{ "ro", "ron" },
+{ "ru", "rus" },
+{ "rw", "kin" },
+{ "sa", "san" },
+{ "sd", "snd" },
+{ "se", "sme" },
+{ "sg", "sag" },
+{ "si", "sin" },
+{ "sk", "slk" },
+{ "sl", "slv" },
+{ "sn", "sna" },
+{ "so", "som" },
+{ "sq", "sqi" },
+{ "sr", "srp" },
+{ "ss", "ssw" },
+{ "st", "sot" },
+{ "sv", "swe" },
+{ "sw", "swa" },
+{ "ta", "tam" },
+{ "te", "tel" },
+{ "tg", "tgk" },
+{ "th", "tha" },
+{ "ti", "tir" },
+{ "tk", "tuk" },
+{ "tl", "tgl" },
+{ "tn", "tsn" },
+{ "to", "ton" },
+{ "tr", "tur" },
+{ "ts", "tso" },
+{ "tt", "tat" },
+{ "ug", "uig" },
+{ "uk", "ukr" },
+{ "ur", "urd" },
+{ "uz", "uzb" },
+{ "ve", "ven" },
+{ "vi", "vie" },
+{ "wa", "wln" },
+{ "wo", "wol" },
+{ "xh", "xho" },
+{ "yo", "yor" },
+{ "zh", "zho" },
+{ "zu", "zul" }
+};
+
+void kiwix::sleep(unsigned int milliseconds)
+{
+#ifdef _WIN32
+  Sleep(milliseconds);
+#else
+  usleep(1000 * milliseconds);
+#endif
+}
+
+
+struct XmlStringWriter: pugi::xml_writer
+{
+  std::string result;
+  virtual void write(const void* data, size_t size){
+    result.append(static_cast<const char*>(data), size);
+  }
+};
+
+std::string kiwix::nodeToString(pugi::xml_node node)
+{
+  XmlStringWriter writer;
+  node.print(writer, "  ");
+  return writer.result;
+}
+
+std::string kiwix::converta2toa3(const std::string& a2code){
+  return codeisomapping.at(a2code);
+}

src/tools/pathTools.cpp

@@ -17,7 +17,7 @@
  * MA 02110-1301, USA.
  */
 
-#include <common/pathTools.h>
+#include <tools/pathTools.h>
 
 #ifdef __APPLE__
 #include <limits.h>
@@ -228,25 +228,20 @@ bool makeDirectory(const string& path)
 string makeTmpDirectory()
 {
 #ifdef _WIN32
-  char cbase[MAX_PATH+1];
-  int base_len = GetTempPath(MAX_PATH+1, cbase);
-  UUID uuid;
-  UuidCreate(&uuid);
-  char* dir_name;
-  UuidToString(&uuid, reinterpret_cast<unsigned char**>(&dir_name));
-  string dir(cbase, base_len);
-  dir += dir_name;
-  _mkdir(dir.c_str());
-  RpcStringFree(reinterpret_cast<unsigned char**>(&dir_name));
+  char cbase[MAX_PATH];
+  char ctmp[MAX_PATH];
+  GetTempPath(MAX_PATH-14, cbase);
+  // This create a file for us, ensure it is unique.
+  // So we need to delete it and create the directory using the same name.
+  GetTempFileName(cbase, "kiwix", 0, ctmp);
+  DeleteFile(ctmp);
+  _mkdir(ctmp);
+  return string(ctmp);
 #else
-  string base = "/tmp";
-  auto _template = base + "/kiwix-lib_XXXXXX";
-  char* _template_array = new char[_template.size()+1];
-  memcpy(_template_array, _template.c_str(), _template.size());
+  char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
   string dir = mkdtemp(_template_array);
-  delete[] _template_array;
-#endif
   return dir;
+#endif
 }
 
 /* Try to create a link and if does not work then make a copy */

src/tools/regexTools.cpp

@@ -17,7 +17,7 @@
  * MA 02110-1301, USA.
  */
 
-#include <common/regexTools.h>
+#include <tools/regexTools.h>
 
 std::map<std::string, icu::RegexMatcher*> regexCache;
 

src/tools/stringTools.cpp

@@ -17,7 +17,7 @@
  * MA 02110-1301, USA.
  */
 
-#include <common/stringTools.h>
+#include <tools/stringTools.h>
 
 #include <unicode/normlzr.h>
 #include <unicode/rep.h>
@@ -57,8 +57,6 @@ std::string kiwix::removeAccents(const std::string& text)
   return unaccentedText;
 }
 
-#ifndef __ANDROID__
-
 /* Prepare integer for display */
 std::string kiwix::beautifyInteger(uint64_t number)
 {
@@ -138,8 +136,6 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-#endif
-
 /* urlEncode() based on javascript encodeURI() &
    encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
 

src/xapian/htmlparse.cc

@@ -1,373 +0,0 @@
-/* htmlparse.cc: simple HTML parser for omega indexer
- *
- * Copyright 1999,2000,2001 BrightStation PLC
- * Copyright 2001 Ananova Ltd
- * Copyright 2002,2006,2007,2008 Olly Betts
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
- * USA
- */
-
-// #include <config.h>
-
-#include "htmlparse.h"
-
-#include <xapian.h>
-
-// #include "utf8convert.h"
-
-#include <algorithm>
-
-#include <ctype.h>
-#include <cstring>
-#include <stdio.h>
-#include <stdlib.h>
-
-using namespace std;
-
-inline void
-lowercase_string(string &str)
-{
-    for (string::iterator i = str.begin(); i != str.end(); ++i) {
-	*i = tolower(static_cast<unsigned char>(*i));
-    }
-}
-
-map<string, unsigned int> HtmlParser::named_ents;
-
-inline static bool
-p_notdigit(char c)
-{
-    return !isdigit(static_cast<unsigned char>(c));
-}
-
-inline static bool
-p_notxdigit(char c)
-{
-    return !isxdigit(static_cast<unsigned char>(c));
-}
-
-inline static bool
-p_notalnum(char c)
-{
-    return !isalnum(static_cast<unsigned char>(c));
-}
-
-inline static bool
-p_notwhitespace(char c)
-{
-    return !isspace(static_cast<unsigned char>(c));
-}
-
-inline static bool
-p_nottag(char c)
-{
-    return !isalnum(static_cast<unsigned char>(c)) &&
-	c != '.' && c != '-' && c != ':'; // ':' for XML namespaces.
-}
-
-inline static bool
-p_whitespacegt(char c)
-{
-    return isspace(static_cast<unsigned char>(c)) || c == '>';
-}
-
-inline static bool
-p_whitespaceeqgt(char c)
-{
-    return isspace(static_cast<unsigned char>(c)) || c == '=' || c == '>';
-}
-
-bool
-HtmlParser::get_parameter(const string & param, string & value)
-{
-    map<string, string>::const_iterator i = parameters.find(param);
-    if (i == parameters.end()) return false;
-    value = i->second;
-    return true;
-}
-
-HtmlParser::HtmlParser()
-{
-    static const struct ent { const char *n; unsigned int v; } ents[] = {
-#include "namedentities.h"
-	{ NULL, 0 }
-    };
-    if (named_ents.empty()) {
-	const struct ent *i = ents;
-	while (i->n) {
-	    named_ents[string(i->n)] = i->v;
-	    ++i;
-	}
-    }
-}
-
-void
-HtmlParser::decode_entities(string &s)
-{
-    // We need a const_iterator version of s.end() - otherwise the
-    // find() and find_if() templates don't work...
-    string::const_iterator amp = s.begin(), s_end = s.end();
-    while ((amp = find(amp, s_end, '&')) != s_end) {
-	unsigned int val = 0;
-	string::const_iterator end, p = amp + 1;
-	if (p != s_end && *p == '#') {
-	    p++;
-	    if (p != s_end && (*p == 'x' || *p == 'X')) {
-		// hex
-		p++;
-		end = find_if(p, s_end, p_notxdigit);
-		sscanf(s.substr(p - s.begin(), end - p).c_str(), "%x", &val);
-	    } else {
-		// number
-		end = find_if(p, s_end, p_notdigit);
-		val = atoi(s.substr(p - s.begin(), end - p).c_str());
-	    }
-	} else {
-	    end = find_if(p, s_end, p_notalnum);
-	    string code = s.substr(p - s.begin(), end - p);
-	    map<string, unsigned int>::const_iterator i;
-	    i = named_ents.find(code);
-	    if (i != named_ents.end()) val = i->second;
-	}
-	if (end < s_end && *end == ';') end++;
-	if (val) {
-	    string::size_type amp_pos = amp - s.begin();
-	    if (val < 0x80) {
-		s.replace(amp_pos, end - amp, 1u, char(val));
-	    } else {
-		// Convert unicode value val to UTF-8.
-		char seq[4];
-		unsigned len = Xapian::Unicode::nonascii_to_utf8(val, seq);
-		s.replace(amp_pos, end - amp, seq, len);
-	    }
-	    s_end = s.end();
-	    // We've modified the string, so the iterators are no longer
-	    // valid...
-	    amp = s.begin() + amp_pos + 1;
-	} else {
-	    amp = end;
-	}
-    }
-}
-
-void
-HtmlParser::parse_html(const string &body)
-{
-    in_script = false;
-
-    parameters.clear();
-    string::const_iterator start = body.begin();
-
-    while (true) {
-	// Skip through until we find an HTML tag, a comment, or the end of
-	// document.  Ignore isolated occurrences of `<' which don't start
-	// a tag or comment.
-	string::const_iterator p = start;
-	while (true) {
-	    p = find(p, body.end(), '<');
-	    if (p == body.end()) break;
-	    unsigned char ch = *(p + 1);
-
-	    // Tag, closing tag, or comment (or SGML declaration).
-	    if ((!in_script && isalpha(ch)) || ch == '/' || ch == '!') break;
-
-	    if (ch == '?') {
-		// PHP code or XML declaration.
-		// XML declaration is only valid at the start of the first line.
-		// FIXME: need to deal with BOMs...
-		if (p != body.begin() || body.size() < 20) break;
-
-		// XML declaration looks something like this:
-		// <?xml version="1.0" encoding="UTF-8"?>
-		if (p[2] != 'x' || p[3] != 'm' || p[4] != 'l') break;
-		if (strchr(" \t\r\n", p[5]) == NULL) break;
-
-		string::const_iterator decl_end = find(p + 6, body.end(), '?');
-		if (decl_end == body.end()) break;
-
-		// Default charset for XML is UTF-8.
-		charset = "UTF-8";
-
-		string decl(p + 6, decl_end);
-		size_t enc = decl.find("encoding");
-		if (enc == string::npos) break;
-
-		enc = decl.find_first_not_of(" \t\r\n", enc + 8);
-		if (enc == string::npos || enc == decl.size()) break;
-
-		if (decl[enc] != '=') break;
-		
-		enc = decl.find_first_not_of(" \t\r\n", enc + 1);
-		if (enc == string::npos || enc == decl.size()) break;
-
-		if (decl[enc] != '"' && decl[enc] != '\'') break;
-
-		char quote = decl[enc++];
-		size_t enc_end = decl.find(quote, enc);
-
-		if (enc != string::npos)
-		    charset = decl.substr(enc, enc_end - enc);
-
-		break;
-	    }
-	    p++;
-	}
-
-	// Process text up to start of tag.
-	if (p > start) {
-	    string text = body.substr(start - body.begin(), p - start);
-	    // convert_to_utf8(text, charset);
-	    decode_entities(text);
-	    process_text(text);
-	}
-
-	if (p == body.end()) break;
-
-	start = p + 1;
-
-	if (start == body.end()) break;
-
-	if (*start == '!') {
-	    if (++start == body.end()) break;
-	    if (++start == body.end()) break;
-	    // comment or SGML declaration
-	    if (*(start - 1) == '-' && *start == '-') {
-		++start;
-		string::const_iterator close = find(start, body.end(), '>');
-		// An unterminated comment swallows rest of document
-		// (like Netscape, but unlike MSIE IIRC)
-		if (close == body.end()) break;
-
-		p = close;
-		// look for -->
-		while (p != body.end() && (*(p - 1) != '-' || *(p - 2) != '-'))
-		    p = find(p + 1, body.end(), '>');
-
-		if (p != body.end()) {
-		    // Check for htdig's "ignore this bit" comments.
-		    if (p - start == 15 && string(start, p - 2) == "htdig_noindex") {
-			string::size_type i;
-			i = body.find("<!--/htdig_noindex-->", p + 1 - body.begin());
-			if (i == string::npos) break;
-			start = body.begin() + i + 21;
-			continue;
-		    }
-		    // If we found --> skip to there.
-		    start = p;
-		} else {
-		    // Otherwise skip to the first > we found (as Netscape does).
-		    start = close;
-		}
-	    } else {
-		// just an SGML declaration, perhaps giving the DTD - ignore it
-		start = find(start - 1, body.end(), '>');
-		if (start == body.end()) break;
-	    }
-	    ++start;
-	} else if (*start == '?') {
-	    if (++start == body.end()) break;
-	    // PHP - swallow until ?> or EOF
-	    start = find(start + 1, body.end(), '>');
-
-	    // look for ?>
-	    while (start != body.end() && *(start - 1) != '?')
-		start = find(start + 1, body.end(), '>');
-
-	    // unterminated PHP swallows rest of document (rather arbitrarily
-	    // but it avoids polluting the database when things go wrong)
-	    if (start != body.end()) ++start;
-	} else {
-	    // opening or closing tag
-	    int closing = 0;
-
-	    if (*start == '/') {
-		closing = 1;
-		start = find_if(start + 1, body.end(), p_notwhitespace);
-	    }
-
-	    p = start;
-	    start = find_if(start, body.end(), p_nottag);
-	    string tag = body.substr(p - body.begin(), start - p);
-	    // convert tagname to lowercase
-	    lowercase_string(tag);
-
-	    if (closing) {
-		closing_tag(tag);
-		if (in_script && tag == "script") in_script = false;
-
-		/* ignore any bogus parameters on closing tags */
-		p = find(start, body.end(), '>');
-		if (p == body.end()) break;
-		start = p + 1;
-	    } else {
-		// FIXME: parse parameters lazily.
-		while (start < body.end() && *start != '>') {
-		    string name, value;
-
-		    p = find_if(start, body.end(), p_whitespaceeqgt);
-
-		    name.assign(body, start - body.begin(), p - start);
-
-		    p = find_if(p, body.end(), p_notwhitespace);
-
-		    start = p;
-		    if (start != body.end() && *start == '=') {
-			start = find_if(start + 1, body.end(), p_notwhitespace);
-
-			p = body.end();
-
-			int quote = *start;
-			if (quote == '"' || quote == '\'') {
-			    start++;
-			    p = find(start, body.end(), quote);
-			}
-
-			if (p == body.end()) {
-			    // unquoted or no closing quote
-			    p = find_if(start, body.end(), p_whitespacegt);
-			}
-			value.assign(body, start - body.begin(), p - start);
-			start = find_if(p, body.end(), p_notwhitespace);
-
-			if (!name.empty()) {
-			    // convert parameter name to lowercase
-			    lowercase_string(name);
-			    // in case of multiple entries, use the first
-			    // (as Netscape does)
-			    parameters.insert(make_pair(name, value));
-			}
-		    }
-		}
-#if 0
-		cout << "<" << tag;
-		map<string, string>::const_iterator x;
-		for (x = parameters.begin(); x != parameters.end(); x++) {
-		    cout << " " << x->first << "=\"" << x->second << "\"";
-		}
-		cout << ">\n";
-#endif
-		opening_tag(tag);
-		parameters.clear();
-
-		// In <script> tags we ignore opening tags to avoid problems
-		// with "a<b".
-		if (tag == "script") in_script = true;
-
-		if (start != body.end() && *start == '>') ++start;
-	    }
-	}
-    }
-}

src/xapian/htmlparse.h

@@ -1,49 +0,0 @@
-/* htmlparse.h: simple HTML parser for omega indexer
- *
- * Copyright 1999,2000,2001 BrightStation PLC
- * Copyright 2002,2006,2008 Olly Betts
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
- * USA
- */
-
-#ifndef OMEGA_INCLUDED_HTMLPARSE_H
-#define OMEGA_INCLUDED_HTMLPARSE_H
-
-#include <string>
-#include <map>
-
-using std::string;
-using std::map;
-
-class HtmlParser {
-	map<string, string> parameters;
-    protected:
-	void decode_entities(string &s);
-	bool in_script;
-	string charset;
-	static map<string, unsigned int> named_ents;
-
-	bool get_parameter(const string & param, string & value);
-    public:
-	virtual void process_text(const string &/*text*/) { }
-	virtual void opening_tag(const string &/*tag*/) { }
-	virtual void closing_tag(const string &/*tag*/) { }
-	virtual void parse_html(const string &text);
-	HtmlParser();
-	virtual ~HtmlParser() { }
-};
-
-#endif // OMEGA_INCLUDED_HTMLPARSE_H

src/xapian/myhtmlparse.cc

@@ -1,302 +0,0 @@
-/* myhtmlparse.cc: subclass of HtmlParser for extracting text.
- *
- * Copyright 1999,2000,2001 BrightStation PLC
- * Copyright 2002,2003,2004,2006,2007,2008 Olly Betts
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
- * USA
- */
-
-// #include <config.h>
-
-#include "myhtmlparse.h"
-
-// #include "utf8convert.h"
-
-#include <ctype.h>
-#include <string.h>
-
-inline void
-lowercase_string(string &str)
-{
-    for (string::iterator i = str.begin(); i != str.end(); ++i) {
-	*i = tolower(static_cast<unsigned char>(*i));
-    }
-}
-
-void
-MyHtmlParser::parse_html(const string &text, const string &charset_,
-			 bool charset_from_meta_)
-{
-    charset = charset_;
-    charset_from_meta = charset_from_meta_;
-    HtmlParser::parse_html(text);
-}
-
-void
-MyHtmlParser::process_text(const string &text)
-{
-    if (!text.empty() && !in_script_tag && !in_style_tag) {
-	string::size_type b = text.find_first_not_of(WHITESPACE);
-	if (b) pending_space = true;
-	while (b != string::npos) {
-	    if (pending_space && !dump.empty()) dump += ' ';
-	    string::size_type e = text.find_first_of(WHITESPACE, b);
-	    pending_space = (e != string::npos);
-	    if (!pending_space) {
-		dump.append(text.data() + b, text.size() - b);
-		return;
-	    }
-	    dump.append(text.data() + b, e - b);
-	    b = text.find_first_not_of(WHITESPACE, e + 1);
-	}
-    }
-}
-
-void
-MyHtmlParser::opening_tag(const string &tag)
-{
-    if (tag.empty()) return;
-    switch (tag[0]) {
-	case 'a':
-	    if (tag == "address") pending_space = true;
-	    break;
-	case 'b':
-	    if (tag == "body") {
-		dump.resize(0);
-		break;
-	    }
-	    if (tag == "blockquote" || tag == "br") pending_space = true;
-	    break;
-	case 'c':
-	    if (tag == "center") pending_space = true;
-	    break;
-	case 'd':
-	    if (tag == "dd" || tag == "dir" || tag == "div" || tag == "dl" ||
-		tag == "dt") pending_space = true;
-	    break;
-	case 'e':
-	    if (tag == "embed") pending_space = true;
-	    break;
-	case 'f':
-	    if (tag == "fieldset" || tag == "form") pending_space = true;
-	    break;
-	case 'h':
-	    // hr, and h1, ..., h6
-	    if (tag.length() == 2 && strchr("r123456", tag[1]))
-		pending_space = true;
-	    break;
-	case 'i':
-	    if (tag == "iframe" || tag == "img" || tag == "isindex" ||
-		tag == "input") pending_space = true;
-	    break;
-	case 'k':
-	    if (tag == "keygen") pending_space = true;
-	    break;
-	case 'l':
-	    if (tag == "legend" || tag == "li" || tag == "listing")
-		pending_space = true;
-	    break;
-	case 'm':
-	    if (tag == "meta") {
-		string content;
-		if (get_parameter("content", content)) {
-		    string name;
-		    if (get_parameter("name", name)) {
-			lowercase_string(name);
-			if (name == "description") {
-			    if (sample.empty()) {
-				swap(sample, content);
-				// convert_to_utf8(sample, charset);
-				decode_entities(sample);
-			    }
-			} else if (name == "keywords") {
-			    if (!keywords.empty()) keywords += ' ';
-			    // convert_to_utf8(content, charset);
-			    decode_entities(content);
-			    keywords += content;
-			} else if (name == "robots") {
-			    decode_entities(content);
-			    lowercase_string(content);
-			    if (content.find("none") != string::npos ||
-				content.find("noindex") != string::npos) {
-				indexing_allowed = false;
-				throw true;
-			    }
-			}
-			break;
-		    }
-		    // If the current charset came from a meta tag, don't
-		    // force reparsing again!
-		    if (charset_from_meta) break;
-		    string hdr;
-		    if (get_parameter("http-equiv", hdr)) {
-			lowercase_string(hdr);
-			if (hdr == "content-type") {
-			    lowercase_string(content);
-			    size_t start = content.find("charset=");
-			    if (start == string::npos) break;
-			    start += 8;
-			    if (start == content.size()) break;
-			    size_t end = start;
-			    if (content[start] != '"') {
-				while (end < content.size()) {
-				    unsigned char ch = content[end];
-				    if (ch <= 32 || ch >= 127 ||
-					strchr(";()<>@,:\\\"/[]?={}", ch))
-					break;
-				    ++end;
-				}
-			    } else {
-				++start;
-				++end;
-				while (end < content.size()) {
-				    unsigned char ch = content[end];
-				    if (ch == '"') break;
-				    if (ch == '\\') content.erase(end, 1);
-				    ++end;
-				}
-			    }
-			    string newcharset(content, start, end - start);
-			    if (charset != newcharset) {
-				throw newcharset;
-			    }
-			}
-		    }
-		    break;
-		}
-		if (charset_from_meta) break;
-		string newcharset;
-		if (get_parameter("charset", newcharset)) {
-		    // HTML5 added: <meta charset="...">
-		    lowercase_string(newcharset);
-		    if (charset != newcharset) {
-			throw newcharset;
-		    }
-		}
-		break;
-	    }
-	    if (tag == "marquee" || tag == "menu" || tag == "multicol")
-		pending_space = true;
-	    break;
-	case 'o':
-	    if (tag == "ol" || tag == "option") pending_space = true;
-	    break;
-	case 'p':
-	    if (tag == "p" || tag == "pre" || tag == "plaintext")
-		pending_space = true;
-	    break;
-	case 'q':
-	    if (tag == "q") pending_space = true;
-	    break;
-	case 's':
-	    if (tag == "style") {
-		in_style_tag = true;
-		break;
-	    }
-	    if (tag == "script") {
-		in_script_tag = true;
-		break;
-	    }
-	    if (tag == "select") pending_space = true;
-	    break;
-	case 't':
-	    if (tag == "table" || tag == "td" || tag == "textarea" ||
-		tag == "th") pending_space = true;
-	    break;
-	case 'u':
-	    if (tag == "ul") pending_space = true;
-	    break;
-	case 'x':
-	    if (tag == "xmp") pending_space = true;
-	    break;
-    }
-}
-
-void
-MyHtmlParser::closing_tag(const string &tag)
-{
-    if (tag.empty()) return;
-    switch (tag[0]) {
-	case 'a':
-	    if (tag == "address") pending_space = true;
-	    break;
-	case 'b':
-	    if (tag == "body") {
-		throw true;
-	    }
-	    if (tag == "blockquote" || tag == "br") pending_space = true;
-	    break;
-	case 'c':
-	    if (tag == "center") pending_space = true;
-	    break;
-	case 'd':
-	    if (tag == "dd" || tag == "dir" || tag == "div" || tag == "dl" ||
-		tag == "dt") pending_space = true;
-	    break;
-	case 'f':
-	    if (tag == "fieldset" || tag == "form") pending_space = true;
-	    break;
-	case 'h':
-	    // hr, and h1, ..., h6
-	    if (tag.length() == 2 && strchr("r123456", tag[1]))
-		pending_space = true;
-	    break;
-	case 'i':
-	    if (tag == "iframe") pending_space = true;
-	    break;
-	case 'l':
-	    if (tag == "legend" || tag == "li" || tag == "listing")
-		pending_space = true;
-	    break;
-	case 'm':
-	    if (tag == "marquee" || tag == "menu") pending_space = true;
-	    break;
-	case 'o':
-	    if (tag == "ol" || tag == "option") pending_space = true;
-	    break;
-	case 'p':
-	    if (tag == "p" || tag == "pre") pending_space = true;
-	    break;
-	case 'q':
-	    if (tag == "q") pending_space = true;
-	    break;
-	case 's':
-	    if (tag == "style") {
-		in_style_tag = false;
-		break;
-	    }
-	    if (tag == "script") {
-		in_script_tag = false;
-		break;
-	    }
-	    if (tag == "select") pending_space = true;
-	    break;
-	case 't':
-	    if (tag == "title") {
-		if (title.empty()) swap(title, dump);
-		break;
-	    }
-	    if (tag == "table" || tag == "td" || tag == "textarea" ||
-		tag == "th") pending_space = true;
-	    break;
-	case 'u':
-	    if (tag == "ul") pending_space = true;
-	    break;
-	case 'x':
-	    if (tag == "xmp") pending_space = true;
-	    break;
-    }
-}

src/xapian/myhtmlparse.h

@@ -1,66 +0,0 @@
-/* myhtmlparse.h: subclass of HtmlParser for extracting text
- *
- * Copyright 1999,2000,2001 BrightStation PLC
- * Copyright 2002,2003,2004,2006,2008 Olly Betts
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
- * USA
- */
-
-#ifndef OMEGA_INCLUDED_MYHTMLPARSE_H
-#define OMEGA_INCLUDED_MYHTMLPARSE_H
-
-#include "htmlparse.h"
-
-// FIXME: Should we include \xa0 which is non-breaking space in iso-8859-1, but
-// not in all charsets and perhaps spans of all \xa0 should become a single
-// \xa0?
-#define WHITESPACE " \t\n\r"
-
-class MyHtmlParser : public HtmlParser {
-    public:
-	bool in_script_tag;
-	bool in_style_tag;
-	bool pending_space;
-	bool indexing_allowed;
-	bool charset_from_meta;
-	string title, sample, keywords, dump;
-	void process_text(const string &text);
-	void opening_tag(const string &tag);
-	void closing_tag(const string &tag);
-        using HtmlParser::parse_html;
-	void parse_html(const string &text, const string &charset_,
-			bool charset_from_meta_);
-	MyHtmlParser() :
-		in_script_tag(false),
-		in_style_tag(false),
-		pending_space(false),
-		indexing_allowed(true),
-		charset_from_meta(false) { }
-
-	void reset() {
-	    in_script_tag = false;
-	    in_style_tag = false;
-	    pending_space = false;
-	    indexing_allowed = true;
-	    charset_from_meta = false;
-	    title.resize(0);
-	    sample.resize(0);
-	    keywords.resize(0);
-	    dump.resize(0);
-	}
-};
-
-#endif // OMEGA_INCLUDED_MYHTMLPARSE_H

src/xapian/namedentities.h

@@ -1,279 +0,0 @@
-/* namedentities.h: named HTML entities.
- *
- * Copyright (C) 2006,2007 Olly Betts
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- */
-
-#ifndef OMEGA_INCLUDED_NAMEDENTITIES_H
-#define OMEGA_INCLUDED_NAMEDENTITIES_H
-
-// Names and values from: "Character entity references in HTML 4"
-// http://www.w3.org/TR/html4/sgml/entities.html
-{ "quot", 34 },
-{ "amp", 38 },
-{ "apos", 39 }, // Not in HTML 4 list but used in OpenOffice XML.
-{ "lt", 60 },
-{ "gt", 62 },
-{ "nbsp", 160 },
-{ "iexcl", 161 },
-{ "cent", 162 },
-{ "pound", 163 },
-{ "curren", 164 },
-{ "yen", 165 },
-{ "brvbar", 166 },
-{ "sect", 167 },
-{ "uml", 168 },
-{ "copy", 169 },
-{ "ordf", 170 },
-{ "laquo", 171 },
-{ "not", 172 },
-{ "shy", 173 },
-{ "reg", 174 },
-{ "macr", 175 },
-{ "deg", 176 },
-{ "plusmn", 177 },
-{ "sup2", 178 },
-{ "sup3", 179 },
-{ "acute", 180 },
-{ "micro", 181 },
-{ "para", 182 },
-{ "middot", 183 },
-{ "cedil", 184 },
-{ "sup1", 185 },
-{ "ordm", 186 },
-{ "raquo", 187 },
-{ "frac14", 188 },
-{ "frac12", 189 },
-{ "frac34", 190 },
-{ "iquest", 191 },
-{ "Agrave", 192 },
-{ "Aacute", 193 },
-{ "Acirc", 194 },
-{ "Atilde", 195 },
-{ "Auml", 196 },
-{ "Aring", 197 },
-{ "AElig", 198 },
-{ "Ccedil", 199 },
-{ "Egrave", 200 },
-{ "Eacute", 201 },
-{ "Ecirc", 202 },
-{ "Euml", 203 },
-{ "Igrave", 204 },
-{ "Iacute", 205 },
-{ "Icirc", 206 },
-{ "Iuml", 207 },
-{ "ETH", 208 },
-{ "Ntilde", 209 },
-{ "Ograve", 210 },
-{ "Oacute", 211 },
-{ "Ocirc", 212 },
-{ "Otilde", 213 },
-{ "Ouml", 214 },
-{ "times", 215 },
-{ "Oslash", 216 },
-{ "Ugrave", 217 },
-{ "Uacute", 218 },
-{ "Ucirc", 219 },
-{ "Uuml", 220 },
-{ "Yacute", 221 },
-{ "THORN", 222 },
-{ "szlig", 223 },
-{ "agrave", 224 },
-{ "aacute", 225 },
-{ "acirc", 226 },
-{ "atilde", 227 },
-{ "auml", 228 },
-{ "aring", 229 },
-{ "aelig", 230 },
-{ "ccedil", 231 },
-{ "egrave", 232 },
-{ "eacute", 233 },
-{ "ecirc", 234 },
-{ "euml", 235 },
-{ "igrave", 236 },
-{ "iacute", 237 },
-{ "icirc", 238 },
-{ "iuml", 239 },
-{ "eth", 240 },
-{ "ntilde", 241 },
-{ "ograve", 242 },
-{ "oacute", 243 },
-{ "ocirc", 244 },
-{ "otilde", 245 },
-{ "ouml", 246 },
-{ "divide", 247 },
-{ "oslash", 248 },
-{ "ugrave", 249 },
-{ "uacute", 250 },
-{ "ucirc", 251 },
-{ "uuml", 252 },
-{ "yacute", 253 },
-{ "thorn", 254 },
-{ "yuml", 255 },
-{ "OElig", 338 },
-{ "oelig", 339 },
-{ "Scaron", 352 },
-{ "scaron", 353 },
-{ "Yuml", 376 },
-{ "fnof", 402 },
-{ "circ", 710 },
-{ "tilde", 732 },
-{ "Alpha", 913 },
-{ "Beta", 914 },
-{ "Gamma", 915 },
-{ "Delta", 916 },
-{ "Epsilon", 917 },
-{ "Zeta", 918 },
-{ "Eta", 919 },
-{ "Theta", 920 },
-{ "Iota", 921 },
-{ "Kappa", 922 },
-{ "Lambda", 923 },
-{ "Mu", 924 },
-{ "Nu", 925 },
-{ "Xi", 926 },
-{ "Omicron", 927 },
-{ "Pi", 928 },
-{ "Rho", 929 },
-{ "Sigma", 931 },
-{ "Tau", 932 },
-{ "Upsilon", 933 },
-{ "Phi", 934 },
-{ "Chi", 935 },
-{ "Psi", 936 },
-{ "Omega", 937 },
-{ "alpha", 945 },
-{ "beta", 946 },
-{ "gamma", 947 },
-{ "delta", 948 },
-{ "epsilon", 949 },
-{ "zeta", 950 },
-{ "eta", 951 },
-{ "theta", 952 },
-{ "iota", 953 },
-{ "kappa", 954 },
-{ "lambda", 955 },
-{ "mu", 956 },
-{ "nu", 957 },
-{ "xi", 958 },
-{ "omicron", 959 },
-{ "pi", 960 },
-{ "rho", 961 },
-{ "sigmaf", 962 },
-{ "sigma", 963 },
-{ "tau", 964 },
-{ "upsilon", 965 },
-{ "phi", 966 },
-{ "chi", 967 },
-{ "psi", 968 },
-{ "omega", 969 },
-{ "thetasym", 977 },
-{ "upsih", 978 },
-{ "piv", 982 },
-{ "ensp", 8194 },
-{ "emsp", 8195 },
-{ "thinsp", 8201 },
-{ "zwnj", 8204 },
-{ "zwj", 8205 },
-{ "lrm", 8206 },
-{ "rlm", 8207 },
-{ "ndash", 8211 },
-{ "mdash", 8212 },
-{ "lsquo", 8216 },
-{ "rsquo", 8217 },
-{ "sbquo", 8218 },
-{ "ldquo", 8220 },
-{ "rdquo", 8221 },
-{ "bdquo", 8222 },
-{ "dagger", 8224 },
-{ "Dagger", 8225 },
-{ "bull", 8226 },
-{ "hellip", 8230 },
-{ "permil", 8240 },
-{ "prime", 8242 },
-{ "Prime", 8243 },
-{ "lsaquo", 8249 },
-{ "rsaquo", 8250 },
-{ "oline", 8254 },
-{ "frasl", 8260 },
-{ "euro", 8364 },
-{ "image", 8465 },
-{ "weierp", 8472 },
-{ "real", 8476 },
-{ "trade", 8482 },
-{ "alefsym", 8501 },
-{ "larr", 8592 },
-{ "uarr", 8593 },
-{ "rarr", 8594 },
-{ "darr", 8595 },
-{ "harr", 8596 },
-{ "crarr", 8629 },
-{ "lArr", 8656 },
-{ "uArr", 8657 },
-{ "rArr", 8658 },
-{ "dArr", 8659 },
-{ "hArr", 8660 },
-{ "forall", 8704 },
-{ "part", 8706 },
-{ "exist", 8707 },
-{ "empty", 8709 },
-{ "nabla", 8711 },
-{ "isin", 8712 },
-{ "notin", 8713 },
-{ "ni", 8715 },
-{ "prod", 8719 },
-{ "sum", 8721 },
-{ "minus", 8722 },
-{ "lowast", 8727 },
-{ "radic", 8730 },
-{ "prop", 8733 },
-{ "infin", 8734 },
-{ "ang", 8736 },
-{ "and", 8743 },
-{ "or", 8744 },
-{ "cap", 8745 },
-{ "cup", 8746 },
-{ "int", 8747 },
-{ "there4", 8756 },
-{ "sim", 8764 },
-{ "cong", 8773 },
-{ "asymp", 8776 },
-{ "ne", 8800 },
-{ "equiv", 8801 },
-{ "le", 8804 },
-{ "ge", 8805 },
-{ "sub", 8834 },
-{ "sup", 8835 },
-{ "nsub", 8836 },
-{ "sube", 8838 },
-{ "supe", 8839 },
-{ "oplus", 8853 },
-{ "otimes", 8855 },
-{ "perp", 8869 },
-{ "sdot", 8901 },
-{ "lceil", 8968 },
-{ "rceil", 8969 },
-{ "lfloor", 8970 },
-{ "rfloor", 8971 },
-{ "lang", 9001 },
-{ "rang", 9002 },
-{ "loz", 9674 },
-{ "spades", 9824 },
-{ "clubs", 9827 },
-{ "hearts", 9829 },
-{ "diams", 9830 },
-
-#endif // OMEGA_INCLUDED_NAMEDENTITIES_H

src/xapianSearcher.cpp

@@ -1,231 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "xapianSearcher.h"
-#include <sys/types.h>
-#include <unicode/locid.h>
-#ifndef _WIN32
-#  include <unistd.h>
-#endif
-#include <zim/article.h>
-#include <zim/error.h>
-#include <zim/file.h>
-#include <zim/zim.h>
-#include "xapian/myhtmlparse.h"
-
-#include <vector>
-
-namespace kiwix
-{
-std::map<std::string, int> read_valuesmap(const std::string& s)
-{
-  std::map<std::string, int> result;
-  std::vector<std::string> elems = split(s, ";");
-  for (std::vector<std::string>::iterator elem = elems.begin();
-       elem != elems.end();
-       elem++) {
-    std::vector<std::string> tmp_elems = split(*elem, ":");
-    result.insert(
-        std::pair<std::string, int>(tmp_elems[0], atoi(tmp_elems[1].c_str())));
-  }
-  return result;
-}
-
-/* Constructor */
-XapianSearcher::XapianSearcher(const string& xapianDirectoryPath,
-                               Reader* reader)
-    : reader(reader)
-{
-  this->openIndex(xapianDirectoryPath);
-}
-
-/* Open Xapian readable database */
-void XapianSearcher::openIndex(const string& directoryPath)
-{
-  this->readableDatabase = Xapian::Database(directoryPath);
-  this->valuesmap
-      = read_valuesmap(this->readableDatabase.get_metadata("valuesmap"));
-  this->language = this->readableDatabase.get_metadata("language");
-  this->stopwords = this->readableDatabase.get_metadata("stopwords");
-  setup_queryParser();
-}
-
-/* Close Xapian writable database */
-void XapianSearcher::closeIndex()
-{
-  return;
-}
-void XapianSearcher::setup_queryParser()
-{
-  queryParser.set_database(readableDatabase);
-  if (!language.empty()) {
-    /* Build ICU Local object to retrieve ISO-639 language code (from
-       ISO-639-3) */
-    icu::Locale languageLocale(language.c_str());
-
-    /* Configuring language base steemming */
-    try {
-      stemmer = Xapian::Stem(languageLocale.getLanguage());
-      queryParser.set_stemmer(stemmer);
-      queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
-    } catch (...) {
-      std::cout << "No steemming for language '" << languageLocale.getLanguage()
-                << "'" << std::endl;
-    }
-  }
-
-  if (!stopwords.empty()) {
-    std::string stopWord;
-    std::istringstream file(this->stopwords);
-    while (std::getline(file, stopWord, '\n')) {
-      this->stopper.add(stopWord);
-    }
-    queryParser.set_stopper(&(this->stopper));
-  }
-}
-
-/* Search strings in the database */
-void XapianSearcher::searchInIndex(string& search,
-                                   const unsigned int resultStart,
-                                   const unsigned int resultEnd,
-                                   const bool verbose)
-{
-  /* Create the query */
-  Xapian::Query query = queryParser.parse_query(search);
-
-  /* Create the enquire object */
-  Xapian::Enquire enquire(this->readableDatabase);
-  enquire.set_query(query);
-
-  /* Get the results */
-  this->results = enquire.get_mset(resultStart, resultEnd - resultStart);
-  this->current_result = this->results.begin();
-}
-
-/* Get next result */
-Result* XapianSearcher::getNextResult()
-{
-  if (this->current_result != this->results.end()) {
-    XapianResult* result = new XapianResult(this, this->current_result);
-    this->current_result++;
-    return result;
-  }
-  return NULL;
-}
-
-void XapianSearcher::restart_search()
-{
-  this->current_result = this->results.begin();
-}
-
-XapianResult::XapianResult(XapianSearcher* searcher,
-                           Xapian::MSetIterator& iterator)
-    : searcher(searcher), iterator(iterator), document(iterator.get_document())
-{
-}
-
-std::string XapianResult::get_url()
-{
-  return document.get_data();
-}
-std::string XapianResult::get_title()
-{
-  if (searcher->valuesmap.empty()) {
-    /* This is the old legacy version. Guess and try */
-    return document.get_value(0);
-  } else if (searcher->valuesmap.find("title") != searcher->valuesmap.end()) {
-    return document.get_value(searcher->valuesmap["title"]);
-  }
-  return "";
-}
-
-int XapianResult::get_score()
-{
-  return iterator.get_percent();
-}
-std::string XapianResult::get_snippet()
-{
-  if (searcher->valuesmap.empty()) {
-    /* This is the old legacy version. Guess and try */
-    std::string stored_snippet = document.get_value(1);
-    if (!stored_snippet.empty()) {
-      return stored_snippet;
-    }
-    /* Let's continue here, and see if we can genenate one */
-  } else if (searcher->valuesmap.find("snippet") != searcher->valuesmap.end()) {
-    return document.get_value(searcher->valuesmap["snippet"]);
-  }
-  /* No reader, no snippet */
-  if (!searcher->reader) {
-    return "";
-  }
-  /* Get the content of the article to generate a snippet.
-     We parse it and use the html dump to avoid remove html tags in the
-     content and be able to nicely cut the text at random place. */
-  MyHtmlParser htmlParser;
-  std::string content = get_content();
-  if (content.empty()) {
-    return content;
-  }
-  try {
-    htmlParser.parse_html(content, "UTF-8", true);
-  } catch (...) {
-  }
-  return searcher->results.snippet(htmlParser.dump, 500);
-}
-
-std::string XapianResult::get_content()
-{
-  if (!searcher->reader) {
-    return "";
-  }
-  auto entry = searcher->reader->getEntryFromEncodedPath(get_url());
-  return entry.getContent();
-}
-
-int XapianResult::get_size()
-{
-  if (searcher->valuesmap.empty()) {
-    /* This is the old legacy version. Guess and try */
-    return document.get_value(2).empty() == true
-               ? -1
-               : atoi(document.get_value(2).c_str());
-  } else if (searcher->valuesmap.find("size") != searcher->valuesmap.end()) {
-    return atoi(document.get_value(searcher->valuesmap["size"]).c_str());
-  }
-  /* The size is never used. Do we really want to get the content and
-     calculate the size ? */
-  return -1;
-}
-
-int XapianResult::get_wordCount()
-{
-  if (searcher->valuesmap.empty()) {
-    /* This is the old legacy version. Guess and try */
-    return document.get_value(3).empty() == true
-               ? -1
-               : atoi(document.get_value(3).c_str());
-  } else if (searcher->valuesmap.find("wordcount")
-             != searcher->valuesmap.end()) {
-    return atoi(document.get_value(searcher->valuesmap["wordcount"]).c_str());
-  }
-  return -1;
-}
-
-}  // Kiwix namespace

src/xmlrpc.h

@@ -3,7 +3,7 @@
 #ifndef KIWIX_XMLRPC_H_
 #define KIWIX_XMLRPC_H_
 
-#include <common/otherTools.h>
+#include <tools/otherTools.h>
 
 namespace kiwix {
 

static/meson.build

@@ -1,15 +1,5 @@
 
-
-ctpp2c = find_program('ctpp2c', required:false)
-
-if ctpp2c.found()
-  search_result_template = custom_target('result_template',
-    input: 'results.tmpl',
-    output: 'results.ct2',
-    command: [intermediate_ctpp2c, ctpp2c, '@INPUT@', '@OUTPUT@']
-  )
-  resources_list = 'resources_list_ctpp2.txt'
-  lib_resources = custom_target('resources',
+lib_resources = custom_target('resources',
     input: 'resources_list.txt',
     output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
     command:[res_compiler,
@@ -17,8 +7,5 @@ if ctpp2c.found()
              '--hfile', '@OUTPUT1@',
              '--source_dir', '@OUTDIR@',
              '@INPUT@'],
-    depends: [search_result_template]
-  )
-else
- lib_resources = []
-endif
+    depend_files: files('search_result.tmpl')
+)

static/resources_list.txt

@@ -1 +1 @@
-results.ct2
+search_result.tmpl

static/search_result.tmpl

@@ -91,68 +91,67 @@
       }
 
     </style>
-    <title>Search: <TMPL_var searchPattern></title>
+    <title>Search: {{searchPattern}}</title>
   </head>
   <body bgcolor="white">
     <div class="header">
-      <TMPL_if results>
+      {{#hasResult}}
         Results
         <b>
-          <TMPL_var resultStart>-<TMPL_var resultEnd>
+          {{resultStart}}-{{resultEnd}}
         </b> of <b>
-          <TMPL_var count>
+          {{count}}
         </b> for <b>
-          <TMPL_var searchPattern>
+          {{searchPattern}}
         </b>
-      <TMPL_else>
-        No results were found for <b><TMPL_var searchPattern></b>
-      </TMPL_if>
+      {{/hasResult}}
+      {{^hasResult}}
+        No results were found for <b>{{searchPattern}}</b>
+      {{/hasResult}}
     </div>
 
     <div class="results">
       <ul>
-        <TMPL_foreach results as result>
+        {{#results}}
           <li>
-            <a href="<TMPL_var protocolPrefix><TMPL_var result.contentId>/<TMPL_var result.url>">
-              <TMPL_var result.title>
+            <a href="{{protocolPrefix}}{{resultContentId}}/{{url}}">
+              {{title}}
             </a>
-            <cite>
-              <TMPL_if result.snippet>
-                <TMPL_var result.snippet>...
-              </TMPL_if>
-            </cite>
-            <TMPL_if wordCount>
-              <div class="informations"><TMPL_var wordCount> words</div>
-            </TMPL_if>
+            {{#snippet}}
+              <cite>{{>snippet}}...</cite>
+            {{/snippet}}
+            {{#wordCount}}
+              <div class="informations">{{wordCount}} words</div>
+            {{/wordCount}}
           </li>
-        </TMPL_foreach>
+        {{/results}}
       </ul>
     </div>
 
     <div class="footer">
       <ul>
-        <TMPL_if (resultLastPageStart>0)>
+        {{#resultLastPageStart}}
           <li>
-            <a href="<TMPL_var searchProtocolPrefix>pattern=<TMPL_var searchPatternEncoded><TMPL_if contentId>&content=<TMPL_var contentId></TMPL_if>&start=0&end=<TMPL_var resultRange>">
+            <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
               ◀
             </a>
           </li>
-        </TMPL_if>
-        <TMPL_foreach pages as page>
+        {{/resultLastPageStart}}
+        {{#pages}}
           <li>
-            <a <TMPL_if page.selected>class="selected"</TMPL_if>
-               href="<TMPL_var searchProtocolPrefix>pattern=<TMPL_var searchPatternEncoded><TMPL_if contentId>&content=<TMPL_var contentId></TMPL_if>&start=<TMPL_var page.start>&end=<TMPL_var page.end>">
-              <TMPL_var page.label>
+            <a {{#selected}}class="selected"{{/selected}}
+               href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
+              {{label}}
             </a>
           </li>
-        </TMPL_foreach>
-        <TMPL_if (resultLastPageStart>0)>
+        {{/pages}}
+        {{#resultLastPageStart}}
           <li>
-            <a href="<TMPL_var searchProtocolPrefix>pattern=<TMPL_var searchPatternEncoded><TMPL_if contentId>&content=<TMPL_var contentId></TMPL_if>&start=<TMPL_var resultLastPageStart>&end=<TMPL_var (resultLastPageStart+resultRange)>">
+            <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
               ▶
             </a>
           </li>
-        </TMPL_if>
+        {{/resultLastPageStart}}
       </ul>
     </div>
   </body>

travis/compile.sh

@@ -35,6 +35,7 @@ then
 else
   export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/x86_64-linux-gnu/pkgconfig
 fi
-meson . build -Dctpp2-install-prefix=${INSTALL_DIR} ${MESON_OPTION}
+export CPPFLAGS="-I${INSTALL_DIR}/include"
+meson . build ${MESON_OPTION}
 cd build
 ninja

travis/install_deps.sh

@@ -3,18 +3,22 @@
 set -e
 
 REPO_NAME=${TRAVIS_REPO_SLUG#*/}
-ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz
+ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.xz
 
 # Ninja
 cd $HOME
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]
 then
-  pip3 install meson==0.43.0
+  pip3 install meson==0.49.2
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip
   unzip ninja-mac.zip ninja
 else
-  pip3 install --user meson==0.43.0
+  wget https://bootstrap.pypa.io/get-pip.py
+  python3.5 get-pip.py --user
+
+  python3.5 -m pip install --user --upgrade pip
+  python3.5 -m pip install --user meson==0.49.2
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
   unzip ninja-linux.zip ninja