Merge pull request #682 from kiwix/fix_test_compilation_windows

They are included by 3dbcbe5 which need them to test permission rights on
posix only.

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install deps
         shell: bash
         run: |
-          ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
+          ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
           wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
       - name: Compile
         shell: bash
@@ -94,7 +94,7 @@ jobs:
       HOME: /home/runner
     runs-on: ubuntu-latest
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
     steps:
     - name: Extract branch name
       shell: bash
@@ -115,7 +115,7 @@ jobs:
     - name: Install deps
       shell: bash
       run: |
-        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
+        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_libkiwix.tar.xz
         wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
     - name: Compile
       shell: bash

.github/workflows/package.yml

@@ -7,7 +7,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        distro: [ubuntu-hirsute, ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
+        distro:
+          - ubuntu-jammy
+          - ubuntu-impish
+          - ubuntu-hirsute
+          - ubuntu-focal
+          - ubuntu-bionic
     steps:
       - uses: actions/checkout@v2
 
@@ -30,18 +35,26 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-groovy
-        if: matrix.distro == 'ubuntu-groovy'
-        name: Build package for ubuntu-groovy
-        id: build-ubuntu-groovy
+      - uses: legoktm/gh-action-build-deb@ubuntu-impish
+        if: matrix.distro == 'ubuntu-impish'
+        name: Build package for ubuntu-impish
+        id: build-ubuntu-impish
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
+        if: matrix.distro == 'ubuntu-hirsute'
+        name: Build package for ubuntu-hirsute
+        id: build-ubuntu-hirsute
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}

.gitignore

@@ -5,3 +5,5 @@ subprojects/googletest-release*
 build/
 .vscode/
 builddir/
+.cache/
+.clangd/

ChangeLog

@@ -1,7 +1,68 @@
 libkiwix 10.0.0
 ===============
 
- * ...
+ This release is huge release.
+ The project has been renamed to libkiwix, it is more coherent with the library name.
+
+ * Server front page :
+  - Use js in the front page to display the available book,
+     using the OPDS stream as source. The front page is now populated only with
+     the visible books and user can search for books. (@MananJethwany #530, #541, #534)
+     (@kelson42 #628)
+  - Revamp css (@MananJethwany #559)
+  - Correctly Convert 3iso language code to 2iso (@juuz0 #672)
+
+ * Server suggestions search :
+  - Add pagination for suggestion search (@maneeshpm #591)
+  - Fix suggestion system (@MananJethwany #498)
+  - Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
+  - The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
+  - Properly handle html encoding of suggestions (@veloman-yunkan #458)
+
+ * Server improvements :
+  - Remove meta endpoints (@mgautier #669)
+  - Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
+  - Add details on 404 error pages (@soumyankar #490)
+  - Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
+  - Better headbar insertion (after charset definition) (@kelson42 #442)
+
+ * New OPDS Stream v2 :
+  - Add a list of categories (@veloman-yunkan)
+  - Support for partial entries (@veloman-yunkan #602)
+  - Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
+  - Add language endpoint to catalog (@veloman-yunkan #553)
+  - Add illustration API to get the illustration of a book (@mgautierfr #645)
+  - OPDS search can now filter books by category (@veloman-yunkan #459)
+
+ * Library improvements :
+  - Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
+  - Properly handle removing of book from the library (@veloman-yunkan #485)
+  - Use xapian to search for books in the library (@veloman-yunkan #460, #488)
+
+ * Added methods/functions :
+  - Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
+  - Add `getVersions` and `printVersions` functions (@kelson42 #665)
+  - Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
+  - Add `get_zimid()` method to the search result (@maneeshpm #510)
+
+ * Various improvements :
+  - Better secret value for aria2c rpc (@juuz0 #666)
+  - Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
+  - Add basic documentation (@mgautierfr #640)
+  - Do not use Reader internally (@maneeshpm #536 #576)
+  - Remove dependency headers from our public headers (@mgautierfr #574)
+  - Downloader now don't write metalink on the filesystem (@kelson42 #502)
+  - Support opening a zim file using a fd (@veloman-yukan #429)
+  - Use C++11 std::thread instead of pthread (@mgautierfr #445)
+  - [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
+  - Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
+  - Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
+  - Better parsing of the counterMap (@kelson42 #437)
+  - Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
+  - Remove deprecated methods (@mgautierfr)
+  - CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
+  - Fix compilation for FreeBSD (@swills g#432)
+  - Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
 
 kiwix-lib 9.4.1
 ===============

README.md

@@ -91,6 +91,14 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
+with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
+
+```bash
+meson . build -Dwrapper=android -Dwerror=false
+ninja -C build
+```
+
 Testing
 -------
 
@@ -146,6 +154,53 @@ cp ninja ../bin
 cd ..
 ```
 
+Custom Index Page
+-----------------
+
+to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
+(note - while using custom html file please mention all external links as absolute path.)
+
+to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
+
+To use JS provided by kiwix-serve you can use the following template to start with ->
+
+```
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title><-- Custom Tittle --></title>
+    <script
+      type="text/javascript"
+      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
+    ></script>
+    <script
+      type="text/javascript"
+      src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
+    ></script>
+    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+  </head>
+  <body>
+  </body>
+</html>
+```
+
+- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
+- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
+- To add language select box add - `<select id="languageFilter"></select>` under body tag.
+- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add search box for books use following form -
+    ```
+        <form id='kiwixSearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
+        <input type="submit" class="searchButton" value="Search"/>
+        </form>
+    ```
+
+
 If you compile manually Libmicrohttpd, you might need to compile it
 without GNU TLS, a bug here will empeach further compilation
 otherwise.

debian/control

@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
  meson,
  pkg-config,
- libzim-dev (>= 6.1.8),
+ libzim-dev (>= 7.2.0),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
@@ -23,7 +23,7 @@ Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
- libzim-dev (>= 6.0.0),
+ libzim-dev (>= 7.2.0),
  libicu-dev,
  libpugixml-dev,
  libcurl4-gnutls-dev,

docs/.gitignore

@@ -0,0 +1,2 @@
+api
+xml

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'libkiwix'
+copyright = '2022, libkiwix-team'
+author = 'libzim-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+    'exhale'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+breathe_projects = {
+    "libkiwix": "./xml"
+}
+breathe_default_project = 'libkiwix'
+
+exhale_args = {
+    "containmentFolder":   "./api",
+    "rootFileName":        "ref_api.rst",
+    "rootFileTitle":       "Reference API",
+    "doxygenStripFromPath":"..",
+    "treeViewIsBootstrap": True,
+    "createTreeView" : True,
+    "exhaleExecutesDoxygen": True,
+    "exhaleDoxygenStdin":    "INPUT = ../include"
+}
+
+primary_domain = 'cpp'
+
+highlight_language = 'cpp'

docs/index.rst

@@ -0,0 +1,14 @@
+.. libkiwix documentation master file, created by
+   sphinx-quickstart on Fri Jul 24 15:40:50 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libzim's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage
+   api/ref_api

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,3 @@
+breathe
+exhale
+sphinx<4

docs/usage.rst

@@ -0,0 +1,17 @@
+Libkiwix programming
+====================
+
+Introduction
+------------
+
+libkiwix is written in C++. To use the library, you need the include files of libkiwix have
+to link against libzim.
+
+Errors are handled with exceptions. When something goes wrong, libzim throws an error,
+which is always derived from std::exception.
+
+All classes are defined in the namespace kiwix.
+
+libkiwix is a set of tools to manage zim files and provide some common functionnality.
+While libkiwix has some wrappers around libzim classes, they are deprecated and will be removed
+in the future.

format_code.sh

@@ -7,6 +7,7 @@ files=(
 "include/common/otherTools.h"
 "include/common/regexTools.h"
 "include/common/networkTools.h"
+"include/common/archiveTools.h"
 "include/manager.h"
 "include/reader.h"
 "include/kiwix.h"
@@ -22,6 +23,7 @@ files=(
 "src/common/pathTools.cpp"
 "src/common/regexTools.cpp"
 "src/common/otherTools.cpp"
+"src/common/archiveTools.cpp"
 "src/common/networkTools.cpp"
 "src/common/stringTools.cpp"
 "src/xapianSearcher.cpp"

include/book.h

@@ -21,11 +21,19 @@
 #define KIWIX_BOOK_H
 
 #include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include "common.h"
 
 namespace pugi {
 class xml_node;
 }
 
+namespace zim {
+class Archive;
+}
+
 namespace kiwix
 {
 
@@ -37,12 +45,32 @@ class Reader;
  */
 class Book
 {
- public:
+ public: // types
+  class Illustration
+  {
+    friend class Book;
+   public:
+    uint16_t width = 48;
+    uint16_t height = 48;
+    std::string mimeType;
+    std::string url;
+
+    const std::string& getData() const;
+
+   private:
+    mutable std::string data;
+    mutable std::mutex mutex;
+  };
+
+  typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
+
+ public: // functions
   Book();
   ~Book();
 
   bool update(const Book& other);
-  void update(const Reader& reader);
+  DEPRECATED void update(const Reader& reader);
+  void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
   std::string getHumanReadableIdFromPath() const;
@@ -68,9 +96,13 @@ class Book
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const;
-  const std::string& getFaviconUrl() const { return m_faviconUrl; }
-  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
+  DEPRECATED const std::string& getFavicon() const;
+  DEPRECATED const std::string& getFaviconUrl() const;
+  DEPRECATED const std::string& getFaviconMimeType() const;
+
+  Illustrations getIllustrations() const;
+  std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
+
   const std::string& getDownloadId() const { return m_downloadId; }
 
   void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
@@ -91,14 +123,13 @@ class Book
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
   void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
   void setSize(uint64_t size) { m_size = size; }
-  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
-  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
   void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
 
- private:
+ private: // functions
   std::string getCategoryFromTags() const;
+  const Illustration& getDefaultIllustration() const;
 
- protected:
+ protected: // data
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
@@ -119,9 +150,11 @@ class Book
   uint64_t m_mediaCount = 0;
   bool m_readOnly = false;
   uint64_t m_size = 0;
-  mutable std::string m_favicon;
-  std::string m_faviconUrl;
-  std::string m_faviconMimeType;
+  Illustrations m_illustrations;
+
+  // Used as the return value of getDefaultIllustration() when no default
+  // illustration is found in the book
+  static const Illustration missingDefaultIllustration;
 };
 
 }

include/entry.h

@@ -47,7 +47,7 @@ class Entry
      *
      * @param article a zim::Article object
      */
-    Entry(zim::Entry entry);
+    DEPRECATED Entry(zim::Entry entry) : Entry(entry, true) {};
     virtual ~Entry() = default;
 
     /**
@@ -176,6 +176,16 @@ class Entry
 
   private:
     zim::Entry entry;
+
+  private:
+    // Entry is deprecated, so we've marked the constructor as deprecated.
+    // But we still need to construct the entry (in our deprecated code)
+    // To avoid warning because we use deprecated function, we create a second
+    // constructor not deprecated. The `bool marker` is unused, it sole purpose
+    // is to change the signature to have two different constructor.
+    // This one is not deprecated and we must use it in our private code.
+    Entry(zim::Entry entry, bool marker);
+    friend class Reader;
 };
 
 }

include/kiwix.h

@@ -22,4 +22,4 @@
 
 #include "library.h"
 
-#endif
+#endif

include/library.h

@@ -24,6 +24,8 @@
 #include <vector>
 #include <map>
 #include <memory>
+#include <mutex>
+#include <zim/archive.h>
 
 #include "book.h"
 #include "bookmark.h"
@@ -138,20 +140,53 @@ private: // functions
     bool accept(const Book& book) const;
 };
 
-
 /**
- * A Library store several books.
+ * This class is not part of the libkiwix API. Its only purpose is
+ * to simplify the implementation of the Library's move operations
+ * and avoid bugs should new data members be added to Library.
  */
-class Library
+class LibraryBase
 {
-  std::map<std::string, kiwix::Book> m_books;
+protected: // types
+  typedef uint64_t LibraryRevision;
+
+  struct Entry : Book
+  {
+    LibraryRevision lastUpdatedRevision = 0;
+
+    // May also keep the Archive and Reader pointers here and get
+    // rid of the m_readers and m_archives data members in Library
+  };
+
+protected: // data
+  LibraryRevision m_revision;
+  std::map<std::string, Entry> m_books;
   std::map<std::string, std::shared_ptr<Reader>> m_readers;
+  std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
   std::vector<kiwix::Bookmark> m_bookmarks;
   class BookDB;
   std::unique_ptr<BookDB> m_bookDB;
 
+protected: // functions
+  LibraryBase();
+  ~LibraryBase();
+
+  LibraryBase(LibraryBase&& );
+  LibraryBase& operator=(LibraryBase&& );
+};
+
+/**
+ * A Library store several books.
+ */
+class Library : private LibraryBase
+{
+  // all data fields must be added in LibraryBase
+  mutable std::mutex m_mutex;
+
  public:
+  typedef LibraryRevision Revision;
   typedef std::vector<std::string> BookIdCollection;
+  typedef std::map<std::string, int> AttributeCounts;
 
  public:
   Library();
@@ -177,6 +212,11 @@ class Library
    */
   bool addBook(const Book& book);
 
+  /**
+   * A self-explanatory alias for addBook()
+   */
+  bool addOrUpdateBook(const Book& book) { return addBook(book); }
+
   /**
    * Add a bookmark to the library.
    *
@@ -193,11 +233,15 @@ class Library
    */
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
+  // XXX: This is a non-thread-safe operation
   const Book& getBookById(const std::string& id) const;
-  Book& getBookById(const std::string& id);
+  // XXX: This is a non-thread-safe operation
   const Book& getBookByPath(const std::string& path) const;
-  Book& getBookByPath(const std::string& path);
-  std::shared_ptr<Reader> getReaderById(const std::string& id);
+
+  Book getBookByIdThreadSafe(const std::string& id) const;
+
+  DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
+  std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
 
   /**
    * Remove a book from the library.
@@ -239,6 +283,13 @@ class Library
    */
   std::vector<std::string> getBooksLanguages() const;
 
+  /**
+   * Get all languagues of the books in the library with counts.
+   *
+   * @return A list of languages with the count of books in each language.
+   */
+  AttributeCounts getBooksLanguagesWithCounts() const;
+
   /**
    * Get all categories of the books in the library.
    *
@@ -274,17 +325,6 @@ class Library
    */
   BookIdCollection getBooksIds() const;
 
-  /**
-   * Filter the library and generate a new one with the keep elements.
-   *
-   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
-   *
-   * @param search List only books with search in the title or description.
-   * @return The list of bookIds corresponding to the query.
-   */
-  DEPRECATED BookIdCollection filter(const std::string& search) const;
-
-
   /**
    * Filter the library and return the id of the keep elements.
    *
@@ -304,43 +344,35 @@ class Library
   void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
 
   /**
-   * List books in the library.
+   * Return the current revision of the library.
    *
-   * @param mode The mode of listing :
-   *             - LOCAL    : list only local books (with a path).
-   *             - REMOTE   : list only remote books (with an url).
-   *             - VALID    : list only valid books (without a path or with a
-   *                          path pointing to a valid zim file).
-   *             - NOLOCAL  : list only books without valid path.
-   *             - NOREMOTE : list only books without url.
-   *             - NOVALID  : list only books not valid.
-   *             - ALL : Do not do any filter (LOCAL or REMOTE)
-   *             - Flags can be combined.
-   * @param sortBy Attribute to sort by the book list.
-   * @param search List only books with search in the title, description.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param maxSize Do not list book bigger than maxSize.
-   *                Set to 0 to cancel this filter.
-   * @return The list of bookIds corresponding to the query.
+   * The revision of the library is updated (incremented by one) only by
+   * the addBook() operation.
+   *
+   * @return Current revision of the library.
    */
-  DEPRECATED BookIdCollection listBooksIds(
-    int supportedListMode = ALL,
-    supportedListSortBy sortBy = UNSORTED,
-    const std::string& search = "",
-    const std::string& language = "",
-    const std::string& creator = "",
-    const std::string& publisher = "",
-    const std::vector<std::string>& tags = {},
-    size_t maxSize = 0) const;
+  LibraryRevision getRevision() const;
+
+  /**
+   * Remove books that have not been updated since the specified revision.
+   *
+   * @param  rev the library revision to use
+   * @return Count of books that were removed by this operation.
+   */
+  uint32_t removeBooksNotUpdatedSince(LibraryRevision rev);
 
   friend class OPDSDumper;
   friend class libXMLDumper;
 
+private: // types
+  typedef const std::string& (Book::*BookStrPropMemFn)() const;
+
 private: // functions
+  AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
+  std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
+  void dropReader(const std::string& bookId);
 };
 
 }

include/manager.h

@@ -26,6 +26,7 @@
 
 #include <string>
 #include <vector>
+#include <memory>
 
 namespace pugi {
 class xml_document;
@@ -34,26 +35,25 @@ class xml_document;
 namespace kiwix
 {
 
-class LibraryManipulator {
- public:
-  virtual ~LibraryManipulator() {}
-  virtual bool addBookToLibrary(Book book) = 0;
-  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
-};
+class LibraryManipulator
+{
+ public: // functions
+  explicit LibraryManipulator(Library* library);
+  virtual ~LibraryManipulator();
 
-class DefaultLibraryManipulator : public LibraryManipulator {
- public:
-  DefaultLibraryManipulator(Library* library) :
-    library(library) {}
-  virtual ~DefaultLibraryManipulator() {}
-  bool addBookToLibrary(Book book) {
-    return library->addBook(book);
-  }
-  void addBookmarkToLibrary(Bookmark bookmark) {
-    library->addBookmark(bookmark);
-  }
- private:
-   kiwix::Library* library;
+  Library& getLibrary() const { return library; }
+
+  bool addBookToLibrary(const Book& book);
+  void addBookmarkToLibrary(const Bookmark& bookmark);
+  uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
+
+ protected: // overrides
+  virtual void bookWasAddedToLibrary(const Book& book);
+  virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
+  virtual void booksWereRemovedFromLibrary();
+
+ private: // data
+  kiwix::Library& library;
 };
 
 /**
@@ -61,10 +61,12 @@ class DefaultLibraryManipulator : public LibraryManipulator {
  */
 class Manager
 {
- public:
-  Manager(LibraryManipulator* manipulator);
-  Manager(Library* library);
-  ~Manager();
+ public: // types
+  typedef std::vector<std::string> Paths;
+
+ public: // functions
+  explicit Manager(LibraryManipulator* manipulator);
+  explicit Manager(Library* library);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
@@ -72,10 +74,22 @@ class Manager
    * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
+   * @param trustLibrary use book metadata coming from XML.
    * @return True if file has been properly parsed.
    */
   bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
+  /**
+   * Sync the contents of the library with one or more `library.xml` files.
+   *
+   * The metadata of the library files is trusted unconditionally.
+   * Any books not present in the input library.xml files are removed
+   * from the library.
+   *
+   * @param paths The (utf8) paths to the `library.xml` files.
+   */
+  void reload(const Paths& paths);
+
   /**
    * Load a library content store in the string.
    *
@@ -150,8 +164,7 @@ class Manager
   uint64_t m_itemsPerPage = 0;
 
  protected:
-  kiwix::LibraryManipulator* manipulator;
-  bool mustDeleteManipulator;
+  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,

include/meson.build

@@ -13,18 +13,9 @@ headers = [
   'search_renderer.h',
   'server.h',
   'kiwixserve.h',
-  'name_mapper.h'
+  'name_mapper.h',
+  'tools.h',
+  'version.h'
 ]
 
 install_headers(headers, subdir:'kiwix')
-
-install_headers(
-  'tools/base64.h',
-  'tools/networkTools.h',
-  'tools/otherTools.h',
-  'tools/pathTools.h',
-  'tools/regexTools.h',
-  'tools/stringTools.h',
-  subdir:'kiwix/tools'
-)
-

include/name_mapper.h

@@ -22,6 +22,8 @@
 
 #include <string>
 #include <map>
+#include <memory>
+#include <mutex>
 
 namespace kiwix
 {
@@ -31,15 +33,15 @@ class Library;
 class NameMapper {
   public:
     virtual ~NameMapper() = default;
-    virtual std::string getNameForId(const std::string& id) = 0;
-    virtual std::string getIdForName(const std::string& name) = 0;
+    virtual std::string getNameForId(const std::string& id) const = 0;
+    virtual std::string getIdForName(const std::string& name) const = 0;
 };
 
 
 class IdNameMapper : public NameMapper {
   public:
-    virtual std::string getNameForId(const std::string& id) { return id; };
-    virtual std::string getIdForName(const std::string& name) { return name; };
+    virtual std::string getNameForId(const std::string& id) const { return id; };
+    virtual std::string getIdForName(const std::string& name) const { return name; };
 };
 
 class HumanReadableNameMapper : public NameMapper {
@@ -50,11 +52,29 @@ class HumanReadableNameMapper : public NameMapper {
   public:
     HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
     virtual ~HumanReadableNameMapper() = default;
-    virtual std::string getNameForId(const std::string& id);
-    virtual std::string getIdForName(const std::string& name);
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
 };
 
+class UpdatableNameMapper : public NameMapper {
+    typedef std::shared_ptr<NameMapper> NameMapperHandle;
+  public:
+    UpdatableNameMapper(Library& library, bool withAlias);
 
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+
+    void update();
+
+  private:
+    NameMapperHandle currentNameMapper() const;
+
+  private:
+    mutable std::mutex mutex;
+    Library& library;
+    NameMapperHandle nameMapper;
+    const bool withAlias;
+};
 
 }
 

include/opds_catalog.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_OPDS_CATALOG_H
+#define KIWIX_OPDS_CATALOG_H
+
+
+#include "library.h"
+
+namespace kiwix
+{
+
+std::string getSearchUrl(const Filter& f);
+
+} // namespace kiwix
+
+#endif // KIWIX_OPDS_CATALOG_H

include/opds_dumper.h

@@ -26,9 +26,6 @@
 
 #include <pugixml.hpp>
 
-#include "tools/base64.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
 #include "library.h"
 #include "reader.h"
 
@@ -62,17 +59,32 @@ class OPDSDumper
    *
    * @param bookIds the ids of the books to include in the feed
    * @param query the query used to obtain the list of book ids
+   * @param partial whether the feed should include partial or complete entries
    * @return The OPDS feed.
    */
-  std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const;
+  std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
+
+  /**
+   * Dump the OPDS complete entry document.
+   *
+   * @param bookId the id of the book
+   * @return The OPDS complete entry document.
+   */
+  std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
 
   /**
    * Dump the categories OPDS feed.
    *
-   * @param categories list of category names
    * @return The OPDS feed.
    */
-  std::string categoriesOPDSFeed(const std::vector<std::string>& categories) const;
+  std::string categoriesOPDSFeed() const;
+
+  /**
+   * Dump the languages OPDS feed.
+   *
+   * @return The OPDS feed.
+   */
+  std::string languagesOPDSFeed() const;
 
   /**
    * Set the id of the library.

include/reader.h

@@ -29,8 +29,6 @@
 #include <string>
 #include "common.h"
 #include "entry.h"
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
 
 using namespace std;
 
@@ -41,26 +39,25 @@ namespace kiwix
  * The SuggestionItem is a helper class that contains the info about a single
  * suggestion item.
  */
-
 class SuggestionItem
 {
   // Functions
-  private:
+  public:
     // Create a sugggestion item.
-    explicit SuggestionItem(std::string title, std::string normalizedTitle,
-                            std::string path, std::string snippet = "") :
+    explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
+                            const std::string& path, const std::string& snippet = "") :
       title(title),
       normalizedTitle(normalizedTitle),
       path(path),
       snippet(snippet) {}
 
   public:
-    const std::string getTitle()           {return title;}
-    const std::string getNormalizedTitle() {return normalizedTitle;}
-    const std::string getPath()            {return path;}
-    const std::string getSnippet()         {return snippet;}
+    const std::string& getTitle() const           { return title;}
+    const std::string& getNormalizedTitle() const { return normalizedTitle;}
+    const std::string& getPath() const            { return path;}
+    const std::string& getSnippet() const         { return snippet;}
 
-    const bool hasSnippet()                {return !snippet.empty();}
+    bool hasSnippet() const                { return !snippet.empty();}
 
   // Data
   private:
@@ -68,13 +65,13 @@ class SuggestionItem
     std::string normalizedTitle;
     std::string path;
     std::string snippet;
-
-  friend class Reader;
 };
 
 /**
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
+ *
+ * Reader is now deprecated. Directly use `zim::Archive`.
  */
 
 using SuggestionsList_t = std::vector<SuggestionItem>;
@@ -90,10 +87,18 @@ class Reader
    *                    unsplitted path as if the file were not splitted
    *                    (.zim extesion).
    */
-  explicit Reader(const string zimFilePath);
+  explicit DEPRECATED Reader(const string zimFilePath);
+
+   /**
+   * Create a Reader to read a zim file given by the Archive.
+   *
+   * @param archive The shared pointer to the Archive object.
+   */
+  explicit DEPRECATED Reader(const std::shared_ptr<zim::Archive> archive)
+   : Reader(archive, true) {};
 #ifndef _WIN32
-  explicit Reader(int fd);
-  Reader(int fd, zim::offset_type offset, zim::size_type size);
+  explicit DEPRECATED Reader(int fd);
+  DEPRECATED Reader(int fd, zim::offset_type offset, zim::size_type size);
 #endif
   ~Reader() = default;
 
@@ -287,16 +292,6 @@ class Reader
    */
   string getScraper() const;
 
-  /**
-   * Get the origId of the zim file.
-   *
-   * The origId is only used in the case of patch zim file and is the Id
-   * of the original zim file.
-   *
-   * @return The origId of the zim file as specified in the zim metadata.
-   */
-  string getOrigId() const;
-
   /**
    * Get the favicon of the zim file.
    *
@@ -488,7 +483,7 @@ class Reader
   zim::Archive* getZimArchive() const;
 
  protected:
-  std::unique_ptr<zim::Archive> zimArchive;
+  std::shared_ptr<zim::Archive> zimArchive;
   std::string zimFilePath;
 
   SuggestionsList_t suggestions;
@@ -496,6 +491,15 @@ class Reader
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;
+
+  // Reader is deprecated, so we've marked the constructor as deprecated.
+  // But we still need to construct the reader (in our deprecated code)
+  // To avoid warning because we use deprecated function, we create a
+  // constructor not deprecated. The `bool marker` is unused, it sole purpose
+  // is to change the signature to have a different constructor.
+  // This one is not deprecated and we must use it in our private code.
+  Reader(const std::shared_ptr<zim::Archive> archive, bool marker);
+  friend class Library;
 };
 }
 

include/search_renderer.h

@@ -21,6 +21,7 @@
 #define KIWIX_SEARCH_RENDERER_H
 
 #include <string>
+#include <zim/search.h>
 
 namespace kiwix
 {
@@ -34,12 +35,26 @@ class SearchRenderer
 {
  public:
   /**
-   * The default constructor.
+   * Construct a SearchRenderer from a Searcher.
    *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
+   * This method is now deprecated. Construct the renderer from a
+   * `zim::SearchResultSet`
+   *
+   * @param searcher The `Searcher` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
    */
-  SearchRenderer(Searcher* searcher, NameMapper* mapper);
+  DEPRECATED SearchRenderer(Searcher* searcher, NameMapper* mapper);
+
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                 unsigned int start, unsigned int estimatedResultCount);
 
   ~SearchRenderer();
 
@@ -74,7 +89,7 @@ class SearchRenderer
 
  protected:
   std::string beautifyInteger(const unsigned int number);
-  Searcher* mp_searcher;
+  zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
   std::string searchContent;
   std::string searchPattern;

include/searcher.h

@@ -29,8 +29,8 @@
 #include <string>
 #include <memory>
 #include <vector>
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
+
+#include <zim/search.h>
 
 using namespace std;
 
@@ -52,9 +52,12 @@ class Result
 };
 
 struct SearcherInternal;
+struct SuggestionInternal;
 /**
  * The Searcher class is reponsible to do different kind of search using the
  * fulltext index.
+ *
+ * The Searcher is now deprecated. Use libzim search feature.
  */
 class Searcher
 {
@@ -62,7 +65,7 @@ class Searcher
   /**
    * The default constructor.
    */
-  Searcher();
+  DEPRECATED Searcher();
 
   ~Searcher();
 
@@ -85,12 +88,12 @@ class Searcher
    *
    * @param search The search query.
    * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
+   * @param maxResultCount Maximum results to get from start (used for pagination).
    * @param verbose print some info on stdout if true.
    */
   void search(const std::string& search,
               unsigned int resultStart,
-              unsigned int resultEnd,
+              unsigned int maxResultCount,
               const bool verbose = false);
 
   /**
@@ -104,12 +107,12 @@ class Searcher
    * @param longitude The longitude of the center point.
    * @param distance The radius of the disc.
    * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
+   * @param maxResultCount Maximum number of results to get from start (used for pagination).
    * @param verbose print some info on stdout if true.
    */
   void geo_search(float latitude, float longitude, float distance,
                   unsigned int resultStart,
-                  unsigned int resultEnd,
+                  unsigned int maxResultCount,
                   const bool verbose = false);
 
   /**
@@ -142,23 +145,29 @@ class Searcher
    */
   unsigned int getEstimatedResultCount();
 
+  /**
+   * Get a SearchResultSet object for current search
+   */
+  zim::SearchResultSet getSearchResultSet();
+
   unsigned int getResultStart() { return resultStart; }
-  unsigned int getResultEnd() { return resultEnd; }
+  unsigned int getMaxResultCount() { return maxResultCount; }
 
  protected:
   std::string beautifyInteger(const unsigned int number);
   void closeIndex();
   void searchInIndex(string& search,
                      const unsigned int resultStart,
-                     const unsigned int resultEnd,
+                     const unsigned int maxResultCount,
                      const bool verbose = false);
 
   std::vector<Reader*> readers;
   std::unique_ptr<SearcherInternal> internal;
+  std::unique_ptr<SuggestionInternal> suggestionInternal;
   std::string searchPattern;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
-  unsigned int resultEnd;
+  unsigned int maxResultCount;
 
  private:
   void reset();

include/server.h

@@ -55,16 +55,20 @@ namespace kiwix
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
+       void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
        void setBlockExternalLinks(bool blockExternalLinks)
         { m_blockExternalLinks = blockExternalLinks; }
-
+       int getPort();
+       std::string getAddress();
+       
      protected:
        Library* mp_library;
        NameMapper* mp_nameMapper;
        std::string m_root = "";
        std::string m_addr = "";
+       std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
        bool m_verbose = false;

include/tools.h

@@ -0,0 +1,220 @@
+/*
+ * Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_TOOLS_H
+#define KIWIX_TOOLS_H
+
+#include <string>
+#include <vector>
+#include <map>
+
+namespace kiwix {
+
+/**
+ * Return the current directory.
+ *
+ * @return the current directory (utf8 encoded)
+ */
+std::string getCurrentDirectory();
+
+/**
+ * Return the data directory.
+ *
+ * The data directory is a directory where to put data (zim files, ...)
+ * It depends of the platform and it may be changed by user using environment variable.
+ *
+ * The resolution order is :
+ * - `KIWIX_DATA_DIR` env variable (if set).
+ * - On Windows :
+ *    . `$APPDATA/kiwix` if $APPDATA is set
+ *    . `$USERPROFILE/kiwix` if $USERPROFILE is set
+ * - Else :
+ *    . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
+ *    . `$HOME/.local/share/kiwx` if $HOWE is set
+ * - current directory
+ *
+ * @return the path of the data directory (utf8 encoded)
+ */
+std::string getDataDirectory();
+
+/** Return the path of the executable
+ *
+ * Some application may be packaged in auto extractible archive (Appimage) and the
+ * real executable is different of the path of the archive.
+ * If `realPathOnly` is true, return the path of the real executable instead of the
+ * archive launched by the user.
+ *
+ * @param realPathOnly If we must return the real path of the executable.
+ * @return the path of the executable (utf8 encoded)
+ */
+std::string getExecutablePath(bool realPathOnly = false);
+
+/** Tell if the path is a relative path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path A utf8 encoded path.
+ * @return true if the path is relative.
+ */
+bool isRelativePath(const std::string& path);
+
+/** Append a path to another one.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param basePath the base path.
+ * @param relativePath a path to add to the base path, must be a relative path.
+ * @return The concatenation of the paths, using the right separator.
+ */
+std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
+
+/** Remove the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The parent directory (or empty string if none).
+ */
+std::string removeLastPathElement(const std::string& path);
+
+/** Get the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The base name of the path or empty string if none (ending with a separator).
+ */
+std::string getLastPathElement(const std::string& path);
+
+/** Compute the absolute path of a relative path based on another one
+ *
+ * Equivalent to appendToDirectory followed by a normalization of the path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path (if empty, current directory is taken).
+ * @param relativePath the relative path.
+ * @return a absolute path.
+ */
+std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
+
+/** Compute the relative path of a path relative to another one
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path.
+ * @param absolutePath the absolute path to find the relative path for.
+ * @return a relative path (pointing to absolutePath, relative to path).
+ */
+std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
+
+/** Sleep the current thread.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * @param milliseconds The number of milliseconds to wait for.
+ */
+void sleep(unsigned int milliseconds);
+
+/** Split a string
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * Assuming text = "foo:;bar;baz,oups;"
+ *
+ * split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
+ * split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
+ * split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
+ * split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
+ *
+ * @param str The string to split.
+ * @param delims A string of potential delimiters.
+ *               Each charater in the string can be a individual delimiters.
+ * @param dropEmpty true if empty part must be dropped from the result.
+ * @param keepDelim true if delimiter must be included from the result.
+ * @return a list of part (potentially containing delimiters)
+ */
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
+
+/** Convert language code from iso2 code to iso3
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate locales.
+ *
+ * @param a2code a iso2 code string.
+ * @return the corresponding iso3 code.
+ * @throw std::out_of_range if iso2 code is not known.
+ */
+std::string converta2toa3(const std::string& a2code);
+
+/** Extracts content from given file.
+ *
+ * This function provides content of a file provided it's path.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Content of corresponding file in string format.
+ */
+std::string getFileContent(const std::string& path);
+
+/** Checks if file exists.
+ *
+ * This function returns boolean stating if file exists.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file exists or not.
+ */
+bool fileExists(const std::string& path);
+
+/** Checks if file is readable.
+ *
+ * This function returns boolean stating if file is readable.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file is readale or not.
+ */
+bool fileReadable(const std::string& path);
+
+/** Provides mimetype from filename.
+ *
+ * This function provides mimetype from file-name.
+ *
+ * @param filename string containing filename.
+ * @return mimetype from filename in string format.
+ */
+std::string getMimeTypeForFile(const std::string& filename);
+
+/** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 network interfaces
+ */
+std::map<std::string, std::string> getNetworkInterfaces();
+
+/** Provides the best IP address
+ * This function provides the best IP address from the list given by getNetworkInterfaces
+ */
+std::string getBestPublicIp();
+
+}
+#endif // KIWIX_TOOLS_H

include/version.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_VERSION_H
+#define KIWIX_VERSION_H
+
+#include <string>
+#include <vector>
+
+namespace kiwix
+{
+  typedef std::vector<std::pair<std::string, std::string>> LibVersions;
+  LibVersions getVersions();
+  void printVersions(std::ostream& out = std::cout);
+}
+
+#endif // KIWIX_VERSION_H

meson.build

@@ -44,7 +44,7 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif
@@ -65,7 +65,7 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microh
 inc = include_directories('include', extra_include)
 
 conf = configuration_data()
-conf.set('VERSION', '"@0@"'.format(meson.project_version()))
+conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -78,6 +78,9 @@ subdir('scripts')
 subdir('static')
 subdir('src')
 subdir('test')
+if get_option('doc')
+  subdir('docs')
+endif
 
 pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
 

meson_options.txt

@@ -1,2 +1,4 @@
 option('wrapper', type:'array', choices:['java', 'android'], value:[],
   description: 'The wrapper to generate.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')

src/aria2.cpp

@@ -3,13 +3,15 @@
 #include "aria2.h"
 #include "xmlrpc.h"
 #include <iostream>
+#include <algorithm>
 #include <sstream>
 #include <thread>
 #include <chrono>
-#include <tools/otherTools.h>
-#include <tools/pathTools.h>
-#include <tools/stringTools.h>
-#include <downloader.h> // For AriaError
+#include "tools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "downloader.h" // For AriaError
 
 #ifdef _WIN32
 # define ARIA2_CMD "aria2c.exe"
@@ -30,7 +32,7 @@ namespace kiwix {
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret("kiwixariarpc"),
+  m_secret(getNewRpcSecret()),
   m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
   mp_curl(nullptr)
 {
@@ -68,7 +70,7 @@ Aria2::Aria2():
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
   callCmd.push_back(download_dir.c_str());
-  if (fileExists(session_file)) {
+  if (fileReadable(session_file)) {
     callCmd.push_back(inputFile.c_str());
   }
   callCmd.push_back(session.c_str());
@@ -195,6 +197,13 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
   return doRequest(methodCall);
 }
 
+std::string Aria2::getNewRpcSecret()
+{
+  std::string uuid = gen_uuid("");
+  uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
+  return uuid.substr(0, 9);
+}
+
 std::vector<std::string> Aria2::tellActive()
 {
   MethodCall methodCall("aria2.tellActive", m_secret);

src/aria2.h

@@ -37,6 +37,7 @@ class Aria2
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    static std::string getNewRpcSecret();
     std::vector<std::string> tellActive();
     std::vector<std::string> tellWaiting();
     void saveSession();

src/book.cpp

@@ -20,10 +20,16 @@
 #include "book.h"
 #include "reader.h"
 
+#include "tools.h"
 #include "tools/base64.h"
 #include "tools/regexTools.h"
 #include "tools/networkTools.h"
 #include "tools/otherTools.h"
+#include "tools/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/archiveTools.h"
+
+#include <zim/archive.h>
 
 #include <pugixml.hpp>
 
@@ -35,11 +41,17 @@ Book::Book() :
   m_readOnly(false)
 {
 }
+
 /* Destructor */
 Book::~Book()
 {
 }
 
+Book::Illustrations Book::getIllustrations() const
+{
+  return m_illustrations;
+}
+
 bool Book::update(const kiwix::Book& other)
 {
   if (m_readOnly)
@@ -48,55 +60,43 @@ bool Book::update(const kiwix::Book& other)
   if (m_id != other.m_id)
     return false;
 
-  m_readOnly = other.m_readOnly;
-  m_path = other.m_path;
-  m_pathValid = other.m_pathValid;
-  m_title = other.m_title;
-  m_description = other.m_description;
-  m_language = other.m_language;
-  m_creator = other.m_creator;
-  m_publisher = other.m_publisher;
-  m_date = other.m_date;
-  m_url = other.m_url;
-  m_name = other.m_name;
-  m_flavour = other.m_flavour;
-  m_tags = other.m_tags;
-  m_category = other.m_category;
-  m_origId = other.m_origId;
-  m_articleCount = other.m_articleCount;
-  m_mediaCount = other.m_mediaCount;
-  m_size = other.m_size;
-  m_favicon = other.m_favicon;
-  m_faviconMimeType = other.m_faviconMimeType;
-  m_faviconUrl = other.m_faviconUrl;
-
-  m_downloadId = other.m_downloadId;
-
+  *this = other;
   return true;
 }
 
 void Book::update(const kiwix::Reader& reader)
 {
-  m_path = reader.getZimFilePath();
-  m_pathValid = true;
-  m_id = reader.getId();
-  m_title = reader.getTitle();
-  m_description = reader.getDescription();
-  m_language = reader.getLanguage();
-  m_creator = reader.getCreator();
-  m_publisher = reader.getPublisher();
-  m_date = reader.getDate();
-  m_name = reader.getName();
-  m_flavour = reader.getFlavour();
-  m_tags = reader.getTags();
-  m_category = getCategoryFromTags();
-  m_origId = reader.getOrigId();
-  m_articleCount = reader.getArticleCount();
-  m_mediaCount = reader.getMediaCount();
-  m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
-  m_pathValid = true;
+  update(*reader.getZimArchive());
+}
 
-  reader.getFavicon(m_favicon, m_faviconMimeType);
+void Book::update(const zim::Archive& archive) {
+  m_path = archive.getFilename();
+  m_pathValid = true;
+  m_id = getArchiveId(archive);
+  m_title = getArchiveTitle(archive);
+  m_description = getMetaDescription(archive);
+  m_language = getMetaLanguage(archive);
+  m_creator = getMetaCreator(archive);
+  m_publisher = getMetaPublisher(archive);
+  m_date = getMetaDate(archive);
+  m_name = getMetaName(archive);
+  m_flavour = getMetaFlavour(archive);
+  m_tags = getMetaTags(archive);
+  m_category = getCategoryFromTags();
+  m_articleCount = archive.getArticleCount();
+  m_mediaCount = getArchiveMediaCount(archive);
+  m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
+
+  m_illustrations.clear();
+  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+    const auto illustration = std::make_shared<Illustration>();
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
+    illustration->width = illustration->height = illustrationSize;
+    illustration->mimeType = illustrationItem.getMimetype();
+    illustration->data = illustrationItem.getData();
+    // NOTE: illustration->url is left uninitialized
+    m_illustrations.push_back(illustration);
+  }
 }
 
 #define ATTR(name) node.attribute(name).value()
@@ -108,7 +108,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  m_pathValid = fileExists(path);
+  m_pathValid = fileReadable(path);
   m_title = ATTR("title");
   m_description = ATTR("description");
   m_language = ATTR("language");
@@ -123,9 +123,14 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  m_favicon = base64_decode(ATTR("favicon"));
-  m_faviconMimeType = ATTR("faviconMimeType");
-  m_faviconUrl = ATTR("faviconUrl");
+  std::string favicon_mimetype = ATTR("faviconMimeType");
+  if (! favicon_mimetype.empty()) {
+    const auto favicon = std::make_shared<Illustration>();
+    favicon->data = base64_decode(ATTR("favicon"));
+    favicon->mimeType = favicon_mimetype;
+    favicon->url = ATTR("faviconUrl");
+    m_illustrations.assign(1, favicon);
+  }
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -173,8 +178,11 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
       m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
     }
     if (rel == "http://opds-spec.org/image/thumbnail") {
-      m_faviconUrl = urlHost + linkNode.attribute("href").value();
-      m_faviconMimeType = linkNode.attribute("type").value();
+      const auto favicon = std::make_shared<Illustration>();
+      favicon->data.clear();
+      favicon->url = urlHost + linkNode.attribute("href").value();
+      favicon->mimeType = linkNode.attribute("type").value();
+      m_illustrations.assign(1, favicon);
     }
  }
 
@@ -185,7 +193,7 @@ std::string Book::getHumanReadableIdFromPath() const
 {
   std::string id = m_path;
   if (!id.empty()) {
-    kiwix::removeAccents(id);
+    id = kiwix::removeAccents(id);
 
 #ifdef _WIN32
     id = replaceRegex(id, "", "^.*\\\\");
@@ -207,15 +215,54 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-const std::string& Book::getFavicon() const {
-  if (m_favicon.empty() && !m_faviconUrl.empty()) {
-    try {
-      m_favicon = download(m_faviconUrl);
-    } catch(...) {
-      std::cerr << "Cannot download favicon from " << m_faviconUrl;
+const Book::Illustration Book::missingDefaultIllustration;
+
+std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
+{
+  for ( const auto& ilPtr : m_illustrations ) {
+    if (ilPtr->width == size && ilPtr->height == size) {
+      return ilPtr;
     }
   }
-  return m_favicon;
+  throw std::runtime_error("Cannot find illustration");
+}
+
+const Book::Illustration& Book::getDefaultIllustration() const
+{
+  try {
+    return *getIllustration(48);
+  } catch (...) {
+    return missingDefaultIllustration;
+  }
+}
+
+const std::string& Book::Illustration::getData() const
+{
+  if (data.empty() && !url.empty()) {
+    const std::lock_guard<std::mutex> l(mutex);
+    if ( data.empty() ) {
+      try {
+        data = download(url);
+      } catch(...) {
+        std::cerr << "Cannot download favicon from " << url;
+      }
+    }
+  }
+  return data;
+}
+
+const std::string& Book::getFavicon() const {
+  return getDefaultIllustration().getData();
+}
+
+const std::string& Book::getFaviconUrl() const
+{
+  return getDefaultIllustration().url;
+}
+
+const std::string& Book::getFaviconMimeType() const
+{
+  return getDefaultIllustration().mimeType;
 }
 
 std::string Book::getTagStr(const std::string& tagName) const {

src/config.h.in

@@ -1,3 +1,3 @@
 
-#mesondefine VERSION
+#mesondefine LIBKIWIX_VERSION
 

src/entry.cpp

@@ -23,7 +23,7 @@
 namespace kiwix
 {
 
-Entry::Entry(zim::Entry entry)
+Entry::Entry(zim::Entry entry, bool _marker)
   : entry(entry)
 {
 }
@@ -53,7 +53,7 @@ Entry Entry::getRedirectEntry() const
     throw NoEntry();
   }
 
-  return entry.getRedirectEntry();
+  return Entry(entry.getRedirectEntry(), true);
 }
 
 Entry Entry::getFinalEntry() const
@@ -67,7 +67,7 @@ Entry Entry::getFinalEntry() const
   if (final_entry.isRedirect()) {
     throw NoEntry();
   }
-  return final_entry;
+  return Entry(final_entry, true);
 }
 
 }

src/kiwixserve.cpp

@@ -9,6 +9,7 @@
     # include <unistd.h>
 #endif
 
+#include "tools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
 

src/library.cpp

@@ -22,6 +22,7 @@
 #include "reader.h"
 #include "libxml_dumper.h"
 
+#include "tools.h"
 #include "tools/base64.h"
 #include "tools/regexTools.h"
 #include "tools/pathTools.h"
@@ -48,23 +49,48 @@ std::string normalizeText(const std::string& text)
   return removeAccents(text);
 }
 
+bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
+{
+  return book1.isPathValid()
+      && book2.isPathValid()
+      && book1.getPath() == book2.getPath();
+}
+
 } // unnamed namespace
 
-class Library::BookDB : public Xapian::WritableDatabase
+class LibraryBase::BookDB : public Xapian::WritableDatabase
 {
 public:
   BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
 };
 
-/* Constructor */
-Library::Library()
+LibraryBase::LibraryBase()
   : m_bookDB(new BookDB)
 {
 }
 
-Library::Library(Library&& ) = default;
+LibraryBase::~LibraryBase()
+{
+}
 
-Library& Library::operator=(Library&& ) = default;
+LibraryBase::LibraryBase(LibraryBase&& ) = default;
+LibraryBase& LibraryBase::operator=(LibraryBase&& ) = default;
+
+/* Constructor */
+Library::Library()
+{
+}
+
+Library::Library(Library&& other)
+  : LibraryBase(std::move(other))
+{
+}
+
+Library& Library::operator=(Library&& other)
+{
+  LibraryBase::operator=(std::move(other));
+  return *this;
+}
 
 /* Destructor */
 Library::~Library()
@@ -74,25 +100,37 @@ Library::~Library()
 
 bool Library::addBook(const Book& book)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
+  ++m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
     auto& oldbook = m_books.at(book.getId());
-    oldbook.update(book);
+    if ( ! booksReferToTheSameArchive(oldbook, book) ) {
+      dropReader(book.getId());
+    }
+    oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
+                          // XXX: Then m_bookDB will become out-of-sync with
+                          // XXX: the real contents of the library.
+    oldbook.lastUpdatedRevision = m_revision;
     return false;
   } catch (std::out_of_range&) {
-    m_books[book.getId()] = book;
+    Entry& newEntry = m_books[book.getId()];
+    static_cast<Book&>(newEntry) = book;
+    newEntry.lastUpdatedRevision = m_revision;
     return true;
   }
 }
 
 void Library::addBookmark(const Bookmark& bookmark)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
       m_bookmarks.erase(it);
@@ -103,27 +141,63 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
+void Library::dropReader(const std::string& id)
+{
+  m_readers.erase(id);
+  m_archives.erase(id);
+}
 
 bool Library::removeBookById(const std::string& id)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   m_bookDB->delete_document("Q" + id);
-  m_readers.erase(id);
+  dropReader(id);
   return m_books.erase(id) == 1;
 }
 
+Library::Revision Library::getRevision() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return m_revision;
+}
+
+uint32_t Library::removeBooksNotUpdatedSince(LibraryRevision libraryRevision)
+{
+  BookIdCollection booksToRemove;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    for ( const auto& entry : m_books) {
+      if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
+        booksToRemove.push_back(entry.first);
+      }
+    }
+  }
+
+  uint32_t countOfRemovedBooks = 0;
+  for ( const auto& id : booksToRemove ) {
+    if ( removeBookById(id) )
+      ++countOfRemovedBooks;
+  }
+  return countOfRemovedBooks;
+}
+
 const Book& Library::getBookById(const std::string& id) const
 {
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
   return m_books.at(id);
 }
 
-Book& Library::getBookById(const std::string& id)
+Book Library::getBookByIdThreadSafe(const std::string& id) const
 {
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookById(id));
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return getBookById(id);
 }
 
 const Book& Library::getBookByPath(const std::string& path) const
 {
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
   for(auto& it: m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
@@ -134,29 +208,43 @@ const Book& Library::getBookByPath(const std::string& path) const
   throw std::out_of_range(ss.str());
 }
 
-Book& Library::getBookByPath(const std::string& path)
-{
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookByPath(path));
-}
-
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {
+    std::lock_guard<std::mutex> lock(m_mutex);
     return m_readers.at(id);
   } catch (std::out_of_range& e) {}
 
+  const auto archive = getArchiveById(id);
+  if ( !archive )
+    return nullptr;
+
+  const shared_ptr<Reader> reader(new Reader(archive, true));
+  std::lock_guard<std::mutex> lock(m_mutex);
+  m_readers[id] = reader;
+  return reader;
+}
+
+std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  try {
+    return m_archives.at(id);
+  } catch (std::out_of_range& e) {}
+
   auto book = getBookById(id);
   if (!book.isPathValid())
     return nullptr;
-  auto sptr = make_shared<Reader>(book.getPath());
-  m_readers[id] = sptr;
+
+  auto sptr = make_shared<zim::Archive>(book.getPath());
+  m_archives[id] = sptr;
   return sptr;
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   unsigned int result = 0;
   for (auto& pair: m_books) {
     auto& book = pair.second;
@@ -170,39 +258,63 @@ unsigned int Library::getBookCount(const bool localBooks,
 
 bool Library::writeToFile(const std::string& path) const
 {
+  const auto allBookIds = getBooksIds();
+
   auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
   dumper.setBaseDir(baseDir);
-  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
+  std::string xml;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    xml = dumper.dumpLibXMLContent(allBookIds);
+  };
+  return writeTextFile(path, xml);
 }
 
 bool Library::writeBookmarksToFile(const std::string& path) const
 {
   LibXMLDumper dumper(this);
-  return writeTextFile(path, dumper.dumpLibXMLBookmark());
+  // NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
+  const std::string xml = dumper.dumpLibXMLBookmark();
+  return writeTextFile(path, xml);
+}
+
+Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  AttributeCounts propValueCounts;
+
+  for (const auto& pair: m_books) {
+    const auto& book = pair.second;
+    if (book.getOrigId().empty()) {
+      propValueCounts[(book.*p)()] += 1;
+    }
+  }
+  return propValueCounts;
+}
+
+std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
+{
+  std::vector<std::string> result;
+  for ( const auto& kv : getBookAttributeCounts(p) ) {
+    result.push_back(kv.first);
+  }
+  return result;
 }
 
 std::vector<std::string> Library::getBooksLanguages() const
 {
-  std::vector<std::string> booksLanguages;
-  std::map<std::string, bool> booksLanguagesMap;
+  return getBookPropValueSet(&Book::getLanguage);
+}
 
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    auto& language = book.getLanguage();
-    if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksLanguagesMap[language] = true;
-        booksLanguages.push_back(language);
-      }
-    }
-  }
-
-  return booksLanguages;
+Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
+{
+  return getBookAttributeCounts(&Book::getLanguage);
 }
 
 std::vector<std::string> Library::getBooksCategories() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   std::set<std::string> categories;
 
   for (const auto& pair: m_books) {
@@ -218,40 +330,12 @@ std::vector<std::string> Library::getBooksCategories() const
 
 std::vector<std::string> Library::getBooksCreators() const
 {
-  std::vector<std::string> booksCreators;
-  std::map<std::string, bool> booksCreatorsMap;
-
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    auto& creator = book.getCreator();
-    if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksCreatorsMap[creator] = true;
-        booksCreators.push_back(creator);
-      }
-    }
-  }
-
-  return booksCreators;
+  return getBookPropValueSet(&Book::getCreator);
 }
 
 std::vector<std::string> Library::getBooksPublishers() const
 {
-  std::vector<std::string> booksPublishers;
-  std::map<std::string, bool> booksPublishersMap;
-
-  for (auto& pair:m_books) {
-    auto& book = pair.second;
-    auto& publisher = book.getPublisher();
-    if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksPublishersMap[publisher] = true;
-        booksPublishers.push_back(publisher);
-      }
-    }
-  }
-
-  return booksPublishers;
+  return getBookPropValueSet(&Book::getPublisher);
 }
 
 const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
@@ -261,6 +345,7 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
+  std::lock_guard<std::mutex> lock(m_mutex);
   for(auto& bookmark:m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
@@ -271,6 +356,7 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
 
 Library::BookIdCollection Library::getBooksIds() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
   for (auto& pair: m_books) {
@@ -280,15 +366,6 @@ Library::BookIdCollection Library::getBooksIds() const
   return bookIds;
 }
 
-Library::BookIdCollection Library::filter(const std::string& search) const
-{
-  if (search.empty()) {
-    return getBooksIds();
-  }
-
-  return filter(Filter().query(search));
-}
-
 
 void Library::updateBookDB(const Book& book)
 {
@@ -460,6 +537,7 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 
   BookIdCollection bookIds;
 
+  std::lock_guard<std::mutex> lock(m_mutex);
   Xapian::Enquire enquire(*m_bookDB);
   enquire.set_query(query);
   const auto results = enquire.get_mset(0, m_books.size());
@@ -473,7 +551,9 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 Library::BookIdCollection Library::filter(const Filter& filter) const
 {
   BookIdCollection result;
-  for(auto id : filterViaBookDB(filter)) {
+  const auto preliminaryResult = filterViaBookDB(filter);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto id : preliminaryResult) {
     if(filter.accept(m_books.at(id))) {
       result.push_back(id);
     }
@@ -542,6 +622,11 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
 
 void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
 {
+  // NOTE: Can reimplement this method in a way that doesn't require locking
+  // NOTE: for the entire duration of the sort. Will need to obtain (under a
+  // NOTE: lock) the required atributes from the books once, and then the
+  // NOTE: sorting will run on a copy of data without locking.
+  std::lock_guard<std::mutex> lock(m_mutex);
   switch(sort) {
     case TITLE:
       std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
@@ -564,48 +649,6 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
 }
 
 
-Library::BookIdCollection Library::listBooksIds(
-    int mode,
-    supportedListSortBy sortBy,
-    const std::string& search,
-    const std::string& language,
-    const std::string& creator,
-    const std::string& publisher,
-    const std::vector<std::string>& tags,
-    size_t maxSize) const {
-
-  Filter _filter;
-  if (mode & LOCAL)
-    _filter.local(true);
-  if (mode & NOLOCAL)
-    _filter.local(false);
-  if (mode & VALID)
-    _filter.valid(true);
-  if (mode & NOVALID)
-    _filter.valid(false);
-  if (mode & REMOTE)
-    _filter.remote(true);
-  if (mode & NOREMOTE)
-    _filter.remote(false);
-  if (!tags.empty())
-    _filter.acceptTags(tags);
-  if (maxSize != 0)
-    _filter.maxSize(maxSize);
-  if (!language.empty())
-    _filter.lang(language);
-  if (!publisher.empty())
-    _filter.publisher(publisher);
-  if (!creator.empty())
-    _filter.creator(creator);
-  if (!search.empty())
-    _filter.query(search);
-
-  auto bookIds = filter(_filter);
-
-  sort(bookIds, sortBy, true);
-  return bookIds;
-}
-
 Filter::Filter()
   : activeFilters(0),
     _maxSize(0)

src/libxml_dumper.cpp

@@ -20,10 +20,10 @@
 #include "libxml_dumper.h"
 #include "book.h"
 
+#include "tools.h"
 #include "tools/base64.h"
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
-#include "tools/pathTools.h"
 
 namespace kiwix
 {
@@ -60,10 +60,13 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
     ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
     ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
-    if (!book.getFavicon().empty())
-      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
+    try {
+      auto defaultIllustration = book.getIllustration(48);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
+      if (!defaultIllustration->getData().empty())
+        ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
+    } catch(...) {}
   } else {
     ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
   }
@@ -91,7 +94,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
   auto book_node = entry_node.append_child("book");
 
   try {
-    auto book = library->getBookById(bookmark.getBookId());
+    auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
     ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());

src/manager.cpp

@@ -19,34 +19,88 @@
 
 #include "manager.h"
 
+#include "tools.h"
 #include "tools/pathTools.h"
 
 #include <pugixml.hpp>
 
 namespace kiwix
 {
+
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+////////////////////////////////////////////////////////////////////////////////
+// LibraryManipulator
+////////////////////////////////////////////////////////////////////////////////
+
+LibraryManipulator::LibraryManipulator(Library* library)
+  : library(*library)
+{}
+
+LibraryManipulator::~LibraryManipulator()
+{}
+
+bool LibraryManipulator::addBookToLibrary(const Book& book)
+{
+  const auto ret = library.addBook(book);
+  if ( ret ) {
+    bookWasAddedToLibrary(book);
+  }
+  return ret;
+}
+
+void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
+{
+  library.addBookmark(bookmark);
+  bookmarkWasAddedToLibrary(bookmark);
+}
+
+uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
+{
+  const auto n = library.removeBooksNotUpdatedSince(rev);
+  if ( n != 0 ) {
+    booksWereRemovedFromLibrary();
+  }
+  return n;
+}
+
+void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
+{
+}
+
+void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
+{
+}
+
+void LibraryManipulator::booksWereRemovedFromLibrary()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Manager
+////////////////////////////////////////////////////////////////////////////////
+
 /* Constructor */
 Manager::Manager(LibraryManipulator* manipulator):
   writableLibraryPath(""),
-  manipulator(manipulator),
-  mustDeleteManipulator(false)
+  manipulator(manipulator, NoDelete())
 {
 }
 
 Manager::Manager(Library* library) :
   writableLibraryPath(""),
-  manipulator(new DefaultLibraryManipulator(library)),
-  mustDeleteManipulator(true)
+  manipulator(new LibraryManipulator(library))
 {
 }
 
-/* Destructor */
-Manager::~Manager()
-{
-  if (mustDeleteManipulator) {
-    delete manipulator;
-  }
-}
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
                           bool readOnly,
                           const std::string& libraryPath,
@@ -214,8 +268,8 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
     tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
   }
   try {
-    kiwix::Reader reader(tmp_path);
-    book->update(reader);
+    zim::Archive archive(tmp_path);
+    book->update(archive);
     book->setPathValid(true);
   } catch (const std::exception& e) {
     book->setPathValid(false);
@@ -248,4 +302,21 @@ bool Manager::readBookmarkFile(const std::string& path)
   return true;
 }
 
+void Manager::reload(const Paths& paths)
+{
+  const auto libRevision = manipulator->getLibrary().getRevision();
+  for (std::string path : paths) {
+    if (!path.empty()) {
+      if ( kiwix::isRelativePath(path) )
+        path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
+
+      if (!readFile(path, false, true)) {
+        throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
+      }
+    }
+  }
+
+  manipulator->removeBooksNotUpdatedSince(libRevision);
+}
+
 }

src/meson.build

@@ -19,6 +19,7 @@ kiwix_sources = [
   'tools/stringTools.cpp',
   'tools/networkTools.cpp',
   'tools/otherTools.cpp',
+  'tools/archiveTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
   'server/byte_range.cpp',
@@ -26,7 +27,9 @@ kiwix_sources = [
   'server/request_context.cpp',
   'server/response.cpp',
   'server/internalServer.cpp',
-  'server/internalServer_catalog_v2.cpp'
+  'server/internalServer_catalog_v2.cpp',
+  'opds_catalog.cpp',
+  'version.cpp'
 ]
 kiwix_sources += lib_resources
 

src/name_mapper.cpp

@@ -51,12 +51,54 @@ HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool w
   }
 }
 
-std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
   return m_idToName.at(id);
 }
 
-std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
   return m_nameToId.at(name);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// UpdatableNameMapper
+////////////////////////////////////////////////////////////////////////////////
+
+UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+  : library(lib)
+  , withAlias(withAlias)
+{
+  update();
+}
+
+void UpdatableNameMapper::update()
+{
+  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  std::lock_guard<std::mutex> lock(mutex);
+  nameMapper.reset(newNameMapper);
+}
+
+UpdatableNameMapper::NameMapperHandle
+UpdatableNameMapper::currentNameMapper() const
+{
+  // Return a copy of the handle to the current NameMapper object. It will
+  // ensure that the object survives any call to UpdatableNameMapper::update()
+  // made before the completion of any pending operation on that object.
+  std::lock_guard<std::mutex> lock(mutex);
+  return nameMapper;
+}
+
+std::string UpdatableNameMapper::getNameForId(const std::string& id) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getNameForId(id);
+}
+
+std::string UpdatableNameMapper::getIdForName(const std::string& name) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getIdForName(name);
+}
+
 }

src/opds_catalog.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "opds_catalog.h"
+#include "tools/stringTools.h"
+
+#include <sstream>
+
+namespace kiwix
+{
+
+namespace
+{
+const char opdsSearchEndpoint[] = "/catalog/v2/entries";
+
+enum Separator { AMP };
+
+std::ostringstream& operator<<(std::ostringstream& oss, Separator sep)
+{
+  if ( oss.tellp() > 0 )
+    oss << "&";
+  return oss;
+}
+
+std::string buildSearchString(const Filter& f)
+{
+  std::ostringstream oss;
+  if ( f.hasQuery() )
+    oss << AMP << "q=" << urlEncode(f.getQuery());
+
+  if ( f.hasCategory() )
+    oss << AMP << "category=" << urlEncode(f.getCategory());
+
+  if ( f.hasLang() )
+    oss << AMP << "lang=" << urlEncode(f.getLang());
+
+  if ( f.hasName() )
+    oss << AMP << "name=" << urlEncode(f.getName());
+
+  if ( !f.getAcceptTags().empty() )
+    oss << AMP << "tag=" << urlEncode(join(f.getAcceptTags(), ";"));
+
+  return oss.str();
+}
+
+} // unnamed namespace
+
+std::string getSearchUrl(const Filter& f)
+{
+  const std::string searchString = buildSearchString(f);
+
+  if ( searchString.empty() )
+    return opdsSearchEndpoint;
+  else
+    return opdsSearchEndpoint + ("?" + searchString);
+}
+
+} // namespace kiwix

src/opds_dumper.cpp

@@ -20,10 +20,12 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include "tools/otherTools.h"
-
 #include "kiwixlib-resources.h"
 #include <mustache.hpp>
+#include <unicode/locid.h>
+
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
 
 namespace kiwix
 {
@@ -49,23 +51,37 @@ namespace
 {
 
 typedef kainjow::mustache::data MustacheData;
-typedef kainjow::mustache::list BookData;
+typedef kainjow::mustache::list BooksData;
+typedef kainjow::mustache::list IllustrationInfo;
 
-BookData getBookData(const Library* library, const std::vector<std::string>& bookIds)
+IllustrationInfo getBookIllustrationInfo(const Book& book)
+{
+    kainjow::mustache::list illustrations;
+    if ( book.isPathValid() ) {
+      for ( const auto& illustration : book.getIllustrations() ) {
+        // For now, we are handling only sizexsize@1 illustration.
+        // So we can simply pass one size to mustache.
+        illustrations.push_back(kainjow::mustache::object{
+          {"icon_size", to_string(illustration->width)},
+          {"icon_mimetype", illustration->mimeType}
+        });
+      }
+    }
+    return illustrations;
+}
+
+kainjow::mustache::object getSingleBookData(const Book& book)
 {
-  BookData bookData;
-  for ( const auto& bookId : bookIds ) {
-    const Book& book = library->getBookById(bookId);
     const MustacheData bookUrl = book.getUrl().empty()
                                ? MustacheData(false)
                                : MustacheData(book.getUrl());
-    bookData.push_back(kainjow::mustache::object{
-      {"id", "urn:uuid:"+book.getId()},
+    return kainjow::mustache::object{
+      {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
       {"language", book.getLanguage()},
-      {"content_id",  book.getHumanReadableIdFromPath()},
+      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
       {"updated", book.getDate() + "T00:00:00Z"},
       {"category", book.getCategory()},
       {"flavour", book.getFlavour()},
@@ -76,17 +92,52 @@ BookData getBookData(const Library* library, const std::vector<std::string>& boo
       {"publisher_name", book.getPublisher()},
       {"url", bookUrl},
       {"size", to_string(book.getSize())},
-    });
+      {"icons", getBookIllustrationInfo(book)},
+    };
+}
+
+std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+{
+  auto data = getSingleBookData(book);
+  data["with_xml_header"] = MustacheData(withXMLHeader);
+  data["dump_partial_entries"] = MustacheData(partial);
+  data["endpoint_root"] = endpointRoot;
+  data["root"] = rootLocation;
+  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+}
+
+BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+{
+  BooksData booksData;
+  for ( const auto& bookId : bookIds ) {
+    try {
+      const Book book = library->getBookByIdThreadSafe(bookId);
+      booksData.push_back(kainjow::mustache::object{
+          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
+      });
+    } catch ( const std::out_of_range& ) {
+      // the book was removed from the library since its id was obtained
+      // ignore it
+    }
   }
 
-  return bookData;
+  return booksData;
 }
 
+std::string getLanguageSelfName(const std::string& lang) {
+  const icu::Locale locale(lang.c_str());
+  icu::UnicodeString ustring;
+  locale.getDisplayLanguage(locale, ustring);
+  std::string result;
+  ustring.toUTF8String(result);
+  return result;
+};
+
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto bookData = getBookData(library, bookIds);
+  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
@@ -95,36 +146,44 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
-     {"books", bookData }
+     {"books", booksData }
   };
 
   return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
 }
 
-string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const
+string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
-  const auto bookData = getBookData(library, bookIds);
+  const auto endpointRoot = rootLocation + "/catalog/v2";
+  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
 
+  const char* const endpoint = partial ? "/partial_entries" : "/entries";
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
-     {"endpoint_root", rootLocation + "/catalog/v2"},
-     {"feed_id", gen_uuid(libraryId + "/entries?"+query)},
+     {"endpoint_root", endpointRoot},
+     {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
      {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
      {"query", query.empty() ? "" : "?" + urlEncode(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
-     {"books", bookData }
+     {"books", booksData },
+     {"dump_partial_entries", MustacheData(partial)}
   };
 
   return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
 }
 
-std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categories) const
+std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
+{
+  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+}
+
+std::string OPDSDumper::categoriesOPDSFeed() const
 {
   const auto now = gen_date_str();
   kainjow::mustache::list categoryData;
-  for ( const auto& category : categories ) {
+  for ( const auto& category : library->getBooksCategories() ) {
     const auto urlencodedCategoryName = urlEncode(category);
     categoryData.push_back(kainjow::mustache::object{
       {"name", category},
@@ -145,4 +204,32 @@ std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categ
   );
 }
 
+std::string OPDSDumper::languagesOPDSFeed() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+
+  return render_template(
+             RESOURCE::templates::catalog_v2_languages_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/languages")},
+               {"languages", languageData }
+             }
+  );
+}
+
 }

src/reader.cpp

@@ -21,48 +21,14 @@
 #include <time.h>
 
 #include <zim/search.h>
+#include <zim/suggestion.h>
 #include <zim/item.h>
 #include <zim/error.h>
 
+#include "tools.h"
+#include "tools/stringTools.h"
 #include "tools/otherTools.h"
-
-inline char hi(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[(v >> 4) & 0xf];
-}
-
-inline char lo(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[v & 0xf];
-}
-
-std::string hexUUID(std::string in)
-{
-  std::ostringstream out;
-  for (unsigned n = 0; n < 4; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 4; n < 6; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 6; n < 8; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 8; n < 10; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 10; n < 16; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  std::string op = out.str();
-  return op;
-}
+#include "tools/archiveTools.h"
 
 namespace kiwix
 {
@@ -86,6 +52,11 @@ Reader::Reader(const string zimFilePath)
   srand(time(nullptr));
 }
 
+Reader::Reader(const std::shared_ptr<zim::Archive> archive, bool _marker)
+  : zimArchive(archive),
+    zimFilePath(archive->getFilename())
+  {}
+
 #ifndef _WIN32
 Reader::Reader(int fd)
   :  zimArchive(new zim::Archive(fd)),
@@ -111,12 +82,7 @@ zim::Archive* Reader::getZimArchive() const
 
 MimeCounterType Reader::parseCounterMetadata() const
 {
-  try {
-    auto counterContent = zimArchive->getMetadata("Counter");
-    return parseMimetypeCounter(counterContent);
-  } catch (zim::EntryNotFound& e) {
-    return {};
-  }
+  return kiwix::parseArchiveCounter(*zimArchive);
 }
 
 /* Get the count of articles which can be indexed/displayed */
@@ -138,19 +104,7 @@ unsigned int Reader::getArticleCount() const
 /* Get the count of medias content in the ZIM file */
 unsigned int Reader::getMediaCount() const
 {
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  for (auto &pair:counterMap) {
-    if (startsWith(pair.first, "image/") ||
-        startsWith(pair.first, "video/") ||
-        startsWith(pair.first, "audio/")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
+  return kiwix::getArchiveMediaCount(*zimArchive);
 }
 
 /* Get the total of all items of a ZIM file, redirects included */
@@ -162,15 +116,13 @@ unsigned int Reader::getGlobalCount() const
 /* Return the UID of the ZIM file */
 string Reader::getId() const
 {
-  std::ostringstream s;
-  s << zimArchive->getUuid();
-  return s.str();
+  return kiwix::getArchiveId(*zimArchive);
 }
 
 Entry Reader::getRandomPage() const
 {
   try {
-    return zimArchive->getRandomEntry();
+    return Entry(zimArchive->getRandomEntry(), true);
   } catch(...) {
     throw NoEntry();
   }
@@ -178,19 +130,12 @@ Entry Reader::getRandomPage() const
 
 Entry Reader::getMainPage() const
 {
-  return zimArchive->getMainEntry();
+  return Entry(zimArchive->getMainEntry(), true);
 }
 
 bool Reader::getFavicon(string& content, string& mimeType) const
 {
-  try {
-    auto item = zimArchive->getIllustrationItem();
-    content = item.getData();
-    mimeType = item.getMimetype();
-    return true;
-  } catch(zim::EntryNotFound& e) {};
-
-  return false;
+  return kiwix::getArchiveFavicon(*zimArchive, 48, content, mimeType);
 }
 
 string Reader::getZimFilePath() const
@@ -212,47 +157,32 @@ bool Reader::getMetadata(const string& name, string& value) const
 
 string Reader::getName() const
 {
-  METADATA("Name")
+  return kiwix::getMetaName(*zimArchive);
 }
 
 string Reader::getTitle() const
 {
-  string value = zimArchive->getMetadata("Title");
-  if (value.empty()) {
-    value = getLastPathElement(zimFilePath);
-    std::replace(value.begin(), value.end(), '_', ' ');
-    size_t pos = value.find(".zim");
-    value = value.substr(0, pos);
-  }
-  return value;
+  return kiwix::getArchiveTitle(*zimArchive);
 }
 
 string Reader::getCreator() const
 {
-  METADATA("Creator")
+  return kiwix::getMetaCreator(*zimArchive);
 }
 
 string Reader::getPublisher() const
 {
-  METADATA("Publisher")
+  return kiwix::getMetaPublisher(*zimArchive);
 }
 
 string Reader::getDate() const
 {
-  METADATA("Date")
+  return kiwix::getMetaDate(*zimArchive);
 }
 
 string Reader::getDescription() const
 {
-  string value;
-  this->getMetadata("Description", value);
-
-  /* Mediawiki Collection tends to use the "Subtitle" name */
-  if (value.empty()) {
-    this->getMetadata("Subtitle", value);
-  }
-
-  return value;
+  return kiwix::getMetaDescription(*zimArchive);
 }
 
 string Reader::getLongDescription() const
@@ -262,7 +192,7 @@ string Reader::getLongDescription() const
 
 string Reader::getLanguage() const
 {
-  METADATA("Language")
+  return kiwix::getMetaLanguage(*zimArchive);
 }
 
 string Reader::getLicense() const
@@ -272,13 +202,7 @@ string Reader::getLicense() const
 
 string Reader::getTags(bool original) const
 {
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  if (original) {
-    return tags_str;
-  }
-  auto tags = convertTags(tags_str);
-  return join(tags, ";");
+  return kiwix::getMetaTags(*zimArchive, original);
 }
 
 
@@ -301,7 +225,7 @@ string Reader::getRelation() const
 
 string Reader::getFlavour() const
 {
-  METADATA("Flavour")
+  return kiwix::getMetaFlavour(*zimArchive);
 }
 
 string Reader::getSource() const
@@ -315,39 +239,10 @@ string Reader::getScraper() const
 }
 #undef METADATA
 
-string Reader::getOrigId() const
-{
-  string value;
-  this->getMetadata("startfileuid", value);
-  if (value.empty()) {
-    return "";
-  }
-  std::string id = value;
-  std::string origID;
-  std::string temp = "";
-  unsigned int k = 0;
-  char tempArray[16] = "";
-  for (unsigned int i = 0; i < id.size(); i++) {
-    if (id[i] == '\n') {
-      tempArray[k] = atoi(temp.c_str());
-      temp = "";
-      k++;
-    } else {
-      temp += id[i];
-    }
-  }
-  origID = hexUUID(tempArray);
-  return origID;
-}
-
 Entry Reader::getEntryFromPath(const std::string& path) const
 {
-  if (path.empty() || path == "/") {
-    return getMainPage();
-  }
-
   try {
-    return zimArchive->getEntryByPath(path);
+    return Entry(kiwix::getEntryFromPath(*zimArchive, path), true);
   } catch (zim::EntryNotFound& e) {
     throw NoEntry();
   }
@@ -361,7 +256,7 @@ Entry Reader::getEntryFromEncodedPath(const std::string& path) const
 Entry Reader::getEntryFromTitle(const std::string& title) const
 {
   try {
-    return zimArchive->getEntryByTitle(title);
+    return Entry(zimArchive->getEntryByTitle(title), true);
   } catch(zim::EntryNotFound& e) {
     throw NoEntry();
   }
@@ -460,12 +355,7 @@ bool Reader::searchSuggestions(const string& prefix,
 std::vector<std::string> Reader::getTitleVariants(
     const std::string& title) const
 {
-  std::vector<std::string> variants;
-  variants.push_back(title);
-  variants.push_back(kiwix::ucFirst(title));
-  variants.push_back(kiwix::lcFirst(title));
-  variants.push_back(kiwix::toTitle(title));
-  return variants;
+  return kiwix::getTitleVariants(title);
 }
 
 
@@ -488,35 +378,36 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
                                     SuggestionsList_t& results)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal = false;
 
-  /* Try to search in the title using fulltext search database */
+  auto suggestionSearcher = zim::SuggestionSearcher(*zimArchive);
 
-  auto suggestionSearcher = zim::Searcher(*zimArchive);
-  zim::Query suggestionQuery;
-  suggestionQuery.setQuery(prefix, true);
-  auto suggestionSearch = suggestionSearcher.search(suggestionQuery);
-
-  if (suggestionSearch.getEstimatedMatches()) {
+  if (zimArchive->hasTitleIndex()) {
+    auto suggestionSearch = suggestionSearcher.suggest(prefix);
     const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
-    for (auto current = suggestions.begin();
-         current != suggestions.end();
-         current++) {
+    for (auto current : suggestions) {
       SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
                                 current.getPath(), current.getSnippet());
       results.push_back(suggestion);
     }
-    retVal = true;
   } else {
+    // Check some of the variants of the prefix
     for (std::vector<std::string>::iterator variantsItr = variants.begin();
          variantsItr != variants.end();
          variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
-               || retVal;
+      auto suggestionSearch = suggestionSearcher.suggest(*variantsItr);
+      for (auto current : suggestionSearch.getResults(0, suggestionsCount)) {
+        if (results.size() >= suggestionsCount) {
+          break;
+        }
+
+        SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
+                                current.getPath(), current.getSnippet());
+        results.push_back(suggestion);
+      }
     }
   }
 
-  return retVal;
+  return results.size() > 0;
 }
 
 /* Get next suggestion */
@@ -575,7 +466,7 @@ bool Reader::isCorrupted() const
 /* Return the file size, works also for splitted files */
 unsigned int Reader::getFileSize() const
 {
-  return zimArchive->getFilesize() / 1024;
+  return kiwix::getArchiveFileSize(*zimArchive);
 }
 
 }

src/search_renderer.cpp

@@ -30,17 +30,29 @@
 
 #include <mustache.hpp>
 #include "kiwixlib-resources.h"
-
+#include "tools/stringTools.h"
 
 namespace kiwix
 {
 
 /* Constructor */
 SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : mp_searcher(searcher),
+    : m_srs(searcher->getSearchResultSet()),
       mp_nameMapper(mapper),
       protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?")
+      searchProtocolPrefix("search://?"),
+      estimatedResultCount(searcher->getEstimatedResultCount()),
+      resultStart(searcher->getResultStart())
+{}
+
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : m_srs(srs),
+      mp_nameMapper(mapper),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://?"),
+      estimatedResultCount(estimatedResultCount),
+      resultStart(start)
 {}
 
 /* Destructor */
@@ -70,29 +82,26 @@ std::string SearchRenderer::getHtml()
 {
   kainjow::mustache::data results{kainjow::mustache::data::type::list};
 
-  mp_searcher->restart_search();
-  Result* p_result = NULL;
-  while ((p_result = mp_searcher->getNextResult())) {
+  for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
-    result.set("title", p_result->get_title());
-    result.set("url", p_result->get_url());
-    result.set("snippet", p_result->get_snippet());
-    result.set("resultContentId", mp_nameMapper->getNameForId(p_result->get_zimId()));
+    result.set("title", it.getTitle());
+    result.set("url", it.getPath());
+    result.set("snippet", it.getSnippet());
+    std::ostringstream s;
+    s << it.getZimId();
+    result.set("resultContentId", mp_nameMapper->getNameForId(s.str()));
 
-    if (p_result->get_wordCount() >= 0) {
-      result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
+    if (it.getWordCount() >= 0) {
+      result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
     }
 
     results.push_back(result);
-    delete p_result;
   }
 
   // pages
   kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
-  auto resultStart = mp_searcher->getResultStart();
   auto resultEnd = 0U;
-  auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
   auto currentPage = 0U;
   auto pageStart = 0U;
   auto pageEnd = 0U;
@@ -146,4 +155,4 @@ std::string SearchRenderer::getHtml()
   return ss.str();
 }
 
-}
+}

src/searcher.cpp

@@ -18,14 +18,16 @@
  */
 
 
-#include <cmath>
 
 #include "searcher.h"
 #include "reader.h"
 
 #include <zim/search.h>
+#include <zim/suggestion.h>
 
 #include <mustache.hpp>
+#include <cmath>
+#include "tools/stringTools.h"
 #include "kiwixlib-resources.h"
 
 #define MAX_SEARCH_LEN 140
@@ -36,6 +38,7 @@ class _Result : public Result
 {
  public:
   _Result(zim::SearchResultSet::iterator iterator);
+  _Result(SuggestionItem suggestionItem);
   virtual ~_Result(){};
 
   virtual std::string get_url();
@@ -49,6 +52,8 @@ class _Result : public Result
 
  private:
   zim::SearchResultSet::iterator iterator;
+  SuggestionItem suggestionItem;
+  bool isSuggestion;
 };
 
 struct SearcherInternal : zim::SearchResultSet {
@@ -61,12 +66,20 @@ struct SearcherInternal : zim::SearchResultSet {
   zim::SearchResultSet::iterator current_iterator;
 };
 
+struct SuggestionInternal : zim::SuggestionResultSet {
+  explicit SuggestionInternal(const zim::SuggestionResultSet& srs)
+    : zim::SuggestionResultSet(srs),
+      currentIterator(srs.begin()) {}
+
+  zim::SuggestionResultSet::iterator currentIterator;
+};
+
 /* Constructor */
 Searcher::Searcher()
     : searchPattern(""),
       estimatedResultCount(0),
       resultStart(0),
-      resultEnd(0)
+      maxResultCount(0)
 {
   loadICUExternalTables();
 }
@@ -81,6 +94,12 @@ bool Searcher::add_reader(Reader* reader)
   if (!reader->hasFulltextIndex()) {
       return false;
   }
+
+  for ( const Reader* const existing_reader : readers ) {
+    if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
+      return false;
+  }
+
   this->readers.push_back(reader);
   return true;
 }
@@ -94,7 +113,7 @@ Reader* Searcher::get_reader(int readerIndex)
 /* Search strings in the database */
 void Searcher::search(const std::string& search,
                       unsigned int resultStart,
-                      unsigned int resultEnd,
+                      unsigned int maxResultCount,
                       const bool verbose)
 {
   this->reset();
@@ -105,9 +124,9 @@ void Searcher::search(const std::string& search,
 
   this->searchPattern = search;
   this->resultStart = resultStart;
-  this->resultEnd = resultEnd;
+  this->maxResultCount = maxResultCount;
   /* Try to find results */
-  if (resultStart != resultEnd) {
+  if (maxResultCount != 0) {
     /* Perform the search */
     string unaccentedSearch = removeAccents(search);
     std::vector<zim::Archive> archives;
@@ -118,11 +137,11 @@ void Searcher::search(const std::string& search,
       }
     }
     zim::Searcher searcher(archives);
+    searcher.setVerbose(verbose);
     zim::Query query;
-    query.setQuery(unaccentedSearch, false);
-    query.setVerbose(verbose);
+    query.setQuery(unaccentedSearch);
     zim::Search search = searcher.search(query);
-    internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+    internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
     this->estimatedResultCount = search.getEstimatedMatches();
   }
 
@@ -132,7 +151,7 @@ void Searcher::search(const std::string& search,
 
 void Searcher::geo_search(float latitude, float longitude, float distance,
                           unsigned int resultStart,
-                          unsigned int resultEnd,
+                          unsigned int maxResultCount,
                           const bool verbose)
 {
   this->reset();
@@ -146,10 +165,10 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
   oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
   this->searchPattern = oss.str();
   this->resultStart = resultStart;
-  this->resultEnd = resultEnd;
+  this->maxResultCount = maxResultCount;
 
   /* Try to find results */
-  if (resultStart == resultEnd) {
+  if (maxResultCount == 0) {
     return;
   }
 
@@ -159,12 +178,12 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     archives.push_back(*(*current)->getZimArchive());
   }
   zim::Searcher searcher(archives);
+  searcher.setVerbose(verbose);
   zim::Query query;
-  query.setVerbose(verbose);
-  query.setQuery("", false);
+  query.setQuery("");
   query.setGeorange(latitude, longitude, distance);
   zim::Search search = searcher.search(query);
-  internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+  internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
   this->estimatedResultCount = search.getEstimatedMatches();
 }
 
@@ -178,11 +197,21 @@ void Searcher::restart_search()
 
 Result* Searcher::getNextResult()
 {
-  if (internal.get() &&
-             internal->current_iterator != internal->end()) {
+  if (internal.get() && internal->current_iterator != internal->end()) {
     Result* result = new _Result(internal->current_iterator);
     internal->current_iterator++;
     return result;
+  } else if (suggestionInternal.get() &&
+             suggestionInternal->currentIterator != suggestionInternal->end()) {
+    SuggestionItem item(
+                        suggestionInternal->currentIterator->getTitle(),
+                        normalize(suggestionInternal->currentIterator->getTitle()),
+                        suggestionInternal->currentIterator->getPath(),
+                        suggestionInternal->currentIterator->getSnippet()
+                      );
+    Result* result = new _Result(item);
+    suggestionInternal->currentIterator++;
+    return result;
   }
   return NULL;
 }
@@ -205,20 +234,15 @@ void Searcher::suggestions(std::string& searchPattern, const bool verbose)
 
   this->searchPattern = searchPattern;
   this->resultStart = 0;
-  this->resultEnd = 10;
+  this->maxResultCount = 10;
   string unaccentedSearch = removeAccents(searchPattern);
 
-  std::vector<zim::Archive> archives;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    archives.push_back(*(*current)->getZimArchive());
-  }
-  zim::Searcher searcher(archives);
-  zim::Query query;
-  query.setVerbose(verbose);
-  query.setQuery(unaccentedSearch, true);
-  zim::Search search = searcher.search(query);
-  internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+  // Multizim suggestion is not supported as of now! taking only one archive
+  zim::Archive archive = *(*this->readers.begin())->getZimArchive();
+  zim::SuggestionSearcher searcher(archive);
+  searcher.setVerbose(verbose);
+  zim::SuggestionSearch search = searcher.suggest(searchPattern);
+  suggestionInternal.reset(new SuggestionInternal(search.getResults(resultStart, maxResultCount)));
   this->estimatedResultCount = search.getEstimatedMatches();
 }
 
@@ -228,41 +252,75 @@ unsigned int Searcher::getEstimatedResultCount()
   return this->estimatedResultCount;
 }
 
-_Result::_Result(zim::SearchResultSet::iterator iterator)
-    : iterator(iterator)
+zim::SearchResultSet Searcher::getSearchResultSet()
 {
+  return *(this->internal);
 }
 
+_Result::_Result(zim::SearchResultSet::iterator iterator)
+    : iterator(iterator),
+      suggestionItem("", "", ""),
+      isSuggestion(false)
+{}
+
+_Result::_Result(SuggestionItem item)
+    : iterator(),
+      suggestionItem(item.getTitle(), item.getNormalizedTitle(), item.getPath(), item.getSnippet()),
+      isSuggestion(true)
+{}
+
 std::string _Result::get_url()
 {
+  if (isSuggestion) {
+    return suggestionItem.getPath();
+  }
   return iterator.getPath();
 }
 std::string _Result::get_title()
 {
+  if (isSuggestion) {
+    return suggestionItem.getTitle();
+  }
   return iterator.getTitle();
 }
 int _Result::get_score()
 {
+  if (isSuggestion) {
+    return 0;
+  }
   return iterator.getScore();
 }
 std::string _Result::get_snippet()
 {
+  if (isSuggestion) {
+    return suggestionItem.getSnippet();
+  }
   return iterator.getSnippet();
 }
 std::string _Result::get_content()
 {
+  if (isSuggestion) return "";
   return iterator->getItem(true).getData();
 }
 int _Result::get_size()
 {
+  if (isSuggestion) {
+    return 0;
+  }
   return iterator.getSize();
 }
 int _Result::get_wordCount()
 {
+  if (isSuggestion) {
+    return 0;
+  }
   return iterator.getWordCount();
 }
 std::string _Result::get_zimId()
 {
+  if (isSuggestion) {
+    return "";
+  }
   std::ostringstream s;
   s << iterator.getZimId();
   return s.str();

src/server.cpp

@@ -48,7 +48,8 @@ bool Server::start() {
     m_verbose,
     m_withTaskbar,
     m_withLibraryButton,
-    m_blockExternalLinks));
+    m_blockExternalLinks,
+    m_indexTemplateString));
   return mp_server->start();
 }
 
@@ -70,4 +71,14 @@ void Server::setRoot(const std::string& root)
   }
 }
 
+int Server::getPort()
+{
+  return mp_server->getPort();
+}
+
+std::string Server::getAddress()
+{
+  return mp_server->getAddress();
+}
+
 }

src/server/internalServer.cpp

@@ -43,10 +43,12 @@ extern "C" {
 #include "microhttpd_wrapper.h"
 }
 
-#include "tools/otherTools.h"
+#include "tools.h"
 #include "tools/pathTools.h"
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
+#include "tools/archiveTools.h"
+#include "tools/networkTools.h"
 #include "library.h"
 #include "name_mapper.h"
 #include "entry.h"
@@ -55,6 +57,11 @@ extern "C" {
 #include "opds_dumper.h"
 
 #include <zim/uuid.h>
+#include <zim/error.h>
+#include <zim/search.h>
+#include <zim/suggestion.h>
+#include <zim/entry.h>
+#include <zim/item.h>
 
 #include <mustache.hpp>
 
@@ -112,7 +119,8 @@ InternalServer::InternalServer(Library* library,
                                bool verbose,
                                bool withTaskbar,
                                bool withLibraryButton,
-                               bool blockExternalLinks) :
+                               bool blockExternalLinks,
+                               std::string indexTemplateString) :
   m_addr(addr),
   m_port(port),
   m_root(normalizeRootUrl(root)),
@@ -121,6 +129,7 @@ InternalServer::InternalServer(Library* library,
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
+  m_indexTemplateString(indexTemplateString.empty() ? RESOURCE::templates::index_html : indexTemplateString),
   mp_daemon(nullptr),
   mp_library(library),
   mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
@@ -141,15 +150,16 @@ bool InternalServer::start() {
   sockAddr.sin_family = AF_INET;
   sockAddr.sin_port = htons(m_port);
   if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
+    if (0 != INADDR_ANY) {
       sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+    }
+    m_addr = kiwix::getBestPublicIp();
   } else {
     if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
       std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
       return false;
     }
   }
-
   mp_daemon = MHD_start_daemon(flags,
                             m_port,
                             NULL,
@@ -253,7 +263,7 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
 {
   try {
     if (! request.is_valid_url())
-      return Response::build_404(*this, request, "", "");
+      return Response::build_404(*this, request.get_full_url(), "", "");
 
     const ETag etag = get_matching_if_none_match_etag(request);
     if ( etag )
@@ -265,8 +275,8 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (startsWith(request.get_url(), "/catalog/"))
       return handle_catalog(request);
 
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
+    if (startsWith(request.get_url(), "/raw/"))
+      return handle_raw(request);
 
     if (request.get_url() == "/search")
       return handle_search(request);
@@ -320,116 +330,107 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
 
 std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
 {
-  return ContentResponse::build(*this, RESOURCE::templates::index_html, get_default_data(), "text/html; charset=utf-8", true);
+  return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8", true);
 }
 
-std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+/**
+ * Archive and Zim handlers begin
+ **/
+
+// TODO: retrieve searcher from caching mechanism
+SuggestionsList_t getSuggestions(const zim::Archive* const archive,
+                  const std::string& queryString, int start, int suggestionCount)
 {
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return Response::build_404(*this, request, bookName, "");
-  }
+  SuggestionsList_t suggestions;
+  auto searcher = zim::SuggestionSearcher(*archive);
+  if (archive->hasTitleIndex()) {
+    auto search = searcher.suggest(queryString);
+    auto srs = search.getResults(start, suggestionCount);
 
-  if (reader == nullptr) {
-    return Response::build_404(*this, request, bookName, "");
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
+    for (auto it : srs) {
+      SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
+                                it.getPath(), it.getSnippet());
+      suggestions.push_back(suggestion);
+    }
   } else {
-    return Response::build_404(*this, request, bookName, "");
+    // TODO: This case should be handled by libzim
+    std::vector<std::string> variants = getTitleVariants(queryString);
+    int currCount = 0;
+    for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
+      auto search = searcher.suggest(queryString);
+      auto srs = search.getResults(0, suggestionCount);
+      for (auto it : srs) {
+        SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
+                                  it.getPath());
+        suggestions.push_back(suggestion);
+        currCount++;
+      }
+    }
   }
-
-  auto response = ContentResponse::build(*this, content, mimeType);
-  response->set_cacheable();
-  return std::move(response);
+  return suggestions;
 }
 
+
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_suggest\n");
   }
 
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-
   std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
+  std::shared_ptr<zim::Archive> archive;
   try {
     bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, bookName, "");
+    // error handled by the archive == nullptr check below
+  }
+
+  if (archive == nullptr) {
+    const std::string error_details = "No such book: " + bookName;
+    return Response::build_404(*this, "", bookName, "", error_details);
+  }
+
+  const auto queryString = request.get_optional_param("term", std::string());
+  const auto start = request.get_optional_param<unsigned int>("start", 0);
+  unsigned int count = request.get_optional_param<unsigned int>("count", 10);
+  if (count == 0) {
+    count = 10;
   }
 
   if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+    printf("Searching suggestions for: \"%s\"\n", queryString.c_str());
   }
 
   MustacheData results{MustacheData::type::list};
 
   bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    SuggestionsList_t suggestions;
-    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
-    for(auto& suggestion:suggestions) {
-      MustacheData result;
-      result.set("label", suggestion.getTitle());
 
-      if (suggestion.hasSnippet()) {
-        result.set("label", suggestion.getSnippet());
-      }
+  /* Get the suggestions */
+  SuggestionsList_t suggestions = getSuggestions(archive.get(), queryString, start, count);
+  for(auto& suggestion:suggestions) {
+    MustacheData result;
+    result.set("label", suggestion.getTitle());
 
-      result.set("value", suggestion.getTitle());
-      result.set("kind", "path");
-      result.set("path", suggestion.getPath());
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
+    if (suggestion.hasSnippet()) {
+      result.set("label", suggestion.getSnippet());
     }
+
+    result.set("value", suggestion.getTitle());
+    result.set("kind", "path");
+    result.set("path", suggestion.getPath());
+    result.set("first", first);
+    first = false;
+    results.push_back(result);
   }
 
+
   /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
+  if (archive->hasFulltextIndex()) {
     MustacheData result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term + " ");
+    result.set("label", "containing '" + queryString + "'...");
+    result.set("value", queryString + " ");
     result.set("kind", "pattern");
     result.set("first", first);
     results.push_back(result);
@@ -457,7 +458,7 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
     response->set_cacheable();
     return std::move(response);
   } catch (const ResourceNotFound& e) {
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
   }
 }
 
@@ -467,13 +468,6 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     printf("** running handle_search\n");
   }
 
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
   std::string patternString;
   try {
     patternString = request.get_argument("pattern");
@@ -492,30 +486,37 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
   } catch(const std::out_of_range&) {}
     catch(const std::invalid_argument&) {}
 
-  std::shared_ptr<Reader> reader(nullptr);
+  std::string bookName;
+  std::shared_ptr<zim::Archive> archive;
   try {
-    reader = mp_library->getReaderById(bookId);
+    bookName = request.get_argument("content");
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {}
 
   /* Make the search */
-  if ( (!reader && !bookName.empty())
+  if ( (!archive && !bookName.empty())
     || (patternString.empty() && ! has_geo_query) ) {
     auto data = get_default_data();
     data.set("pattern", encodeDiples(patternString));
     auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
-    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
     response->set_code(MHD_HTTP_NOT_FOUND);
     return std::move(response);
   }
 
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
+  std::shared_ptr<zim::Searcher> searcher;
+  if (archive) {
+    searcher = std::make_shared<zim::Searcher>(*archive);
   } else {
     for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
+      auto currentArchive = mp_library->getArchiveById(bookId);
+      if (currentArchive) {
+        if (! searcher) {
+          searcher = std::make_shared<zim::Searcher>(*currentArchive);
+        } else {
+          searcher->addArchive(*currentArchive);
+        }
       }
     }
   }
@@ -536,25 +537,37 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     pageLength = 25;
   }
 
-  auto end = start + pageLength;
-
   /* Get the results */
   try {
+    zim::Query query;
     if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
+      // Execute geo-search
+      if (m_verbose.load()) {
+        cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
+      }
+
+      query.setQuery("");
+      query.setGeorange(latitude, longitude, distance);
     } else {
-      searcher.search(patternString,
-                       start, end, m_verbose.load());
+      // Execute Ft search
+      if (m_verbose.load()) {
+          cout << "Performing query `" << patternString << "'" << endl;
+      }
+
+      std::string queryString = removeAccents(patternString);
+      query.setQuery(queryString);
     }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
+
+    zim::Search search = searcher->search(query);
+    SearchRenderer renderer(search.getResults(start, pageLength), mp_nameMapper, start,
+                            search.getEstimatedMatches());
     renderer.setSearchPattern(patternString);
     renderer.setSearchContent(bookName);
     renderer.setProtocolPrefix(m_root + "/");
     renderer.setSearchProtocolPrefix(m_root + "/search?");
     renderer.setPageLength(pageLength);
     auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
 
     return std::move(response);
   } catch (const std::exception& e) {
@@ -570,25 +583,26 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
   }
 
   std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
+  std::shared_ptr<zim::Archive> archive;
   try {
     bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, bookName, "");
+    // error handled by the archive == nullptr check below
   }
 
-  if (reader == nullptr) {
-    return Response::build_404(*this, request, bookName, "");
+  if (archive == nullptr) {
+    const std::string error_details = "No such book: " + bookName;
+    return Response::build_404(*this, "", bookName, "", error_details);
   }
 
   try {
-    auto entry = reader->getRandomPage();
-    return build_redirect(bookName, entry.getFinalEntry());
-  } catch(kiwix::NoEntry& e) {
-    return Response::build_404(*this, request, bookName, "");
+    auto entry = archive->getRandomEntry();
+    return build_redirect(bookName, getFinalItem(*archive, entry));
+  } catch(zim::EntryNotFound& e) {
+    const std::string error_details = "Oops! Failed to pick a random article :(";
+    return Response::build_404(*this, "", bookName, getArchiveTitle(*archive), error_details);
   }
 }
 
@@ -600,7 +614,7 @@ std::unique_ptr<Response> InternalServer::handle_captured_external(const Request
   } catch (const std::out_of_range& e) {}
 
   if (source.empty())
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
 
   auto data = get_default_data();
   data.set("source", source);
@@ -619,7 +633,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
     host = request.get_header("Host");
     url  = request.get_url_part(1);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
   }
 
   if (url == "v2") {
@@ -627,7 +641,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   }
 
   if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
   }
 
   if (url == "searchdescription.xml") {
@@ -734,22 +748,10 @@ std::string searchSuggestionHTML(const std::string& searchURL, const std::string
 
 } // unnamed namespace
 
-std::shared_ptr<Reader>
-InternalServer::get_reader(const std::string& bookName) const
-{
-  std::shared_ptr<Reader> reader;
-  try {
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-  }
-  return reader;
-}
-
 std::unique_ptr<Response>
-InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+InternalServer::build_redirect(const std::string& bookName, const zim::Item& item) const
 {
-  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
+  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(item.getPath());
   return Response::build_redirect(*this, redirectUrl);
 }
 
@@ -765,12 +767,17 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   if (bookName.empty())
     return build_homepage(request);
 
-  const std::shared_ptr<Reader> reader = get_reader(bookName);
-  if (reader == nullptr) {
+  std::shared_ptr<zim::Archive> archive;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
+  } catch (const std::out_of_range& e) {}
+
+  if (archive == nullptr) {
     std::string searchURL = m_root+"/search?pattern="+pattern; // Make a full search on the entire library.
     const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
 
-    return Response::build_404(*this, request, bookName, "", details);
+    return Response::build_404(*this, request.get_full_url(), bookName, "", details);
   }
 
   auto urlStr = request.get_url().substr(bookName.size()+1);
@@ -779,31 +786,89 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   }
 
   try {
-    auto entry = reader->getEntryFromPath(urlStr);
+    auto entry = getEntryFromPath(*archive, urlStr);
     if (entry.isRedirect() || urlStr.empty()) {
       // If urlStr is empty, we want to mainPage.
       // We must do a redirection to the real page.
-      return build_redirect(bookName, entry.getFinalEntry());
+      return build_redirect(bookName, getFinalItem(*archive, entry));
     }
-    auto response = ItemResponse::build(*this, request, entry.getZimEntry().getItem());
+    auto response = ItemResponse::build(*this, request, entry.getItem());
     try {
-      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
+      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, getArchiveTitle(*archive));
     } catch (std::bad_cast& e) {}
 
     if (m_verbose.load()) {
       printf("Found %s\n", entry.getPath().c_str());
-      printf("mimeType: %s\n", entry.getMimetype().c_str());
+      printf("mimeType: %s\n", entry.getItem(true).getMimetype().c_str());
     }
 
     return response;
-  } catch(kiwix::NoEntry& e) {
+  } catch(zim::EntryNotFound& e) {
     if (m_verbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
 
     std::string searchURL = m_root+"/search?content="+bookName+"&pattern="+pattern; // Make a search on this specific book only.
     const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
 
-    return Response::build_404(*this, request, bookName, reader->getTitle(), details);
+    return Response::build_404(*this, request.get_full_url(), bookName, getArchiveTitle(*archive), details);
+  }
+}
+
+
+std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_raw\n");
+  }
+
+  std::string bookName;
+  std::string kind;
+  try {
+     bookName = request.get_url_part(1);
+     kind = request.get_url_part(2);
+  } catch (const std::out_of_range& e) {
+     return Response::build_404(*this, request.get_full_url(), bookName, "", "");
+  }
+
+  if (kind != "meta" && kind!= "content") {
+    const std::string error_details = kind + " is not a valid request for raw content.";
+    return Response::build_404(*this, request.get_full_url(), bookName, "", error_details);
+  }
+
+  std::shared_ptr<zim::Archive> archive;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
+  } catch (const std::out_of_range& e) {}
+
+  if (archive == nullptr) {
+    const std::string error_details = "No such book: " + bookName;
+    return Response::build_404(*this, request.get_full_url(), bookName, "", error_details);
+  }
+
+  // Remove the beggining of the path:
+  // /raw/<bookName>/<kind>/foo
+  // ^^^^^          ^      ^
+  //   5      +     1  +   1   = 7
+  auto itemPath = request.get_url().substr(bookName.size()+kind.size()+7);
+
+  try {
+    if (kind == "meta") {
+      auto item = archive->getMetadataItem(itemPath);
+      return ItemResponse::build(*this, request, item, /*raw=*/true);
+    } else {
+      auto entry = archive->getEntryByPath(itemPath);
+      if (entry.isRedirect()) {
+        return build_redirect(bookName, entry.getItem(true));
+      }
+      return ItemResponse::build(*this, request, entry.getItem(), /*raw=*/true);
+    }
+  } catch (zim::EntryNotFound& e ) {
+    if (m_verbose.load()) {
+      printf("Failed to find %s\n", itemPath.c_str());
+    }
+    const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
+    return Response::build_404(*this, request.get_full_url(), bookName, getArchiveTitle(*archive), error_details);
   }
 }
 

src/server/internalServer.h

@@ -54,7 +54,8 @@ class InternalServer {
                    bool verbose,
                    bool withTaskbar,
                    bool withLibraryButton,
-                   bool blockExternalLinks);
+                   bool blockExternalLinks,
+                   std::string indexTemplateString);
     virtual ~InternalServer() = default;
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
@@ -66,30 +67,34 @@ class InternalServer {
                                void** cont_cls);
     bool start();
     void stop();
+    std::string getAddress() { return m_addr; }
+    int getPort() { return m_port; } 
 
   private: // functions
     std::unique_ptr<Response> handle_request(const RequestContext& request);
-    std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
     std::unique_ptr<Response> build_homepage(const RequestContext& request);
     std::unique_ptr<Response> handle_skin(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
-    std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
+    std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
-    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
     std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
     std::unique_ptr<Response> handle_content(const RequestContext& request);
+    std::unique_ptr<Response> handle_raw(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
 
     MustacheData get_default_data() const;
 
-    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
     bool etag_not_needed(const RequestContext& r) const;
     ETag get_matching_if_none_match_etag(const RequestContext& request) const;
 
@@ -102,6 +107,7 @@ class InternalServer {
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
+    std::string m_indexTemplateString;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
@@ -111,8 +117,8 @@ class InternalServer {
     std::string m_library_id;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
     friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
 
 };

src/server/internalServer_catalog_v2.cpp

@@ -43,7 +43,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
   }
 
   if (url == "root.xml") {
@@ -55,12 +55,21 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
         kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
         "application/opensearchdescription+xml"
     );
+  } else if (url == "entry") {
+    const std::string entryId  = request.get_url_part(3);
+    return handle_catalog_v2_complete_entry(request, entryId);
   } else if (url == "entries") {
-    return handle_catalog_v2_entries(request);
+    return handle_catalog_v2_entries(request, /*partial=*/false);
+  } else if (url == "partial_entries") {
+    return handle_catalog_v2_entries(request, /*partial=*/true);
   } else if (url == "categories") {
     return handle_catalog_v2_categories(request);
+  } else if (url == "languages") {
+    return handle_catalog_v2_languages(request);
+  } else if (url == "illustration") {
+    return handle_catalog_v2_illustration(request);
   } else {
-    return Response::build_404(*this, request, "", "");
+    return Response::build_404(*this, request.get_full_url(), "", "");
   }
 }
 
@@ -74,19 +83,21 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
                {"endpoint_root", m_root + "/catalog/v2"},
                {"feed_id", gen_uuid(m_library_id)},
                {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")}
+               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
+               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
+               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
              },
              "application/atom+xml;profile=opds-catalog;kind=navigation"
   );
 }
 
-std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request)
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
 {
   OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
   opdsDumper.setLibraryId(m_library_id);
   const auto bookIds = search_catalog(request, opdsDumper);
-  const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query());
+  const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
              *this,
              opdsFeed,
@@ -94,6 +105,25 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
   );
 }
 
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId)
+{
+  try {
+    mp_library->getBookById(entryId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request.get_full_url(), "", "");
+  }
+
+  OPDSDumper opdsDumper(mp_library);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(m_library_id);
+  const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
+  return ContentResponse::build(
+             *this,
+             opdsFeed,
+             "application/atom+xml;type=entry;profile=opds-catalog"
+  );
+}
+
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
 {
   OPDSDumper opdsDumper(mp_library);
@@ -101,9 +131,35 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
   opdsDumper.setLibraryId(m_library_id);
   return ContentResponse::build(
              *this,
-             opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()),
+             opdsDumper.categoriesOPDSFeed(),
              "application/atom+xml;profile=opds-catalog;kind=navigation"
   );
 }
 
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
+{
+  OPDSDumper opdsDumper(mp_library);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(m_library_id);
+  return ContentResponse::build(
+             *this,
+             opdsDumper.languagesOPDSFeed(),
+             "application/atom+xml;profile=opds-catalog;kind=navigation"
+  );
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
+{
+  try {
+    const auto bookName  = request.get_url_part(3);
+    const auto bookId = mp_nameMapper->getIdForName(bookName);
+    auto book = mp_library->getBookByIdThreadSafe(bookId);
+    auto size = request.get_argument<unsigned int>("size");
+    auto illustration = book.getIllustration(size);
+    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+  } catch(...) {
+    return Response::build_404(*this, request.get_full_url(), "", "");
+  }
+}
+
 } // namespace kiwix

src/server/response.cpp

@@ -83,10 +83,12 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   return response;
 }
 
-std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
+std::unique_ptr<Response> Response::build_404(const InternalServer& server, const std::string& url, const std::string& bookName, const std::string& bookTitle, const std::string& details)
 {
   MustacheData results;
-  results.set("url", request.get_full_url());
+  if ( !url.empty() ) {
+    results.set("url", url);
+  }
   results.set("details", details);
 
   auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
@@ -115,7 +117,16 @@ std::unique_ptr<Response> Response::build_500(const InternalServer& server, cons
   data.set("error", msg);
   auto content = render_template(RESOURCE::templates::_500_html, data);
   std::unique_ptr<Response> response (
-      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
+      new ContentResponse(
+        server.m_root, //root
+        true, //verbose
+        true, //raw
+        false, //withTaskbar
+        false, //withLibraryButton
+        false, //blockExternalLinks
+        content, //content
+        "text/html" //mimetype
+  ));
   response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
   return response;
 }
@@ -236,8 +247,11 @@ ContentResponse::can_compress(const RequestContext& request) const
 bool
 ContentResponse::contentDecorationAllowed() const
 {
-    return (startsWith(m_mimeType, "text/html")
-         && m_mimeType.find(";raw=true") == std::string::npos);
+  if (m_raw) {
+    return false;
+  }
+  return (startsWith(m_mimeType, "text/html")
+    && m_mimeType.find(";raw=true") == std::string::npos);
 }
 
 MHD_Response*
@@ -325,11 +339,12 @@ void ContentResponse::set_taskbar(const std::string& bookName, const std::string
 }
 
 
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
   m_mimeType(mimetype),
+  m_raw(raw),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
@@ -339,11 +354,17 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool wit
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& content,
+  const std::string& mimetype,
+  bool isHomePage,
+  bool raw)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
+        raw,
         server.m_withTaskbar && !isHomePage,
         server.m_withLibraryButton,
         server.m_blockExternalLinks,
@@ -351,7 +372,13 @@ std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& se
         mimetype));
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& template_str,
+  kainjow::mustache::data data,
+  const std::string& mimetype,
+  bool isHomePage)
+{
   auto content = render_template(template_str, data);
   return ContentResponse::build(server, content, mimetype, isHomePage);
 }
@@ -366,14 +393,14 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
     response->set_cacheable();
     response->m_byteRange = byteRange;
     return std::move(response);

src/server/response.h

@@ -47,7 +47,7 @@ class Response {
 
     static std::unique_ptr<Response> build(const InternalServer& server);
     static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
-    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
+    static std::unique_ptr<Response> build_404(const InternalServer& server, const std::string& url, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
     static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
     static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
     static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
@@ -78,9 +78,27 @@ class Response {
 
 class ContentResponse : public Response {
   public:
-    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
+    ContentResponse(
+      const std::string& root,
+      bool verbose,
+      bool raw,
+      bool withTaskbar,
+      bool withLibraryButton,
+      bool blockExternalLinks,
+      const std::string& content,
+      const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& content,
+      const std::string& mimetype,
+      bool isHomePage = false,
+      bool raw = false);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& template_str,
+      kainjow::mustache::data data,
+      const std::string& mimetype,
+      bool isHomePage = false);
 
     void set_taskbar(const std::string& bookName, const std::string& bookTitle);
 
@@ -98,6 +116,7 @@ class ContentResponse : public Response {
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
+    bool m_raw;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
@@ -108,7 +127,7 @@ class ContentResponse : public Response {
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "archiveTools.h"
+
+#include "tools.h"
+#include "pathTools.h"
+#include "otherTools.h"
+#include "stringTools.h"
+
+#include <zim/error.h>
+#include <zim/item.h>
+
+namespace kiwix
+{
+std::string getMetadata(const zim::Archive& archive, const std::string& name) {
+    try {
+        return archive.getMetadata(name);
+    } catch (zim::EntryNotFound& e) {
+        return "";
+    }
+}
+
+std::string getArchiveTitle(const zim::Archive& archive) {
+  std::string value = getMetadata(archive, "Title");
+  if (value.empty()) {
+    value = getLastPathElement(archive.getFilename());
+    std::replace(value.begin(), value.end(), '_', ' ');
+    size_t pos = value.find(".zim");
+    value = value.substr(0, pos);
+  }
+  return value;
+}
+
+std::string getMetaDescription(const zim::Archive& archive) {
+  std::string value;
+  value = getMetadata(archive, "Description");
+
+  /* Mediawiki Collection tends to use the "Subtitle" name */
+  if (value.empty()) {
+    value = getMetadata(archive, "Subtitle");
+  }
+
+  return value;
+}
+
+std::string getMetaTags(const zim::Archive& archive, bool original) {
+  std::string tags_str = getMetadata(archive, "Tags");
+  if (original) {
+    return tags_str;
+  }
+  auto tags = convertTags(tags_str);
+  return join(tags, ";");
+}
+
+std::string getMetaLanguage(const zim::Archive& archive) {
+  return getMetadata(archive, "Language");
+}
+
+std::string getMetaName(const zim::Archive& archive) {
+  return getMetadata(archive, "Name");
+}
+
+std::string getMetaDate(const zim::Archive& archive) {
+  return getMetadata(archive, "Date");
+}
+
+std::string getMetaCreator(const zim::Archive& archive) {
+  return getMetadata(archive, "Creator");
+}
+
+std::string getMetaPublisher(const zim::Archive& archive) {
+  return getMetadata(archive, "Publisher");
+}
+
+std::string getMetaFlavour(const zim::Archive& archive) {
+  return getMetadata(archive, "Flavour");
+}
+
+std::string getArchiveId(const zim::Archive& archive) {
+  return (std::string) archive.getUuid();
+}
+
+bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
+                           std::string& content, std::string& mimeType){
+  try {
+    auto item = archive.getIllustrationItem(size);
+    content = item.getData();
+    mimeType = item.getMimetype();
+    return true;
+  } catch(zim::EntryNotFound& e) {};
+
+  return false;
+}
+
+// should this be in libzim
+unsigned int getArchiveMediaCount(const zim::Archive& archive) {
+  std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
+  unsigned int counter = 0;
+
+  for (auto &pair:counterMap) {
+    if (startsWith(pair.first, "image/") ||
+        startsWith(pair.first, "video/") ||
+        startsWith(pair.first, "audio/")) {
+      counter += pair.second;
+    }
+  }
+
+  return counter;
+}
+
+unsigned int getArchiveFileSize(const zim::Archive& archive) {
+  return archive.getFilesize() / 1024;
+}
+
+zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry)
+{
+  return entry.getItem(true);
+}
+
+zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path)
+{
+  try {
+    return archive.getEntryByPath(path);
+  } catch (zim::EntryNotFound& e) {
+    if (path.empty() || path == "/") {
+      return archive.getMainEntry();
+    }
+  }
+  throw zim::EntryNotFound("Cannot find entry for non empty path");
+}
+
+MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
+  try {
+    auto counterContent = archive.getMetadata("Counter");
+    return parseMimetypeCounter(counterContent);
+  } catch (zim::EntryNotFound& e) {
+    return {};
+  }
+}
+
+} // kiwix

src/tools/archiveTools.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_ARCHIVETOOLS_H
+#define KIWIX_ARCHIVETOOLS_H
+
+#include <zim/archive.h>
+#include <tools/otherTools.h>
+
+/**
+ * This file contains all the functions that would make handling data related to
+ * an archive easier.
+ **/
+
+namespace kiwix
+{
+    std::string getMetadata(const zim::Archive& archive, const std::string& name);
+    std::string getArchiveTitle(const zim::Archive& archive);
+    std::string getMetaDescription(const zim::Archive& archive);
+    std::string getMetaTags(const zim::Archive& archive, bool original = false);
+    std::string getMetaLanguage(const zim::Archive& archive);
+    std::string getMetaName(const zim::Archive& archive);
+    std::string getMetaDate(const zim::Archive& archive);
+    std::string getMetaCreator(const zim::Archive& archive);
+    std::string getMetaPublisher(const zim::Archive& archive);
+    std::string getMetaFlavour(const zim::Archive& archive);
+    std::string getArchiveId(const zim::Archive& archive);
+
+    bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
+                           std::string& content, std::string& mimeType);
+
+    unsigned int getArchiveMediaCount(const zim::Archive& archive);
+    unsigned int getArchiveFileSize(const zim::Archive& archive);
+
+    zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
+
+    zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
+
+    MimeCounterType parseArchiveCounter(const zim::Archive& archive);
+
+}
+
+#endif

src/tools/networkTools.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,6 +18,7 @@
  * MA 02110-1301, USA.
  */
 
+#include "tools.h"
 #include <tools/networkTools.h>
 
 #include <stdio.h>
@@ -29,6 +31,17 @@
 #include <iostream>
 #include <stdexcept>
 
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <iostream>
+#else
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <net/if.h>
+#include <netdb.h>
+#endif
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
@@ -57,3 +70,104 @@ std::string kiwix::download(const std::string& url) {
   }
   return ss.str();
 }
+
+std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
+  std::map<std::string, std::string> interfaces;
+
+#ifdef _WIN32
+  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
+  if (sd == INVALID_SOCKET) {
+    std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  INTERFACE_INFO InterfaceList[20];
+  unsigned long nBytesReturned;
+  if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
+      sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
+    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
+  for (int i = 0; i < nNumInterfaces; ++i) {
+    sockaddr_in *pAddress;
+    pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
+    if(pAddress->sin_family == AF_INET) {
+    /* Add to the map */
+    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
+    interfaces[interfaceName] = interfaceName;
+    }
+  }
+#else
+  /* Get Network interfaces information */
+  char buf[16384];
+  struct ifconf ifconf;
+  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
+  ifconf.ifc_len = sizeof(buf);
+  ifconf.ifc_buf=buf;
+  if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
+    perror("ioctl(SIOCGIFCONF)");
+  }
+
+  /* Go through each interface */
+  struct ifreq *ifreq;
+  ifreq = ifconf.ifc_req;
+  for (int i = 0; i < ifconf.ifc_len; ) {
+    if (ifreq->ifr_addr.sa_family == AF_INET) {
+      /* Get the network interface ip */
+      char host[128] = { 0 };
+      const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
+                                    host, sizeof(host),
+                                    0, 0, NI_NUMERICHOST);
+      if (!error) {
+        std::string interfaceName = std::string(ifreq->ifr_name);
+        std::string interfaceIp = std::string(host);
+        /* Add to the map */
+        interfaces[interfaceName] = interfaceIp;
+      } else {
+        perror("getnameinfo()");
+      }
+    }
+
+    /* some systems have ifr_addr.sa_len and adjust the length that
+     * way, but not mine. weird */
+    size_t len;
+#ifndef __linux__
+    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
+#else
+    len = sizeof(*ifreq);
+#endif
+    ifreq = (struct ifreq*)((char*)ifreq+len);
+    i += len;
+  }
+#endif
+  return interfaces;
+}
+
+std::string kiwix::getBestPublicIp() {
+  auto interfaces = getNetworkInterfaces();
+
+#ifndef _WIN32
+  const char* const prioritizedNames[] =
+      { "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
+  for(auto name: prioritizedNames) {
+    auto it = interfaces.find(name);
+    if(it != interfaces.end()) {
+      return it->second;
+    }
+  }
+#endif
+
+  const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
+  for(auto prefix : prefixes){
+    for(auto& itr : interfaces) {
+      auto interfaceIp = itr.second;
+      if (interfaceIp.find(prefix) == 0) {
+        return interfaceIp;
+      }
+    }
+  }
+
+  return "127.0.0.1";
+}

src/tools/otherTools.cpp

@@ -17,7 +17,11 @@
  * MA 02110-1301, USA.
  */
 
+// Implement function declared in tools.h and tools/otherTools.h
+#include "tools.h"
 #include "tools/otherTools.h"
+
+
 #include <algorithm>
 #include <iomanip>
 

src/tools/otherTools.h

@@ -32,9 +32,7 @@ namespace pugi {
 
 namespace kiwix
 {
-  void sleep(unsigned int milliseconds);
   std::string nodeToString(const pugi::xml_node& node);
-  std::string converta2toa3(const std::string& a2code);
 
   /*
    * Convert all format tag string to new format

src/tools/pathTools.cpp

@@ -17,7 +17,10 @@
  * MA 02110-1301, USA.
  */
 
+// Implement method defined in <kiwix/tools.h> and "tools/pathTools.h"
+#include "tools.h"
 #include "tools/pathTools.h"
+
 #include <stdexcept>
 
 #ifdef __APPLE__
@@ -59,7 +62,6 @@
 #define PATH_MAX 1024
 #endif
 
-
 #ifdef _WIN32
 std::string WideToUtf8(const std::wstring& wstr)
 {
@@ -78,7 +80,7 @@ std::wstring Utf8ToWide(const std::string& str)
 }
 #endif
 
-bool isRelativePath(const std::string& path)
+bool kiwix::isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
   if (path.size() < 3 ) {
@@ -173,7 +175,7 @@ std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool ab
   return ret;
 }
 
-std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
+std::string kiwix::computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
   auto parts = kiwix::split(path, SEPARATOR, false);
   auto pathParts = normalizeParts(parts, false);
@@ -198,11 +200,11 @@ std::string computeRelativePath(const std::string& path, const std::string& abso
   return ret;
 }
 
-std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
+std::string kiwix::computeAbsolutePath(const std::string& path, const std::string& relativePath)
 {
   std::string absolutePath = path;
   if (path.empty()) {
-    absolutePath = getCurrentDirectory();
+    absolutePath = kiwix::getCurrentDirectory();
   }
 
   auto parts = kiwix::split(absolutePath, SEPARATOR, false);
@@ -215,7 +217,7 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
   return ret;
 }
 
-std::string removeLastPathElement(const std::string& path)
+std::string kiwix::removeLastPathElement(const std::string& path)
 {
   auto parts_ = kiwix::split(path, SEPARATOR, false);
   auto parts = normalizeParts(parts_, false);
@@ -226,7 +228,7 @@ std::string removeLastPathElement(const std::string& path)
   return ret;
 }
 
-std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
+std::string kiwix::appendToDirectory(const std::string& directoryPath, const std::string& filename)
 {
   std::string newPath = directoryPath;
   if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
@@ -236,7 +238,7 @@ std::string appendToDirectory(const std::string& directoryPath, const std::strin
   return newPath;
 }
 
-std::string getLastPathElement(const std::string& path)
+std::string kiwix::getLastPathElement(const std::string& path)
 {
   auto parts_ = kiwix::split(path, SEPARATOR);
   auto parts = normalizeParts(parts_, false);
@@ -267,7 +269,7 @@ std::string getFileSizeAsString(const std::string& path)
   return convert.str();
 }
 
-std::string getFileContent(const std::string& path)
+std::string kiwix::getFileContent(const std::string& path)
 {
 #ifdef _WIN32
   auto wpath = Utf8ToWide(path);
@@ -300,19 +302,21 @@ std::string getFileContent(const std::string& path)
   return content;
 }
 
-bool fileExists(const std::string& path)
+bool kiwix::fileExists(const std::string& path)
 {
 #ifdef _WIN32
-  return PathFileExistsW(Utf8ToWide(path).c_str());
+  return (_waccess_s(Utf8ToWide(path).c_str(), 0) == 0);
 #else
-  bool flag = false;
-  std::fstream fin;
-  fin.open(path.c_str(), std::ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
+  return (access(path.c_str(), F_OK) == 0);
+#endif
+}
+
+bool kiwix::fileReadable(const std::string& path)
+{
+#ifdef _WIN32
+  return (_waccess_s(Utf8ToWide(path).c_str(), 4) == 0);
+#else
+  return (access(path.c_str(), R_OK) == 0);
 #endif
 }
 
@@ -366,7 +370,7 @@ bool copyFile(const std::string& sourcePath, const std::string& destPath)
 #endif
 }
 
-std::string getExecutablePath(bool realPathOnly)
+std::string kiwix::getExecutablePath(bool realPathOnly)
 {
   if (!realPathOnly) {
     char* cAppImage = ::getenv("APPIMAGE");
@@ -420,7 +424,7 @@ bool writeTextFile(const std::string& path, const std::string& content)
   return true;
 }
 
-std::string getCurrentDirectory()
+std::string kiwix::getCurrentDirectory()
 {
 #ifdef _WIN32
   wchar_t* a_cwd = _wgetcwd(NULL, 0);
@@ -434,7 +438,7 @@ std::string getCurrentDirectory()
   return ret;
 }
 
-std::string getDataDirectory()
+std::string kiwix::getDataDirectory()
 {
 // Try to get the dataDir from the `KIWIX_DATA_DIR` env var
 #ifdef _WIN32
@@ -503,7 +507,7 @@ static std::map<std::string, std::string> extMimeTypes = {
 };
 
 /* Try to get the mimeType from the file extension */
-std::string getMimeTypeForFile(const std::string& filename)
+std::string kiwix::getMimeTypeForFile(const std::string& filename)
 {
   std::string mimeType = "text/plain";
   auto pos = filename.find_last_of(".");
@@ -524,4 +528,3 @@ std::string getMimeTypeForFile(const std::string& filename)
 
   return mimeType;
 }
-

src/tools/pathTools.h

@@ -26,23 +26,13 @@
 std::string WideToUtf8(const std::wstring& wstr);
 std::wstring Utf8ToWide(const std::string& str);
 #endif
-bool isRelativePath(const std::string& path);
-std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
-std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
-std::string removeLastPathElement(const std::string& path);
-std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
 
 unsigned int getFileSize(const std::string& path);
 std::string getFileSizeAsString(const std::string& path);
-std::string getFileContent(const std::string& path);
-bool fileExists(const std::string& path);
 bool makeDirectory(const std::string& path);
 std::string makeTmpDirectory();
 bool copyFile(const std::string& sourcePath, const std::string& destPath);
-std::string getLastPathElement(const std::string& path);
-std::string getExecutablePath(bool realPathOnly = false);
-std::string getCurrentDirectory();
-std::string getDataDirectory();
 bool writeTextFile(const std::string& path, const std::string& content);
-std::string getMimeTypeForFile(const std::string& filename);
+
 #endif
+

src/tools/stringTools.cpp

@@ -17,9 +17,11 @@
  * MA 02110-1301, USA.
  */
 
-#include <tools/stringTools.h>
+// Implement function declared in tools.h and tools/stringTools.h
+#include "tools.h"
+#include "tools/stringTools.h"
 
-#include <tools/pathTools.h>
+#include "tools/pathTools.h"
 #include <unicode/normlzr.h>
 #include <unicode/rep.h>
 #include <unicode/translit.h>
@@ -268,7 +270,7 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
                                       const std::string& delims,
-                                      bool trimEmpty,
+                                      bool dropEmpty,
                                       bool keepDelim)
 {
   std::string::size_type lastPos = 0;
@@ -277,7 +279,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
   while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
   {
     auto token = str.substr(lastPos, pos - lastPos);
-    if (!trimEmpty || !token.empty()) {
+    if (!dropEmpty || !token.empty()) {
       tokens.push_back(token);
     }
     if (keepDelim) {
@@ -287,7 +289,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
   }
 
   auto token = str.substr(lastPos);
-  if (!trimEmpty || !token.empty()) {
+  if (!dropEmpty || !token.empty()) {
     tokens.push_back(token);
   }
   return tokens;
@@ -395,3 +397,11 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
         && std::equal(start.begin(), start.end(), base.begin());
 }
 
+std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
+  std::vector<std::string> variants;
+  variants.push_back(title);
+  variants.push_back(kiwix::ucFirst(title));
+  variants.push_back(kiwix::lcFirst(title));
+  variants.push_back(kiwix::toTitle(title));
+  return variants;
+}

src/tools/stringTools.h

@@ -43,7 +43,6 @@ void loadICUExternalTables();
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
 std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);
@@ -70,5 +69,7 @@ T extractFromString(const std::string& str) {
 }
 
 bool startsWith(const std::string& base, const std::string& start);
+
+std::vector<std::string> getTitleVariants(const std::string& title);
 } //namespace kiwix
 #endif

src/version.cpp

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <iostream>
+#include <sstream>
+
+#include <version.h>
+#include <zim/zim.h>
+#include <kiwix_config.h>
+#include <unicode/uversion.h>
+#include <pugixml.hpp>
+#include <curl/curl.h>
+#include <microhttpd.h>
+#include <xapian.h>
+#include <mustache.hpp>
+#include <zlib.h>
+
+namespace kiwix
+{
+  LibVersions getVersions() {
+    LibVersions versions = {
+      { "libkiwix",      LIBKIWIX_VERSION    },
+      { "libzim",        LIBZIM_VERSION      },
+      { "libxapian",     XAPIAN_VERSION      },
+      { "libcurl",       LIBCURL_VERSION     },
+      { "libmicrohttpd", MHD_get_version()   },
+      { "libz",          ZLIB_VERSION        }
+    };
+
+    // U_ICU_VERSION does not include the patch level if 0
+    std::ostringstream libicu_version;
+    libicu_version << U_ICU_VERSION_MAJOR_NUM << "." << U_ICU_VERSION_MINOR_NUM << "." << U_ICU_VERSION_PATCHLEVEL_NUM;
+    versions.push_back({ "libicu", libicu_version.str() });
+
+    // No human readable version string for pugixml
+    const unsigned pugixml_major = (PUGIXML_VERSION - PUGIXML_VERSION % 1000) / 1000;
+    const unsigned pugixml_minor = (PUGIXML_VERSION - pugixml_major * 1000 - PUGIXML_VERSION % 10) / 10;
+    const unsigned pugixml_patch = PUGIXML_VERSION - pugixml_major * 1000 - pugixml_minor * 10;
+    std::ostringstream libpugixml_version;
+    libpugixml_version << pugixml_major << "." << pugixml_minor << "." << pugixml_patch;
+    versions.push_back({ "libpugixml", libpugixml_version.str() });
+
+    // Needs version 5.0 of Mustache
+#if defined(KAINJOW_MUSTACHE_VERSION_MAJOR)
+    std::ostringstream libmustache_version;
+    libmustache_version << KAINJOW_MUSTACHE_VERSION_MAJOR << "." <<
+      KAINJOW_MUSTACHE_VERSION_MINOR << "." << KAINJOW_MUSTACHE_VERSION_PATCH;
+    versions.push_back({ "libmustache", libmustache_version.str() });
+#endif
+
+    return versions;
+  }
+
+  void printVersions(std::ostream& out) {
+    LibVersions versions = getVersions();
+    for (const auto& iter : versions) {
+      out << (iter != versions.front() ? "+ " : "")
+                << iter.first << " " << iter.second << std::endl;
+    }
+  }
+} //namespace kiwix

static/resources_list.txt

@@ -18,13 +18,19 @@ skin/jquery-ui/images/ui-bg_glass_95_fef1ec_1x400.png
 skin/jquery-ui/jquery-ui.theme.min.css
 skin/jquery-ui/jquery-ui.min.css
 skin/caret.png
+skin/bittorrent.png
+skin/magnet.png
+skin/download.png
+skin/hash.png
+skin/search-icon.svg
 skin/taskbar.js
-skin/langList.js
-skin/categoryList.js
 skin/iso6391To3.js
 skin/isotope.pkgd.min.js
 skin/index.js
 skin/taskbar.css
+skin/index.css
+skin/fonts/Poppins.ttf
+skin/fonts/Roboto.ttf
 skin/block_external.js
 templates/search_result.html
 templates/no_search_result.html
@@ -39,6 +45,8 @@ templates/captured_external.html
 templates/catalog_entries.xml
 templates/catalog_v2_root.xml
 templates/catalog_v2_entries.xml
+templates/catalog_v2_entry.xml
 templates/catalog_v2_categories.xml
+templates/catalog_v2_languages.xml
 opensearchdescription.xml
 catalog_v2_searchdescription.xml

static/skin/categoryList.js

@@ -1,19 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-const categoryList = {
-  "other": "Other",
-  "gutenberg": "Gutenberg",
-  "mooc": "Mooc",
-  "phet": "Phet",
-  "psiram": "Psiram",
-  "stack_exchange": "Stack Exchange",
-  "ted": "Ted",
-  "vikidia": "Vikidia",
-  "wikibooks": "Wikibooks",
-  "wikinews": "Wikinews",
-  "wikipedia": "Wikipedia",
-  "wikiquote": "Wikiquote",
-  "wikisource": "Wikisource",
-  "wikiversity": "Wikiversity",
-  "wikivoyage": "Wikivoyage",
-  "wiktionary": "Wiktionary"
-}

static/skin/index.js

@@ -4,10 +4,10 @@
         start: 0,
         count: viewPortToCount()
     };
-    const filterTypes = ['lang', 'category', 'q'];
     const bookOrderMap = new Map();
     const filterCookieName = 'filters';
     const oneDayDelta = 86400000;
+    let loader;
     let footer;
     let fadeOutDiv;
     let iso;
@@ -41,40 +41,89 @@
         return result;
     }
 
+    const humanFriendlySize = (fileSize) => {
+        if (fileSize === 0) {
+            return '';
+        }
+        const units = ['bytes', 'kB', 'MB', 'GB', 'TB'];
+        let quotient = Math.floor(Math.log10(fileSize) / 3);
+        quotient = quotient < units.length ? quotient : units.length - 1;
+        fileSize /= (1000 ** quotient);
+        return `${+fileSize.toFixed(2)} ${units[quotient]}`;
+    };
+
+    const humanFriendlyTitle = (title) => {
+        title = title.replace(/_/g, ' ');
+        return htmlEncode(title[0].toUpperCase() + title.slice(1));
+    }
+
     function htmlEncode(str) {
         return str.replace(/[\u00A0-\u9999<>\&]/gim, (i) => `&#${i.charCodeAt(0)};`);
     }
 
     function viewPortToCount(){
-        return Math.floor(window.innerHeight/100 + 1)*(window.innerWidth>1000 ? 3 : 2);
+        const zoom = Math.floor((( window.outerWidth - 10 ) / window.innerWidth) * 100);
+        return Math.floor(window.innerHeight/(3*zoom) + 1)*(window.innerWidth/(2.5*zoom) + 1);
     }
 
     function getInnerHtml(node, query) {
-        return node.querySelector(query).innerHTML;
+        const queryNode = node.querySelector(query);
+        return queryNode != null ? queryNode.innerHTML : "";
     }
 
     function generateBookHtml(book, sort = false) {
-        const link = book.querySelector('link').getAttribute('href');
+        const link = book.querySelector('link[type="text/html"]').getAttribute('href');
+        let iconUrl;
+        book.querySelectorAll('link[rel="http://opds-spec.org/image/thumbnail"]').forEach(link => {
+            if (link.getAttribute('type').split(';')[1] == 'width=48' && !iconUrl) {
+                iconUrl = link.getAttribute('href');
+            }
+        });
         const title =  getInnerHtml(book, 'title');
         const description = getInnerHtml(book, 'summary');
         const id = getInnerHtml(book, 'id');
-        const iconUrl = getInnerHtml(book, 'icon');
-        const articleCount = getInnerHtml(book, 'articleCount');
-        const mediaCount = getInnerHtml(book, 'mediaCount');
-
-        const linkTag = document.createElement('a');
-        linkTag.setAttribute('class', 'book');
-        linkTag.setAttribute('data-id', id);
-        linkTag.setAttribute('href', link);
-        if (sort) {
-            linkTag.setAttribute('data-idx', bookOrderMap.get(id));
+        const language = getInnerHtml(book, 'language');
+        const tags = getInnerHtml(book, 'tags');
+        let tagHtml = tags.split(';').filter(tag => {return !(tag.split(':')[0].startsWith('_'))})
+            .map((tag) => {return tag.charAt(0).toUpperCase() + tag.slice(1)})
+            .join(' | ').replace(/_/g, ' ');
+        let downloadLink;
+        let zimSize = 0;
+        try {
+            const downloadBookLink = book.querySelector('link[type="application/x-zim"]')
+            zimSize = parseInt(downloadBookLink.getAttribute('length'));
+            downloadLink = downloadBookLink.getAttribute('href').split('.meta4')[0];
+        } catch {
+            downloadLink = '';
         }
-        linkTag.innerHTML = `<div class='book__background' style="background-image: url('${iconUrl}');">
-            <div class='book__title' title='${title}'>${title}</div>
-            <div class='book__description' title='${description}'>${description}</div>
-            <div class='book__info'>${articleCount} articles, ${mediaCount} medias</div>
-        </div>`;
-        return linkTag;
+        const humanFriendlyZimSize = humanFriendlySize(zimSize);
+
+        const divTag = document.createElement('div');
+        divTag.setAttribute('class', 'book');
+        divTag.setAttribute('data-id', id);
+        if (sort) {
+            divTag.setAttribute('data-idx', bookOrderMap.get(id));
+        }
+        const faviconAttr = iconUrl != undefined ? `style="background-image: url('${iconUrl}')"` : '';
+        const languageAttr = language != '' ? '' : 'style="background-color: transparent"';
+        divTag.innerHTML = `<a class="book__link" href="${link}" data-hover="Preview">
+            <div class="book__wrapper">
+            <div class="book__icon" ${faviconAttr}></div>
+            <div class="book__header">
+                <div id="book__title">${title}</div>
+                ${downloadLink ? `<div class="book__download"><span data-link="${downloadLink}">Download ${humanFriendlyZimSize ? ` - ${humanFriendlyZimSize}</span></div>`: ''}` : ''}
+            </div>
+            <div class="book__description" title="${description}">${description}</div>
+            <div class="book__languageTag" ${languageAttr}>${getLanguageCodeToDisplay(language)}</div>
+            <div class="book__tags"><div class="book__tags--wrapper">${tagHtml}</div></div>
+            </div></div></a>`;
+        return divTag;
+    }
+
+    function getLanguageCodeToDisplay(langCode3Letter) {
+        const langCode2Letter = (Object.keys(iso6391To3).find(key => iso6391To3[key] === langCode3Letter));
+        const res = (langCode2Letter != undefined) ? langCode2Letter : langCode3Letter;
+        return res.toUpperCase();
     }
 
     function toggleFooter(show=false) {
@@ -86,8 +135,72 @@
         }
     }
 
+    function insertModal(button) {
+        const downloadLink = button.getAttribute('data-link');
+        button.addEventListener('click', (event) => {
+            event.preventDefault();
+            document.body.insertAdjacentHTML('beforeend', `<div class="modal-wrapper">
+                <div class="modal">
+                    <div class="modal-heading">
+                        <div class="modal-title">
+                            <div>
+                                Download
+                            </div>
+                        </div>
+                        <div onclick="closeModal()" class="modal-close-button">
+                            <div>
+                                <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 14 14" fill="none">
+                                    <path fill-rule="evenodd" clip-rule="evenodd" d="M13.7071 1.70711C14.0976 1.31658 14.0976 
+                                    0.683417 13.7071 0.292893C13.3166 -0.0976311 12.6834 -0.0976311 12.2929 0.292893L7 5.58579L1.70711
+                                    0.292893C1.31658 -0.0976311 0.683417 -0.0976311 0.292893 0.292893C-0.0976311 0.683417
+                                    -0.0976311 1.31658 0.292893 1.70711L5.58579 7L0.292893 12.2929C-0.0976311 12.6834
+                                    -0.0976311 13.3166 0.292893 13.7071C0.683417 14.0976 1.31658 14.0976 1.70711 13.7071L7
+                                    8.41421L12.2929 13.7071C12.6834 14.0976 13.3166 14.0976 13.7071 13.7071C14.0976 13.3166 
+                                    14.0976 12.6834 13.7071 12.2929L8.41421 7L13.7071 1.70711Z" fill="black" />
+                                </svg>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="modal-content">
+                        <div class="modal-regular-download">
+                            <a href="${downloadLink}" download>
+                                <img src="../skin/download.png" alt="direct download" />
+                                <div>Direct</div>
+                            </a>
+                        </div>
+                        <div class="modal-regular-download">
+                            <a href="${downloadLink}.sha256" download>
+                                <img src="../skin/hash.png" alt="download hash" />
+                                <div>Sha256 hash</div>
+                            </a>
+                        </div>
+                        <div class="modal-regular-download">
+                            <a href="${downloadLink}.magnet" target="_blank">
+                                <img src="../skin/magnet.png" alt="download magnet" />
+                                <div>Magnet link</div>
+                            </a>
+                        </div>
+                        <div class="modal-regular-download">
+                            <a href="${downloadLink}.torrent" download>
+                                <img src="../skin/bittorrent.png" alt="download torrent" />
+                                <div>Torrent file</div>
+                            </a>
+                        </div>
+                    </div>
+                </div>
+            </div>`);
+        })
+    }
+
+    async function getBookCount(query) {
+        const url = `${root}/catalog/search?${query}`;
+        return await fetch(url).then(async (resp) => {
+            const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
+            return parseInt(data.querySelector('totalResults').innerHTML);
+        });
+    }
+
     async function loadBooks() {
-        const loader = document.querySelector('.loader');
         loader.style.display = 'block';
         return await fetch(queryUrlBuilder()).then(async (resp) => {
             const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
@@ -96,47 +209,78 @@
                 bookOrderMap.set(getInnerHtml(book, 'id'), idx);
             });
             incrementalLoadingParams.start += books.length;
-            if (parseInt(data.querySelector('totalResults').innerHTML) === bookOrderMap.size) {
+            const results = parseInt(data.querySelector('totalResults').innerHTML)
+            if (results === bookOrderMap.size) {
                 incrementalLoadingParams.count = 0;
                 toggleFooter(true);
             } else {
                 toggleFooter();
             }
+            const kiwixResultText = document.querySelector('.kiwixHomeBody__results')
+            if (results) {
+                let resultText = `${results} books`;
+                if (results === 1) {
+                    resultText = `${results} book`;
+                }
+                kiwixResultText.innerHTML = resultText;
+            } else {
+                kiwixResultText.innerHTML = ``;
+            }
             loader.style.display = 'none';
             return books;
         });
     }
 
-    async function loadAndDisplayOptions(nodeQuery, query) {
-        // currently taking an object in place of query, will replace it with query while fetching data from backend later on.
-        document.querySelector(nodeQuery).innerHTML += Object.keys(query)
-            .map((option) => {return `<option value='${option}'>${htmlEncode(query[option])}</option>`})
-        .join('');
+    async function loadAndDisplayOptions(nodeQuery, query, valueEntryNode) {
+        await fetch(query).then(async (resp) => {
+            const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
+            let optionStr = '';
+            data.querySelectorAll('entry').forEach(entry => {
+                const title = getInnerHtml(entry, 'title');
+                const value = getInnerHtml(entry, valueEntryNode);
+                optionStr += `<option value="${value}">${humanFriendlyTitle(title)}</option>`;
+            });
+            document.querySelector(nodeQuery).innerHTML += optionStr;
+        });
     }
 
     function checkAndInjectEmptyMessage() {
+        const kiwixHomeBody = document.querySelector('.kiwixHomeBody');
         if (!bookOrderMap.size) {
             if (!noResultInjected) {
                 noResultInjected = true;
-                iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('a'));
+                iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('div'));
                 iso.layout();
-                const spanTag = document.createElement('span');
-                spanTag.setAttribute('class', 'noResults');
-                spanTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter?</a>`;
-                document.querySelector('body').append(spanTag);
-                spanTag.getElementsByTagName('a')[0].onclick = (event) => {
-                    event.preventDefault();
-                    window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
-                    setCookie(filterCookieName, 'lang=');
-                    resetAndFilter();
-                    filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
-                };
+                setTimeout(() => {
+                    const divTag = document.createElement('div');
+                    divTag.setAttribute('class', 'noResults');
+                    divTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter</a>?`;
+                    kiwixHomeBody.append(divTag);
+                    kiwixHomeBody.setAttribute('style', 'display: flex; justify-content: center; align-items: center');
+                    divTag.getElementsByTagName('a')[0].onclick = (event) => {
+                        event.preventDefault();
+                        window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
+                        setCookie(filterCookieName, 'lang=');
+                        resetAndFilter();
+                        document.querySelectorAll('.filter').forEach(filter => {
+                            filter.value = params.get(filter.name) || '';
+                            if (filter.value) {
+                                filter.style = 'background-color: #858585; color: #fff';
+                            } else {
+                                filter.style = 'background-color: #ffffff; color: black';
+                            }
+                        })
+                    };
+                    loader.setAttribute('style', 'position: absolute; top: 50%');
+                }, 300);
             }
             return true;
         } else if (noResultInjected) {
             noResultInjected = false;
             document.getElementsByClassName('noResults')[0].remove();
+            kiwixHomeBody.removeAttribute('style');
         }
+        loader.removeAttribute('style');
         return false;
     }
 
@@ -170,7 +314,13 @@
         });
         books = [...books].filter((book) => {return !booksToFilter.has(getInnerHtml(book, 'id'))});
         booksToDelete.forEach(book => {iso.remove(book);});
-        books.forEach((book) => {iso.insert(generateBookHtml(book, sort))});
+        books.forEach((book) => {
+            iso.insert(generateBookHtml(book, sort))
+            const downloadButton = document.querySelector(`[data-id="${getInnerHtml(book, 'id')}"] .book__download`);
+            if (downloadButton) {
+                insertModal(downloadButton);
+            }
+        });
     }
 
     async function resetAndFilter(filterType = '', filterValue = '') {
@@ -185,16 +335,23 @@
             window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
             setCookie(filterCookieName, params.toString());
         }
+        document.querySelectorAll('.filter').forEach(filter => {
+            if (filter.value) {
+                filter.style = 'background-color: #858585; color: #fff';
+            } else {
+                filter.style = 'background-color: #ffffff; color: black';
+            }
+        });
         await loadAndDisplayBooks(true);
     }
 
     window.addEventListener('popstate', async () => {
         await resetAndFilter();
-        filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
+        document.querySelectorAll('.filter').forEach(filter => {filter.value = params.get(filter.name) || ''});
     });
 
     async function loadSubset() {
-        if (window.innerHeight + window.scrollY >= document.body.offsetHeight) {
+        if (window.innerHeight + window.scrollY >= (document.body.offsetHeight * 0.98)) {
             if (incrementalLoadingParams.count) {
                 loadAndDisplayBooks();
             }
@@ -223,28 +380,48 @@
                     return index ? parseInt(index) : Infinity;
                 }
             },
-            sortBy: 'weight'
+            sortBy: 'weight',
+            layoutMode: 'cellsByRow',
+            cellsByRow: {
+                columnWidth: '.book',
+                rowHeight: '.book'
+            }
         });
         footer = document.getElementById('kiwixfooter');
         fadeOutDiv = document.getElementById('fadeOut');
+        loader = document.querySelector('.loader');
         await loadAndDisplayBooks();
-        await loadAndDisplayOptions('#languageFilter', langList);
-        await loadAndDisplayOptions('#categoryFilter', categoryList);
-        filterTypes.forEach((filter) => {
-            const filterTag = document.getElementsByName(filter)[0];
-            filterTag.addEventListener('change', () => {resetAndFilter(filterTag.name, filterTag.value)});
+        await loadAndDisplayOptions('#languageFilter', `${root}/catalog/v2/languages`, 'language');
+        await loadAndDisplayOptions('#categoryFilter', `${root}/catalog/v2/categories`, 'title');
+        document.querySelectorAll('.filter').forEach(filter => {
+            filter.addEventListener('change', () => {resetAndFilter(filter.name, filter.value)});
         });
         if (filters) {
             window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
         }
-        params.forEach((value, key) => {document.getElementsByName(key)[0].value = value});
+        params.forEach((value, key) => {
+            const selectBox = document.getElementsByName(key)[0];
+            if (selectBox) {
+                selectBox.value = value
+            }
+        });
         document.getElementById('kiwixSearchForm').onsubmit = (event) => {event.preventDefault()};
         if (!window.location.search) {
             const browserLang = navigator.language.split('-')[0];
             const langFilter = document.getElementById('languageFilter');
-            langFilter.value = browserLang.length === 3 ? browserLang : iso6391To3[browserLang];
-            langFilter.dispatchEvent(new Event('change'));
+            const lang = browserLang.length === 3 ? browserLang : iso6391To3[browserLang];
+            if (await getBookCount(`lang=${lang}`)) {
+                langFilter.value = lang;
+                langFilter.dispatchEvent(new Event('change'));
+            }
         }
+        document.querySelectorAll('.filter').forEach(filter => {
+            if (filter.value) {
+                filter.style = 'background-color: #858585; color: #fff';
+            } else {
+                filter.style = 'background-color: #ffffff; color: black';
+            }
+        });
         setCookie(filterCookieName, params.toString());
     }
 })();

static/skin/iso6391To3.js

@@ -1,15 +1,22 @@
+// Source: https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
 // eslint-disable-next-line no-unused-vars
 const iso6391To3 = {
     "aa": "aar",
+    "ab": "abk",
+    "ae": "ave",
     "af": "afr",
     "ak": "aka",
     "am": "amh",
+    "an": "arg",
     "ar": "ara",
     "as": "asm",
+    "av": "ava",
+    "ay": "aym",
     "az": "aze",
     "ba": "bak",
     "be": "bel",
     "bg": "bul",
+    "bi": "bis",
     "bm": "bam",
     "bn": "ben",
     "bo": "bod",
@@ -17,24 +24,33 @@ const iso6391To3 = {
     "bs": "bos",
     "ca": "cat",
     "ce": "che",
+    "ch": "cha",
     "co": "cos",
+    "cr": "cre",
     "cs": "ces",
+    "cu": "chu",
     "cv": "chv",
     "cy": "cym",
     "da": "dan",
     "de": "deu",
+    "dv": "div",
     "dz": "dzo",
+    "el": "ell",
     "ee": "ewe",
     "en": "eng",
+    "eo": "epo",
     "es": "spa",
     "et": "est",
     "eu": "eus",
     "fa": "fas",
     "ff": "ful",
     "fi": "fin",
+    "fj": "fij",
     "fo": "fao",
     "fr": "fra",
+    "fy": "fry",
     "ga": "gle",
+    "gd": "gla",
     "gl": "glg",
     "gn": "grn",
     "gu": "guj",
@@ -42,44 +58,78 @@ const iso6391To3 = {
     "ha": "hau",
     "he": "heb",
     "hi": "hin",
+    "ho": "hmo",
     "hr": "hrv",
+    "ht": "hat",
     "hu": "hun",
     "hy": "hye",
+    "hz": "her",
+    "ia": "ina",
     "id": "ind",
+    "ie": "ile",
     "ig": "ibo",
+    "ii": "iii",
+    "ik": "ipk",
+    "io": "ido",
     "is": "isl",
     "it": "ita",
     "iu": "iku",
     "ja": "jpn",
     "jv": "jav",
     "ka": "kat",
+    "kg": "kon",
     "ki": "kik",
+    "kj": "kua",
+    "kl": "kal",
     "kk": "kaz",
     "km": "khm",
     "kn": "kan",
     "ko": "kor",
+    "kr": "kau",
     "ks": "kas",
     "ku": "kur",
+    "kv": "kom",
     "kw": "cor",
     "ky": "kir",
+    "la": "lat",
     "lb": "ltz",
     "lg": "lug",
+    "li": "lim",
     "ln": "lin",
     "lo": "lao",
     "lt": "lit",
+    "lu": "lub",
     "lv": "lav",
     "mg": "mlg",
+    "mh": "mah",
     "mi": "mri",
     "mk": "mkd",
     "ml": "mal",
     "mn": "mon",
     "mr": "mar",
+    "ms": "msa",
     "mt": "mlt",
     "my": "mya",
+    "na": "nau",
+    "nb": "nob",
+    "nd": "nde",
+    "ne": "nep",
+    "ng": "ndo",
     "nl": "nld",
+    "nn": "nno",
+    "no": "nor",
+    "nr": "nbl",
+    "nv": "nav",
     "ny": "nya",
+    "oc": "oci",
+    "oj": "oji",
     "om": "orm",
+    "or": "ori",
+    "os": "oss",
+    "pa": "pan",
+    "pi": "pli",
     "pl": "pol",
+    "ps": "pus",
     "pt": "por",
     "qu": "que",
     "rm": "roh",
@@ -88,7 +138,10 @@ const iso6391To3 = {
     "ru": "rus",
     "rw": "kin",
     "sa": "san",
+    "sc": "srd",
     "sd": "snd",
+    "se": "sme",
+    "sm": "smo",
     "sg": "sag",
     "si": "sin",
     "sk": "slk",
@@ -98,27 +151,37 @@ const iso6391To3 = {
     "sq": "sqi",
     "sr": "srp",
     "ss": "ssw",
+    "st": "sot",
+    "su": "sun",
     "sv": "swe",
+    "sw": "swa",
     "ta": "tam",
     "te": "tel",
     "tg": "tgk",
     "th": "tha",
     "ti": "tir",
     "tk": "tuk",
+    "tl": "tgl",
     "tn": "tsn",
+    "to": "ton",
     "tr": "tur",
     "ts": "tso",
     "tt": "tat",
+    "tw": "twi",
+    "ty": "tah",
     "ug": "uig",
     "uk": "ukr",
     "ur": "urd",
     "uz": "uzb",
     "ve": "ven",
     "vi": "vie",
+    "vo": "vol",
     "wa": "wln",
     "wo": "wol",
     "xh": "xho",
+    "yi": "yid",
     "yo": "yor",
+    "za": "zha",
     "zh": "zho",
     "zu": "zul"
 }

static/skin/langList.js

@@ -1,124 +0,0 @@
-// eslint-disable-next-line no-unused-vars
-const langList = {
-  "aar": "Afaraf",
-  "afr": "Afrikaans",
-  "aka": "Akan",
-  "amh": "አማርኛ",
-  "ara": "اللغة العربية",
-  "asm": "অসমীয়া",
-  "aze": "azərbaycan dili",
-  "bak": "башҡорт теле",
-  "bel": "беларуская мова",
-  "bul": "български език",
-  "bam": "bamanankan",
-  "ben": "বাংলা",
-  "bod": "བོད་ཡིག",
-  "bre": "brezhoneg",
-  "bos": "bosanski jezik",
-  "cat": "Català",
-  "che": "нохчийн мотт",
-  "cos": "corsu",
-  "ces": "čeština",
-  "chv": "чӑваш чӗлхи",
-  "cym": "Cymraeg",
-  "dan": "dansk",
-  "deu": "Deutsch",
-  "dzo": "རྫོང་ཁ",
-  "ewe": "Eʋegbe",
-  "eng": "English",
-  "spa": "Español",
-  "est": "eesti",
-  "eus": "euskara",
-  "fas": "فارسی",
-  "ful": "Fulfulde",
-  "fin": "suomi",
-  "fao": "føroyskt",
-  "fra": "Français",
-  "gle": "Gaeilge",
-  "glg": "galego",
-  "grn": "Avañe'ẽ",
-  "guj": "ગુજરાતી",
-  "glv": "Gaelg",
-  "hau": "هَوُسَ",
-  "heb": "עברית",
-  "hin": "हिन्दी",
-  "hrv": "hrvatski jezik",
-  "hun": "magyar",
-  "hye": "Հայերեն",
-  "ind": "Bahasa Indonesia",
-  "ibo": "Asụsụ Igbo",
-  "isl": "Íslenska",
-  "ita": "Italiano",
-  "iku": "ᐃᓄᒃᑎᑐᑦ",
-  "jpn": "日本語",
-  "jav": "basa Jawa",
-  "kat": "ქართული",
-  "kik": "Gĩkũyũ",
-  "kaz": "қазақ тілі",
-  "khm": "ខេមរភាសា",
-  "kan": "ಕನ್ನಡ",
-  "kor": "한국어",
-  "kas": "कश्मीरी",
-  "kur": "Kurdî",
-  "cor": "Kernewek",
-  "kir": "Кыргызча",
-  "ltz": "Lëtzebuergesch",
-  "lug": "Luganda",
-  "lin": "Lingála",
-  "lao": "ພາສາ",
-  "lit": "lietuvių kalba",
-  "lav": "latviešu valoda",
-  "mlg": "fiteny malagasy",
-  "mri": "te reo Māori",
-  "mkd": "македонски јазик",
-  "mal": "മലയാളം",
-  "mon": "Монгол хэл",
-  "mar": "मराठी",
-  "mlt": "Malti",
-  "mya": "ဗမာစာ",
-  "nld": "Nederlands",
-  "nya": "chiCheŵa",
-  "orm": "Afaan Oromoo",
-  "pol": "język polski",
-  "por": "Português",
-  "que": "Runa Simi",
-  "roh": "rumantsch grischun",
-  "run": "Ikirundi",
-  "ron": "Română",
-  "rus": "Русский",
-  "kin": "Ikinyarwanda",
-  "san": "संस्कृतम्",
-  "snd": "सिन्धी",
-  "sag": "yângâ tî sängö",
-  "sin": "සිංහල",
-  "slk": "slovenčina",
-  "slv": "slovenski jezik",
-  "sna": "chiShona",
-  "som": "Soomaaliga",
-  "sqi": "Shqip",
-  "srp": "српски језик",
-  "ssw": "SiSwati",
-  "swe": "svenska",
-  "tam": "தமிழ்",
-  "tel": "తెలుగు",
-  "tgk": "тоҷикӣ",
-  "tha": "ไทย",
-  "tir": "ትግርኛ",
-  "tuk": "Türkmen",
-  "tsn": "Setswana",
-  "tur": "Türkçe",
-  "tso": "Xitsonga",
-  "tat": "татар теле",
-  "uig": "ئۇيغۇرچە‎",
-  "ukr": "Українська",
-  "urd": "اردو",
-  "uzb": "Ўзбек",
-  "ven": "Tshivenḓa",
-  "vie": "Tiếng Việt",
-  "wln": "walon",
-  "wol": "Wollof",
-  "xho": "isiXhosa",
-  "yor": "Yorùbá",
-  "zho": "中文",
-  "zul": "isiZulu"
-}

static/skin/search-icon.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" version="1.0" width="512.000000" height="512.000000" preserveAspectRatio="xMidYMid meet" style=""><rect id="backgroundrect" width="100%" height="100%" x="0" y="0" fill="none" stroke="none"/>
+
+
+<g class="currentLayer" style=""><title>Layer 1</title><g transform="translate(0,512) scale(0.10000000149011612,-0.10000000149011612) " fill="#484848" stroke="none" id="svg_1" class="selected" fill-opacity="1">
+<path d="M1858 5104 c-341 -50 -678 -192 -963 -408 -104 -78 -304 -278 -384 -384 -124 -164 -256 -409 -315 -583 -311 -929 47 -1926 877 -2442 662 -410 1535 -405 2188 14 l76 49 125 -128 c69 -70 352 -364 630 -653 574 -597 548 -575 688 -567 85 5 123 21 176 75 52 52 77 114 77 193 0 98 -24 142 -151 273 -420 434 -1132 1177 -1132 1182 0 3 22 33 49 68 164 209 291 471 362 746 159 614 28 1255 -363 1774 -78 104 -279 305 -386 386 -288 218 -618 355 -974 406 -130 19 -448 18 -580 -1z m502 -545 c308 -38 607 -184 844 -409 476 -455 606 -1146 327 -1745 -128 -273 -356 -519 -626 -674 -387 -222 -857 -259 -1282 -102 -626 231 -1040 879 -983 1540 13 146 33 241 77 371 214 634 830 1060 1493 1032 36 -1 103 -7 150 -13z" id="svg_2" fill="#484848" fill-opacity="1"/>
+</g></g></svg>

static/templates/404.html

@@ -6,9 +6,11 @@
   </head>
   <body>
     <h1>Not Found</h1>
+    {{#url}}
     <p>
       The requested URL "{{url}}" was not found on this server.
     </p>
+    {{/url}}
     {{#details}}
     <p>
       {{{details}}}

static/templates/catalog_entries.xml

@@ -9,30 +9,4 @@
 {{/filter}}
   <link rel="self" href="" type="application/atom+xml" />
   <link rel="search" type="application/opensearchdescription+xml" href="{{root}}/catalog/searchdescription.xml" />
-  {{#books}}
-  <entry>
-    <id>{{id}}</id>
-    <title>{{title}}</title>
-    <summary>{{description}}</summary>
-    <language>{{language}}</language>
-    <updated>{{updated}}</updated>
-    <name>{{name}}</name>
-    <flavour>{{flavour}}</flavour>
-    <category>{{category}}</category>
-    <tags>{{tags}}</tags>
-    <articleCount>{{article_count}}</articleCount>
-    <mediaCount>{{media_count}}</mediaCount>
-    <icon>/meta?name=favicon&amp;content={{{content_id}}}</icon>
-    <link type="text/html" href="/{{{content_id}}}" />
-    <author>
-      <name>{{author_name}}</name>
-    </author>
-    <publisher>
-      <name>{{publisher_name}}</name>
-    </publisher>
-    {{#url}}
-    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
-    {{/url}}
-  </entry>
-  {{/books}}
-</feed>
+{{#books}}{{{entry}}}{{/books}}</feed>

static/templates/catalog_v2_entries.xml

@@ -5,7 +5,7 @@
   <id>{{feed_id}}</id>
 
   <link rel="self"
-        href="{{endpoint_root}}/entries{{{query}}}"
+        href="{{endpoint_root}}/{{#dump_partial_entries}}partial_{{/dump_partial_entries}}entries{{{query}}}"
         type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
   <link rel="start"
         href="{{endpoint_root}}/root.xml"
@@ -21,30 +21,4 @@
   <startIndex>{{startIndex}}</startIndex>
   <itemsPerPage>{{itemsPerPage}}</itemsPerPage>
 {{/filter}}
-  {{#books}}
-  <entry>
-    <id>{{id}}</id>
-    <title>{{title}}</title>
-    <summary>{{description}}</summary>
-    <language>{{language}}</language>
-    <updated>{{updated}}</updated>
-    <name>{{name}}</name>
-    <flavour>{{flavour}}</flavour>
-    <category>{{category}}</category>
-    <tags>{{tags}}</tags>
-    <articleCount>{{article_count}}</articleCount>
-    <mediaCount>{{media_count}}</mediaCount>
-    <icon>/meta?name=favicon&amp;content={{{content_id}}}</icon>
-    <link type="text/html" href="/{{{content_id}}}" />
-    <author>
-      <name>{{author_name}}</name>
-    </author>
-    <publisher>
-      <name>{{publisher_name}}</name>
-    </publisher>
-    {{#url}}
-    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
-    {{/url}}
-  </entry>
-  {{/books}}
-</feed>
+{{#books}}{{{entry}}}{{/books}}</feed>

static/templates/catalog_v2_entry.xml

@@ -0,0 +1,32 @@
+{{#with_xml_header}}<?xml version="1.0" encoding="UTF-8"?>
+{{/with_xml_header}}  <entry>
+    <id>urn:uuid:{{id}}</id>
+    <title>{{title}}</title>
+    <updated>{{updated}}</updated>
+{{#dump_partial_entries}}
+    <link rel="alternate"
+          href="{{endpoint_root}}/entry/{{{id}}}"
+          type="application/atom+xml;type=entry;profile=opds-catalog"/>
+{{/dump_partial_entries}}{{^dump_partial_entries}}    <summary>{{description}}</summary>
+    <language>{{language}}</language>
+    <name>{{name}}</name>
+    <flavour>{{flavour}}</flavour>
+    <category>{{category}}</category>
+    <tags>{{tags}}</tags>
+    <articleCount>{{article_count}}</articleCount>
+    <mediaCount>{{media_count}}</mediaCount>
+    {{#icons}}<link rel="http://opds-spec.org/image/thumbnail"
+          href="{{root}}/catalog/v2/illustration/{{{content_id}}}/?size={{icon_size}}"
+          type="{{icon_mimetype}};width={{icon_size}};height={{icon_size}};scale=1"/>
+    {{/icons}}<link type="text/html" href="{{root}}/{{{content_id}}}" />
+    <author>
+      <name>{{author_name}}</name>
+    </author>
+    <publisher>
+      <name>{{publisher_name}}</name>
+    </publisher>
+    {{#url}}
+    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
+    {{/url}}
+{{/dump_partial_entries}}
+  </entry>

static/templates/catalog_v2_languages.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:dc="http://purl.org/dc/terms/"
+      xmlns:opds="https://specs.opds.io/opds-1.2"
+      xmlns:thr="http://purl.org/syndication/thread/1.0">
+  <id>{{feed_id}}</id>
+  <link rel="self"
+        href="{{endpoint_root}}/languages"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <link rel="start"
+        href="{{endpoint_root}}/root.xml"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <title>List of languages</title>
+  <updated>{{date}}</updated>
+
+  {{#languages}}
+  <entry>
+    <title>{{lang_self_name}}</title>
+    <dc:language>{{{lang_code}}}</dc:language>
+    <thr:count>{{book_count}}</thr:count>
+    <link rel="subsection"
+          href="{{endpoint_root}}/entries?lang={{{lang_code}}}"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>{{updated}}</updated>
+    <id>{{id}}</id>
+  </entry>
+  {{/languages}}
+</feed>

static/templates/catalog_v2_root.xml

@@ -23,6 +23,15 @@
     <id>{{all_entries_feed_id}}</id>
     <content type="text">All entries from this catalog.</content>
   </entry>
+  <entry>
+    <title>All entries (partial)</title>
+    <link rel="subsection"
+          href="{{endpoint_root}}/partial_entries"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>{{date}}</updated>
+    <id>{{partial_entries_feed_id}}</id>
+    <content type="text">All entries from this catalog in partial format.</content>
+  </entry>
   <entry>
     <title>List of categories</title>
     <link rel="subsection"
@@ -32,4 +41,13 @@
     <id>{{category_list_feed_id}}</id>
     <content type="text">List of all categories in this catalog.</content>
   </entry>
+  <entry>
+    <title>List of languages</title>
+    <link rel="subsection"
+          href="{{endpoint_root}}/languages"
+          type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+    <updated>{{date}}</updated>
+    <id>{{language_list_feed_id}}</id>
+    <content type="text">List of all languages in this catalog.</content>
+  </entry>
 </feed>

static/templates/index.html

@@ -2,6 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
     <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
     <title>Welcome to Kiwix Server</title>
     <script
       type="text/javascript"
@@ -21,186 +22,58 @@
       href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css"
       rel="Stylesheet"
     />
+    <link
+      type="text/css"
+      href="{{root}}/skin/index.css"
+      rel="Stylesheet"
+    />
     <style>
-      html {
-        min-height: 100%;
-        position: relative;
+      @font-face {
+        font-family: "poppins";
+        src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype");
       }
-      body {
-        background: radial-gradient(#eeeeee 15%, transparent 16%) 0 0,
-          radial-gradient(#eeeeee 15%, transparent 16%) 8px 8px,
-          radial-gradient(rgba(255, 255, 255, 0.1) 15%, transparent 20%) 0 1px,
-          radial-gradient(rgba(255, 255, 255, 0.1) 15%, transparent 20%) 8px 9px;
-        background-color: #e8e8e8;
-        background-size: 16px 16px;
-        margin-left: auto;
-        margin-right: auto;
-        max-width: 1100px;
-        min-height: 100%;
-      }
-      .book__list {
-        text-align: center;
-      }
-      .kiwixHomeBody {
-        position: relative;
-        text-align: center;
-        min-height: 100%;
-        margin: 0 0 15px;
-      }
-      .book {
-        display: inline-block;
-        vertical-align: bottom;
-        margin: 8px;
-        padding: 12px 15px;
-        width: 300px;
-        border: 1px solid #ccc;
-        border-radius: 8px;
-        text-align: left;
-        color: #000;
-        font-family: sans-serif;
-        font-size: 13px;
-        background-color: #f1f1f1;
-        box-shadow: 2px 2px 5px 0 #ccc;
-      }
-      #kiwixfooter {
-        text-align: center;
-        margin: 0.5em;
-        position: absolute;
-        bottom: 0;
-        left: 46%;
-      }
-      .kiwixHomeNavbar {
-        display: flex;
-        justify-content: center;
-      }
-      .kiwixFilter {
-        margin: 8px 10px;
-      }
-      .kiwixSearch, .searchButton {
-        margin: 0 13px 0 0;
-      }
-      .kiwixSearchForm {
-        margin: 8px 10px;
-        float: right;
-      }
-      @media (max-width: 1100px) {
-        .kiwixHomeBody {
-          padding: 0 125px;
-        }
-      }
-      .book:hover {
-        background-color: #f9f9f9;
-        box-shadow: none;
-      }
-      .book__background {
-        background-repeat: no-repeat;
-        background-size: 48px 48px;
-        background-position: top right;
-      }
-      .book__title {
-        padding: 0 55px 0 0;
-        overflow: hidden;
-        text-overflow: ellipsis;
-        white-space: nowrap;
-        font-size: 18px;
-        color: #0645ad;
-        line-height: 1em;
-      }
-      .book__description {
-        padding: 5px 55px 5px 0;
-        overflow: hidden;
-        text-overflow: ellipsis;
-        white-space: nowrap;
-        font-size: 15px;
-        line-height: 1em;
-      }
-      .book__info {
-        color: #777;
-        font-weight: bold;
-        font-size: 13px;
-        line-height: 1em;
-      }
-      a:link {
-        text-decoration: none;
-      }
-      a:visited {
-        text-decoration: none;
-      }
-      .noResults {
-        position: absolute;
-        top: 48%;
-        left: 42%;
-      }
-      .loader-spinner {
-        position: absolute;
-        top: -50%;
-        left: 50%;
-        border: 5px solid #f3f3f3;
-        border-radius: 50%;
-        border-top: 5px solid #3498db;
-        width: 40px;
-        height: 40px;
-        margin: auto;
-        -webkit-animation: spin 1s linear infinite; /* Safari */
-        animation: spin 1s linear infinite;
-        margin-top: 35px;
-        margin-bottom: -35px;
-        z-index: 2;
-      }
-      /* Safari */
-      @-webkit-keyframes spin {
-        0% { -webkit-transform: rotate(0deg); }
-        100% { -webkit-transform: rotate(360deg); }
-      }
-      @keyframes spin {
-        0% { transform: rotate(0deg); }
-        100% { transform: rotate(360deg); }
-      }
-      .loader {
-        position: relative;
-        height: 70px;
-        width: 100%;
-      }
-      .fadeOut {
-        position: fixed;
-        display: none;
-        bottom: 0;
-        left: 0;
-        z-index: 1;
-        background: linear-gradient(180deg, rgba(232, 232, 232, 0) 0%, rgb(232, 232, 232) 100%);
-        height: 80px;
-        width: 100%;
-      }
-      .spacer {
-        height: 20px;
-        background: transparent;
+
+      @font-face {
+          font-family: "roboto";
+          src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype");
       }
     </style>
     <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
-    <script src="{{root}}/skin/categoryList.js"></script>
-    <script src="{{root}}/skin/langList.js"></script>
     <script src="{{root}}/skin/iso6391To3.js"></script>
     <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
   </head>
-  <body class="kiwix">
-    <div class='kiwixHomeNavbar'>
-      <select name="lang" id="languageFilter" class='kiwixFilter'>
-        <option value="" selected>All languages</option>
-      </select>
-      <select name="category" id="categoryFilter" class='kiwixFilter'>
-        <option value="" selected>All categories</option>
-      </select>
-      <form id='kiwixSearchForm' class='kiwixSearchForm'>
-        <input type="text" name="q" id="searchFilter" class='kiwixSearch'>
+  <body>
+    <div class='kiwixNav'>
+      <div class="kiwixNav__filters">
+        <div class="kiwixNav__select">
+          <select name="lang" id="languageFilter" class='kiwixNav__kiwixFilter filter'>
+            <option value="" selected>All languages</option>
+          </select>
+        </div>
+        <div class="kiwixNav__select">
+          <select name="category" id="categoryFilter" class='kiwixNav__kiwixFilter filter'>
+            <option value="" selected>All categories</option>
+          </select>
+        </div>
+      </div>
+      <form id='kiwixSearchForm' class='kiwixNav__SearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
         <input type="submit" class="searchButton" value="Search"/>
       </form>
     </div>
     <div class="kiwixHomeBody">
+      <h3 class="kiwixHomeBody__results"></h3>
       <div class="book__list"></div>
       <div id="fadeOut" class="fadeOut"></div>
     </div>
-    <div class="loader"><div class="loader-spinner"></div></div>
-    <div class="spacer"></div>
-    <div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>
+    <div class="loader" style="position: absolute; top: 50%"><div class="loader-spinner"></div></div>
+    <div id="kiwixfooter" class="kiwixfooter">Powered by&nbsp;<a href="https://kiwix.org">Kiwix</a></div>
   </body>
+  <script>
+    function closeModal() {
+      for(modal of document.getElementsByClassName('modal-wrapper')) {
+        modal.remove();
+      }
+    }
+  </script>
 </html>

static/templates/taskbar_part.html

@@ -5,18 +5,18 @@
         <form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
           {{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
           <label for="kiwixsearchbox">&#x1f50d;</label>
-          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
+          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search '{{title}}'" aria-label="Search '{{title}}'">
         </form>
       </div>
         <input type="checkbox" id="kiwix_button_show_toggle">
         <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
         <div class="kiwix_button_cont">
           {{#withlibrarybutton}}
-            <a id="kiwix_serve_taskbar_library_button" href="{{root}}/"><button>&#x1f3e0;</button></a>
+            <a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="{{root}}/"><button>&#x1f3e0;</button></a>
           {{/withlibrarybutton}}
           {{#hascontent}}
-          <a id="kiwix_serve_taskbar_home_button" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
-          <a id="kiwix_serve_taskbar_random_button"
+          <a id="kiwix_serve_taskbar_home_button" title="Go to the main page of '{{title}}'" aria-label="Go to the main page of '{{title}}'" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
+          <a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
             href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
           {{/hascontent}}
         </div>

test/book.cpp

@@ -2,42 +2,6 @@
 #include "../include/book.h"
 #include <pugixml.hpp>
 
-TEST(BookTest, updateTest)
-{
-    kiwix::Book book;
-
-    book.setId("xyz");
-    book.setReadOnly(false);
-    book.setPath("/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim");
-    book.setPathValid(true);
-    book.setUrl("book-url");
-    book.setTags("youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes");
-    book.setName("skin-of-color-society_en_all");
-    book.setFavicon("book-favicon");
-    book.setFaviconMimeType("book-favicon-mimetype");
-
-    kiwix::Book newBook;
-
-    newBook.setReadOnly(true);
-    EXPECT_FALSE(newBook.update(book));
-
-    newBook.setReadOnly(false);
-    EXPECT_FALSE(newBook.update(book));
-
-    newBook.setId("xyz");
-    EXPECT_TRUE(newBook.update(book));
-
-    EXPECT_EQ(newBook.readOnly(), book.readOnly());
-    EXPECT_EQ(newBook.getPath(), book.getPath());
-    EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
-    EXPECT_EQ(newBook.getUrl(), book.getUrl());
-    EXPECT_EQ(newBook.getTags(), book.getTags());
-    EXPECT_EQ(newBook.getCategory(), book.getCategory());
-    EXPECT_EQ(newBook.getName(), book.getName());
-    EXPECT_EQ(newBook.getFavicon(), book.getFavicon());
-    EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType());
-}
-
 namespace
 {
 
@@ -68,6 +32,9 @@ TEST(BookTest, updateFromXMLTest)
             articleCount="123456"
             mediaCount="234567"
             size="345678"
+            favicon="ZmFrZS1ib29rLWZhdmljb24tZGF0YQ=="
+            faviconMimeType="text/plain"
+            faviconUrl="http://who.org/zara.fav"
           >
       </book>
     )");
@@ -85,6 +52,10 @@ TEST(BookTest, updateFromXMLTest)
     EXPECT_EQ(book.getArticleCount(), 123456U);
     EXPECT_EQ(book.getMediaCount(), 234567U);
     EXPECT_EQ(book.getSize(), 345678U*1024U);
+    auto defaultIllustration = book.getIllustration(48);
+    EXPECT_EQ(defaultIllustration->getData(), "fake-book-favicon-data");
+    EXPECT_EQ(defaultIllustration->mimeType, "text/plain");
+    EXPECT_EQ(defaultIllustration->url, "http://who.org/zara.fav");
 }
 
 TEST(BookTest, updateFromXMLCategoryHandlingTest)
@@ -166,3 +137,76 @@ TEST(BookTest, updateCopiesCategory)
     newBook.update(book);
     EXPECT_EQ(newBook.getCategory(), "ted");
 }
+
+TEST(BookTest, updateTest)
+{
+    const XMLDoc xml(R"(
+      <book id="xyz"
+            path="/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim"
+            url="book-url"
+            name="skin-of-color-society_en_all"
+            tags="youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes"
+            favicon="Ym9vay1mYXZpY29u"
+            faviconMimeType="book-favicon-mimetype"
+          >
+      </book>
+    )");
+
+    kiwix::Book book;
+    book.updateFromXml(xml.child("book"), "/data/zim");
+
+    book.setReadOnly(false);
+    book.setPathValid(true);
+
+    kiwix::Book newBook;
+
+    newBook.setReadOnly(true);
+    EXPECT_FALSE(newBook.update(book));
+
+    newBook.setReadOnly(false);
+    EXPECT_FALSE(newBook.update(book));
+
+    newBook.setId("xyz");
+    EXPECT_TRUE(newBook.update(book));
+
+    EXPECT_EQ(newBook.readOnly(), book.readOnly());
+    EXPECT_EQ(newBook.getPath(), book.getPath());
+    EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
+    EXPECT_EQ(newBook.getUrl(), book.getUrl());
+    EXPECT_EQ(newBook.getTags(), book.getTags());
+    EXPECT_EQ(newBook.getCategory(), book.getCategory());
+    EXPECT_EQ(newBook.getName(), book.getName());
+    auto defaultIllustration = book.getIllustration(48);
+    auto newDefaultIllustration = newBook.getIllustration(48);
+    EXPECT_EQ(newDefaultIllustration->getData(), defaultIllustration->getData());
+    EXPECT_EQ(newDefaultIllustration->mimeType, defaultIllustration->mimeType);
+}
+
+namespace
+{
+
+std::string path2HumanReadableId(const std::string& path)
+{
+    const XMLDoc xml("<book id=\"xyz\" path=\"" + path + "\"></book>");
+
+    kiwix::Book book;
+    book.updateFromXml(xml.child("book"), "/data/zim");
+    return book.getHumanReadableIdFromPath();
+}
+
+} // unnamed namespace
+
+TEST(BookTest, getHumanReadableIdFromPath)
+{
+  EXPECT_EQ("abc",     path2HumanReadableId("abc.zim"));
+  EXPECT_EQ("abc",     path2HumanReadableId("ABC.zim"));
+  EXPECT_EQ("abc",     path2HumanReadableId("âbç.zim"));
+  EXPECT_EQ("ancient", path2HumanReadableId("ancient.zimbabwe"));
+  EXPECT_EQ("ab_cd",   path2HumanReadableId("ab cd.zim"));
+#ifdef _WIN32
+  EXPECT_EQ("abc",     path2HumanReadableId("C:\\Data\\ZIM\\abc.zim"));
+#else
+  EXPECT_EQ("abc",     path2HumanReadableId("/Data/ZIM/abc.zim"));
+#endif
+  EXPECT_EQ("3plus2",  path2HumanReadableId("3+2.zim"));
+}

test/data/library.xml

@@ -14,6 +14,8 @@
         articleCount="284"
         mediaCount="2"
         size="556"
+        faviconMimeType="image/png"
+        favicon="SOME DATA"
       ></book>
   <book
         id="raycharles_uncategorized"
@@ -21,11 +23,11 @@
         url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
         title="Ray (uncategorized) Charles"
         description="No category is assigned to this library entry."
-        language="eng"
+        language="rus"
         creator="Wikipedia"
         publisher="Kiwix"
         date="2020-03-31"
-        name="wikipedia_en_ray_charles"
+        name="wikipedia_ru_ray_charles"
         tags="unittest;wikipedia;_pictures:no;_videos:no;_details:no"
         articleCount="284"
         mediaCount="2"
@@ -33,15 +35,15 @@
       ></book>
   <book
         id="charlesray"
-        path="./zimfile.zim"
-        url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
+        path="./zimfile&amp;other.zim"
+        url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim"
         title="Charles, Ray"
         description="Wikipedia articles about Ray Charles"
-        language="eng"
+        language="fra"
         creator="Wikipedia"
         publisher="Kiwix"
         date="2020-03-31"
-        name="wikipedia_en_ray_charles"
+        name="wikipedia_fr_ray_charles"
         tags="unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes"
         articleCount="284"
         mediaCount="2"

test/data/zimfile&other.zim

@@ -0,0 +1 @@
+zimfile.zim

test/httplib.h

@@ -2889,7 +2889,7 @@ inline ssize_t Stream::write(const std::string &s) {
 
 template <typename... Args>
 inline ssize_t Stream::write_format(const char *fmt, const Args &... args) {
-  std::array<char, 2048> buf;
+  std::array<char, 2048> buf{};
 
 #if defined(_MSC_VER) && _MSC_VER < 1900
   auto sn = _snprintf_s(buf, bufsiz, buf.size() - 1, fmt, args...);
@@ -3756,7 +3756,7 @@ inline socket_t Client::create_client_socket() const {
 }
 
 inline bool Client::read_response_line(Stream &strm, Response &res) {
-  std::array<char, 2048> buf;
+  std::array<char, 2048> buf{};
 
   detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
 

test/library.cpp

@@ -232,7 +232,7 @@ class LibraryTest : public ::testing::Test {
   void SetUp() override {
      kiwix::Manager manager(&lib);
      manager.readOpds(sampleOpdsStream, "foo.urlHost");
-     manager.readXml(sampleLibraryXML, true, "./test/library.xml", true);
+     manager.readXml(sampleLibraryXML, false, "./test/library.xml", true);
   }
 
     kiwix::Bookmark createBookmark(const std::string &id) {
@@ -275,9 +275,25 @@ TEST_F(LibraryTest, getBookMarksTest)
 TEST_F(LibraryTest, sanityCheck)
 {
   EXPECT_EQ(lib.getBookCount(true, true), 12U);
-  EXPECT_EQ(lib.getBooksLanguages().size(), 3U);
-  EXPECT_EQ(lib.getBooksCreators().size(), 9U);
-  EXPECT_EQ(lib.getBooksPublishers().size(), 3U);
+  EXPECT_EQ(lib.getBooksLanguages(),
+            std::vector<std::string>({"deu", "eng", "fra"})
+  );
+  EXPECT_EQ(lib.getBooksCreators(), std::vector<std::string>({
+      "Islam Stack Exchange",
+      "Movies & TV Stack Exchange",
+      "Mythology & Folklore Stack Exchange",
+      "TED",
+      "Tania Louis",
+      "Wiki",
+      "Wikibooks",
+      "Wikipedia",
+      "Wikiquote"
+  }));
+  EXPECT_EQ(lib.getBooksPublishers(), std::vector<std::string>({
+      "",
+      "Kiwix",
+      "Kiwix & Some Enthusiasts"
+  }));
 }
 
 TEST_F(LibraryTest, categoryHandling)
@@ -644,13 +660,14 @@ TEST_F(LibraryTest, filterByMultipleCriteria)
 
 TEST_F(LibraryTest, getBookByPath)
 {
-  auto& book = lib.getBookById(lib.getBooksIds()[0]);
+  kiwix::Book book = lib.getBookById(lib.getBooksIds()[0]);
 #ifdef _WIN32
   auto path = "C:\\some\\abs\\path.zim";
 #else
   auto path = "/some/abs/path.zim";
 #endif
   book.setPath(path);
+  lib.addBook(book);
   EXPECT_EQ(lib.getBookByPath(path).getId(), book.getId());
   EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
 }
@@ -667,9 +684,9 @@ TEST_F(LibraryTest, removeBookByIdRemovesTheBook)
 
 TEST_F(LibraryTest, removeBookByIdDropsTheReader)
 {
-  EXPECT_NE(nullptr, lib.getReaderById("raycharles"));
+  EXPECT_NE(nullptr, lib.getArchiveById("raycharles"));
   lib.removeBookById("raycharles");
-  EXPECT_THROW(lib.getReaderById("raycharles"), std::out_of_range);
+  EXPECT_THROW(lib.getArchiveById("raycharles"), std::out_of_range);
 };
 
 TEST_F(LibraryTest, removeBookByIdUpdatesTheSearchDB)
@@ -690,4 +707,35 @@ TEST_F(LibraryTest, removeBookByIdUpdatesTheSearchDB)
   EXPECT_THROW(lib.getBookById("raycharles"), std::out_of_range);
 };
 
+TEST_F(LibraryTest, removeBooksNotUpdatedSince)
+{
+  EXPECT_FILTER_RESULTS(kiwix::Filter(),
+    "An example ZIM archive",
+    "Encyclopédie de la Tunisie",
+    "Granblue Fantasy Wiki",
+    "Géographie par Wikipédia",
+    "Islam Stack Exchange",
+    "Mathématiques",
+    "Movies & TV Stack Exchange",
+    "Mythology & Folklore Stack Exchange",
+    "Ray Charles",
+    "TED talks - Business",
+    "Tania Louis",
+    "Wikiquote"
+  );
+
+  const uint64_t rev = lib.getRevision();
+  for ( const auto& id : lib.filter(kiwix::Filter().query("exchange")) ) {
+    lib.addBook(lib.getBookByIdThreadSafe(id));
+  }
+
+  EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev));
+
+  EXPECT_FILTER_RESULTS(kiwix::Filter(),
+    "Islam Stack Exchange",
+    "Movies & TV Stack Exchange",
+    "Mythology & Folklore Stack Exchange",
+  );
+};
+
 };

test/manager.cpp

@@ -2,6 +2,7 @@
 #include "../include/manager.h"
 #include "../include/library.h"
 #include "../include/book.h"
+#include "../include/tools.h"
 #include <iostream>
 #include <fstream>
 
@@ -13,13 +14,13 @@ TEST(ManagerTest, addBookFromPathAndGetIdTest)
     auto bookId = manager.addBookFromPathAndGetId("./test/example.zim");
     ASSERT_NE(bookId, "");
     kiwix::Book book = lib.getBookById(bookId);
-    EXPECT_EQ(book.getPath(), computeAbsolutePath("", "./test/example.zim"));
+    EXPECT_EQ(book.getPath(), kiwix::computeAbsolutePath("", "./test/example.zim"));
 
     const std::string pathToSave = "./pathToSave";
     const std::string url = "url";
     bookId = manager.addBookFromPathAndGetId("./test/example.zim", pathToSave, url, true);
     book = lib.getBookById(bookId);
-    auto savedPath = computeAbsolutePath(removeLastPathElement(manager.writableLibraryPath), pathToSave);
+    auto savedPath = kiwix::computeAbsolutePath(kiwix::removeLastPathElement(manager.writableLibraryPath), pathToSave);
     EXPECT_EQ(book.getPath(), savedPath);
     EXPECT_EQ(book.getUrl(), url);
 }
@@ -66,3 +67,29 @@ TEST(ManagerTest, readXml)
     EXPECT_EQ(45U, book.getMediaCount());
     EXPECT_EQ(678U*1024, book.getSize());
 }
+
+TEST(Manager, reload)
+{
+  kiwix::Library lib;
+  kiwix::Manager manager(&lib);
+
+  manager.reload({ "./test/library.xml" });
+  EXPECT_EQ(lib.getBooksIds(), (kiwix::Library::BookIdCollection{
+        "charlesray",
+        "raycharles",
+        "raycharles_uncategorized"
+  }));
+
+  lib.removeBookById("raycharles");
+  EXPECT_EQ(lib.getBooksIds(), (kiwix::Library::BookIdCollection{
+        "charlesray",
+        "raycharles_uncategorized"
+  }));
+
+  manager.reload({ "./test/library.xml" });
+  EXPECT_EQ(lib.getBooksIds(), kiwix::Library::BookIdCollection({
+        "charlesray",
+        "raycharles",
+        "raycharles_uncategorized"
+  }));
+}

test/meson.build

@@ -8,6 +8,8 @@ tests = [
     'kiwixserve',
     'book',
     'manager',
+    'name_mapper',
+    'opds_catalog'
 ]
 
 if build_machine.system() != 'windows'
@@ -25,6 +27,7 @@ if gtest_dep.found() and not meson.is_cross_build()
     data_files = [
       'example.zim',
       'zimfile.zim',
+      'zimfile&other.zim',
       'corner_cases.zim',
       'library.xml'
     ]

test/name_mapper.cpp

@@ -0,0 +1,147 @@
+#include "../include/name_mapper.h"
+
+#include "../include/library.h"
+#include "../include/manager.h"
+#include "gtest/gtest.h"
+
+namespace
+{
+
+const char libraryXML[] = R"(
+<library version="1.0">
+  <book id="01"         path="/data/zero_one.zim">         </book>
+  <book id="02"         path="/data/zero two.zim">         </book>
+  <book id="03"         path="/data/ZERO thrêë.zim">       </book>
+  <book id="04-2021-10" path="/data/zero_four_2021-10.zim"></book>
+  <book id="04-2021-11" path="/data/zero_four_2021-11.zim"></book>
+</library>
+)";
+
+class NameMapperTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+     kiwix::Manager manager(&lib);
+     manager.readXml(libraryXML, false, "./library.xml", true);
+     for ( const std::string& id : lib.getBooksIds() ) {
+       kiwix::Book bookCopy = lib.getBookById(id);
+       bookCopy.setPathValid(true);
+       lib.addBook(bookCopy);
+     }
+  }
+
+  kiwix::Library lib;
+};
+
+class CapturedStderr
+{
+  std::ostringstream buffer;
+  std::streambuf* const sbuf;
+public:
+  CapturedStderr()
+    : sbuf(std::cerr.rdbuf())
+  {
+    std::cerr.rdbuf(buffer.rdbuf());
+  }
+
+  CapturedStderr(const CapturedStderr&) = delete;
+
+  ~CapturedStderr()
+  {
+    std::cerr.rdbuf(sbuf);
+  }
+
+  operator std::string() const { return buffer.str(); }
+};
+
+} // unnamed namespace
+
+void checkUnaliasedEntriesInNameMapper(const kiwix::NameMapper& nm)
+{
+  EXPECT_EQ("zero_one",           nm.getNameForId("01"));
+  EXPECT_EQ("zero_two",           nm.getNameForId("02"));
+  EXPECT_EQ("zero_three",         nm.getNameForId("03"));
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("zero_four_2021-11",  nm.getNameForId("04-2021-11"));
+
+  EXPECT_EQ("01",         nm.getIdForName("zero_one"));
+  EXPECT_EQ("02",         nm.getIdForName("zero_two"));
+  EXPECT_EQ("03",         nm.getIdForName("zero_three"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_EQ("04-2021-11", nm.getIdForName("zero_four_2021-11"));
+}
+
+TEST_F(NameMapperTest, HumanReadableNameMapperWithoutAliases)
+{
+  CapturedStderr stderror;
+  kiwix::HumanReadableNameMapper nm(lib, false);
+  EXPECT_EQ("", std::string(stderror));
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+
+  lib.removeBookById("04-2021-10");
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+}
+
+TEST_F(NameMapperTest, HumanReadableNameMapperWithAliases)
+{
+  CapturedStderr stderror;
+  kiwix::HumanReadableNameMapper nm(lib, true);
+  EXPECT_EQ(
+      "Path collision: /data/zero_four_2021-10.zim and"
+      " /data/zero_four_2021-11.zim can't share the same URL path 'zero_four'."
+      " Therefore, only /data/zero_four_2021-10.zim will be served.\n"
+      , std::string(stderror)
+  );
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+
+  lib.removeBookById("04-2021-10");
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+}
+
+TEST_F(NameMapperTest, UpdatableNameMapperWithoutAliases)
+{
+  CapturedStderr stderror;
+  kiwix::UpdatableNameMapper nm(lib, false);
+  EXPECT_EQ("", std::string(stderror));
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+
+  lib.removeBookById("04-2021-10");
+  nm.update();
+  EXPECT_THROW(nm.getNameForId("04-2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four_2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+}
+
+TEST_F(NameMapperTest, UpdatableNameMapperWithAliases)
+{
+  CapturedStderr stderror;
+  kiwix::UpdatableNameMapper nm(lib, true);
+  EXPECT_EQ(
+      "Path collision: /data/zero_four_2021-10.zim and"
+      " /data/zero_four_2021-11.zim can't share the same URL path 'zero_four'."
+      " Therefore, only /data/zero_four_2021-10.zim will be served.\n"
+      , std::string(stderror)
+  );
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+
+  {
+    CapturedStderr nmUpdateStderror;
+    lib.removeBookById("04-2021-10");
+    nm.update();
+    EXPECT_EQ("", std::string(nmUpdateStderror));
+  }
+  EXPECT_EQ("04-2021-11", nm.getIdForName("zero_four"));
+  EXPECT_THROW(nm.getNameForId("04-2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four_2021-10"), std::out_of_range);
+}

test/opds_catalog.cpp

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
+ * NON-INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#include "../include/opds_catalog.h"
+
+#include "gtest/gtest.h"
+
+using namespace kiwix;
+
+TEST(OpdsCatalog, getSearchUrl)
+{
+  #define EXPECT_SEARCH_URL(url) EXPECT_EQ(url, getSearchUrl(f))
+  {
+    Filter f;
+    EXPECT_SEARCH_URL("/catalog/v2/entries");
+  }
+  {
+    Filter f;
+    f.query("abc");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?q=abc");
+  }
+  {
+    Filter f;
+    f.query("abc def");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?q=abc%20def");
+  }
+  {
+    Filter f;
+    f.category("ted");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?category=ted");
+  }
+  {
+    Filter f;
+    f.lang("eng");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?lang=eng");
+  }
+  {
+    Filter f;
+    f.name("second");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?name=second");
+  }
+  {
+    Filter f;
+    f.acceptTags({"paper", "plastic"});
+    EXPECT_SEARCH_URL("/catalog/v2/entries?tag=paper;plastic");
+  }
+  {
+    Filter f;
+    f.query("abc");
+    f.category("ted");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?q=abc&category=ted");
+  }
+  {
+    Filter f;
+    f.category("ted");
+    f.query("abc");
+    EXPECT_SEARCH_URL("/catalog/v2/entries?q=abc&category=ted");
+  }
+  {
+    Filter f;
+    f.query("peru");
+    f.category("scifi");
+    f.lang("html");
+    f.name("edsonarantesdonascimento");
+    f.acceptTags({"body", "script"});
+    EXPECT_SEARCH_URL("/catalog/v2/entries?q=peru&category=scifi&lang=html&name=edsonarantesdonascimento&tag=body;script");
+  }
+  #undef EXPECT_SEARCH_URL
+}

test/pathTools.cpp

@@ -20,7 +20,12 @@
 #include "gtest/gtest.h"
 #include <string>
 #include <vector>
-#include "../include/tools/pathTools.h"
+#ifndef _WIN32
+# include <unistd.h>
+# include <fcntl.h>
+#endif
+#include "../include/tools.h"
+#include "../src/tools/pathTools.h"
 
 #ifdef _WIN32
 # define S "\\"
@@ -101,61 +106,61 @@ TEST(pathTools, normalizePartsRelative)
 
 TEST(pathTools, isRelativePath)
 {
-  ASSERT_TRUE(isRelativePath("foo"));
-  ASSERT_TRUE(isRelativePath(P2("foo","bar")));
-  ASSERT_TRUE(isRelativePath(P3(".","foo","bar")));
-  ASSERT_TRUE(isRelativePath(P2("..","foo")));
-  ASSERT_TRUE(isRelativePath(P4("foo","","bar","")));
-  ASSERT_FALSE(isRelativePath(A1("foo")));
-  ASSERT_FALSE(isRelativePath(A2("foo", "bar")));
+  ASSERT_TRUE(kiwix::isRelativePath("foo"));
+  ASSERT_TRUE(kiwix::isRelativePath(P2("foo","bar")));
+  ASSERT_TRUE(kiwix::isRelativePath(P3(".","foo","bar")));
+  ASSERT_TRUE(kiwix::isRelativePath(P2("..","foo")));
+  ASSERT_TRUE(kiwix::isRelativePath(P4("foo","","bar","")));
+  ASSERT_FALSE(kiwix::isRelativePath(A1("foo")));
+  ASSERT_FALSE(kiwix::isRelativePath(A2("foo", "bar")));
 #ifdef _WIN32
-  ASSERT_FALSE(isRelativePath(P2(A_SAMBA, "foo")));
-  ASSERT_FALSE(isRelativePath(P3(A_SAMBA, "foo", "bar")));
+  ASSERT_FALSE(kiwix::isRelativePath(P2(A_SAMBA, "foo")));
+  ASSERT_FALSE(kiwix::isRelativePath(P3(A_SAMBA, "foo", "bar")));
 #endif
 }
 
 TEST(pathTools, computeAbsolutePath)
 {
-  ASSERT_EQ(computeAbsolutePath(A2("a","b"), "foo"),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A2("a","b"), "foo"),
             A3("a","b","foo"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b",""), "foo"),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b",""), "foo"),
             A3("a","b","foo"));
-  ASSERT_EQ(computeAbsolutePath(A2("a","b"), P2(".","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A2("a","b"), P2(".","foo")),
             A3("a","b","foo"));
-  ASSERT_EQ(computeAbsolutePath(A2("a","b"), P2("..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A2("a","b"), P2("..","foo")),
             A2("a","foo"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b",""), P2("..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b",""), P2("..","foo")),
             A2("a","foo"));
-  ASSERT_EQ(computeAbsolutePath(A5("a","b","c","d","e"), P2("..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A5("a","b","c","d","e"), P2("..","foo")),
             A5("a","b","c","d","foo"));
-  ASSERT_EQ(computeAbsolutePath(A5("a","b","c","d","e"), P5("..","..","..","g","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A5("a","b","c","d","e"), P5("..","..","..","g","foo")),
             A4("a","b","g","foo"));
 #ifdef _WIN32
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b",""), P2("..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b",""), P2("..","foo")),
             P3(A_SAMBA,"a","foo"));
-  ASSERT_EQ(computeAbsolutePath(P6(A_SAMBA,"a","b","c","d","e"), P5("..","..","..","g","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P6(A_SAMBA,"a","b","c","d","e"), P5("..","..","..","g","foo")),
             P5(A_SAMBA,"a","b","g","foo"));
 #endif
 }
 
 TEST(pathTools, computeRelativePath)
 {
-  ASSERT_EQ(computeRelativePath(A2("a","b"), A3("a","b","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A2("a","b"), A3("a","b","foo")),
             "foo");
-  ASSERT_EQ(computeRelativePath(A3("a","b",""), A3("a","b","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A3("a","b",""), A3("a","b","foo")),
             "foo");
-  ASSERT_EQ(computeRelativePath(A2("a","b"), A2("a","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A2("a","b"), A2("a","foo")),
             P2("..","foo"));
-  ASSERT_EQ(computeRelativePath(A3("a","b",""), A2("a","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A3("a","b",""), A2("a","foo")),
             P2("..","foo"));
-  ASSERT_EQ(computeRelativePath(A5("a","b","c","d","e"), A5("a","b","c","d","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A5("a","b","c","d","e"), A5("a","b","c","d","foo")),
             P2("..","foo"));
-  ASSERT_EQ(computeRelativePath(A5("a","b","c","d","e"), A4("a","b","g","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(A5("a","b","c","d","e"), A4("a","b","g","foo")),
             P5("..","..","..","g","foo"));
 #ifdef _WIN32
-  ASSERT_EQ(computeRelativePath(P3(A_SAMBA,"a","b"), P3(A_SAMBA,"a","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(P3(A_SAMBA,"a","b"), P3(A_SAMBA,"a","foo")),
             P2("..","foo"));
-  ASSERT_EQ(computeRelativePath(P6(A_SAMBA,"a","b","c","d","e"), P5(A_SAMBA,"a","b","g","foo")),
+  ASSERT_EQ(kiwix::computeRelativePath(P6(A_SAMBA,"a","b","c","d","e"), P5(A_SAMBA,"a","b","g","foo")),
             P5("..","..","..","g","foo"));
 
 #endif
@@ -163,73 +168,101 @@ TEST(pathTools, computeRelativePath)
 
 TEST(pathTools, removeLastPathElement)
 {
-  ASSERT_EQ(removeLastPathElement(P3("a","b","c")),
+  ASSERT_EQ(kiwix::removeLastPathElement(P3("a","b","c")),
             P2("a","b"));
-  ASSERT_EQ(removeLastPathElement(A3("a","b","c")),
+  ASSERT_EQ(kiwix::removeLastPathElement(A3("a","b","c")),
             A2("a","b"));
-  ASSERT_EQ(removeLastPathElement(P4("a","b","c","")),
+  ASSERT_EQ(kiwix::removeLastPathElement(P4("a","b","c","")),
             P2("a","b"));
-  ASSERT_EQ(removeLastPathElement(A4("a","b","c","")),
+  ASSERT_EQ(kiwix::removeLastPathElement(A4("a","b","c","")),
             A2("a","b"));
 }
 
 TEST(pathTools, appendToDirectory)
 {
-  ASSERT_EQ(appendToDirectory(P3("a","b","c"), "foo.xml"),
+  ASSERT_EQ(kiwix::appendToDirectory(P3("a","b","c"), "foo.xml"),
             P4("a","b","c","foo.xml"));
-  ASSERT_EQ(appendToDirectory(P4("a","b","c",""), "foo.xml"),
+  ASSERT_EQ(kiwix::appendToDirectory(P4("a","b","c",""), "foo.xml"),
             P4("a","b","c","foo.xml"));
-  ASSERT_EQ(appendToDirectory(P3("a","b","c"), P2("d","foo.xml")),
+  ASSERT_EQ(kiwix::appendToDirectory(P3("a","b","c"), P2("d","foo.xml")),
             P5("a","b","c","d","foo.xml"));
-  ASSERT_EQ(appendToDirectory(P4("a","b","c",""), P2("d","foo.xml")),
+  ASSERT_EQ(kiwix::appendToDirectory(P4("a","b","c",""), P2("d","foo.xml")),
             P5("a","b","c","d","foo.xml"));
-  ASSERT_EQ(appendToDirectory(P3("a","b","c"), P2(".","foo.xml")),
+  ASSERT_EQ(kiwix::appendToDirectory(P3("a","b","c"), P2(".","foo.xml")),
             P5("a","b","c",".","foo.xml"));
-  ASSERT_EQ(appendToDirectory(P4("a","b","c",""), P2(".","foo.xml")),
+  ASSERT_EQ(kiwix::appendToDirectory(P4("a","b","c",""), P2(".","foo.xml")),
             P5("a","b","c",".","foo.xml"));
 }
 
+TEST(pathTools, fileExists)
+{
+  ASSERT_TRUE(kiwix::fileExists(P3(".","test","example.zim")));
+  ASSERT_FALSE(kiwix::fileExists(P3(".","test","noFile.zim")));
+}
+
+TEST(pathTools, fileReadable)
+{
+  ASSERT_TRUE(kiwix::fileReadable(P3(".","test","example.zim")));
+  ASSERT_FALSE(kiwix::fileReadable(P3(".","test","noFile.zim")));
+#ifdef _POSIX_SOURCE
+  std::string path = P3(".","test","example.zim");
+  auto myprivs = geteuid();
+  if (myprivs != 0) {
+    int fd = open(path.c_str(), O_RDONLY);
+    if (fd != -1) {
+      int wResp = fchmod(fd, ~(S_IRWXU | S_IRWXG | S_IRWXO)); // remove all permissions
+      if (wResp == 0) {
+        EXPECT_FALSE(kiwix::fileReadable(P3(".","test","example.zim")));
+      }
+      int resetResp = fchmod(fd, S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR | S_IWGRP); // reset back permissions to -rw-rw-r--
+      if (resetResp == 0) {
+        EXPECT_TRUE(kiwix::fileReadable(P3(".","test","example.zim")));
+      }
+    }
+  }
+#endif
+}
 
 TEST(pathTools, goUp)
 {
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), ".."),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), ".."),
             A2("a", "b"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P2("..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P2("..","..")),
             A1("a"));
 #ifdef _WIN32
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
             "c:");
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
             "c:");
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), ".."),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), ".."),
             P3(A_SAMBA,"a", "b"));
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P2("..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P2("..","..")),
             P2(A_SAMBA,"a"));
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P3("..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P3("..","..","..")),
             A_SAMBA);
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","..")),
             A_SAMBA);
 
 #else
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
             "/");
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
             "/");
 #endif
 
 
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P2("..", "foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P2("..", "foo")),
             A3("a", "b","foo"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P3("..","..","foo")),
             A2("a","foo"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P4("..","..","..","foo")),
             A1("foo"));
-  ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P5("..","..","..","..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(A3("a","b","c"), P5("..","..","..","..","foo")),
             A1("foo"));
 #ifdef _WIN32
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","foo")),
             P2(A_SAMBA,"foo"));
-  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P5("..","..","..","..","foo")),
+  ASSERT_EQ(kiwix::computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P5("..","..","..","..","foo")),
             P2(A_SAMBA,"foo"));
 #endif
 }
@@ -241,20 +274,20 @@ TEST(pathTools, dirChange)
   std::string p1("c:\\a\\b\\c");
   std::string p2("d:\\d\\e\\foo.xml");
   {
-    std::string relative_path = computeRelativePath(p1, p2);
+    std::string relative_path = kiwix::computeRelativePath(p1, p2);
     ASSERT_EQ(relative_path, p2);
-    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    std::string abs_path = kiwix::computeAbsolutePath(p1, relative_path);
     ASSERT_EQ(abs_path, p2);
-    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
+    ASSERT_EQ(kiwix::computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
   }
   std::string ps("\\\\samba\\d\\e\\foo.xml");
   {
-    std::string relative_path = computeRelativePath(p1, ps);
+    std::string relative_path = kiwix::computeRelativePath(p1, ps);
     ASSERT_EQ(relative_path, ps);
-    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    std::string abs_path = kiwix::computeAbsolutePath(p1, relative_path);
     ASSERT_EQ(abs_path, ps);
     // I'm not sure this test is valid on windows :/
-//    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\\\samba\\d\\e\\foo.xml"), ps);
+//    ASSERT_EQ(kiwix::computeAbsolutePath(p1, "..\\..\\..\\..\\..\\\\samba\\d\\e\\foo.xml"), ps);
   }
 }
 

test/regex.cpp

@@ -21,7 +21,7 @@
 #include <string>
 
 
-#include "../include/tools/regexTools.h"
+#include "../src/tools/regexTools.h"
 
 namespace
 {

test/server.cpp

@@ -5,7 +5,7 @@
 #include "../include/manager.h"
 #include "../include/server.h"
 #include "../include/name_mapper.h"
-
+#include "../include/tools.h"
 
 using TestContextImpl = std::vector<std::pair<std::string, std::string> >;
 struct TestContext : TestContextImpl {
@@ -54,7 +54,7 @@ public: // types
 
 public: // functions
   ZimFileServer(int serverPort, std::string libraryFilePath);
-  ZimFileServer(int serverPort, const FilePathCollection& zimpaths);
+  ZimFileServer(int serverPort, const FilePathCollection& zimpaths, std::string indexTemplateString = "");
   ~ZimFileServer();
 
   Response GET(const char* path, const Headers& headers = Headers())
@@ -68,7 +68,7 @@ public: // functions
   }
 
 private:
-  void run(int serverPort);
+  void run(int serverPort, std::string indexTemplateString = "");
 
 private: // data
   kiwix::Library library;
@@ -81,33 +81,35 @@ private: // data
 ZimFileServer::ZimFileServer(int serverPort, std::string libraryFilePath)
 : manager(&this->library)
 {
-  if ( isRelativePath(libraryFilePath) )
-    libraryFilePath = computeAbsolutePath(getCurrentDirectory(), libraryFilePath);
+  if ( kiwix::isRelativePath(libraryFilePath) )
+    libraryFilePath = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryFilePath);
   manager.readFile(libraryFilePath, true, true);
-
   run(serverPort);
 }
 
-ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths)
+ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths, std::string indexTemplateString)
 : manager(&this->library)
 {
   for ( const auto& zimpath : zimpaths ) {
     if (!manager.addBookFromPath(zimpath, zimpath, "", false))
       throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
   }
-
-  run(serverPort);
+  run(serverPort, indexTemplateString);
 }
 
-void ZimFileServer::run(int serverPort)
+void ZimFileServer::run(int serverPort, std::string indexTemplateString)
 {
   const std::string address = "127.0.0.1";
   nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false));
   server.reset(new kiwix::Server(&library, nameMapper.get()));
+  server->setRoot("ROOT");
   server->setAddress(address);
   server->setPort(serverPort);
   server->setNbThreads(2);
   server->setVerbose(false);
+  if (!indexTemplateString.empty()) {
+    server->setIndexTemplateString(indexTemplateString);
+  }
 
   if ( !server->start() )
     throw std::runtime_error("ZimFileServer failed to start");
@@ -159,50 +161,59 @@ std::ostream& operator<<(std::ostream& out, const Resource& r)
 typedef std::vector<Resource> ResourceCollection;
 
 const ResourceCollection resources200Compressible{
-  { WITH_ETAG, "/" },
+  { WITH_ETAG, "/ROOT/" },
 
-  { WITH_ETAG, "/skin/jquery-ui/jquery-ui.structure.min.css" },
-  { WITH_ETAG, "/skin/jquery-ui/jquery-ui.min.js" },
-  { WITH_ETAG, "/skin/jquery-ui/external/jquery/jquery.js" },
-  { WITH_ETAG, "/skin/jquery-ui/jquery-ui.theme.min.css" },
-  { WITH_ETAG, "/skin/jquery-ui/jquery-ui.min.css" },
-  { WITH_ETAG, "/skin/taskbar.js" },
-  { WITH_ETAG, "/skin/taskbar.css" },
-  { WITH_ETAG, "/skin/block_external.js" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.structure.min.css" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.min.js" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/external/jquery/jquery.js" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.min.css" },
+  { WITH_ETAG, "/ROOT/skin/taskbar.js" },
+  { WITH_ETAG, "/ROOT/skin/taskbar.css" },
+  { WITH_ETAG, "/ROOT/skin/block_external.js" },
 
-  { NO_ETAG,   "/catalog/root.xml" },
-  { NO_ETAG,   "/catalog/searchdescription.xml" },
-  { NO_ETAG,   "/catalog/search" },
+  { NO_ETAG,   "/ROOT/catalog/root.xml" },
+  { NO_ETAG,   "/ROOT/catalog/searchdescription.xml" },
+  { NO_ETAG,   "/ROOT/catalog/search" },
 
-  { NO_ETAG,   "/search?content=zimfile&pattern=a" },
+  { NO_ETAG,   "/ROOT/search?content=zimfile&pattern=a" },
 
-  { NO_ETAG,   "/suggest?content=zimfile&term=ray" },
+  { NO_ETAG,   "/ROOT/suggest?content=zimfile" },
+  { NO_ETAG,   "/ROOT/suggest?content=zimfile&term=ray" },
 
-  { NO_ETAG,   "/catch/external?source=www.example.com" },
+  { NO_ETAG,   "/ROOT/catch/external?source=www.example.com" },
 
-  { WITH_ETAG, "/zimfile/A/index" },
-  { WITH_ETAG, "/zimfile/A/Ray_Charles" },
+  { WITH_ETAG, "/ROOT/zimfile/A/index" },
+  { WITH_ETAG, "/ROOT/zimfile/A/Ray_Charles" },
+
+  { WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" },
 };
 
 const ResourceCollection resources200Uncompressible{
-  { WITH_ETAG, "/skin/jquery-ui/images/animated-overlay.gif" },
-  { WITH_ETAG, "/skin/caret.png" },
+  { WITH_ETAG, "/ROOT/skin/jquery-ui/images/animated-overlay.gif" },
+  { WITH_ETAG, "/ROOT/skin/caret.png" },
 
-  { WITH_ETAG, "/meta?content=zimfile&name=title" },
-  { WITH_ETAG, "/meta?content=zimfile&name=description" },
-  { WITH_ETAG, "/meta?content=zimfile&name=language" },
-  { WITH_ETAG, "/meta?content=zimfile&name=name" },
-  { WITH_ETAG, "/meta?content=zimfile&name=tags" },
-  { WITH_ETAG, "/meta?content=zimfile&name=date" },
-  { WITH_ETAG, "/meta?content=zimfile&name=creator" },
-  { WITH_ETAG, "/meta?content=zimfile&name=publisher" },
-  { WITH_ETAG, "/meta?content=zimfile&name=favicon" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" },
 
-  { WITH_ETAG, "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
+  { NO_ETAG, "/ROOT/catalog/v2/illustration/zimfile?size=48" },
 
-  { WITH_ETAG, "/corner_cases/A/empty.html" },
-  { WITH_ETAG, "/corner_cases/-/empty.css" },
-  { WITH_ETAG, "/corner_cases/-/empty.js" },
+  { WITH_ETAG, "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
+
+  { WITH_ETAG, "/ROOT/corner_cases/A/empty.html" },
+  { WITH_ETAG, "/ROOT/corner_cases/-/empty.css" },
+  { WITH_ETAG, "/ROOT/corner_cases/-/empty.js" },
+
+  // The title and creator are too small to be compressed
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
+  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
 };
 
 ResourceCollection all200Resources()
@@ -210,6 +221,35 @@ ResourceCollection all200Resources()
   return concat(resources200Compressible, resources200Uncompressible);
 }
 
+TEST(indexTemplateStringTest, emptyIndexTemplate) {
+  const int PORT = 8001;
+  const ZimFileServer::FilePathCollection ZIMFILES {
+    "./test/zimfile.zim",
+    "./test/corner_cases.zim"
+  };
+
+  ZimFileServer zfs(PORT, ZIMFILES, "");
+  EXPECT_EQ(200, zfs.GET("/ROOT/")->status);
+}
+
+TEST(indexTemplateStringTest, indexTemplateCheck) {
+  const int PORT = 8001;
+  const ZimFileServer::FilePathCollection ZIMFILES {
+    "./test/zimfile.zim",
+    "./test/corner_cases.zim"
+  };
+
+  ZimFileServer zfs(PORT, ZIMFILES, "<!DOCTYPE html><head>"
+      "<title>Welcome to kiwix library</title>"
+    "</head>"
+  "</html>");
+  EXPECT_EQ("<!DOCTYPE html><head>"
+    "<title>Welcome to kiwix library</title>"
+    "<link type=\"root\" href=\"/ROOT\">"
+    "</head>"
+  "</html>", zfs.GET("/ROOT/")->body);
+}
+
 TEST_F(ServerTest, 200)
 {
   for ( const Resource& res : all200Resources() )
@@ -236,23 +276,32 @@ TEST_F(ServerTest, UncompressibleContentIsNotCompressed)
 }
 
 const char* urls404[] = {
-  "/non-existent-item",
-  "/skin/non-existent-skin-resource",
-  "/catalog",
-  "/catalog/non-existent-item",
-  "/catalogBLABLABLA/root.xml",
-  "/meta",
-  "/meta?content=zimfile",
-  "/meta?content=zimfile&name=non-existent-item",
-  "/meta?content=non-existent-book&name=title",
-  "/random",
-  "/random?content=non-existent-book",
-  "/search",
-  "/suggest",
-  "/suggest?content=zimfile",
-  "/suggest?content=non-existent-book&term=abcd",
-  "/catch/external",
-  "/zimfile/A/non-existent-article",
+  "/",
+  "/zimfile",
+  "/ROOT/non-existent-item",
+  "/ROOT/skin/non-existent-skin-resource",
+  "/ROOT/catalog",
+  "/ROOT/catalog/non-existent-item",
+  "/ROOT/catalogBLABLABLA/root.xml",
+  "/ROOT/catalog/v2/illustration/zimfile?size=96",
+  "/ROOT/meta",
+  "/ROOT/meta?content=zimfile",
+  "/ROOT/meta?content=zimfile&name=non-existent-item",
+  "/ROOT/meta?content=non-existent-book&name=title",
+  "/ROOT/random",
+  "/ROOT/random?content=non-existent-book",
+  "/ROOT/search",
+  "/ROOT/suggest",
+  "/ROOT/suggest?content=non-existent-book&term=abcd",
+  "/ROOT/catch/external",
+  "/ROOT/zimfile/A/non-existent-article",
+
+  "/ROOT/raw/non-existent-book/meta/Title",
+  "/ROOT/raw/zimfile/wrong-kind/Foo",
+
+  // zimfile has no Favicon nor Illustration_48x48@1 meta item
+  "/ROOT/raw/zimfile/meta/Favicon",
+  "/ROOT/raw/zimfile/meta/Illustration_48x48@1",
 };
 
 TEST_F(ServerTest, 404)
@@ -263,7 +312,7 @@ TEST_F(ServerTest, 404)
 
 TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
 {
-  auto g = zfs1_->GET("/random?content=zimfile");
+  auto g = zfs1_->GET("/ROOT/random?content=zimfile");
   ASSERT_EQ(302, g->status);
   ASSERT_TRUE(g->has_header("Location"));
   ASSERT_TRUE(g->get_header_value("Location").find("/zimfile/A/") != std::string::npos);
@@ -271,10 +320,36 @@ TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
 
 TEST_F(ServerTest, BookMainPageIsRedirectedToArticleIndex)
 {
-  auto g = zfs1_->GET("/zimfile");
+  auto g = zfs1_->GET("/ROOT/zimfile");
   ASSERT_EQ(302, g->status);
   ASSERT_TRUE(g->has_header("Location"));
-  ASSERT_EQ("/zimfile/A/index", g->get_header_value("Location"));
+  ASSERT_EQ("/ROOT/zimfile/A/index", g->get_header_value("Location"));
+}
+
+
+TEST_F(ServerTest, RawEntry)
+{
+  auto p = zfs1_->GET("/ROOT/raw/zimfile/meta/Title");
+  EXPECT_EQ(200, p->status);
+  EXPECT_EQ(p->body, std::string("Ray Charles"));
+
+  p = zfs1_->GET("/ROOT/raw/zimfile/meta/Creator");
+  EXPECT_EQ(200, p->status);
+  EXPECT_EQ(p->body, std::string("Wikipedia"));
+
+  // The raw content of Ray_Charles returned by the server is
+  // the same as the one in the zim file.
+  auto archive = zim::Archive("./test/zimfile.zim");
+  auto entry = archive.getEntryByPath("A/Ray_Charles");
+  p = zfs1_->GET("/ROOT/raw/zimfile/content/A/Ray_Charles");
+  EXPECT_EQ(200, p->status);
+  EXPECT_EQ(std::string(p->body), std::string(entry.getItem(true).getData()));
+
+  // ... but the "normal" content is not
+  p = zfs1_->GET("/ROOT/zimfile/A/Ray_Charles");
+  EXPECT_EQ(200, p->status);
+  EXPECT_NE(std::string(p->body), std::string(entry.getItem(true).getData()));
+  EXPECT_TRUE(p->body.find("taskbar") != std::string::npos);
 }
 
 TEST_F(ServerTest, HeadMethodIsSupported)
@@ -433,7 +508,7 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
 
 TEST_F(ServerTest, ValidSingleRangeByteRangeRequestsAreHandledProperly)
 {
-  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const char url[] = "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
   const auto full = zfs1_->GET(url);
   EXPECT_FALSE(full->has_header("Content-Range"));
   EXPECT_EQ("bytes", full->get_header_value("Accept-Ranges"));
@@ -483,7 +558,7 @@ TEST_F(ServerTest, ValidSingleRangeByteRangeRequestsAreHandledProperly)
 
 TEST_F(ServerTest, InvalidAndMultiRangeByteRangeRequestsResultIn416Responses)
 {
-  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const char url[] = "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
 
   const char* invalidRanges[] = {
     "0-10", "bytes=", "bytes=123", "bytes=-10-20", "bytes=10-20xxx",
@@ -504,7 +579,7 @@ TEST_F(ServerTest, InvalidAndMultiRangeByteRangeRequestsResultIn416Responses)
 
 TEST_F(ServerTest, ValidByteRangeRequestsOfZeroSizedEntriesResultIn416Responses)
 {
-  const char url[] = "/corner_cases/-/empty.js";
+  const char url[] = "/ROOT/corner_cases/-/empty.js";
 
   const char* ranges[] = {
     "bytes=0-",
@@ -523,7 +598,7 @@ TEST_F(ServerTest, ValidByteRangeRequestsOfZeroSizedEntriesResultIn416Responses)
 
 TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
 {
-  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const char url[] = "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
 
   const Headers onlyRange{ {"Range", "bytes=123-456"} };
   Headers rangeAndCompression(onlyRange);
@@ -538,7 +613,7 @@ TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
 
 TEST_F(ServerTest, RangeHeaderIsCaseInsensitive)
 {
-  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const char url[] = "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
   const auto r0 = zfs1_->GET(url, { {"Range", "bytes=100-200"} } );
 
   const char* header_variations[] = { "RANGE", "range", "rAnGe", "RaNgE" };
@@ -611,47 +686,48 @@ std::string maskVariableOPDSFeedData(std::string s)
     "  <link rel=\"self\" href=\"\" type=\"application/atom+xml\" />\n" \
     "  <link rel=\"search\""                                            \
            " type=\"application/opensearchdescription+xml\""            \
-           " href=\"/catalog/searchdescription.xml\" />\n"
+           " href=\"/ROOT/catalog/searchdescription.xml\" />\n"
 
 #define CHARLES_RAY_CATALOG_ENTRY \
     "  <entry>\n"                                                       \
     "    <id>urn:uuid:charlesray</id>\n"                                \
     "    <title>Charles, Ray</title>\n"                                 \
-    "    <summary>Wikipedia articles about Ray Charles</summary>\n"    \
-    "    <language>eng</language>\n"                                    \
     "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"                     \
-    "    <name>wikipedia_en_ray_charles</name>\n"                       \
+    "    <summary>Wikipedia articles about Ray Charles</summary>\n"     \
+    "    <language>fra</language>\n"                                    \
+    "    <name>wikipedia_fr_ray_charles</name>\n"                       \
     "    <flavour></flavour>\n"                                         \
     "    <category>jazz</category>\n"                                   \
     "    <tags>unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes</tags>\n" \
     "    <articleCount>284</articleCount>\n"                            \
     "    <mediaCount>2</mediaCount>\n"                                  \
-    "    <icon>/meta?name=favicon&amp;content=zimfile</icon>\n"         \
-    "    <link type=\"text/html\" href=\"/zimfile\" />\n"               \
+    "    <link type=\"text/html\" href=\"/ROOT/zimfile%26other\" />\n"               \
     "    <author>\n"                                                    \
     "      <name>Wikipedia</name>\n"                                    \
     "    </author>\n"                                                   \
     "    <publisher>\n"                                                 \
     "      <name>Kiwix</name>\n"                                        \
     "    </publisher>\n"                                                \
-    "    <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim\" length=\"569344\" />\n" \
+    "    <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim\" length=\"569344\" />\n" \
     "  </entry>\n"
 
 #define RAY_CHARLES_CATALOG_ENTRY \
     "  <entry>\n"                                                       \
     "    <id>urn:uuid:raycharles</id>\n"                                \
     "    <title>Ray Charles</title>\n"                                  \
+    "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"                     \
     "    <summary>Wikipedia articles about Ray Charles</summary>\n"     \
     "    <language>eng</language>\n"                                    \
-    "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"                     \
     "    <name>wikipedia_en_ray_charles</name>\n"                       \
     "    <flavour></flavour>\n"                                         \
     "    <category>wikipedia</category>\n"                              \
     "    <tags>unittest;wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes</tags>\n" \
     "    <articleCount>284</articleCount>\n"                            \
     "    <mediaCount>2</mediaCount>\n"                                  \
-    "    <icon>/meta?name=favicon&amp;content=zimfile</icon>\n"         \
-    "    <link type=\"text/html\" href=\"/zimfile\" />\n"               \
+    "    <link rel=\"http://opds-spec.org/image/thumbnail\"\n"          \
+    "          href=\"/ROOT/catalog/v2/illustration/zimfile/?size=48\"\n" \
+    "          type=\"image/png;width=48;height=48;scale=1\"/>\n"               \
+    "    <link type=\"text/html\" href=\"/ROOT/zimfile\" />\n"               \
     "    <author>\n"                                                    \
     "      <name>Wikipedia</name>\n"                                    \
     "    </author>\n"                                                   \
@@ -665,17 +741,16 @@ std::string maskVariableOPDSFeedData(std::string s)
     "  <entry>\n"                                                       \
     "    <id>urn:uuid:raycharles_uncategorized</id>\n"                  \
     "    <title>Ray (uncategorized) Charles</title>\n"                  \
-    "    <summary>No category is assigned to this library entry.</summary>\n" \
-    "    <language>eng</language>\n"                                    \
     "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"                     \
-    "    <name>wikipedia_en_ray_charles</name>\n"                       \
+    "    <summary>No category is assigned to this library entry.</summary>\n" \
+    "    <language>rus</language>\n"                                    \
+    "    <name>wikipedia_ru_ray_charles</name>\n"                       \
     "    <flavour></flavour>\n"                                         \
     "    <category></category>\n"                                \
     "    <tags>unittest;wikipedia;_pictures:no;_videos:no;_details:no</tags>\n" \
     "    <articleCount>284</articleCount>\n"                            \
     "    <mediaCount>2</mediaCount>\n"                                  \
-    "    <icon>/meta?name=favicon&amp;content=zimfile</icon>\n"         \
-    "    <link type=\"text/html\" href=\"/zimfile\" />\n"               \
+    "    <link type=\"text/html\" href=\"/ROOT/zimfile\" />\n"               \
     "    <author>\n"                                                    \
     "      <name>Wikipedia</name>\n"                                    \
     "    </author>\n"                                                   \
@@ -687,7 +762,7 @@ std::string maskVariableOPDSFeedData(std::string s)
 
 TEST_F(LibraryServerTest, catalog_root_xml)
 {
-  const auto r = zfs1_->GET("/catalog/root.xml");
+  const auto r = zfs1_->GET("/ROOT/catalog/root.xml");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -705,7 +780,7 @@ TEST_F(LibraryServerTest, catalog_root_xml)
 
 TEST_F(LibraryServerTest, catalog_searchdescription_xml)
 {
-  const auto r = zfs1_->GET("/catalog/searchdescription.xml");
+  const auto r = zfs1_->GET("/ROOT/catalog/searchdescription.xml");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(r->body,
     "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
@@ -716,14 +791,14 @@ TEST_F(LibraryServerTest, catalog_searchdescription_xml)
     "       xmlns:atom=\"http://www.w3.org/2005/Atom\"\n"
     "       xmlns:k=\"http://kiwix.org/opensearchextension/1.0\"\n"
     "       indexOffset=\"0\"\n"
-    "       template=\"/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&notag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n"
+    "       template=\"/ROOT/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&notag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n"
     "</OpenSearchDescription>\n"
   );
 }
 
 TEST_F(LibraryServerTest, catalog_search_by_phrase)
 {
-  const auto r = zfs1_->GET("/catalog/search?q=\"ray%20charles\"");
+  const auto r = zfs1_->GET("/ROOT/catalog/search?q=\"ray%20charles\"");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -742,7 +817,7 @@ TEST_F(LibraryServerTest, catalog_search_by_phrase)
 
 TEST_F(LibraryServerTest, catalog_search_by_words)
 {
-  const auto r = zfs1_->GET("/catalog/search?q=ray%20charles");
+  const auto r = zfs1_->GET("/ROOT/catalog/search?q=ray%20charles");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -763,7 +838,7 @@ TEST_F(LibraryServerTest, catalog_search_by_words)
 TEST_F(LibraryServerTest, catalog_prefix_search)
 {
   {
-    const auto r = zfs1_->GET("/catalog/search?q=description:ray%20description:charles");
+    const auto r = zfs1_->GET("/ROOT/catalog/search?q=description:ray%20description:charles");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       OPDS_FEED_TAG
@@ -780,7 +855,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search)
     );
   }
   {
-    const auto r = zfs1_->GET("/catalog/search?q=title:\"ray%20charles\"");
+    const auto r = zfs1_->GET("/ROOT/catalog/search?q=title:\"ray%20charles\"");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       OPDS_FEED_TAG
@@ -799,7 +874,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search)
 
 TEST_F(LibraryServerTest, catalog_search_with_word_exclusion)
 {
-  const auto r = zfs1_->GET("/catalog/search?q=ray%20-uncategorized");
+  const auto r = zfs1_->GET("/ROOT/catalog/search?q=ray%20-uncategorized");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -818,7 +893,7 @@ TEST_F(LibraryServerTest, catalog_search_with_word_exclusion)
 
 TEST_F(LibraryServerTest, catalog_search_by_tag)
 {
-  const auto r = zfs1_->GET("/catalog/search?tag=_category:jazz");
+  const auto r = zfs1_->GET("/ROOT/catalog/search?tag=_category:jazz");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -836,7 +911,7 @@ TEST_F(LibraryServerTest, catalog_search_by_tag)
 
 TEST_F(LibraryServerTest, catalog_search_by_category)
 {
-  const auto r = zfs1_->GET("/catalog/search?category=jazz");
+  const auto r = zfs1_->GET("/ROOT/catalog/search?category=jazz");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     OPDS_FEED_TAG
@@ -855,7 +930,7 @@ TEST_F(LibraryServerTest, catalog_search_by_category)
 TEST_F(LibraryServerTest, catalog_search_results_pagination)
 {
   {
-    const auto r = zfs1_->GET("/catalog/search?count=1");
+    const auto r = zfs1_->GET("/ROOT/catalog/search?count=1");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       OPDS_FEED_TAG
@@ -871,7 +946,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination)
     );
   }
   {
-    const auto r = zfs1_->GET("/catalog/search?start=1&count=1");
+    const auto r = zfs1_->GET("/ROOT/catalog/search?start=1&count=1");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       OPDS_FEED_TAG
@@ -887,7 +962,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination)
     );
   }
   {
-    const auto r = zfs1_->GET("/catalog/search?start=100&count=10");
+    const auto r = zfs1_->GET("/ROOT/catalog/search?start=100&count=10");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       OPDS_FEED_TAG
@@ -898,7 +973,6 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination)
       "  <startIndex>100</startIndex>\n"
       "  <itemsPerPage>0</itemsPerPage>\n"
       CATALOG_LINK_TAGS
-      "  \n"
       "</feed>\n"
     );
   }
@@ -906,20 +980,20 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination)
 
 TEST_F(LibraryServerTest, catalog_v2_root)
 {
-  const auto r = zfs1_->GET("/catalog/v2/root.xml");
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/root.xml");
   EXPECT_EQ(r->status, 200);
   const char expected_output[] = R"(<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom"
       xmlns:opds="https://specs.opds.io/opds-1.2">
   <id>12345678-90ab-cdef-1234-567890abcdef</id>
   <link rel="self"
-        href="/catalog/v2/root.xml"
+        href="/ROOT/catalog/v2/root.xml"
         type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
   <link rel="start"
-        href="/catalog/v2/root.xml"
+        href="/ROOT/catalog/v2/root.xml"
         type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
   <link rel="search"
-        href="/catalog/v2/searchdescription.xml"
+        href="/ROOT/catalog/v2/searchdescription.xml"
         type="application/opensearchdescription+xml"/>
   <title>OPDS Catalog Root</title>
   <updated>YYYY-MM-DDThh:mm:ssZ</updated>
@@ -927,21 +1001,39 @@ TEST_F(LibraryServerTest, catalog_v2_root)
   <entry>
     <title>All entries</title>
     <link rel="subsection"
-          href="/catalog/v2/entries"
+          href="/ROOT/catalog/v2/entries"
           type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
     <updated>YYYY-MM-DDThh:mm:ssZ</updated>
     <id>12345678-90ab-cdef-1234-567890abcdef</id>
     <content type="text">All entries from this catalog.</content>
   </entry>
+  <entry>
+    <title>All entries (partial)</title>
+    <link rel="subsection"
+          href="/ROOT/catalog/v2/partial_entries"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+    <id>12345678-90ab-cdef-1234-567890abcdef</id>
+    <content type="text">All entries from this catalog in partial format.</content>
+  </entry>
   <entry>
     <title>List of categories</title>
     <link rel="subsection"
-          href="/catalog/v2/categories"
+          href="/ROOT/catalog/v2/categories"
           type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
     <updated>YYYY-MM-DDThh:mm:ssZ</updated>
     <id>12345678-90ab-cdef-1234-567890abcdef</id>
     <content type="text">List of all categories in this catalog.</content>
   </entry>
+  <entry>
+    <title>List of languages</title>
+    <link rel="subsection"
+          href="/ROOT/catalog/v2/languages"
+          type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+    <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+    <id>12345678-90ab-cdef-1234-567890abcdef</id>
+    <content type="text">List of all languages in this catalog.</content>
+  </entry>
 </feed>
 )";
   EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output);
@@ -949,7 +1041,7 @@ TEST_F(LibraryServerTest, catalog_v2_root)
 
 TEST_F(LibraryServerTest, catalog_v2_searchdescription_xml)
 {
-  const auto r = zfs1_->GET("/catalog/v2/searchdescription.xml");
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/searchdescription.xml");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(r->body,
     "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
@@ -960,24 +1052,24 @@ TEST_F(LibraryServerTest, catalog_v2_searchdescription_xml)
     "       xmlns:atom=\"http://www.w3.org/2005/Atom\"\n"
     "       xmlns:k=\"http://kiwix.org/opensearchextension/1.0\"\n"
     "       indexOffset=\"0\"\n"
-    "       template=\"/catalog/v2/entries?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n"
+    "       template=\"/ROOT/catalog/v2/entries?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n"
     "</OpenSearchDescription>\n"
   );
 }
 
 TEST_F(LibraryServerTest, catalog_v2_categories)
 {
-  const auto r = zfs1_->GET("/catalog/v2/categories");
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/categories");
   EXPECT_EQ(r->status, 200);
   const char expected_output[] = R"(<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom"
       xmlns:opds="https://specs.opds.io/opds-1.2">
   <id>12345678-90ab-cdef-1234-567890abcdef</id>
   <link rel="self"
-        href="/catalog/v2/categories"
+        href="/ROOT/catalog/v2/categories"
         type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
   <link rel="start"
-        href="/catalog/v2/root.xml"
+        href="/ROOT/catalog/v2/root.xml"
         type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
   <title>List of categories</title>
   <updated>YYYY-MM-DDThh:mm:ssZ</updated>
@@ -985,7 +1077,7 @@ TEST_F(LibraryServerTest, catalog_v2_categories)
   <entry>
     <title>jazz</title>
     <link rel="subsection"
-          href="/catalog/v2/entries?category=jazz"
+          href="/ROOT/catalog/v2/entries?category=jazz"
           type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
     <updated>YYYY-MM-DDThh:mm:ssZ</updated>
     <id>12345678-90ab-cdef-1234-567890abcdef</id>
@@ -994,7 +1086,7 @@ TEST_F(LibraryServerTest, catalog_v2_categories)
   <entry>
     <title>wikipedia</title>
     <link rel="subsection"
-          href="/catalog/v2/entries?category=wikipedia"
+          href="/ROOT/catalog/v2/entries?category=wikipedia"
           type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
     <updated>YYYY-MM-DDThh:mm:ssZ</updated>
     <id>12345678-90ab-cdef-1234-567890abcdef</id>
@@ -1005,7 +1097,61 @@ TEST_F(LibraryServerTest, catalog_v2_categories)
   EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output);
 }
 
-#define CATALOG_V2_ENTRIES_PREAMBLE(q)                        \
+TEST_F(LibraryServerTest, catalog_v2_languages)
+{
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/languages");
+  EXPECT_EQ(r->status, 200);
+  const char expected_output[] = R"(<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:dc="http://purl.org/dc/terms/"
+      xmlns:opds="https://specs.opds.io/opds-1.2"
+      xmlns:thr="http://purl.org/syndication/thread/1.0">
+  <id>12345678-90ab-cdef-1234-567890abcdef</id>
+  <link rel="self"
+        href="/ROOT/catalog/v2/languages"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <link rel="start"
+        href="/ROOT/catalog/v2/root.xml"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <title>List of languages</title>
+  <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+
+  <entry>
+    <title>English</title>
+    <dc:language>eng</dc:language>
+    <thr:count>1</thr:count>
+    <link rel="subsection"
+          href="/ROOT/catalog/v2/entries?lang=eng"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+    <id>12345678-90ab-cdef-1234-567890abcdef</id>
+  </entry>
+  <entry>
+    <title>français</title>
+    <dc:language>fra</dc:language>
+    <thr:count>1</thr:count>
+    <link rel="subsection"
+          href="/ROOT/catalog/v2/entries?lang=fra"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+    <id>12345678-90ab-cdef-1234-567890abcdef</id>
+  </entry>
+  <entry>
+    <title>русский</title>
+    <dc:language>rus</dc:language>
+    <thr:count>1</thr:count>
+    <link rel="subsection"
+          href="/ROOT/catalog/v2/entries?lang=rus"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>YYYY-MM-DDThh:mm:ssZ</updated>
+    <id>12345678-90ab-cdef-1234-567890abcdef</id>
+  </entry>
+</feed>
+)";
+  EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output);
+}
+
+#define CATALOG_V2_ENTRIES_PREAMBLE0(x)                       \
     "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"            \
     "<feed xmlns=\"http://www.w3.org/2005/Atom\"\n"           \
     "      xmlns:opds=\"https://specs.opds.io/opds-1.2\"\n"   \
@@ -1013,20 +1159,25 @@ TEST_F(LibraryServerTest, catalog_v2_categories)
     "  <id>12345678-90ab-cdef-1234-567890abcdef</id>\n"       \
     "\n"                                                      \
     "  <link rel=\"self\"\n"                                  \
-    "        href=\"/catalog/v2/entries" q "\"\n"             \
+    "        href=\"/ROOT/catalog/v2/" x "\"\n"                    \
     "        type=\"application/atom+xml;profile=opds-catalog;kind=acquisition\"/>\n" \
     "  <link rel=\"start\"\n"                                 \
-    "        href=\"/catalog/v2/root.xml\"\n"              \
+    "        href=\"/ROOT/catalog/v2/root.xml\"\n"              \
     "        type=\"application/atom+xml;profile=opds-catalog;kind=navigation\"/>\n" \
     "  <link rel=\"up\"\n"                                    \
-    "        href=\"/catalog/v2/root.xml\"\n"              \
+    "        href=\"/ROOT/catalog/v2/root.xml\"\n"              \
     "        type=\"application/atom+xml;profile=opds-catalog;kind=navigation\"/>\n" \
     "\n"                                                      \
 
+#define CATALOG_V2_ENTRIES_PREAMBLE(q) \
+            CATALOG_V2_ENTRIES_PREAMBLE0("entries" q)
+
+#define CATALOG_V2_PARTIAL_ENTRIES_PREAMBLE(q) \
+            CATALOG_V2_ENTRIES_PREAMBLE0("partial_entries" q)
 
 TEST_F(LibraryServerTest, catalog_v2_entries)
 {
-  const auto r = zfs1_->GET("/catalog/v2/entries");
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/entries");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     CATALOG_V2_ENTRIES_PREAMBLE("")
@@ -1043,7 +1194,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries)
 TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range)
 {
   {
-    const auto r = zfs1_->GET("/catalog/v2/entries?start=1");
+    const auto r = zfs1_->GET("/ROOT/catalog/v2/entries?start=1");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       CATALOG_V2_ENTRIES_PREAMBLE("?start=1")
@@ -1059,7 +1210,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range)
   }
 
   {
-    const auto r = zfs1_->GET("/catalog/v2/entries?count=2");
+    const auto r = zfs1_->GET("/ROOT/catalog/v2/entries?count=2");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       CATALOG_V2_ENTRIES_PREAMBLE("?count=2")
@@ -1075,7 +1226,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range)
   }
 
   {
-    const auto r = zfs1_->GET("/catalog/v2/entries?start=1&count=1");
+    const auto r = zfs1_->GET("/ROOT/catalog/v2/entries?start=1&count=1");
     EXPECT_EQ(r->status, 200);
     EXPECT_EQ(maskVariableOPDSFeedData(r->body),
       CATALOG_V2_ENTRIES_PREAMBLE("?count=1&start=1")
@@ -1092,7 +1243,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range)
 
 TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms)
 {
-  const auto r = zfs1_->GET("/catalog/v2/entries?q=\"ray%20charles\"");
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/entries?q=\"ray%20charles\"");
   EXPECT_EQ(r->status, 200);
   EXPECT_EQ(maskVariableOPDSFeedData(r->body),
     CATALOG_V2_ENTRIES_PREAMBLE("?q=%22ray%20charles%22")
@@ -1106,3 +1257,117 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms)
     "</feed>\n"
   );
 }
+
+TEST_F(LibraryServerTest, suggestions_in_range)
+{
+  /**
+   * Attempt to get 50 suggestions in steps of 5
+   * The suggestions are returned in the json format
+   * [{sugg1}, {sugg2}, ... , {suggN}, {suggest ft search}]
+   * Assuming the number of suggestions = (occurance of "{" - 1)
+   */
+  {
+    int suggCount = 0;
+    for (int i = 0; i < 10; i++) {
+      std::string url = "/ROOT/suggest?content=zimfile&term=ray&start=" + std::to_string(i*5) + "&count=5";
+      const auto r = zfs1_->GET(url.c_str());
+      std::string body = r->body;
+      int currCount = std::count(body.begin(), body.end(), '{') - 1;
+      ASSERT_EQ(currCount, 5);
+      suggCount += currCount;
+    }
+    ASSERT_EQ(suggCount, 50);
+  }
+
+  // Attempt to get 10 suggestions in steps of 5 even though there are only 8
+  {
+    std::string url = "/ROOT/suggest?content=zimfile&term=song+for+you&start=0&count=5";
+    const auto r1 = zfs1_->GET(url.c_str());
+    std::string body = r1->body;
+    int currCount = std::count(body.begin(), body.end(), '{') - 1;
+    ASSERT_EQ(currCount, 5);
+
+    url = "/ROOT/suggest?content=zimfile&term=song+for+you&start=5&count=5";
+    const auto r2 = zfs1_->GET(url.c_str());
+    body = r2->body;
+    currCount = std::count(body.begin(), body.end(), '{') - 1;
+    ASSERT_EQ(currCount, 3);
+  }
+
+  // Attempt to get 10 suggestions even though there is only 1
+  {
+    std::string url = "/ROOT/suggest?content=zimfile&term=strong&start=0&count=5";
+    const auto r = zfs1_->GET(url.c_str());
+    std::string body = r->body;
+    int currCount = std::count(body.begin(), body.end(), '{') - 1;
+    ASSERT_EQ(currCount, 1);
+  }
+
+  // No Suggestion
+  {
+    std::string url = "/ROOT/suggest?content=zimfile&term=oops&start=0&count=5";
+    const auto r = zfs1_->GET(url.c_str());
+    std::string body = r->body;
+    int currCount = std::count(body.begin(), body.end(), '{') - 1;
+    ASSERT_EQ(currCount, 0);
+  }
+
+  // Out of bound value
+  {
+    std::string url = "/ROOT/suggest?content=zimfile&term=ray&start=-2&count=-1";
+    const auto r = zfs1_->GET(url.c_str());
+    std::string body = r->body;
+    int currCount = std::count(body.begin(), body.end(), '{') - 1;
+    ASSERT_EQ(currCount, 0);
+  }
+}
+
+TEST_F(LibraryServerTest, catalog_v2_individual_entry_access)
+{
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/entry/raycharles");
+  EXPECT_EQ(r->status, 200);
+  EXPECT_EQ(maskVariableOPDSFeedData(r->body),
+    "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+    RAY_CHARLES_CATALOG_ENTRY
+  );
+
+  const auto r1 = zfs1_->GET("/ROOT/catalog/v2/entry/non-existent-entry");
+  EXPECT_EQ(r1->status, 404);
+}
+
+TEST_F(LibraryServerTest, catalog_v2_partial_entries)
+{
+  const auto r = zfs1_->GET("/ROOT/catalog/v2/partial_entries");
+  EXPECT_EQ(r->status, 200);
+  EXPECT_EQ(maskVariableOPDSFeedData(r->body),
+    CATALOG_V2_PARTIAL_ENTRIES_PREAMBLE("")
+    "  <title>All Entries</title>\n"
+    "  <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
+    "\n"
+    "  <entry>\n"
+    "    <id>urn:uuid:charlesray</id>\n"
+    "    <title>Charles, Ray</title>\n"
+    "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
+    "    <link rel=\"alternate\"\n"
+    "          href=\"/ROOT/catalog/v2/entry/charlesray\"\n"
+    "          type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
+    "  </entry>\n"
+    "  <entry>\n"
+    "    <id>urn:uuid:raycharles</id>\n"
+    "    <title>Ray Charles</title>\n"
+    "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
+    "    <link rel=\"alternate\"\n"
+    "          href=\"/ROOT/catalog/v2/entry/raycharles\"\n"
+    "          type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
+    "  </entry>\n"
+    "  <entry>\n"
+    "    <id>urn:uuid:raycharles_uncategorized</id>\n"
+    "    <title>Ray (uncategorized) Charles</title>\n"
+    "    <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
+    "    <link rel=\"alternate\"\n"
+    "          href=\"/ROOT/catalog/v2/entry/raycharles_uncategorized\"\n"
+    "          type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
+    "  </entry>\n"
+    "</feed>\n"
+  );
+}