Merge pull request #747 from kiwix/version_10.1.1

Correctly detect the number of article for zim version <= 6

.github/workflows/ci.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v1
-      - name: Setup python 3.5
-        uses: actions/setup-python@v1
+      - name: Setup python 3.10
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.5'
+          python-version: '3.10'
       - name: Install packages
         run: |
           brew update
@@ -37,17 +37,8 @@ jobs:
           export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
           cd build
           meson test --verbose
-          ninja coverage
         env:
           SKIP_BIG_MEMORY_TEST: 1
-      - name: Publish coverage
-        shell: bash
-        run: |
-          curl https://codecov.io/bash -o codecov.sh
-          bash codecov.sh -n osx_native_dyn -Z
-          rm codecov.sh
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   Linux:
     strategy:

.github/workflows/package.yml

@@ -8,8 +8,8 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-jammy
           - ubuntu-impish
-          - ubuntu-hirsute
           - ubuntu-focal
           - ubuntu-bionic
     steps:
@@ -34,18 +34,18 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
+      - uses: legoktm/gh-action-build-deb@ubuntu-impish
+        if: matrix.distro == 'ubuntu-impish'
+        name: Build package for ubuntu-impish
+        id: build-ubuntu-impish
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}

.gitignore

@@ -5,3 +5,5 @@ subprojects/googletest-release*
 build/
 .vscode/
 builddir/
+.cache/
+.clangd/

ChangeLog

@@ -1,7 +1,109 @@
+libkiwix 10.1.1
+===============
+
+ * Correctly detect the number of article for older zims (<=6) (@mgautier #743)
+ * [server] Fix fulltext search (@mgautierfr #724)
+ * [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
+ * Fix CI (@mgautierfr #736)
+
+libkiwix 10.1.0
+===============
+
+ This release is an important one as it fixes a Xss vulnerability introduced
+ in libkiwix 10.0.0
+
+ * [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
+ * [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
+ * [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
+ * [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
+ * Add `dc:issued` to opds output stream (@veloman-yunkan #715)
+ * Add handling of several languages not provided by ICU (@juuz0 #701)
+ * [server] Add a caching system for search and suggestion (@maneeshpm #620)
+ * Fix cross-compilation (@kelson42 #703)
+ * Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
+ * Better testing system of html response (@veloman-yunkan #725)
+
+libkiwix 10.0.1
+===============
+
+ * [server] The catalog search interpret `count=0` as no limit.
+   This was the case for a long time. This was changed unintentionally
+   (@veloman-yunkan #686)
+ * [server] Correctly generere a human friendly title in the server frontend.
+   (@juuz0 #687, @kelson42 #689)
+ * [server] Fix download button if there is no url do download from.
+   (@juuz0 #691)
+ * Add non-minified isotope.pkdg.js
+   Needed for debian packaging as we need the source and minified version is
+   not the source (@legoktm #693)
+ * [server] Add a tooltip with the full language for the lang tag.
+ * CI fixes (@kelson42 @legoktm)
+
 libkiwix 10.0.0
 ===============
 
- * ...
+ This release is huge release.
+ The project has been renamed to libkiwix, it is more coherent with the library name.
+
+ * Server front page :
+  - Use js in the front page to display the available book,
+     using the OPDS stream as source. The front page is now populated only with
+     the visible books and user can search for books. (@MananJethwany #530, #541, #534)
+     (@kelson42 #628)
+  - Revamp css (@MananJethwany #559)
+  - Correctly Convert 3iso language code to 2iso (@juuz0 #672)
+
+ * Server suggestions search :
+  - Add pagination for suggestion search (@maneeshpm #591)
+  - Fix suggestion system (@MananJethwany #498)
+  - Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
+  - The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
+  - Properly handle html encoding of suggestions (@veloman-yunkan #458)
+
+ * Server improvements :
+  - Remove meta endpoints (@mgautier #669)
+  - Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
+  - Add details on 404 error pages (@soumyankar #490)
+  - Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
+  - Better headbar insertion (after charset definition) (@kelson42 #442)
+
+ * New OPDS Stream v2 :
+  - Add a list of categories (@veloman-yunkan)
+  - Support for partial entries (@veloman-yunkan #602)
+  - Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
+  - Add language endpoint to catalog (@veloman-yunkan #553)
+  - Add illustration API to get the illustration of a book (@mgautierfr #645)
+  - OPDS search can now filter books by category (@veloman-yunkan #459)
+
+ * Library improvements :
+  - Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
+  - Properly handle removing of book from the library (@veloman-yunkan #485)
+  - Use xapian to search for books in the library (@veloman-yunkan #460, #488)
+
+ * Added methods/functions :
+  - Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
+  - Add `getVersions` and `printVersions` functions (@kelson42 #665)
+  - Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
+  - Add `get_zimid()` method to the search result (@maneeshpm #510)
+
+ * Various improvements :
+  - Better secret value for aria2c rpc (@juuz0 #666)
+  - Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
+  - Add basic documentation (@mgautierfr #640)
+  - Do not use Reader internally (@maneeshpm #536 #576)
+  - Remove dependency headers from our public headers (@mgautierfr #574)
+  - Downloader now don't write metalink on the filesystem (@kelson42 #502)
+  - Support opening a zim file using a fd (@veloman-yukan #429)
+  - Use C++11 std::thread instead of pthread (@mgautierfr #445)
+  - [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
+  - Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
+  - Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
+  - Better parsing of the counterMap (@kelson42 #437)
+  - Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
+  - Remove deprecated methods (@mgautierfr)
+  - CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
+  - Fix compilation for FreeBSD (@swills g#432)
+  - Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
 
 kiwix-lib 9.4.1
 ===============

README.md

@@ -5,6 +5,7 @@ The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
 core. It contains the code shared by all Kiwix ports (Windows,
 GNU/Linux, macOS, Android, iOS, ...).
 
+[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
 [![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
@@ -91,6 +92,14 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
+with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
+
+```bash
+meson . build -Dwrapper=android -Dwerror=false
+ninja -C build
+```
+
 Testing
 -------
 
@@ -154,7 +163,7 @@ to use custom welcome page mention `customIndexPage` argument in `kiwix::interna
 
 to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
 
-To use JS provided by kiwix-serve you can use the following template to start with -> 
+To use JS provided by kiwix-serve you can use the following template to start with ->
 
 ```
 <!DOCTYPE html>
@@ -184,7 +193,7 @@ To use JS provided by kiwix-serve you can use the following template to start wi
 - To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
 - To add language select box add - `<select id="languageFilter"></select>` under body tag.
 - To add language select box add - `<select id="categoryFilter"></select>` under body tag.
-- To add search box for books use following form - 
+- To add search box for books use following form -
     ```
         <form id='kiwixSearchForm'>
         <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/libkiwix'

debian/control

@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
  meson,
  pkg-config,
- libzim-dev (>= 6.1.8),
+ libzim-dev (>= 7.2.0~),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
@@ -23,7 +23,7 @@ Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
- libzim-dev (>= 6.0.0),
+ libzim-dev (>= 7.2.0~),
  libicu-dev,
  libpugixml-dev,
  libcurl4-gnutls-dev,

docs/.gitignore

@@ -0,0 +1,2 @@
+api
+xml

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'libkiwix'
+copyright = '2022, libkiwix-team'
+author = 'libzim-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+    'exhale'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+breathe_projects = {
+    "libkiwix": "./xml"
+}
+breathe_default_project = 'libkiwix'
+
+exhale_args = {
+    "containmentFolder":   "./api",
+    "rootFileName":        "ref_api.rst",
+    "rootFileTitle":       "Reference API",
+    "doxygenStripFromPath":"..",
+    "treeViewIsBootstrap": True,
+    "createTreeView" : True,
+    "exhaleExecutesDoxygen": True,
+    "exhaleDoxygenStdin":    "INPUT = ../include"
+}
+
+primary_domain = 'cpp'
+
+highlight_language = 'cpp'

docs/index.rst

@@ -0,0 +1,14 @@
+.. libkiwix documentation master file, created by
+   sphinx-quickstart on Fri Jul 24 15:40:50 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libzim's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage
+   api/ref_api

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+breathe
+exhale

docs/usage.rst

@@ -0,0 +1,17 @@
+Libkiwix programming
+====================
+
+Introduction
+------------
+
+libkiwix is written in C++. To use the library, you need the include files of libkiwix have
+to link against libzim.
+
+Errors are handled with exceptions. When something goes wrong, libzim throws an error,
+which is always derived from std::exception.
+
+All classes are defined in the namespace kiwix.
+
+libkiwix is a set of tools to manage zim files and provide some common functionnality.
+While libkiwix has some wrappers around libzim classes, they are deprecated and will be removed
+in the future.

include/book.h

@@ -21,6 +21,10 @@
 #define KIWIX_BOOK_H
 
 #include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include "common.h"
 
 namespace pugi {
 class xml_node;
@@ -41,12 +45,31 @@ class Reader;
  */
 class Book
 {
- public:
+ public: // types
+  class Illustration
+  {
+    friend class Book;
+   public:
+    uint16_t width = 48;
+    uint16_t height = 48;
+    std::string mimeType;
+    std::string url;
+
+    const std::string& getData() const;
+
+   private:
+    mutable std::string data;
+    mutable std::mutex mutex;
+  };
+
+  typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
+
+ public: // functions
   Book();
   ~Book();
 
   bool update(const Book& other);
-  void update(const Reader& reader);
+  DEPRECATED void update(const Reader& reader);
   void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
@@ -73,9 +96,13 @@ class Book
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const;
-  const std::string& getFaviconUrl() const { return m_faviconUrl; }
-  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
+  DEPRECATED const std::string& getFavicon() const;
+  DEPRECATED const std::string& getFaviconUrl() const;
+  DEPRECATED const std::string& getFaviconMimeType() const;
+
+  Illustrations getIllustrations() const;
+  std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
+
   const std::string& getDownloadId() const { return m_downloadId; }
 
   void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
@@ -96,14 +123,13 @@ class Book
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
   void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
   void setSize(uint64_t size) { m_size = size; }
-  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
-  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
   void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
 
- private:
+ private: // functions
   std::string getCategoryFromTags() const;
+  const Illustration& getDefaultIllustration() const;
 
- protected:
+ protected: // data
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
@@ -124,9 +150,11 @@ class Book
   uint64_t m_mediaCount = 0;
   bool m_readOnly = false;
   uint64_t m_size = 0;
-  mutable std::string m_favicon;
-  std::string m_faviconUrl;
-  std::string m_faviconMimeType;
+  Illustrations m_illustrations;
+
+  // Used as the return value of getDefaultIllustration() when no default
+  // illustration is found in the book
+  static const Illustration missingDefaultIllustration;
 };
 
 }

include/entry.h

@@ -47,7 +47,7 @@ class Entry
      *
      * @param article a zim::Article object
      */
-    Entry(zim::Entry entry);
+    DEPRECATED Entry(zim::Entry entry) : Entry(entry, true) {};
     virtual ~Entry() = default;
 
     /**
@@ -176,6 +176,16 @@ class Entry
 
   private:
     zim::Entry entry;
+
+  private:
+    // Entry is deprecated, so we've marked the constructor as deprecated.
+    // But we still need to construct the entry (in our deprecated code)
+    // To avoid warning because we use deprecated function, we create a second
+    // constructor not deprecated. The `bool marker` is unused, it sole purpose
+    // is to change the signature to have two different constructor.
+    // This one is not deprecated and we must use it in our private code.
+    Entry(zim::Entry entry, bool marker);
+    friend class Reader;
 };
 
 }

include/kiwix.h

@@ -22,4 +22,4 @@
 
 #include "library.h"
 
-#endif
+#endif

include/library.h

@@ -24,6 +24,7 @@
 #include <vector>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <zim/archive.h>
 
 #include "book.h"
@@ -139,20 +140,51 @@ private: // functions
     bool accept(const Book& book) const;
 };
 
-
 /**
- * A Library store several books.
+ * This class is not part of the libkiwix API. Its only purpose is
+ * to simplify the implementation of the Library's move operations
+ * and avoid bugs should new data members be added to Library.
  */
-class Library
+class LibraryBase
 {
-  std::map<std::string, kiwix::Book> m_books;
+protected: // types
+  typedef uint64_t LibraryRevision;
+
+  struct Entry : Book
+  {
+    LibraryRevision lastUpdatedRevision = 0;
+
+    // May also keep the Archive and Reader pointers here and get
+    // rid of the m_readers and m_archives data members in Library
+  };
+
+protected: // data
+  LibraryRevision m_revision;
+  std::map<std::string, Entry> m_books;
   std::map<std::string, std::shared_ptr<Reader>> m_readers;
   std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
   std::vector<kiwix::Bookmark> m_bookmarks;
   class BookDB;
   std::unique_ptr<BookDB> m_bookDB;
 
+protected: // functions
+  LibraryBase();
+  ~LibraryBase();
+
+  LibraryBase(LibraryBase&& );
+  LibraryBase& operator=(LibraryBase&& );
+};
+
+/**
+ * A Library store several books.
+ */
+class Library : private LibraryBase
+{
+  // all data fields must be added in LibraryBase
+  mutable std::mutex m_mutex;
+
  public:
+  typedef LibraryRevision Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
 
@@ -180,6 +212,11 @@ class Library
    */
   bool addBook(const Book& book);
 
+  /**
+   * A self-explanatory alias for addBook()
+   */
+  bool addOrUpdateBook(const Book& book) { return addBook(book); }
+
   /**
    * Add a bookmark to the library.
    *
@@ -196,11 +233,14 @@ class Library
    */
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
+  // XXX: This is a non-thread-safe operation
   const Book& getBookById(const std::string& id) const;
-  Book& getBookById(const std::string& id);
+  // XXX: This is a non-thread-safe operation
   const Book& getBookByPath(const std::string& path) const;
-  Book& getBookByPath(const std::string& path);
-  std::shared_ptr<Reader> getReaderById(const std::string& id);
+
+  Book getBookByIdThreadSafe(const std::string& id) const;
+
+  DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
 
   /**
@@ -285,17 +325,6 @@ class Library
    */
   BookIdCollection getBooksIds() const;
 
-  /**
-   * Filter the library and generate a new one with the keep elements.
-   *
-   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
-   *
-   * @param search List only books with search in the title or description.
-   * @return The list of bookIds corresponding to the query.
-   */
-  DEPRECATED BookIdCollection filter(const std::string& search) const;
-
-
   /**
    * Filter the library and return the id of the keep elements.
    *
@@ -315,36 +344,22 @@ class Library
   void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
 
   /**
-   * List books in the library.
+   * Return the current revision of the library.
    *
-   * @param mode The mode of listing :
-   *             - LOCAL    : list only local books (with a path).
-   *             - REMOTE   : list only remote books (with an url).
-   *             - VALID    : list only valid books (without a path or with a
-   *                          path pointing to a valid zim file).
-   *             - NOLOCAL  : list only books without valid path.
-   *             - NOREMOTE : list only books without url.
-   *             - NOVALID  : list only books not valid.
-   *             - ALL : Do not do any filter (LOCAL or REMOTE)
-   *             - Flags can be combined.
-   * @param sortBy Attribute to sort by the book list.
-   * @param search List only books with search in the title, description.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param maxSize Do not list book bigger than maxSize.
-   *                Set to 0 to cancel this filter.
-   * @return The list of bookIds corresponding to the query.
+   * The revision of the library is updated (incremented by one) only by
+   * the addBook() operation.
+   *
+   * @return Current revision of the library.
    */
-  DEPRECATED BookIdCollection listBooksIds(
-    int supportedListMode = ALL,
-    supportedListSortBy sortBy = UNSORTED,
-    const std::string& search = "",
-    const std::string& language = "",
-    const std::string& creator = "",
-    const std::string& publisher = "",
-    const std::vector<std::string>& tags = {},
-    size_t maxSize = 0) const;
+  LibraryRevision getRevision() const;
+
+  /**
+   * Remove books that have not been updated since the specified revision.
+   *
+   * @param  rev the library revision to use
+   * @return Count of books that were removed by this operation.
+   */
+  uint32_t removeBooksNotUpdatedSince(LibraryRevision rev);
 
   friend class OPDSDumper;
   friend class libXMLDumper;
@@ -357,6 +372,7 @@ private: // functions
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
+  void dropReader(const std::string& bookId);
 };
 
 }

include/manager.h

@@ -26,6 +26,7 @@
 
 #include <string>
 #include <vector>
+#include <memory>
 
 namespace pugi {
 class xml_document;
@@ -34,26 +35,25 @@ class xml_document;
 namespace kiwix
 {
 
-class LibraryManipulator {
- public:
-  virtual ~LibraryManipulator() {}
-  virtual bool addBookToLibrary(Book book) = 0;
-  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
-};
+class LibraryManipulator
+{
+ public: // functions
+  explicit LibraryManipulator(Library* library);
+  virtual ~LibraryManipulator();
 
-class DefaultLibraryManipulator : public LibraryManipulator {
- public:
-  DefaultLibraryManipulator(Library* library) :
-    library(library) {}
-  virtual ~DefaultLibraryManipulator() {}
-  bool addBookToLibrary(Book book) {
-    return library->addBook(book);
-  }
-  void addBookmarkToLibrary(Bookmark bookmark) {
-    library->addBookmark(bookmark);
-  }
- private:
-   kiwix::Library* library;
+  Library& getLibrary() const { return library; }
+
+  bool addBookToLibrary(const Book& book);
+  void addBookmarkToLibrary(const Bookmark& bookmark);
+  uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
+
+ protected: // overrides
+  virtual void bookWasAddedToLibrary(const Book& book);
+  virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
+  virtual void booksWereRemovedFromLibrary();
+
+ private: // data
+  kiwix::Library& library;
 };
 
 /**
@@ -61,10 +61,12 @@ class DefaultLibraryManipulator : public LibraryManipulator {
  */
 class Manager
 {
- public:
-  Manager(LibraryManipulator* manipulator);
-  Manager(Library* library);
-  ~Manager();
+ public: // types
+  typedef std::vector<std::string> Paths;
+
+ public: // functions
+  explicit Manager(LibraryManipulator* manipulator);
+  explicit Manager(Library* library);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
@@ -72,10 +74,22 @@ class Manager
    * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
+   * @param trustLibrary use book metadata coming from XML.
    * @return True if file has been properly parsed.
    */
   bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
+  /**
+   * Sync the contents of the library with one or more `library.xml` files.
+   *
+   * The metadata of the library files is trusted unconditionally.
+   * Any books not present in the input library.xml files are removed
+   * from the library.
+   *
+   * @param paths The (utf8) paths to the `library.xml` files.
+   */
+  void reload(const Paths& paths);
+
   /**
    * Load a library content store in the string.
    *
@@ -150,8 +164,7 @@ class Manager
   uint64_t m_itemsPerPage = 0;
 
  protected:
-  kiwix::LibraryManipulator* manipulator;
-  bool mustDeleteManipulator;
+  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,

include/meson.build

@@ -14,7 +14,8 @@ headers = [
   'server.h',
   'kiwixserve.h',
   'name_mapper.h',
-  'tools.h'
+  'tools.h',
+  'version.h'
 ]
 
 install_headers(headers, subdir:'kiwix')

include/name_mapper.h

@@ -22,6 +22,8 @@
 
 #include <string>
 #include <map>
+#include <memory>
+#include <mutex>
 
 namespace kiwix
 {
@@ -31,15 +33,15 @@ class Library;
 class NameMapper {
   public:
     virtual ~NameMapper() = default;
-    virtual std::string getNameForId(const std::string& id) = 0;
-    virtual std::string getIdForName(const std::string& name) = 0;
+    virtual std::string getNameForId(const std::string& id) const = 0;
+    virtual std::string getIdForName(const std::string& name) const = 0;
 };
 
 
 class IdNameMapper : public NameMapper {
   public:
-    virtual std::string getNameForId(const std::string& id) { return id; };
-    virtual std::string getIdForName(const std::string& name) { return name; };
+    virtual std::string getNameForId(const std::string& id) const { return id; };
+    virtual std::string getIdForName(const std::string& name) const { return name; };
 };
 
 class HumanReadableNameMapper : public NameMapper {
@@ -50,11 +52,29 @@ class HumanReadableNameMapper : public NameMapper {
   public:
     HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
     virtual ~HumanReadableNameMapper() = default;
-    virtual std::string getNameForId(const std::string& id);
-    virtual std::string getIdForName(const std::string& name);
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
 };
 
+class UpdatableNameMapper : public NameMapper {
+    typedef std::shared_ptr<NameMapper> NameMapperHandle;
+  public:
+    UpdatableNameMapper(Library& library, bool withAlias);
 
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+
+    void update();
+
+  private:
+    NameMapperHandle currentNameMapper() const;
+
+  private:
+    mutable std::mutex mutex;
+    Library& library;
+    NameMapperHandle nameMapper;
+    const bool withAlias;
+};
 
 }
 

include/reader.h

@@ -42,7 +42,6 @@ namespace kiwix
 class SuggestionItem
 {
   // Functions
-  // Temporarily making the constructor public until the code move is complete
   public:
     // Create a sugggestion item.
     explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
@@ -66,13 +65,13 @@ class SuggestionItem
     std::string normalizedTitle;
     std::string path;
     std::string snippet;
-
-  friend class Reader;
 };
 
 /**
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
+ *
+ * Reader is now deprecated. Directly use `zim::Archive`.
  */
 
 using SuggestionsList_t = std::vector<SuggestionItem>;
@@ -88,17 +87,18 @@ class Reader
    *                    unsplitted path as if the file were not splitted
    *                    (.zim extesion).
    */
-  explicit Reader(const string zimFilePath);
+  explicit DEPRECATED Reader(const string zimFilePath);
 
    /**
    * Create a Reader to read a zim file given by the Archive.
    *
    * @param archive The shared pointer to the Archive object.
    */
-  explicit Reader(const std::shared_ptr<zim::Archive> archive);
+  explicit DEPRECATED Reader(const std::shared_ptr<zim::Archive> archive)
+   : Reader(archive, true) {};
 #ifndef _WIN32
-  explicit Reader(int fd);
-  Reader(int fd, zim::offset_type offset, zim::size_type size);
+  explicit DEPRECATED Reader(int fd);
+  DEPRECATED Reader(int fd, zim::offset_type offset, zim::size_type size);
 #endif
   ~Reader() = default;
 
@@ -491,6 +491,15 @@ class Reader
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;
+
+  // Reader is deprecated, so we've marked the constructor as deprecated.
+  // But we still need to construct the reader (in our deprecated code)
+  // To avoid warning because we use deprecated function, we create a
+  // constructor not deprecated. The `bool marker` is unused, it sole purpose
+  // is to change the signature to have a different constructor.
+  // This one is not deprecated and we must use it in our private code.
+  Reader(const std::shared_ptr<zim::Archive> archive, bool marker);
+  friend class Library;
 };
 }
 

include/search_renderer.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <zim/search.h>
+#include "library.h"
 
 namespace kiwix
 {
@@ -35,15 +36,43 @@ class SearchRenderer
 {
  public:
   /**
-   * The default constructor.
+   * Construct a SearchRenderer from a Searcher.
    *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
+   * This method is now deprecated. Construct the renderer from a
+   * `zim::SearchResultSet`
+   *
+   * @param searcher The `Searcher` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   */
+  DEPRECATED SearchRenderer(Searcher* searcher, NameMapper* mapper);
+
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * The constructed version of the SearchRenderer will not introduce
+   * the book name for each result. It is better to use the other constructor
+   * with a Library pointer to have a better html page.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
    */
-  SearchRenderer(Searcher* searcher, NameMapper* mapper);
   SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                  unsigned int start, unsigned int estimatedResultCount);
 
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                 unsigned int start, unsigned int estimatedResultCount);
+
   ~SearchRenderer();
 
   void setSearchPattern(const std::string& pattern);
@@ -79,6 +108,7 @@ class SearchRenderer
   std::string beautifyInteger(const unsigned int number);
   zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
+  Library* mp_library;
   std::string searchContent;
   std::string searchPattern;
   std::string protocolPrefix;

include/searcher.h

@@ -56,6 +56,8 @@ struct SuggestionInternal;
 /**
  * The Searcher class is reponsible to do different kind of search using the
  * fulltext index.
+ *
+ * The Searcher is now deprecated. Use libzim search feature.
  */
 class Searcher
 {
@@ -63,7 +65,7 @@ class Searcher
   /**
    * The default constructor.
    */
-  Searcher();
+  DEPRECATED Searcher();
 
   ~Searcher();
 

include/server.h

@@ -54,13 +54,16 @@ namespace kiwix
        void setAddress(const std::string& addr) { m_addr = addr; }
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
        void setBlockExternalLinks(bool blockExternalLinks)
         { m_blockExternalLinks = blockExternalLinks; }
-
+       int getPort();
+       std::string getAddress();
+       
      protected:
        Library* mp_library;
        NameMapper* mp_nameMapper;
@@ -73,6 +76,7 @@ namespace kiwix
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
        bool m_blockExternalLinks = false;
+       int m_ipConnectionLimit = 0;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

include/tools.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <vector>
+#include <map>
 
 namespace kiwix {
 
@@ -177,16 +178,25 @@ std::string converta2toa3(const std::string& a2code);
  */
 std::string getFileContent(const std::string& path);
 
-/** checks if file exists.
+/** Checks if file exists.
  *
- * This function returns boolean stating if file exists or not.
+ * This function returns boolean stating if file exists.
  *
  * @param path The absolute path provided in string format.
  * @return Boolean representing if file exists or not.
  */
 bool fileExists(const std::string& path);
 
-/** provides mimetype from filename.
+/** Checks if file is readable.
+ *
+ * This function returns boolean stating if file is readable.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file is readale or not.
+ */
+bool fileReadable(const std::string& path);
+
+/** Provides mimetype from filename.
  *
  * This function provides mimetype from file-name.
  *
@@ -194,5 +204,17 @@ bool fileExists(const std::string& path);
  * @return mimetype from filename in string format.
  */
 std::string getMimeTypeForFile(const std::string& filename);
+
+/** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 network interfaces
+ */
+std::map<std::string, std::string> getNetworkInterfaces();
+
+/** Provides the best IP address
+ * This function provides the best IP address from the list given by getNetworkInterfaces
+ */
+std::string getBestPublicIp();
+
 }
 #endif // KIWIX_TOOLS_H

include/version.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_VERSION_H
+#define KIWIX_VERSION_H
+
+#include <string>
+#include <vector>
+
+namespace kiwix
+{
+  typedef std::vector<std::pair<std::string, std::string>> LibVersions;
+  LibVersions getVersions();
+  void printVersions(std::ostream& out = std::cout);
+}
+
+#endif // KIWIX_VERSION_H

meson.build

@@ -1,5 +1,5 @@
 project('libkiwix', 'cpp',
-  version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '10.1.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -19,11 +19,11 @@ if wrapper.contains('java')
 endif
 
 # See https://github.com/kiwix/libkiwix/issues/371
-if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
   extra_libs += '-latomic'
 endif
 
-if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
   # C++ std::thread is implemented using pthread on linux by gcc
   thread_dep = dependency('threads')
 else
@@ -44,19 +44,19 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif
 
 
 extra_cflags = ''
-if target_machine.system() == 'windows' and static_deps
+if host_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-if target_machine.system() == 'windows'
+if host_machine.system() == 'windows'
   add_project_arguments('-DNOMINMAX', language: 'cpp')
 endif
 
@@ -65,7 +65,7 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microh
 inc = include_directories('include', extra_include)
 
 conf = configuration_data()
-conf.set('VERSION', '"@0@"'.format(meson.project_version()))
+conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -78,6 +78,9 @@ subdir('scripts')
 subdir('static')
 subdir('src')
 subdir('test')
+if get_option('doc')
+  subdir('docs')
+endif
 
 pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
 

meson_options.txt

@@ -1,2 +1,4 @@
 option('wrapper', type:'array', choices:['java', 'android'], value:[],
   description: 'The wrapper to generate.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')

src/aria2.cpp

@@ -3,12 +3,14 @@
 #include "aria2.h"
 #include "xmlrpc.h"
 #include <iostream>
+#include <algorithm>
 #include <sstream>
 #include <thread>
 #include <chrono>
 #include "tools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
 #include "downloader.h" // For AriaError
 
 #ifdef _WIN32
@@ -30,7 +32,7 @@ namespace kiwix {
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret("kiwixariarpc"),
+  m_secret(getNewRpcSecret()),
   m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
   mp_curl(nullptr)
 {
@@ -68,7 +70,7 @@ Aria2::Aria2():
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
   callCmd.push_back(download_dir.c_str());
-  if (fileExists(session_file)) {
+  if (fileReadable(session_file)) {
     callCmd.push_back(inputFile.c_str());
   }
   callCmd.push_back(session.c_str());
@@ -195,6 +197,13 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
   return doRequest(methodCall);
 }
 
+std::string Aria2::getNewRpcSecret()
+{
+  std::string uuid = gen_uuid("");
+  uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
+  return uuid.substr(0, 9);
+}
+
 std::vector<std::string> Aria2::tellActive()
 {
   MethodCall methodCall("aria2.tellActive", m_secret);

src/aria2.h

@@ -37,6 +37,7 @@ class Aria2
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    static std::string getNewRpcSecret();
     std::vector<std::string> tellActive();
     std::vector<std::string> tellWaiting();
     void saveSession();

src/book.cpp

@@ -41,11 +41,17 @@ Book::Book() :
   m_readOnly(false)
 {
 }
+
 /* Destructor */
 Book::~Book()
 {
 }
 
+Book::Illustrations Book::getIllustrations() const
+{
+  return m_illustrations;
+}
+
 bool Book::update(const kiwix::Book& other)
 {
   if (m_readOnly)
@@ -54,30 +60,7 @@ bool Book::update(const kiwix::Book& other)
   if (m_id != other.m_id)
     return false;
 
-  m_readOnly = other.m_readOnly;
-  m_path = other.m_path;
-  m_pathValid = other.m_pathValid;
-  m_title = other.m_title;
-  m_description = other.m_description;
-  m_language = other.m_language;
-  m_creator = other.m_creator;
-  m_publisher = other.m_publisher;
-  m_date = other.m_date;
-  m_url = other.m_url;
-  m_name = other.m_name;
-  m_flavour = other.m_flavour;
-  m_tags = other.m_tags;
-  m_category = other.m_category;
-  m_origId = other.m_origId;
-  m_articleCount = other.m_articleCount;
-  m_mediaCount = other.m_mediaCount;
-  m_size = other.m_size;
-  m_favicon = other.m_favicon;
-  m_faviconMimeType = other.m_faviconMimeType;
-  m_faviconUrl = other.m_faviconUrl;
-
-  m_downloadId = other.m_downloadId;
-
+  *this = other;
   return true;
 }
 
@@ -100,11 +83,20 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = archive.getArticleCount();
+  m_articleCount = getArchiveArticleCount(archive);
   m_mediaCount = getArchiveMediaCount(archive);
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
-  getArchiveFavicon(archive, 48, m_favicon, m_faviconMimeType);
+  m_illustrations.clear();
+  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+    const auto illustration = std::make_shared<Illustration>();
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
+    illustration->width = illustration->height = illustrationSize;
+    illustration->mimeType = illustrationItem.getMimetype();
+    illustration->data = illustrationItem.getData();
+    // NOTE: illustration->url is left uninitialized
+    m_illustrations.push_back(illustration);
+  }
 }
 
 #define ATTR(name) node.attribute(name).value()
@@ -116,7 +108,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  m_pathValid = fileExists(path);
+  m_pathValid = fileReadable(path);
   m_title = ATTR("title");
   m_description = ATTR("description");
   m_language = ATTR("language");
@@ -131,9 +123,14 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  m_favicon = base64_decode(ATTR("favicon"));
-  m_faviconMimeType = ATTR("faviconMimeType");
-  m_faviconUrl = ATTR("faviconUrl");
+  std::string favicon_mimetype = ATTR("faviconMimeType");
+  if (! favicon_mimetype.empty()) {
+    const auto favicon = std::make_shared<Illustration>();
+    favicon->data = base64_decode(ATTR("favicon"));
+    favicon->mimeType = favicon_mimetype;
+    favicon->url = ATTR("faviconUrl");
+    m_illustrations.assign(1, favicon);
+  }
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -164,7 +161,9 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_language = VALUE("language");
   m_creator = node.child("author").child("name").child_value();
   m_publisher = node.child("publisher").child("name").child_value();
-  m_date = fromOpdsDate(VALUE("updated"));
+  const std::string dcIssuedDate = VALUE("dc:issued");
+  m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
+  m_date = fromOpdsDate(m_date);
   m_name = VALUE("name");
   m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
@@ -181,8 +180,11 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
       m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
     }
     if (rel == "http://opds-spec.org/image/thumbnail") {
-      m_faviconUrl = urlHost + linkNode.attribute("href").value();
-      m_faviconMimeType = linkNode.attribute("type").value();
+      const auto favicon = std::make_shared<Illustration>();
+      favicon->data.clear();
+      favicon->url = urlHost + linkNode.attribute("href").value();
+      favicon->mimeType = linkNode.attribute("type").value();
+      m_illustrations.assign(1, favicon);
     }
  }
 
@@ -193,7 +195,7 @@ std::string Book::getHumanReadableIdFromPath() const
 {
   std::string id = m_path;
   if (!id.empty()) {
-    kiwix::removeAccents(id);
+    id = kiwix::removeAccents(id);
 
 #ifdef _WIN32
     id = replaceRegex(id, "", "^.*\\\\");
@@ -215,15 +217,54 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-const std::string& Book::getFavicon() const {
-  if (m_favicon.empty() && !m_faviconUrl.empty()) {
-    try {
-      m_favicon = download(m_faviconUrl);
-    } catch(...) {
-      std::cerr << "Cannot download favicon from " << m_faviconUrl;
+const Book::Illustration Book::missingDefaultIllustration;
+
+std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
+{
+  for ( const auto& ilPtr : m_illustrations ) {
+    if (ilPtr->width == size && ilPtr->height == size) {
+      return ilPtr;
     }
   }
-  return m_favicon;
+  throw std::runtime_error("Cannot find illustration");
+}
+
+const Book::Illustration& Book::getDefaultIllustration() const
+{
+  try {
+    return *getIllustration(48);
+  } catch (...) {
+    return missingDefaultIllustration;
+  }
+}
+
+const std::string& Book::Illustration::getData() const
+{
+  if (data.empty() && !url.empty()) {
+    const std::lock_guard<std::mutex> l(mutex);
+    if ( data.empty() ) {
+      try {
+        data = download(url);
+      } catch(...) {
+        std::cerr << "Cannot download favicon from " << url;
+      }
+    }
+  }
+  return data;
+}
+
+const std::string& Book::getFavicon() const {
+  return getDefaultIllustration().getData();
+}
+
+const std::string& Book::getFaviconUrl() const
+{
+  return getDefaultIllustration().url;
+}
+
+const std::string& Book::getFaviconMimeType() const
+{
+  return getDefaultIllustration().mimeType;
 }
 
 std::string Book::getTagStr(const std::string& tagName) const {

src/config.h.in

@@ -1,3 +1,3 @@
 
-#mesondefine VERSION
+#mesondefine LIBKIWIX_VERSION
 

src/entry.cpp

@@ -23,7 +23,7 @@
 namespace kiwix
 {
 
-Entry::Entry(zim::Entry entry)
+Entry::Entry(zim::Entry entry, bool _marker)
   : entry(entry)
 {
 }
@@ -53,7 +53,7 @@ Entry Entry::getRedirectEntry() const
     throw NoEntry();
   }
 
-  return entry.getRedirectEntry();
+  return Entry(entry.getRedirectEntry(), true);
 }
 
 Entry Entry::getFinalEntry() const
@@ -67,7 +67,7 @@ Entry Entry::getFinalEntry() const
   if (final_entry.isRedirect()) {
     throw NoEntry();
   }
-  return final_entry;
+  return Entry(final_entry, true);
 }
 
 }

src/library.cpp

@@ -49,23 +49,48 @@ std::string normalizeText(const std::string& text)
   return removeAccents(text);
 }
 
+bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
+{
+  return book1.isPathValid()
+      && book2.isPathValid()
+      && book1.getPath() == book2.getPath();
+}
+
 } // unnamed namespace
 
-class Library::BookDB : public Xapian::WritableDatabase
+class LibraryBase::BookDB : public Xapian::WritableDatabase
 {
 public:
   BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
 };
 
-/* Constructor */
-Library::Library()
+LibraryBase::LibraryBase()
   : m_bookDB(new BookDB)
 {
 }
 
-Library::Library(Library&& ) = default;
+LibraryBase::~LibraryBase()
+{
+}
 
-Library& Library::operator=(Library&& ) = default;
+LibraryBase::LibraryBase(LibraryBase&& ) = default;
+LibraryBase& LibraryBase::operator=(LibraryBase&& ) = default;
+
+/* Constructor */
+Library::Library()
+{
+}
+
+Library::Library(Library&& other)
+  : LibraryBase(std::move(other))
+{
+}
+
+Library& Library::operator=(Library&& other)
+{
+  LibraryBase::operator=(std::move(other));
+  return *this;
+}
 
 /* Destructor */
 Library::~Library()
@@ -75,25 +100,37 @@ Library::~Library()
 
 bool Library::addBook(const Book& book)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
+  ++m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
     auto& oldbook = m_books.at(book.getId());
-    oldbook.update(book);
+    if ( ! booksReferToTheSameArchive(oldbook, book) ) {
+      dropReader(book.getId());
+    }
+    oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
+                          // XXX: Then m_bookDB will become out-of-sync with
+                          // XXX: the real contents of the library.
+    oldbook.lastUpdatedRevision = m_revision;
     return false;
   } catch (std::out_of_range&) {
-    m_books[book.getId()] = book;
+    Entry& newEntry = m_books[book.getId()];
+    static_cast<Book&>(newEntry) = book;
+    newEntry.lastUpdatedRevision = m_revision;
     return true;
   }
 }
 
 void Library::addBookmark(const Bookmark& bookmark)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
       m_bookmarks.erase(it);
@@ -104,28 +141,63 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
+void Library::dropReader(const std::string& id)
+{
+  m_readers.erase(id);
+  m_archives.erase(id);
+}
 
 bool Library::removeBookById(const std::string& id)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   m_bookDB->delete_document("Q" + id);
-  m_readers.erase(id);
-  m_archives.erase(id);
+  dropReader(id);
   return m_books.erase(id) == 1;
 }
 
+Library::Revision Library::getRevision() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return m_revision;
+}
+
+uint32_t Library::removeBooksNotUpdatedSince(LibraryRevision libraryRevision)
+{
+  BookIdCollection booksToRemove;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    for ( const auto& entry : m_books) {
+      if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
+        booksToRemove.push_back(entry.first);
+      }
+    }
+  }
+
+  uint32_t countOfRemovedBooks = 0;
+  for ( const auto& id : booksToRemove ) {
+    if ( removeBookById(id) )
+      ++countOfRemovedBooks;
+  }
+  return countOfRemovedBooks;
+}
+
 const Book& Library::getBookById(const std::string& id) const
 {
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
   return m_books.at(id);
 }
 
-Book& Library::getBookById(const std::string& id)
+Book Library::getBookByIdThreadSafe(const std::string& id) const
 {
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookById(id));
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return getBookById(id);
 }
 
 const Book& Library::getBookByPath(const std::string& path) const
 {
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
   for(auto& it: m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
@@ -136,37 +208,26 @@ const Book& Library::getBookByPath(const std::string& path) const
   throw std::out_of_range(ss.str());
 }
 
-Book& Library::getBookByPath(const std::string& path)
-{
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookByPath(path));
-}
-
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {
+    std::lock_guard<std::mutex> lock(m_mutex);
     return m_readers.at(id);
   } catch (std::out_of_range& e) {}
 
-  try {
-    auto reader = make_shared<Reader>(m_archives.at(id));
-    m_readers[id] = reader;
-    return reader;
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
+  const auto archive = getArchiveById(id);
+  if ( !archive )
     return nullptr;
 
-  auto archive = make_shared<zim::Archive>(book.getPath());
-  m_archives[id] = archive;
-  auto reader = make_shared<Reader>(archive);
+  const shared_ptr<Reader> reader(new Reader(archive, true));
+  std::lock_guard<std::mutex> lock(m_mutex);
   m_readers[id] = reader;
   return reader;
 }
 
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   try {
     return m_archives.at(id);
   } catch (std::out_of_range& e) {}
@@ -183,6 +244,7 @@ std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   unsigned int result = 0;
   for (auto& pair: m_books) {
     auto& book = pair.second;
@@ -196,20 +258,30 @@ unsigned int Library::getBookCount(const bool localBooks,
 
 bool Library::writeToFile(const std::string& path) const
 {
+  const auto allBookIds = getBooksIds();
+
   auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
   dumper.setBaseDir(baseDir);
-  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
+  std::string xml;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    xml = dumper.dumpLibXMLContent(allBookIds);
+  };
+  return writeTextFile(path, xml);
 }
 
 bool Library::writeBookmarksToFile(const std::string& path) const
 {
   LibXMLDumper dumper(this);
-  return writeTextFile(path, dumper.dumpLibXMLBookmark());
+  // NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
+  const std::string xml = dumper.dumpLibXMLBookmark();
+  return writeTextFile(path, xml);
 }
 
 Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   AttributeCounts propValueCounts;
 
   for (const auto& pair: m_books) {
@@ -242,6 +314,7 @@ Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
 
 std::vector<std::string> Library::getBooksCategories() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   std::set<std::string> categories;
 
   for (const auto& pair: m_books) {
@@ -272,6 +345,7 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
+  std::lock_guard<std::mutex> lock(m_mutex);
   for(auto& bookmark:m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
@@ -282,6 +356,7 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
 
 Library::BookIdCollection Library::getBooksIds() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
   for (auto& pair: m_books) {
@@ -291,15 +366,6 @@ Library::BookIdCollection Library::getBooksIds() const
   return bookIds;
 }
 
-Library::BookIdCollection Library::filter(const std::string& search) const
-{
-  if (search.empty()) {
-    return getBooksIds();
-  }
-
-  return filter(Filter().query(search));
-}
-
 
 void Library::updateBookDB(const Book& book)
 {
@@ -471,6 +537,7 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 
   BookIdCollection bookIds;
 
+  std::lock_guard<std::mutex> lock(m_mutex);
   Xapian::Enquire enquire(*m_bookDB);
   enquire.set_query(query);
   const auto results = enquire.get_mset(0, m_books.size());
@@ -484,7 +551,9 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 Library::BookIdCollection Library::filter(const Filter& filter) const
 {
   BookIdCollection result;
-  for(auto id : filterViaBookDB(filter)) {
+  const auto preliminaryResult = filterViaBookDB(filter);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto id : preliminaryResult) {
     if(filter.accept(m_books.at(id))) {
       result.push_back(id);
     }
@@ -553,6 +622,11 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
 
 void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
 {
+  // NOTE: Can reimplement this method in a way that doesn't require locking
+  // NOTE: for the entire duration of the sort. Will need to obtain (under a
+  // NOTE: lock) the required atributes from the books once, and then the
+  // NOTE: sorting will run on a copy of data without locking.
+  std::lock_guard<std::mutex> lock(m_mutex);
   switch(sort) {
     case TITLE:
       std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
@@ -575,48 +649,6 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
 }
 
 
-Library::BookIdCollection Library::listBooksIds(
-    int mode,
-    supportedListSortBy sortBy,
-    const std::string& search,
-    const std::string& language,
-    const std::string& creator,
-    const std::string& publisher,
-    const std::vector<std::string>& tags,
-    size_t maxSize) const {
-
-  Filter _filter;
-  if (mode & LOCAL)
-    _filter.local(true);
-  if (mode & NOLOCAL)
-    _filter.local(false);
-  if (mode & VALID)
-    _filter.valid(true);
-  if (mode & NOVALID)
-    _filter.valid(false);
-  if (mode & REMOTE)
-    _filter.remote(true);
-  if (mode & NOREMOTE)
-    _filter.remote(false);
-  if (!tags.empty())
-    _filter.acceptTags(tags);
-  if (maxSize != 0)
-    _filter.maxSize(maxSize);
-  if (!language.empty())
-    _filter.lang(language);
-  if (!publisher.empty())
-    _filter.publisher(publisher);
-  if (!creator.empty())
-    _filter.creator(creator);
-  if (!search.empty())
-    _filter.query(search);
-
-  auto bookIds = filter(_filter);
-
-  sort(bookIds, sortBy, true);
-  return bookIds;
-}
-
 Filter::Filter()
   : activeFilters(0),
     _maxSize(0)

src/libxml_dumper.cpp

@@ -60,10 +60,13 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
     ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
     ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
-    if (!book.getFavicon().empty())
-      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
+    try {
+      auto defaultIllustration = book.getIllustration(48);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
+      if (!defaultIllustration->getData().empty())
+        ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
+    } catch(...) {}
   } else {
     ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
   }
@@ -91,7 +94,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
   auto book_node = entry_node.append_child("book");
 
   try {
-    auto book = library->getBookById(bookmark.getBookId());
+    auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
     ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());

src/manager.cpp

@@ -26,28 +26,81 @@
 
 namespace kiwix
 {
+
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+////////////////////////////////////////////////////////////////////////////////
+// LibraryManipulator
+////////////////////////////////////////////////////////////////////////////////
+
+LibraryManipulator::LibraryManipulator(Library* library)
+  : library(*library)
+{}
+
+LibraryManipulator::~LibraryManipulator()
+{}
+
+bool LibraryManipulator::addBookToLibrary(const Book& book)
+{
+  const auto ret = library.addBook(book);
+  if ( ret ) {
+    bookWasAddedToLibrary(book);
+  }
+  return ret;
+}
+
+void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
+{
+  library.addBookmark(bookmark);
+  bookmarkWasAddedToLibrary(bookmark);
+}
+
+uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
+{
+  const auto n = library.removeBooksNotUpdatedSince(rev);
+  if ( n != 0 ) {
+    booksWereRemovedFromLibrary();
+  }
+  return n;
+}
+
+void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
+{
+}
+
+void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
+{
+}
+
+void LibraryManipulator::booksWereRemovedFromLibrary()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Manager
+////////////////////////////////////////////////////////////////////////////////
+
 /* Constructor */
 Manager::Manager(LibraryManipulator* manipulator):
   writableLibraryPath(""),
-  manipulator(manipulator),
-  mustDeleteManipulator(false)
+  manipulator(manipulator, NoDelete())
 {
 }
 
 Manager::Manager(Library* library) :
   writableLibraryPath(""),
-  manipulator(new DefaultLibraryManipulator(library)),
-  mustDeleteManipulator(true)
+  manipulator(new LibraryManipulator(library))
 {
 }
 
-/* Destructor */
-Manager::~Manager()
-{
-  if (mustDeleteManipulator) {
-    delete manipulator;
-  }
-}
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
                           bool readOnly,
                           const std::string& libraryPath,
@@ -249,4 +302,21 @@ bool Manager::readBookmarkFile(const std::string& path)
   return true;
 }
 
+void Manager::reload(const Paths& paths)
+{
+  const auto libRevision = manipulator->getLibrary().getRevision();
+  for (std::string path : paths) {
+    if (!path.empty()) {
+      if ( kiwix::isRelativePath(path) )
+        path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
+
+      if (!readFile(path, false, true)) {
+        throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
+      }
+    }
+  }
+
+  manipulator->removeBooksNotUpdatedSince(libRevision);
+}
+
 }

src/meson.build

@@ -28,7 +28,8 @@ kiwix_sources = [
   'server/response.cpp',
   'server/internalServer.cpp',
   'server/internalServer_catalog_v2.cpp',
-  'opds_catalog.cpp'
+  'opds_catalog.cpp',
+  'version.cpp'
 ]
 kiwix_sources += lib_resources
 

src/name_mapper.cpp

@@ -51,12 +51,54 @@ HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool w
   }
 }
 
-std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
   return m_idToName.at(id);
 }
 
-std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
   return m_nameToId.at(name);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// UpdatableNameMapper
+////////////////////////////////////////////////////////////////////////////////
+
+UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+  : library(lib)
+  , withAlias(withAlias)
+{
+  update();
+}
+
+void UpdatableNameMapper::update()
+{
+  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  std::lock_guard<std::mutex> lock(mutex);
+  nameMapper.reset(newNameMapper);
+}
+
+UpdatableNameMapper::NameMapperHandle
+UpdatableNameMapper::currentNameMapper() const
+{
+  // Return a copy of the handle to the current NameMapper object. It will
+  // ensure that the object survives any call to UpdatableNameMapper::update()
+  // made before the completion of any pending operation on that object.
+  std::lock_guard<std::mutex> lock(mutex);
+  return nameMapper;
+}
+
+std::string UpdatableNameMapper::getNameForId(const std::string& id) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getNameForId(id);
+}
+
+std::string UpdatableNameMapper::getIdForName(const std::string& name) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getIdForName(name);
+}
+
 }

src/opds_dumper.cpp

@@ -58,11 +58,12 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
 {
     kainjow::mustache::list illustrations;
     if ( book.isPathValid() ) {
-      for ( auto illustration_size : zim::Archive(book.getPath()).getIllustrationSizes() ) {
+      for ( const auto& illustration : book.getIllustrations() ) {
+        // For now, we are handling only sizexsize@1 illustration.
+        // So we can simply pass one size to mustache.
         illustrations.push_back(kainjow::mustache::object{
-          {"icon_width", to_string(illustration_size)},
-          {"icon_height", to_string(illustration_size)},
-          {"icon_scale", "1"},
+          {"icon_size", to_string(illustration->width)},
+          {"icon_mimetype", illustration->mimeType}
         });
       }
     }
@@ -71,17 +72,16 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
 
 kainjow::mustache::object getSingleBookData(const Book& book)
 {
-    const MustacheData bookUrl = book.getUrl().empty()
-                               ? MustacheData(false)
-                               : MustacheData(book.getUrl());
+    const auto bookDate = book.getDate() + "T00:00:00Z";
     return kainjow::mustache::object{
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
       {"language", book.getLanguage()},
-      {"content_id",  book.getHumanReadableIdFromPath()},
-      {"updated", book.getDate() + "T00:00:00Z"},
+      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+      {"book_date", bookDate},
       {"category", book.getCategory()},
       {"flavour", book.getFlavour()},
       {"tags", book.getTags()},
@@ -89,53 +89,109 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"media_count", to_string(book.getMediaCount())},
       {"author_name", book.getCreator()},
       {"publisher_name", book.getPublisher()},
-      {"url", bookUrl},
+      {"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& endpointRoot, bool partial)
+std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
 {
   auto data = getSingleBookData(book);
   data["with_xml_header"] = MustacheData(withXMLHeader);
   data["dump_partial_entries"] = MustacheData(partial);
   data["endpoint_root"] = endpointRoot;
+  data["root"] = rootLocation;
   return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
-    const Book& book = library->getBookById(bookId);
-    booksData.push_back(kainjow::mustache::object{
-        {"entry", getSingleBookEntryXML(book, false, endpointRoot, partial)}
-    });
+    try {
+      const Book book = library->getBookByIdThreadSafe(bookId);
+      booksData.push_back(kainjow::mustache::object{
+          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
+      });
+    } catch ( const std::out_of_range& ) {
+      // the book was removed from the library since its id was obtained
+      // ignore it
+    }
   }
 
   return booksData;
 }
 
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"}
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    auto lang = *icuLangPtr;
+    const icu::Locale locale(lang);
+    icu::UnicodeString ustring;
+    locale.getDisplayLanguage(locale, ustring);
+    std::string displayLanguage;
+    ustring.toUTF8String(displayLanguage);
+    std::string iso3LangCode = locale.getISO3Language();
+    iso639_3.insert({iso3LangCode, displayLanguage});
+  }
+}
+
 std::string getLanguageSelfName(const std::string& lang) {
-  const icu::Locale locale(lang.c_str());
-  icu::UnicodeString ustring;
-  locale.getDisplayLanguage(locale, ustring);
-  std::string result;
-  ustring.toUTF8String(result);
-  return result;
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
 };
 
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, "", false);
+  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
      {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
@@ -148,14 +204,14 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, endpointRoot, partial);
+  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"query", query.empty() ? "" : "?" + urlEncode(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
@@ -169,7 +225,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, "", false);
+  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const
@@ -201,6 +257,7 @@ std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
   kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
   for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
     const std::string languageCode = langAndBookCount.first;
     const int bookCount = langAndBookCount.second;

src/reader.cpp

@@ -52,7 +52,7 @@ Reader::Reader(const string zimFilePath)
   srand(time(nullptr));
 }
 
-Reader::Reader(const std::shared_ptr<zim::Archive> archive)
+Reader::Reader(const std::shared_ptr<zim::Archive> archive, bool _marker)
   : zimArchive(archive),
     zimFilePath(archive->getFilename())
   {}
@@ -122,7 +122,7 @@ string Reader::getId() const
 Entry Reader::getRandomPage() const
 {
   try {
-    return zimArchive->getRandomEntry();
+    return Entry(zimArchive->getRandomEntry(), true);
   } catch(...) {
     throw NoEntry();
   }
@@ -130,7 +130,7 @@ Entry Reader::getRandomPage() const
 
 Entry Reader::getMainPage() const
 {
-  return zimArchive->getMainEntry();
+  return Entry(zimArchive->getMainEntry(), true);
 }
 
 bool Reader::getFavicon(string& content, string& mimeType) const
@@ -242,7 +242,7 @@ string Reader::getScraper() const
 Entry Reader::getEntryFromPath(const std::string& path) const
 {
   try {
-    return kiwix::getEntryFromPath(*zimArchive, path);
+    return Entry(kiwix::getEntryFromPath(*zimArchive, path), true);
   } catch (zim::EntryNotFound& e) {
     throw NoEntry();
   }
@@ -256,7 +256,7 @@ Entry Reader::getEntryFromEncodedPath(const std::string& path) const
 Entry Reader::getEntryFromTitle(const std::string& title) const
 {
   try {
-    return zimArchive->getEntryByTitle(title);
+    return Entry(zimArchive->getEntryByTitle(title), true);
   } catch(zim::EntryNotFound& e) {
     throw NoEntry();
   }

src/search_renderer.cpp

@@ -26,6 +26,8 @@
 #include "library.h"
 #include "name_mapper.h"
 
+#include "tools/archiveTools.h"
+
 #include <zim/search.h>
 
 #include <mustache.hpp>
@@ -37,18 +39,24 @@ namespace kiwix
 
 /* Constructor */
 SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : m_srs(searcher->getSearchResultSet()),
-      mp_nameMapper(mapper),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      estimatedResultCount(searcher->getEstimatedResultCount()),
-      resultStart(searcher->getResultStart())
+    : SearchRenderer(
+        searcher->getSearchResultSet(),
+        mapper,
+        nullptr,
+        searcher->getEstimatedResultCount(),
+        searcher->getResultStart())
 {}
 
 SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                       unsigned int start, unsigned int estimatedResultCount)
+    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
+{}
+
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                      unsigned int start, unsigned int estimatedResultCount)
     : m_srs(srs),
       mp_nameMapper(mapper),
+      mp_library(library),
       protocolPrefix("zim://"),
       searchProtocolPrefix("search://?"),
       estimatedResultCount(estimatedResultCount),
@@ -87,9 +95,13 @@ std::string SearchRenderer::getHtml()
     result.set("title", it.getTitle());
     result.set("url", it.getPath());
     result.set("snippet", it.getSnippet());
-    std::ostringstream s;
-    s << it.getZimId();
-    result.set("resultContentId", mp_nameMapper->getNameForId(s.str()));
+    std::string zim_id(it.getZimId());
+    result.set("resultContentId", mp_nameMapper->getNameForId(zim_id));
+    if (!mp_library) {
+      result.set("bookTitle", kainjow::mustache::data(false));
+    } else {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
 
     if (it.getWordCount() >= 0) {
       result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));

src/searcher.cpp

@@ -94,6 +94,12 @@ bool Searcher::add_reader(Reader* reader)
   if (!reader->hasFulltextIndex()) {
       return false;
   }
+
+  for ( const Reader* const existing_reader : readers ) {
+    if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
+      return false;
+  }
+
   this->readers.push_back(reader);
   return true;
 }

src/server.cpp

@@ -49,7 +49,8 @@ bool Server::start() {
     m_withTaskbar,
     m_withLibraryButton,
     m_blockExternalLinks,
-    m_indexTemplateString));
+    m_indexTemplateString,
+    m_ipConnectionLimit));
   return mp_server->start();
 }
 
@@ -71,4 +72,14 @@ void Server::setRoot(const std::string& root)
   }
 }
 
+int Server::getPort()
+{
+  return mp_server->getPort();
+}
+
+std::string Server::getAddress()
+{
+  return mp_server->getAddress();
+}
+
 }

src/server/internalServer.cpp

@@ -48,6 +48,7 @@ extern "C" {
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
 #include "tools/archiveTools.h"
+#include "tools/networkTools.h"
 #include "library.h"
 #include "name_mapper.h"
 #include "entry.h"
@@ -57,8 +58,6 @@ extern "C" {
 
 #include <zim/uuid.h>
 #include <zim/error.h>
-#include <zim/search.h>
-#include <zim/suggestion.h>
 #include <zim/entry.h>
 #include <zim/item.h>
 
@@ -79,6 +78,7 @@ extern "C" {
 
 #define MAX_SEARCH_LEN 140
 #define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#define DEFAULT_CACHE_SIZE 2
 
 namespace kiwix {
 
@@ -95,18 +95,68 @@ inline std::string normalizeRootUrl(std::string rootUrl)
   return rootUrl.empty() ? rootUrl : "/" + rootUrl;
 }
 
-unsigned parseIllustration(const std::string& s)
-{
-  int nw(0), nh(0), nEnd(0);
-  long int w(-1), h(-1);
-  if ( sscanf(s.c_str(), "Illustration_%n%ldx%n%ld@1%n)", &nw, &w, &nh, &h, &nEnd) == 2
-     && nEnd == (int)s.size() && !isspace(s[nw]) && !isspace(s[nh]) && w == h && w >= 0) {
-    return w;
-  }
-  return 0;
+// Returns the value of env var `name` if found, otherwise returns defaultVal
+unsigned int getCacheLength(const char* name, unsigned int defaultVal) {
+  try {
+    const char* envString = std::getenv(name);
+    if (envString == nullptr) {
+      throw std::runtime_error("Environment variable not set");
+    }
+    return extractFromString<unsigned int>(envString);
+  } catch (...) {}
+
+  return defaultVal;
 }
 } // unnamed namespace
 
+SearchInfo::SearchInfo(const std::string& pattern)
+  : pattern(pattern),
+    geoQuery()
+{}
+
+SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery)
+  : pattern(pattern),
+    geoQuery(geoQuery)
+{}
+
+SearchInfo::SearchInfo(const RequestContext& request)
+  : pattern(request.get_optional_param<std::string>("pattern", "")),
+    geoQuery(),
+    bookName(request.get_optional_param<std::string>("content", ""))
+{
+  /* Retrive geo search */
+  try {
+    auto latitude = request.get_argument<float>("latitude");
+    auto longitude = request.get_argument<float>("longitude");
+    auto distance = request.get_argument<float>("distance");
+    geoQuery = GeoQuery(latitude, longitude, distance);
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  if (!geoQuery && pattern.empty()) {
+    throw std::invalid_argument("No query provided.");
+  }
+}
+
+zim::Query SearchInfo::getZimQuery(bool verbose) const {
+  zim::Query query;
+  if (verbose) {
+    std::cout << "Performing query '" << pattern<< "'";
+  }
+  query.setQuery(pattern);
+  if (geoQuery) {
+    if (verbose) {
+      std::cout << " with geo query '" << geoQuery.distance << "&(" << geoQuery.latitude << ";" << geoQuery.longitude << ")'";
+    }
+    query.setGeorange(geoQuery.latitude, geoQuery.longitude, geoQuery.distance);
+  }
+  if (verbose) {
+    std::cout << std::endl;
+  }
+  return query;
+}
+
+
 static IdNameMapper defaultNameMapper;
 
 static MHD_Result staticHandlerCallback(void* cls,
@@ -129,7 +179,8 @@ InternalServer::InternalServer(Library* library,
                                bool withTaskbar,
                                bool withLibraryButton,
                                bool blockExternalLinks,
-                               std::string indexTemplateString) :
+                               std::string indexTemplateString,
+                               int ipConnectionLimit) :
   m_addr(addr),
   m_port(port),
   m_root(normalizeRootUrl(root)),
@@ -139,9 +190,13 @@ InternalServer::InternalServer(Library* library,
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
   m_indexTemplateString(indexTemplateString.empty() ? RESOURCE::templates::index_html : indexTemplateString),
+  m_ipConnectionLimit(ipConnectionLimit),
   mp_daemon(nullptr),
   mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
+  searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
+  searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
+  suggestionSearcherCache(getCacheLength("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
 {}
 
 bool InternalServer::start() {
@@ -153,21 +208,21 @@ bool InternalServer::start() {
   if (m_verbose.load())
     flags |= MHD_USE_DEBUG;
 
-
   struct sockaddr_in sockAddr;
   memset(&sockAddr, 0, sizeof(sockAddr));
   sockAddr.sin_family = AF_INET;
   sockAddr.sin_port = htons(m_port);
   if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
+    if (0 != INADDR_ANY) {
       sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+    }
+    m_addr = kiwix::getBestPublicIp();
   } else {
     if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
       std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
       return false;
     }
   }
-
   mp_daemon = MHD_start_daemon(flags,
                             m_port,
                             NULL,
@@ -176,6 +231,7 @@ bool InternalServer::start() {
                             this,
                             MHD_OPTION_SOCK_ADDR, &sockAddr,
                             MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_PER_IP_CONNECTION_LIMIT, m_ipConnectionLimit,
                             MHD_OPTION_END);
   if (mp_daemon == nullptr) {
     std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
@@ -270,8 +326,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
 std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
 {
   try {
-    if (! request.is_valid_url())
-      return Response::build_404(*this, request, "", "");
+    if (! request.is_valid_url()) {
+      return HTTP404HtmlResponse(*this, request)
+             + urlNotFoundMsg;
+    }
 
     const ETag etag = get_matching_if_none_match_etag(request);
     if ( etag )
@@ -283,8 +341,8 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (startsWith(request.get_url(), "/catalog/"))
       return handle_catalog(request);
 
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
+    if (startsWith(request.get_url(), "/raw/"))
+      return handle_raw(request);
 
     if (request.get_url() == "/search")
       return handle_search(request);
@@ -301,10 +359,12 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     return handle_content(request);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return Response::build_500(*this, e.what());
+    return HTTP500HtmlResponse(*this, request)
+         + e.what();
   } catch (...) {
     fprintf(stderr, "===== Unhandled unknown error\n");
-    return Response::build_500(*this, "Unknown error");
+    return HTTP500HtmlResponse(*this, request)
+         + "Unknown error";
   }
 }
 
@@ -345,14 +405,15 @@ std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& r
  * Archive and Zim handlers begin
  **/
 
-// TODO: retrieve searcher from caching mechanism
-SuggestionsList_t getSuggestions(const zim::Archive* const archive,
-                  const std::string& queryString, int start, int suggestionCount)
+SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Archive* const archive,
+                  const std::string& bookId, const std::string& queryString, int start, int suggestionCount)
 {
   SuggestionsList_t suggestions;
-  auto searcher = zim::SuggestionSearcher(*archive);
+  std::shared_ptr<zim::SuggestionSearcher> searcher;
+  searcher = cache.getOrPut(bookId, [=](){ return make_shared<zim::SuggestionSearcher>(*archive); });
+
   if (archive->hasTitleIndex()) {
-    auto search = searcher.suggest(queryString);
+    auto search = searcher->suggest(queryString);
     auto srs = search.getResults(start, suggestionCount);
 
     for (auto it : srs) {
@@ -365,7 +426,7 @@ SuggestionsList_t getSuggestions(const zim::Archive* const archive,
     std::vector<std::string> variants = getTitleVariants(queryString);
     int currCount = 0;
     for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
-      auto search = searcher.suggest(queryString);
+      auto search = searcher->suggest(queryString);
       auto srs = search.getResults(0, suggestionCount);
       for (auto it : srs) {
         SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
@@ -378,99 +439,48 @@ SuggestionsList_t getSuggestions(const zim::Archive* const archive,
   return suggestions;
 }
 
-/**
- * Archive and Zim handlers end
- **/
-
-std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+namespace
 {
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<zim::Archive> archive;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    archive = mp_library->getArchiveById(bookId);
-  } catch (const std::out_of_range& e) {
-    return Response::build_404(*this, request, bookName, "");
-  }
 
-  if (archive == nullptr) {
-    return Response::build_404(*this, request, bookName, "");
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = getArchiveTitle(*archive);
-  } else if (meta_name == "description") {
-    content = getMetaDescription(*archive);
-  } else if (meta_name == "language") {
-    content = getMetaLanguage(*archive);
-  } else if (meta_name == "name") {
-    content = getMetaName(*archive);
-  } else if (meta_name == "tags") {
-    content = getMetaTags(*archive);
-  } else if (meta_name == "date") {
-    content = getMetaDate(*archive);
-  } else if (meta_name == "creator") {
-    content = getMetaCreator(*archive);
-  } else if (meta_name == "publisher") {
-    content = getMetaPublisher(*archive);
-  } else if (meta_name == "favicon") {
-    getArchiveFavicon(*archive, 48, content, mimeType);
-  } else if (const unsigned illustrationSize = parseIllustration(meta_name)) {
-    getArchiveFavicon(*archive, illustrationSize, content, mimeType);
-  } else {
-    return Response::build_404(*this, request, bookName, "");
-  }
-
-  auto response = ContentResponse::build(*this, content, mimeType);
-  response->set_cacheable();
-  return std::move(response);
+std::string noSuchBookErrorMsg(const std::string& bookName)
+{
+  return "No such book: " + bookName;
 }
 
+std::string noSearchResultsMsg()
+{
+  return "The fulltext search engine is not available for this content.";
+}
+
+} // unnamed namespace
+
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_suggest\n");
   }
 
-  std::string content;
-  std::string mimeType;
-
-  std::string bookName;
-  std::string bookId;
-  std::string queryString;
+  std::string bookName, bookId;
   std::shared_ptr<zim::Archive> archive;
   try {
     bookName = request.get_argument("content");
     bookId = mp_nameMapper->getIdForName(bookName);
-    queryString = request.get_argument("term");
     archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, bookName, "");
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-
-  unsigned int count = 10;
-  try {
-    count = request.get_argument<unsigned int>("count");
-  } catch (const std::exception&) {}
-
-  if (count == 0) {
-    count = 10;
+    // error handled by the archive == nullptr check below
   }
 
   if (archive == nullptr) {
-    return Response::build_404(*this, request, bookName, "");
+    return HTTP404HtmlResponse(*this, request)
+           + noSuchBookErrorMsg(bookName)
+           + TaskbarInfo(bookName);
+  }
+
+  const auto queryString = request.get_optional_param("term", std::string());
+  const auto start = request.get_optional_param<unsigned int>("start", 0);
+  unsigned int count = request.get_optional_param<unsigned int>("count", 10);
+  if (count == 0) {
+    count = 10;
   }
 
   if (m_verbose.load()) {
@@ -482,7 +492,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
   bool first = true;
 
   /* Get the suggestions */
-  SuggestionsList_t suggestions = getSuggestions(archive.get(), queryString, start, count);
+  SuggestionsList_t suggestions = getSuggestions(suggestionSearcherCache, archive.get(),
+                                                  bookId, queryString, start, count);
   for(auto& suggestion:suggestions) {
     MustacheData result;
     result.set("label", suggestion.getTitle());
@@ -532,7 +543,8 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
     response->set_cacheable();
     return std::move(response);
   } catch (const ResourceNotFound& e) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 }
 
@@ -542,115 +554,92 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     printf("** running handle_search\n");
   }
 
-  std::string bookName;
-  std::string bookId;
   try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
+    auto searchInfo = SearchInfo(request);
 
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
+    std::string bookId;
+    std::shared_ptr<zim::Archive> archive;
+    if (!searchInfo.bookName.empty()) {
+      try {
+        bookId = mp_nameMapper->getIdForName(searchInfo.bookName);
+        archive = mp_library->getArchiveById(bookId);
+      } catch (const std::out_of_range&) {
+        throw std::invalid_argument("The requested book doesn't exist.");
+      }
+    }
 
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
 
-  std::shared_ptr<zim::Archive> archive;
-  try {
-    archive = mp_library->getArchiveById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Make the search */
-  if ( (!archive && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
-    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
-    response->set_code(MHD_HTTP_NOT_FOUND);
-    return std::move(response);
-  }
-
-  std::shared_ptr<zim::Searcher> searcher;
-  if (archive) {
-    searcher = std::make_shared<zim::Searcher>(*archive);
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentArchive = mp_library->getArchiveById(bookId);
-      if (currentArchive) {
-        if (! searcher) {
-          searcher = std::make_shared<zim::Searcher>(*currentArchive);
-        } else {
-          searcher->addArchive(*currentArchive);
+    /* Make the search */
+    // Try to get a search from the searchInfo, else build it
+    std::shared_ptr<zim::Search> search;
+    try {
+      search = searchCache.getOrPut(searchInfo,
+        [=](){
+          std::shared_ptr<zim::Searcher> searcher;
+          if (archive) {
+            searcher = searcherCache.getOrPut(bookId, [=](){ return std::make_shared<zim::Searcher>(*archive);});
+          } else {
+            for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+              auto currentArchive = mp_library->getArchiveById(bookId);
+              if (currentArchive) {
+                if (! searcher) {
+                  searcher = std::make_shared<zim::Searcher>(*currentArchive);
+                } else {
+                  searcher->addArchive(*currentArchive);
+                }
+              }
+           }
+          }
+          return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
         }
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-
-  auto pageLength = 25;
-  try {
-    pageLength = request.get_argument<unsigned int>("pageLength");
-  } catch (const std::exception&) {}
-  if (pageLength > MAX_SEARCH_LEN) {
-    pageLength = MAX_SEARCH_LEN;
-  }
-  if (pageLength == 0) {
-    pageLength = 25;
-  }
-
-  /* Get the results */
-  try {
-    zim::Query query;
-    if (patternString.empty()) {
-      // Execute geo-search
-      if (m_verbose.load()) {
-        cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-      }
-
-      query.setQuery("");
-      query.setGeorange(latitude, longitude, distance);
-    } else {
-      // Execute Ft search
-      if (m_verbose.load()) {
-          cout << "Performing query `" << patternString << "'" << endl;
-      }
-
-      std::string queryString = removeAccents(patternString);
-      query.setQuery(queryString);
+      );
+    } catch(std::runtime_error& e) {
+      // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
+      // (in case of zim file not containing a index)
+      return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
+                                   "Fulltext search unavailable",
+                                   "Not Found",
+                                   m_root + "/skin/search_results.css")
+           + noSearchResultsMsg()
+           + TaskbarInfo(searchInfo.bookName, archive.get());
     }
 
-    zim::Search search = searcher->search(query);
-    SearchRenderer renderer(search.getResults(start, pageLength), mp_nameMapper, start,
-                            search.getEstimatedMatches());
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
+
+    auto start = 0;
+    try {
+      start = request.get_argument<unsigned int>("start");
+    } catch (const std::exception&) {}
+
+    auto pageLength = 25;
+    try {
+      pageLength = request.get_argument<unsigned int>("pageLength");
+    } catch (const std::exception&) {}
+    if (pageLength > MAX_SEARCH_LEN) {
+      pageLength = MAX_SEARCH_LEN;
+    }
+    if (pageLength == 0) {
+      pageLength = 25;
+    }
+
+    /* Get the results */
+    SearchRenderer renderer(search->getResults(start, pageLength), mp_nameMapper, mp_library, start,
+                            search->getEstimatedMatches());
+    renderer.setSearchPattern(searchInfo.pattern);
+    renderer.setSearchContent(searchInfo.bookName);
     renderer.setProtocolPrefix(m_root + "/");
     renderer.setSearchProtocolPrefix(m_root + "/search?");
     renderer.setPageLength(pageLength);
     auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
-
+    response->set_taskbar(searchInfo.bookName, archive.get());
     return std::move(response);
+  } catch (const std::invalid_argument& e) {
+    return HTTP400HtmlResponse(*this, request)
+      + invalidUrlMsg
+      + std::string(e.what());
   } catch (const std::exception& e) {
     std::cerr << e.what() << std::endl;
-    return Response::build_500(*this, e.what());
+    return HTTP500HtmlResponse(*this, request)
+         + e.what();
   }
 }
 
@@ -661,25 +650,29 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
   }
 
   std::string bookName;
-  std::string bookId;
   std::shared_ptr<zim::Archive> archive;
   try {
     bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
     archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, bookName, "");
+    // error handled by the archive == nullptr check below
   }
 
   if (archive == nullptr) {
-    return Response::build_404(*this, request, bookName, "");
+    return HTTP404HtmlResponse(*this, request)
+           + noSuchBookErrorMsg(bookName)
+           + TaskbarInfo(bookName);
   }
 
   try {
     auto entry = archive->getRandomEntry();
     return build_redirect(bookName, getFinalItem(*archive, entry));
   } catch(zim::EntryNotFound& e) {
-    return Response::build_404(*this, request, bookName, "");
+    const std::string error_details = "Oops! Failed to pick a random article :(";
+    return HTTP404HtmlResponse(*this, request)
+           + error_details
+           + TaskbarInfo(bookName, archive.get());
   }
 }
 
@@ -690,8 +683,10 @@ std::unique_ptr<Response> InternalServer::handle_captured_external(const Request
     source = kiwix::urlDecode(request.get_argument("source"));
   } catch (const std::out_of_range& e) {}
 
-  if (source.empty())
-    return Response::build_404(*this, request, "", "");
+  if (source.empty()) {
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
+  }
 
   auto data = get_default_data();
   data.set("source", source);
@@ -710,7 +705,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
     host = request.get_header("Host");
     url  = request.get_url_part(1);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "v2") {
@@ -718,7 +714,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   }
 
   if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "searchdescription.xml") {
@@ -797,7 +794,8 @@ InternalServer::search_catalog(const RequestContext& request,
     const auto totalResults = bookIdsToDump.size();
     const size_t count = request.get_optional_param("count", 10UL);
     const size_t startIndex = request.get_optional_param("start", 0UL);
-    bookIdsToDump = subrange(bookIdsToDump, startIndex, count);
+    const size_t intendedCount = count > 0 ? count : bookIdsToDump.size();
+    bookIdsToDump = subrange(bookIdsToDump, startIndex, intendedCount);
     opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
     return bookIdsToDump;
 }
@@ -851,10 +849,11 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   } catch (const std::out_of_range& e) {}
 
   if (archive == nullptr) {
-    std::string searchURL = m_root+"/search?pattern="+pattern; // Make a full search on the entire library.
-    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
-
-    return Response::build_404(*this, request, bookName, "", details);
+    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg
+           + searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+           + TaskbarInfo(bookName);
   }
 
   auto urlStr = request.get_url().substr(bookName.size()+1);
@@ -871,7 +870,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     }
     auto response = ItemResponse::build(*this, request, entry.getItem());
     try {
-      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, getArchiveTitle(*archive));
+      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, archive.get());
     } catch (std::bad_cast& e) {}
 
     if (m_verbose.load()) {
@@ -884,10 +883,75 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     if (m_verbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
 
-    std::string searchURL = m_root+"/search?content="+bookName+"&pattern="+pattern; // Make a search on this specific book only.
-    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
+    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg
+           + searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+           + TaskbarInfo(bookName, archive.get());
+  }
+}
 
-    return Response::build_404(*this, request, bookName, getArchiveTitle(*archive), details);
+
+std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_raw\n");
+  }
+
+  std::string bookName;
+  std::string kind;
+  try {
+     bookName = request.get_url_part(1);
+     kind = request.get_url_part(2);
+  } catch (const std::out_of_range& e) {
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  if (kind != "meta" && kind!= "content") {
+    const std::string error_details = kind + " is not a valid request for raw content.";
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg
+           + error_details;
+  }
+
+  std::shared_ptr<zim::Archive> archive;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    archive = mp_library->getArchiveById(bookId);
+  } catch (const std::out_of_range& e) {}
+
+  if (archive == nullptr) {
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg
+           + noSuchBookErrorMsg(bookName);
+  }
+
+  // Remove the beggining of the path:
+  // /raw/<bookName>/<kind>/foo
+  // ^^^^^          ^      ^
+  //   5      +     1  +   1   = 7
+  auto itemPath = request.get_url().substr(bookName.size()+kind.size()+7);
+
+  try {
+    if (kind == "meta") {
+      auto item = archive->getMetadataItem(itemPath);
+      return ItemResponse::build(*this, request, item, /*raw=*/true);
+    } else {
+      auto entry = archive->getEntryByPath(itemPath);
+      if (entry.isRedirect()) {
+        return build_redirect(bookName, entry.getItem(true));
+      }
+      return ItemResponse::build(*this, request, entry.getItem(), /*raw=*/true);
+    }
+  } catch (zim::EntryNotFound& e ) {
+    if (m_verbose.load()) {
+      printf("Failed to find %s\n", itemPath.c_str());
+    }
+    const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg
+           + error_details;
   }
 }
 

src/server/internalServer.h

@@ -28,6 +28,9 @@ extern "C" {
 #include "library.h"
 #include "name_mapper.h"
 
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
 #include <mustache.hpp>
 
 #include <atomic>
@@ -36,9 +39,58 @@ extern "C" {
 #include "server/request_context.h"
 #include "server/response.h"
 
+#include "tools/concurrent_cache.h"
+
 namespace kiwix {
 
+struct GeoQuery {
+  GeoQuery()
+    : GeoQuery(0, 0, -1)
+  {}
+
+  GeoQuery(float latitude, float longitude, float distance)
+    : latitude(latitude), longitude(longitude), distance(distance)
+  {}
+  float latitude;
+  float longitude;
+  float distance;
+
+  explicit operator bool() const {
+    return distance >= 0;
+  }
+
+  friend bool operator<(const GeoQuery& l, const GeoQuery& r)
+  {
+      return std::tie(l.latitude, l.longitude, l.distance)
+           < std::tie(r.latitude, r.longitude, r.distance); // keep the same order
+  }
+};
+
+class SearchInfo {
+  public:
+    SearchInfo(const std::string& pattern);
+    SearchInfo(const std::string& pattern, GeoQuery geoQuery);
+    SearchInfo(const RequestContext& request);
+
+    zim::Query getZimQuery(bool verbose) const;
+
+    friend bool operator<(const SearchInfo& l, const SearchInfo& r)
+    {
+        return std::tie(l.bookName, l.pattern, l.geoQuery)
+             < std::tie(r.bookName, r.pattern, r.geoQuery); // keep the same order
+    }
+
+  public: //data
+    std::string pattern;
+    GeoQuery geoQuery;
+    std::string bookName;
+};
+
+
 typedef kainjow::mustache::data MustacheData;
+typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
+typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
 
 class Entry;
 class OPDSDumper;
@@ -55,7 +107,8 @@ class InternalServer {
                    bool withTaskbar,
                    bool withLibraryButton,
                    bool blockExternalLinks,
-                   std::string indexTemplateString);
+                   std::string indexTemplateString,
+                   int ipConnectionLimit);
     virtual ~InternalServer() = default;
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
@@ -67,6 +120,8 @@ class InternalServer {
                                void** cont_cls);
     bool start();
     void stop();
+    std::string getAddress() { return m_addr; }
+    int getPort() { return m_port; }
 
   private: // functions
     std::unique_ptr<Response> handle_request(const RequestContext& request);
@@ -80,12 +135,13 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
-    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
     std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
     std::unique_ptr<Response> handle_content(const RequestContext& request);
+    std::unique_ptr<Response> handle_raw(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
@@ -105,19 +161,22 @@ class InternalServer {
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
     std::string m_indexTemplateString;
+    int m_ipConnectionLimit;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
     NameMapper* mp_nameMapper;
 
+    SearcherCache searcherCache;
+    SearchCache searchCache;
+    SuggestionSearcherCache suggestionSearcherCache;
+
     std::string m_server_id;
     std::string m_library_id;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
-    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
-
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
 };
 
 }

src/server/internalServer_catalog_v2.cpp

@@ -43,7 +43,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "root.xml") {
@@ -66,8 +67,11 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
     return handle_catalog_v2_categories(request);
   } else if (url == "languages") {
     return handle_catalog_v2_languages(request);
+  } else if (url == "illustration") {
+    return handle_catalog_v2_illustration(request);
   } else {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 }
 
@@ -108,7 +112,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
   try {
     mp_library->getBookById(entryId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
   }
 
   OPDSDumper opdsDumper(mp_library);
@@ -146,4 +151,19 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const Requ
   );
 }
 
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
+{
+  try {
+    const auto bookName  = request.get_url_part(3);
+    const auto bookId = mp_nameMapper->getIdForName(bookName);
+    auto book = mp_library->getBookByIdThreadSafe(bookId);
+    auto size = request.get_argument<unsigned int>("size");
+    auto illustration = book.getIllustration(size);
+    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+  } catch(...) {
+    return HTTP404HtmlResponse(*this, request)
+           + urlNotFoundMsg;
+  }
+}
+
 } // namespace kiwix

src/server/response.cpp

@@ -25,6 +25,7 @@
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
+#include "tools/archiveTools.h"
 
 #include "string.h"
 #include <mustache.hpp>
@@ -83,17 +84,112 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   return response;
 }
 
-std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
+const UrlNotFoundMsg urlNotFoundMsg;
+const InvalidUrlMsg invalidUrlMsg;
+
+std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
 {
-  MustacheData results;
-  results.set("url", request.get_full_url());
-  results.set("details", details);
+  auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
+  r->set_code(m_httpStatusCode);
+  if ( m_taskbarInfo ) {
+    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
+  }
+  return r;
+}
 
-  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
-  response->set_code(MHD_HTTP_NOT_FOUND);
-  response->set_taskbar(bookName, bookTitle);
+HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
+                                             const RequestContext& request,
+                                             int httpStatusCode,
+                                             const std::string& pageTitleMsg,
+                                             const std::string& headingMsg,
+                                             const std::string& cssUrl)
+  : ContentResponseBlueprint(&server,
+                             &request,
+                             httpStatusCode,
+                             "text/html; charset=utf-8",
+                             RESOURCE::templates::error_html)
+{
+  kainjow::mustache::list emptyList;
+  this->m_data = kainjow::mustache::object{
+                    {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
+                    {"PAGE_TITLE",   pageTitleMsg},
+                    {"PAGE_HEADING", headingMsg},
+                    {"details", emptyList}
+  };
+}
 
-  return std::move(response);
+HTTP404HtmlResponse::HTTP404HtmlResponse(const InternalServer& server,
+                                             const RequestContext& request)
+  : HTTPErrorHtmlResponse(server,
+                          request,
+                          MHD_HTTP_NOT_FOUND,
+                          "Content not found",
+                          "Not Found")
+{
+}
+
+HTTPErrorHtmlResponse& HTTP404HtmlResponse::operator+(UrlNotFoundMsg /*unused*/)
+{
+  const std::string requestUrl = m_request.get_full_url();
+  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{url}}" was not found on this server.)");
+  return *this + msgTmpl.render({"url", requestUrl});
+}
+
+HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
+{
+  m_data["details"].push_back({"p", msg});
+  return *this;
+}
+
+HTTP400HtmlResponse::HTTP400HtmlResponse(const InternalServer& server,
+                                         const RequestContext& request)
+  : HTTPErrorHtmlResponse(server,
+                          request,
+                          MHD_HTTP_BAD_REQUEST,
+                          "Invalid request",
+                          "Invalid request")
+{
+}
+
+HTTPErrorHtmlResponse& HTTP400HtmlResponse::operator+(InvalidUrlMsg /*unused*/)
+{
+  std::string requestUrl = m_request.get_full_url();
+  const auto query = m_request.get_query();
+  if (!query.empty()) {
+    requestUrl += "?" + encodeDiples(query);
+  }
+  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
+  return *this + msgTmpl.render({"url", requestUrl});
+}
+
+HTTP500HtmlResponse::HTTP500HtmlResponse(const InternalServer& server,
+                                         const RequestContext& request)
+  : HTTPErrorHtmlResponse(server,
+                          request,
+                          MHD_HTTP_INTERNAL_SERVER_ERROR,
+                          "Internal Server Error",
+                          "Internal Server Error")
+{
+  // operator+() is a state-modifying operator (akin to operator+=)
+  *this + "An internal server error occured. We are sorry about that :/";
+}
+
+std::unique_ptr<ContentResponse> HTTP500HtmlResponse::generateResponseObject() const
+{
+  // We want a 500 response to be a minimalistic one (so that the server doesn't
+  // have to provide additional resources required for its proper rendering)
+  // ";raw=true" in the MIME-type below disables response decoration
+  // (see ContentResponse::contentDecorationAllowed())
+  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
+  r->set_code(m_httpStatusCode);
+  return r;
+}
+
+ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
+{
+  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
+  return *this;
 }
 
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
@@ -109,17 +205,6 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
   return response;
 }
 
-std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  auto content = render_template(RESOURCE::templates::_500_html, data);
-  std::unique_ptr<Response> response (
-      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
-  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
 
 std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
 {
@@ -236,8 +321,11 @@ ContentResponse::can_compress(const RequestContext& request) const
 bool
 ContentResponse::contentDecorationAllowed() const
 {
-    return (startsWith(m_mimeType, "text/html")
-         && m_mimeType.find(";raw=true") == std::string::npos);
+  if (m_raw) {
+    return false;
+  }
+  return (startsWith(m_mimeType, "text/html")
+    && m_mimeType.find(";raw=true") == std::string::npos);
 }
 
 MHD_Response*
@@ -318,18 +406,19 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
 {
   m_bookName = bookName;
-  m_bookTitle = bookTitle;
+  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
 }
 
 
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
   m_mimeType(mimetype),
+  m_raw(raw),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
@@ -339,11 +428,17 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool wit
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& content,
+  const std::string& mimetype,
+  bool isHomePage,
+  bool raw)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
+        raw,
         server.m_withTaskbar && !isHomePage,
         server.m_withLibraryButton,
         server.m_blockExternalLinks,
@@ -351,7 +446,13 @@ std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& se
         mimetype));
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& template_str,
+  kainjow::mustache::data data,
+  const std::string& mimetype,
+  bool isHomePage)
+{
   auto content = render_template(template_str, data);
   return ContentResponse::build(server, content, mimetype, isHomePage);
 }
@@ -366,14 +467,14 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
     response->set_cacheable();
     response->m_byteRange = byteRange;
     return std::move(response);

src/server/response.h

@@ -33,13 +33,15 @@ extern "C" {
 #include "microhttpd_wrapper.h"
 }
 
+namespace zim {
+class Archive;
+} // namespace zim
+
 namespace kiwix {
 
 class InternalServer;
 class RequestContext;
 
-class EntryResponse;
-
 class Response {
   public:
     Response(bool verbose);
@@ -47,9 +49,7 @@ class Response {
 
     static std::unique_ptr<Response> build(const InternalServer& server);
     static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
-    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
     static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
-    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
     static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
@@ -78,11 +78,29 @@ class Response {
 
 class ContentResponse : public Response {
   public:
-    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
+    ContentResponse(
+      const std::string& root,
+      bool verbose,
+      bool raw,
+      bool withTaskbar,
+      bool withLibraryButton,
+      bool blockExternalLinks,
+      const std::string& content,
+      const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& content,
+      const std::string& mimetype,
+      bool isHomePage = false,
+      bool raw = false);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& template_str,
+      kainjow::mustache::data data,
+      const std::string& mimetype,
+      bool isHomePage = false);
 
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
@@ -98,6 +116,7 @@ class ContentResponse : public Response {
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
+    bool m_raw;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
@@ -105,10 +124,114 @@ class ContentResponse : public Response {
     std::string m_bookTitle;
  };
 
+struct TaskbarInfo
+{
+  const std::string bookName;
+  const zim::Archive* const archive;
+
+  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
+    : bookName(bookName)
+    , archive(a)
+  {}
+};
+
+class ContentResponseBlueprint
+{
+public: // functions
+  ContentResponseBlueprint(const InternalServer* server,
+                           const RequestContext* request,
+                           int httpStatusCode,
+                           const std::string& mimeType,
+                           const std::string& templateStr)
+    : m_server(*server)
+    , m_request(*request)
+    , m_httpStatusCode(httpStatusCode)
+    , m_mimeType(mimeType)
+    , m_template(templateStr)
+  {}
+
+  virtual ~ContentResponseBlueprint() = default;
+
+  operator std::unique_ptr<ContentResponse>() const
+  {
+    return generateResponseObject();
+  }
+
+  operator std::unique_ptr<Response>() const
+  {
+    return operator std::unique_ptr<ContentResponse>();
+  }
+
+
+  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
+
+protected: // functions
+  virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
+
+public: //data
+  const InternalServer& m_server;
+  const RequestContext& m_request;
+  const int m_httpStatusCode;
+  const std::string m_mimeType;
+  const std::string m_template;
+  kainjow::mustache::data m_data;
+  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
+};
+
+struct HTTPErrorHtmlResponse : ContentResponseBlueprint
+{
+  HTTPErrorHtmlResponse(const InternalServer& server,
+                      const RequestContext& request,
+                      int httpStatusCode,
+                      const std::string& pageTitleMsg,
+                      const std::string& headingMsg,
+                      const std::string& cssUrl = "");
+
+  using ContentResponseBlueprint::operator+;
+  HTTPErrorHtmlResponse& operator+(const std::string& msg);
+};
+
+class UrlNotFoundMsg {};
+
+extern const UrlNotFoundMsg urlNotFoundMsg;
+
+struct HTTP404HtmlResponse : HTTPErrorHtmlResponse
+{
+  HTTP404HtmlResponse(const InternalServer& server,
+                      const RequestContext& request);
+
+  using HTTPErrorHtmlResponse::operator+;
+  HTTPErrorHtmlResponse& operator+(UrlNotFoundMsg /*unused*/);
+};
+
+class InvalidUrlMsg {};
+
+extern const InvalidUrlMsg invalidUrlMsg;
+
+struct HTTP400HtmlResponse : HTTPErrorHtmlResponse
+{
+  HTTP400HtmlResponse(const InternalServer& server,
+                      const RequestContext& request);
+
+  using HTTPErrorHtmlResponse::operator+;
+  HTTPErrorHtmlResponse& operator+(InvalidUrlMsg /*unused*/);
+};
+
+struct HTTP500HtmlResponse : HTTPErrorHtmlResponse
+{
+  HTTP500HtmlResponse(const InternalServer& server,
+                      const RequestContext& request);
+
+private: // overrides
+  // generateResponseObject() is overriden in order to produce a minimal
+  // response without any need for additional resources from the server
+  std::unique_ptr<ContentResponse> generateResponseObject() const override;
+};
+
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -125,6 +125,30 @@ unsigned int getArchiveMediaCount(const zim::Archive& archive) {
   return counter;
 }
 
+unsigned int getArchiveArticleCount(const zim::Archive& archive) {
+  // [HACK]
+  // getArticleCount() returns different things depending of the "version" of the zim.
+  // On old zim (<=6), it returns the number of entry in `A` namespace
+  // On recent zim (>=7), it returns:
+  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
+  //  - the number of front article if a frontArticleIndex is present
+  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
+  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
+  if (archive.hasNewNamespaceScheme()) {
+    //The articleCount is "good"
+    return archive.getArticleCount();
+  } else {
+    // We have to parse the `M/Counter` metadata
+    unsigned int counter = 0;
+    for(const auto& pair:parseArchiveCounter(archive)) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
+    }
+    return counter;
+  }
+}
+
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }

src/tools/archiveTools.h

@@ -46,6 +46,7 @@ namespace kiwix
                            std::string& content, std::string& mimeType);
 
     unsigned int getArchiveMediaCount(const zim::Archive& archive);
+    unsigned int getArchiveArticleCount(const zim::Archive& archive);
     unsigned int getArchiveFileSize(const zim::Archive& archive);
 
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);

src/tools/concurrent_cache.h

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (C) 2020 Veloman Yunkan
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
+ * NON-INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifndef ZIM_CONCURRENT_CACHE_H
+#define ZIM_CONCURRENT_CACHE_H
+
+#include "lrucache.h"
+
+#include <future>
+#include <mutex>
+
+namespace kiwix
+{
+
+/**
+   ConcurrentCache implements a concurrent thread-safe cache
+
+   Compared to kiwix::lru_cache, each access operation is slightly more expensive.
+   However, different slots of the cache can be safely accessed concurrently
+   with minimal blocking. Concurrent access to the same element is also
+   safe, and, in case of a cache miss, will block until that element becomes
+   available.
+ */
+template <typename Key, typename Value>
+class ConcurrentCache
+{
+private: // types
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      try {
+        valuePromise.set_value(f());
+      } catch (std::exception& e) {
+        drop(key);
+        throw;
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+private: // data
+  Impl impl_;
+  std::mutex lock_;
+};
+
+} // namespace kiwix
+
+#endif // ZIM_CONCURRENT_CACHE_H
+

src/tools/lrucache.h

@@ -0,0 +1,160 @@
+/*
+ * Copyrigth (c) 2021, Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (c) 2020, Veloman Yunkan
+ * Copyright (c) 2014, lamerman
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of lamerman nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * File:   lrucache.hpp
+ * Author: Alexander Ponomarev
+ *
+ * Created on June 20, 2013, 5:09 PM
+ */
+
+#ifndef _LRUCACHE_HPP_INCLUDED_
+#define _LRUCACHE_HPP_INCLUDED_
+
+#include <map>
+#include <list>
+#include <cstddef>
+#include <stdexcept>
+#include <cassert>
+
+namespace kiwix {
+
+template<typename key_t, typename value_t>
+class lru_cache {
+public: // types
+  typedef typename std::pair<key_t, value_t> key_value_pair_t;
+  typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+
+  enum AccessStatus {
+    HIT, // key was found in the cache
+    PUT, // key was not in the cache but was created by the getOrPut() access
+    MISS // key was not in the cache; get() access failed
+  };
+
+  class AccessResult
+  {
+    const AccessStatus status_;
+    const value_t val_;
+  public:
+    AccessResult(const value_t& val, AccessStatus status)
+      : status_(status), val_(val)
+    {}
+    AccessResult() : status_(MISS), val_() {}
+
+    bool hit() const { return status_ == HIT; }
+    bool miss() const { return !hit(); }
+    const value_t& value() const
+    {
+      if ( status_ == MISS )
+        throw std::range_error("There is no such key in cache");
+      return val_;
+    }
+
+    operator const value_t& () const { return value(); }
+  };
+
+public: // functions
+  explicit lru_cache(size_t max_size) :
+    _max_size(max_size) {
+  }
+
+  // If 'key' is present in the cache, returns the associated value,
+  // otherwise puts the given value into the cache (and returns it with
+  // a status of a cache miss).
+  AccessResult getOrPut(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    } else {
+      putMissing(key, value);
+      return AccessResult(value, PUT);
+    }
+  }
+
+  void put(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      it->second->second = value;
+    } else {
+      putMissing(key, value);
+    }
+  }
+
+  AccessResult get(const key_t& key) {
+    auto it = _cache_items_map.find(key);
+    if (it == _cache_items_map.end()) {
+      return AccessResult();
+    } else {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    }
+  }
+
+  bool drop(const key_t& key) {
+    try {
+      auto list_it = _cache_items_map.at(key);
+      _cache_items_list.erase(list_it);
+      _cache_items_map.erase(key);
+      return true;
+    } catch (std::out_of_range& e) {
+      return false;
+    }
+  }
+
+  bool exists(const key_t& key) const {
+    return _cache_items_map.find(key) != _cache_items_map.end();
+  }
+
+  size_t size() const {
+    return _cache_items_map.size();
+  }
+
+private: // functions
+  void putMissing(const key_t& key, const value_t& value) {
+    assert(_cache_items_map.find(key) == _cache_items_map.end());
+    _cache_items_list.push_front(key_value_pair_t(key, value));
+    _cache_items_map[key] = _cache_items_list.begin();
+    if (_cache_items_map.size() > _max_size) {
+      _cache_items_map.erase(_cache_items_list.back().first);
+      _cache_items_list.pop_back();
+    }
+  }
+
+private: // data
+  std::list<key_value_pair_t> _cache_items_list;
+  std::map<key_t, list_iterator_t> _cache_items_map;
+  size_t _max_size;
+};
+
+} // namespace kiwix
+
+#endif  /* _LRUCACHE_HPP_INCLUDED_ */

src/tools/networkTools.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,6 +18,7 @@
  * MA 02110-1301, USA.
  */
 
+#include "tools.h"
 #include <tools/networkTools.h>
 
 #include <stdio.h>
@@ -29,6 +31,17 @@
 #include <iostream>
 #include <stdexcept>
 
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <iostream>
+#else
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <net/if.h>
+#include <netdb.h>
+#endif
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
@@ -57,3 +70,104 @@ std::string kiwix::download(const std::string& url) {
   }
   return ss.str();
 }
+
+std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
+  std::map<std::string, std::string> interfaces;
+
+#ifdef _WIN32
+  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
+  if (sd == INVALID_SOCKET) {
+    std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  INTERFACE_INFO InterfaceList[20];
+  unsigned long nBytesReturned;
+  if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
+      sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
+    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
+  for (int i = 0; i < nNumInterfaces; ++i) {
+    sockaddr_in *pAddress;
+    pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
+    if(pAddress->sin_family == AF_INET) {
+    /* Add to the map */
+    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
+    interfaces[interfaceName] = interfaceName;
+    }
+  }
+#else
+  /* Get Network interfaces information */
+  char buf[16384];
+  struct ifconf ifconf;
+  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
+  ifconf.ifc_len = sizeof(buf);
+  ifconf.ifc_buf=buf;
+  if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
+    perror("ioctl(SIOCGIFCONF)");
+  }
+
+  /* Go through each interface */
+  struct ifreq *ifreq;
+  ifreq = ifconf.ifc_req;
+  for (int i = 0; i < ifconf.ifc_len; ) {
+    if (ifreq->ifr_addr.sa_family == AF_INET) {
+      /* Get the network interface ip */
+      char host[128] = { 0 };
+      const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
+                                    host, sizeof(host),
+                                    0, 0, NI_NUMERICHOST);
+      if (!error) {
+        std::string interfaceName = std::string(ifreq->ifr_name);
+        std::string interfaceIp = std::string(host);
+        /* Add to the map */
+        interfaces[interfaceName] = interfaceIp;
+      } else {
+        perror("getnameinfo()");
+      }
+    }
+
+    /* some systems have ifr_addr.sa_len and adjust the length that
+     * way, but not mine. weird */
+    size_t len;
+#ifndef __linux__
+    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
+#else
+    len = sizeof(*ifreq);
+#endif
+    ifreq = (struct ifreq*)((char*)ifreq+len);
+    i += len;
+  }
+#endif
+  return interfaces;
+}
+
+std::string kiwix::getBestPublicIp() {
+  auto interfaces = getNetworkInterfaces();
+
+#ifndef _WIN32
+  const char* const prioritizedNames[] =
+      { "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
+  for(auto name: prioritizedNames) {
+    auto it = interfaces.find(name);
+    if(it != interfaces.end()) {
+      return it->second;
+    }
+  }
+#endif
+
+  const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
+  for(auto prefix : prefixes){
+    for(auto& itr : interfaces) {
+      auto interfaceIp = itr.second;
+      if (interfaceIp.find(prefix) == 0) {
+        return interfaceIp;
+      }
+    }
+  }
+
+  return "127.0.0.1";
+}

src/tools/otherTools.cpp

@@ -370,6 +370,13 @@ std::string kiwix::gen_uuid(const std::string& s)
   return kiwix::to_string(zim::Uuid::generate(s));
 }
 
+kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
+{
+  return s.empty()
+       ? kainjow::mustache::data(false)
+       : kainjow::mustache::data(s);
+}
+
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);

src/tools/otherTools.h

@@ -48,6 +48,10 @@ namespace kiwix
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
+  // if s is empty then returns kainjow::mustache::data(false)
+  // otherwise kainjow::mustache::data(value)
+  kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
+
   std::string render_template(const std::string& template_str, kainjow::mustache::data data);
 }
 

src/tools/pathTools.cpp

@@ -305,16 +305,18 @@ std::string kiwix::getFileContent(const std::string& path)
 bool kiwix::fileExists(const std::string& path)
 {
 #ifdef _WIN32
-  return PathFileExistsW(Utf8ToWide(path).c_str());
+  return (_waccess_s(Utf8ToWide(path).c_str(), 0) == 0);
 #else
-  bool flag = false;
-  std::fstream fin;
-  fin.open(path.c_str(), std::ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
+  return (access(path.c_str(), F_OK) == 0);
+#endif
+}
+
+bool kiwix::fileReadable(const std::string& path)
+{
+#ifdef _WIN32
+  return (_waccess_s(Utf8ToWide(path).c_str(), 4) == 0);
+#else
+  return (access(path.c_str(), R_OK) == 0);
 #endif
 }
 

src/version.cpp

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <iostream>
+#include <sstream>
+
+#include <version.h>
+#include <zim/zim.h>
+#include <kiwix_config.h>
+#include <unicode/uversion.h>
+#include <pugixml.hpp>
+#include <curl/curl.h>
+#include <microhttpd.h>
+#include <xapian.h>
+#include <mustache.hpp>
+#include <zlib.h>
+
+namespace kiwix
+{
+  LibVersions getVersions() {
+    LibVersions versions = {
+      { "libkiwix",      LIBKIWIX_VERSION    },
+      { "libzim",        LIBZIM_VERSION      },
+      { "libxapian",     XAPIAN_VERSION      },
+      { "libcurl",       LIBCURL_VERSION     },
+      { "libmicrohttpd", MHD_get_version()   },
+      { "libz",          ZLIB_VERSION        }
+    };
+
+    // U_ICU_VERSION does not include the patch level if 0
+    std::ostringstream libicu_version;
+    libicu_version << U_ICU_VERSION_MAJOR_NUM << "." << U_ICU_VERSION_MINOR_NUM << "." << U_ICU_VERSION_PATCHLEVEL_NUM;
+    versions.push_back({ "libicu", libicu_version.str() });
+
+    // No human readable version string for pugixml
+    const unsigned pugixml_major = (PUGIXML_VERSION - PUGIXML_VERSION % 1000) / 1000;
+    const unsigned pugixml_minor = (PUGIXML_VERSION - pugixml_major * 1000 - PUGIXML_VERSION % 10) / 10;
+    const unsigned pugixml_patch = PUGIXML_VERSION - pugixml_major * 1000 - pugixml_minor * 10;
+    std::ostringstream libpugixml_version;
+    libpugixml_version << pugixml_major << "." << pugixml_minor << "." << pugixml_patch;
+    versions.push_back({ "libpugixml", libpugixml_version.str() });
+
+    // Needs version 5.0 of Mustache
+#if defined(KAINJOW_MUSTACHE_VERSION_MAJOR)
+    std::ostringstream libmustache_version;
+    libmustache_version << KAINJOW_MUSTACHE_VERSION_MAJOR << "." <<
+      KAINJOW_MUSTACHE_VERSION_MINOR << "." << KAINJOW_MUSTACHE_VERSION_PATCH;
+    versions.push_back({ "libmustache", libmustache_version.str() });
+#endif
+
+    return versions;
+  }
+
+  void printVersions(std::ostream& out) {
+    LibVersions versions = getVersions();
+    for (const auto& iter : versions) {
+      out << (iter != versions.front() ? "+ " : "")
+                << iter.first << " " << iter.second << std::endl;
+    }
+  }
+} //namespace kiwix

static/resources_list.txt

@@ -28,14 +28,13 @@ skin/iso6391To3.js
 skin/isotope.pkgd.min.js
 skin/index.js
 skin/taskbar.css
-skin/home.css
+skin/index.css
 skin/fonts/Poppins.ttf
 skin/fonts/Roboto.ttf
 skin/block_external.js
+skin/search_results.css
 templates/search_result.html
-templates/no_search_result.html
-templates/404.html
-templates/500.html
+templates/error.html
 templates/index.html
 templates/suggestion.json
 templates/head_taskbar.html

static/skin/index.js

@@ -16,6 +16,7 @@
     let filters = getCookie(filterCookieName);
     let params = new URLSearchParams(window.location.search || filters || '');
     let timer;
+    let languages = {};
 
     function queryUrlBuilder() {
         let url = `${root}/catalog/search?`;
@@ -51,10 +52,15 @@
         fileSize /= (1000 ** quotient);
         return `${+fileSize.toFixed(2)} ${units[quotient]}`;
     };
-    
+
     const humanFriendlyTitle = (title) => {
-        title = title.replace(/_/g, ' ');
-        return htmlEncode(title[0].toUpperCase() + title.slice(1));
+        if (typeof title === 'string' && title.length > 0) {
+            title = title.replace(/_/g, ' ');
+            if (title.length > 0) {
+                return htmlEncode(title[0].toUpperCase() + title.slice(1));
+            }
+        }
+        return '';
     }
 
     function htmlEncode(str) {
@@ -67,7 +73,8 @@
     }
 
     function getInnerHtml(node, query) {
-        return node.querySelector(query).innerHTML;
+        const queryNode = node.querySelector(query);
+        return queryNode != null ? queryNode.innerHTML : "";
     }
 
     function generateBookHtml(book, sort = false) {
@@ -81,7 +88,8 @@
         const title =  getInnerHtml(book, 'title');
         const description = getInnerHtml(book, 'summary');
         const id = getInnerHtml(book, 'id');
-        const language = getInnerHtml(book, 'language');
+        const langCode = getInnerHtml(book, 'language');
+        const language = languages[langCode];
         const tags = getInnerHtml(book, 'tags');
         let tagHtml = tags.split(';').filter(tag => {return !(tag.split(':')[0].startsWith('_'))})
             .map((tag) => {return tag.charAt(0).toUpperCase() + tag.slice(1)})
@@ -103,18 +111,28 @@
         if (sort) {
             divTag.setAttribute('data-idx', bookOrderMap.get(id));
         }
-        divTag.innerHTML = `<div class="book__wrapper"><div class='book__icon' ><img class="book__icon--image" src='${root}${iconUrl}'></div>
-            <div class='book__title' title='${title}'>
-                <div id="bookTitle">${title}</div>
-                ${humanFriendlyZimSize ? `<div id='bookSize'>${humanFriendlyZimSize}</div>`: ''}
+        const faviconAttr = iconUrl != undefined ? `style="background-image: url('${iconUrl}')"` : '';
+        const languageAttr = langCode != '' ? `title="${language}" aria-label="${language}"` : 'style="background-color: transparent"';
+        divTag.innerHTML = `<a class="book__link" href="${link}" data-hover="Preview">
+            <div class="book__wrapper">
+            <div class="book__icon" ${faviconAttr}></div>
+            <div class="book__header">
+                <div id="book__title">${title}</div>
+                ${downloadLink ? `<div class="book__download"><span data-link="${downloadLink}">Download ${humanFriendlyZimSize ? ` - ${humanFriendlyZimSize}</span></div>`: ''}` : ''}
             </div>
-            <div class='book__description' title='${description}'>${description}</div>
-            <div class='book__languageTag'>${language.substr(0, 2).toUpperCase()}</div>
-            <div class='book__tags'><div class="book__tags--wrapper">${tagHtml}</div></div>
-            <div class='book__links'> <a href="${root}${link}" data-hover="Preview">Preview</a>${downloadLink ? `&nbsp;|&nbsp;<span class="download" data-link=${downloadLink} class="modal-button">Download</span>` : ''} </div></div>`;
+            <div class="book__description" title="${description}">${description}</div>
+            <div class="book__languageTag" ${languageAttr}>${getLanguageCodeToDisplay(langCode)}</div>
+            <div class="book__tags"><div class="book__tags--wrapper">${tagHtml}</div></div>
+            </div></div></a>`;
         return divTag;
     }
 
+    function getLanguageCodeToDisplay(langCode3Letter) {
+        const langCode2Letter = (Object.keys(iso6391To3).find(key => iso6391To3[key] === langCode3Letter));
+        const res = (langCode2Letter != undefined) ? langCode2Letter : langCode3Letter;
+        return res.toUpperCase();
+    }
+
     function toggleFooter(show=false) {
         if (show) {
             footer.style.display = 'block';
@@ -126,7 +144,8 @@
 
     function insertModal(button) {
         const downloadLink = button.getAttribute('data-link');
-        button.addEventListener('click', () => {
+        button.addEventListener('click', (event) => {
+            event.preventDefault();
             document.body.insertAdjacentHTML('beforeend', `<div class="modal-wrapper">
                 <div class="modal">
                     <div class="modal-heading">
@@ -226,7 +245,11 @@
             data.querySelectorAll('entry').forEach(entry => {
                 const title = getInnerHtml(entry, 'title');
                 const value = getInnerHtml(entry, valueEntryNode);
-                optionStr += `<option value='${value}'>${humanFriendlyTitle(title)}</option>`;
+                const hfTitle = humanFriendlyTitle(title);
+                if (valueEntryNode == 'language') {
+                    languages[value] = hfTitle;
+                }
+                optionStr += (hfTitle != '') ? `<option value="${value}">${hfTitle}</option>` : '';
             });
             document.querySelector(nodeQuery).innerHTML += optionStr;
         });
@@ -304,7 +327,7 @@
         booksToDelete.forEach(book => {iso.remove(book);});
         books.forEach((book) => {
             iso.insert(generateBookHtml(book, sort))
-            const downloadButton = document.querySelector(`[data-id="${getInnerHtml(book, 'id')}"] .download`);
+            const downloadButton = document.querySelector(`[data-id="${getInnerHtml(book, 'id')}"] .book__download span`);
             if (downloadButton) {
                 insertModal(downloadButton);
             }
@@ -339,7 +362,7 @@
     });
 
     async function loadSubset() {
-        if (window.innerHeight + window.scrollY >= document.body.offsetHeight) {
+        if (window.innerHeight + window.scrollY >= (document.body.offsetHeight * 0.98)) {
             if (incrementalLoadingParams.count) {
                 loadAndDisplayBooks();
             }
@@ -378,9 +401,9 @@
         footer = document.getElementById('kiwixfooter');
         fadeOutDiv = document.getElementById('fadeOut');
         loader = document.querySelector('.loader');
-        await loadAndDisplayBooks();
         await loadAndDisplayOptions('#languageFilter', `${root}/catalog/v2/languages`, 'language');
         await loadAndDisplayOptions('#categoryFilter', `${root}/catalog/v2/categories`, 'title');
+        await loadAndDisplayBooks();
         document.querySelectorAll('.filter').forEach(filter => {
             filter.addEventListener('change', () => {resetAndFilter(filter.name, filter.value)});
         });

static/skin/iso6391To3.js

@@ -1,15 +1,22 @@
+// Source: https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
 // eslint-disable-next-line no-unused-vars
 const iso6391To3 = {
     "aa": "aar",
+    "ab": "abk",
+    "ae": "ave",
     "af": "afr",
     "ak": "aka",
     "am": "amh",
+    "an": "arg",
     "ar": "ara",
     "as": "asm",
+    "av": "ava",
+    "ay": "aym",
     "az": "aze",
     "ba": "bak",
     "be": "bel",
     "bg": "bul",
+    "bi": "bis",
     "bm": "bam",
     "bn": "ben",
     "bo": "bod",
@@ -17,24 +24,33 @@ const iso6391To3 = {
     "bs": "bos",
     "ca": "cat",
     "ce": "che",
+    "ch": "cha",
     "co": "cos",
+    "cr": "cre",
     "cs": "ces",
+    "cu": "chu",
     "cv": "chv",
     "cy": "cym",
     "da": "dan",
     "de": "deu",
+    "dv": "div",
     "dz": "dzo",
+    "el": "ell",
     "ee": "ewe",
     "en": "eng",
+    "eo": "epo",
     "es": "spa",
     "et": "est",
     "eu": "eus",
     "fa": "fas",
     "ff": "ful",
     "fi": "fin",
+    "fj": "fij",
     "fo": "fao",
     "fr": "fra",
+    "fy": "fry",
     "ga": "gle",
+    "gd": "gla",
     "gl": "glg",
     "gn": "grn",
     "gu": "guj",
@@ -42,44 +58,78 @@ const iso6391To3 = {
     "ha": "hau",
     "he": "heb",
     "hi": "hin",
+    "ho": "hmo",
     "hr": "hrv",
+    "ht": "hat",
     "hu": "hun",
     "hy": "hye",
+    "hz": "her",
+    "ia": "ina",
     "id": "ind",
+    "ie": "ile",
     "ig": "ibo",
+    "ii": "iii",
+    "ik": "ipk",
+    "io": "ido",
     "is": "isl",
     "it": "ita",
     "iu": "iku",
     "ja": "jpn",
     "jv": "jav",
     "ka": "kat",
+    "kg": "kon",
     "ki": "kik",
+    "kj": "kua",
+    "kl": "kal",
     "kk": "kaz",
     "km": "khm",
     "kn": "kan",
     "ko": "kor",
+    "kr": "kau",
     "ks": "kas",
     "ku": "kur",
+    "kv": "kom",
     "kw": "cor",
     "ky": "kir",
+    "la": "lat",
     "lb": "ltz",
     "lg": "lug",
+    "li": "lim",
     "ln": "lin",
     "lo": "lao",
     "lt": "lit",
+    "lu": "lub",
     "lv": "lav",
     "mg": "mlg",
+    "mh": "mah",
     "mi": "mri",
     "mk": "mkd",
     "ml": "mal",
     "mn": "mon",
     "mr": "mar",
+    "ms": "msa",
     "mt": "mlt",
     "my": "mya",
+    "na": "nau",
+    "nb": "nob",
+    "nd": "nde",
+    "ne": "nep",
+    "ng": "ndo",
     "nl": "nld",
+    "nn": "nno",
+    "no": "nor",
+    "nr": "nbl",
+    "nv": "nav",
     "ny": "nya",
+    "oc": "oci",
+    "oj": "oji",
     "om": "orm",
+    "or": "ori",
+    "os": "oss",
+    "pa": "pan",
+    "pi": "pli",
     "pl": "pol",
+    "ps": "pus",
     "pt": "por",
     "qu": "que",
     "rm": "roh",
@@ -88,7 +138,10 @@ const iso6391To3 = {
     "ru": "rus",
     "rw": "kin",
     "sa": "san",
+    "sc": "srd",
     "sd": "snd",
+    "se": "sme",
+    "sm": "smo",
     "sg": "sag",
     "si": "sin",
     "sk": "slk",
@@ -98,27 +151,37 @@ const iso6391To3 = {
     "sq": "sqi",
     "sr": "srp",
     "ss": "ssw",
+    "st": "sot",
+    "su": "sun",
     "sv": "swe",
+    "sw": "swa",
     "ta": "tam",
     "te": "tel",
     "tg": "tgk",
     "th": "tha",
     "ti": "tir",
     "tk": "tuk",
+    "tl": "tgl",
     "tn": "tsn",
+    "to": "ton",
     "tr": "tur",
     "ts": "tso",
     "tt": "tat",
+    "tw": "twi",
+    "ty": "tah",
     "ug": "uig",
     "uk": "ukr",
     "ur": "urd",
     "uz": "uzb",
     "ve": "ven",
     "vi": "vie",
+    "vo": "vol",
     "wa": "wln",
     "wo": "wol",
     "xh": "xho",
+    "yi": "yid",
     "yo": "yor",
+    "za": "zha",
     "zh": "zho",
     "zu": "zul"
 }

static/skin/search_results.css

@@ -0,0 +1,87 @@
+body{
+background-color: white;
+color: #000000;
+font: small/normal Arial,Helvetica,Sans-Serif;
+margin-top: 0.5em;
+font-size: 90%;
+}
+
+a{
+color: #04c;
+}
+
+a:visited {
+color: #639
+}
+
+a:hover {
+text-decoration: underline
+}
+
+h1 {
+font-size: 120%;
+}
+
+ul {
+margin:0;
+padding:0
+}
+
+.results {
+font-size: 110%;
+}
+
+.results li {
+list-style-type:none;
+margin-top: 0.5em;
+}
+
+.results a {
+font-size: 110%;
+text-decoration: underline
+}
+
+cite {
+font-style:normal;
+word-wrap:break-word;
+display: block;
+font-size: 100%;
+}
+
+.informations {
+color: #388222;
+font-size: 100%;
+}
+
+.footer {
+padding: 0;
+margin-top: 1em;
+width: 100%;
+float: left
+}
+
+.footer a, .footer span {
+display: block;
+padding: .3em .7em;
+margin: 0 .38em 0 0;
+text-align:center;
+text-decoration: none;
+}
+
+.footer a:hover {
+background: #ededed;
+}
+
+.footer ul, .footer li {
+list-style:none;
+margin: 0;
+padding: 0;
+}
+
+.footer li {
+float: left;
+}
+
+.selected {
+background: #ededed;
+}

static/templates/404.html

@@ -1,18 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta content="text/html;charset=UTF-8" http-equiv="content-type" />
-    <title>Content not found</title>
-  </head>
-  <body>
-    <h1>Not Found</h1>
-    <p>
-      The requested URL "{{url}}" was not found on this server.
-    </p>
-    {{#details}}
-    <p>
-      {{{details}}}
-    </p>
-    {{/details}}
-  </body>
-</html>

static/templates/500.html

@@ -1,16 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta content="text/html;charset=UTF-8" http-equiv="content-type" />
-    <title>Internal Server Error</title>
-  </head>
-  <body>
-    <h1>Internal Server Error</h1>
-    <p>
-      An internal server error occured. We are sorry about that :/
-    </p>
-    <p>
-      {{ error }}
-    </p>
-  </body>
-</html>

static/templates/catalog_entries.xml

@@ -1,4 +1,6 @@
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:dc="http://purl.org/dc/terms/"
+      xmlns:opds="http://opds-spec.org/2010/catalog">
   <id>{{feed_id}}</id>
   <title>{{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}}</title>
   <updated>{{date}}</updated>

static/templates/catalog_v2_entries.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:dc="http://purl.org/dc/terms/"
       xmlns:opds="https://specs.opds.io/opds-1.2"
       xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">
   <id>{{feed_id}}</id>

static/templates/catalog_v2_entry.xml

@@ -15,18 +15,17 @@
     <tags>{{tags}}</tags>
     <articleCount>{{article_count}}</articleCount>
     <mediaCount>{{media_count}}</mediaCount>
-    {{#icons}}
-    <link rel="http://opds-spec.org/image/thumbnail"
-          href="/meta?name=Illustration_{{icon_width}}x{{icon_height}}@{{icon_scale}}&amp;content={{{content_id}}}"
-          type="image/png;width={{icon_width}};height={{icon_height}};scale={{icon_scale}}"/>
-    {{/icons}}
-    <link type="text/html" href="/{{{content_id}}}" />
+    {{#icons}}<link rel="http://opds-spec.org/image/thumbnail"
+          href="{{root}}/catalog/v2/illustration/{{{content_id}}}/?size={{icon_size}}"
+          type="{{icon_mimetype}};width={{icon_size}};height={{icon_size}};scale=1"/>
+    {{/icons}}<link type="text/html" href="{{root}}/{{{content_id}}}" />
     <author>
       <name>{{author_name}}</name>
     </author>
     <publisher>
       <name>{{publisher_name}}</name>
     </publisher>
+    <dc:issued>{{book_date}}</dc:issued>
     {{#url}}
     <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
     {{/url}}

static/templates/error.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta content="text/html;charset=UTF-8" http-equiv="content-type" />
+    <title>{{PAGE_TITLE}}</title>
+{{#CSS_URL}}
+    <link type="text/css" href="{{{CSS_URL}}}" rel="Stylesheet" />
+{{/CSS_URL}}
+  </head>
+  <body>
+    <h1>{{PAGE_HEADING}}</h1>
+{{#details}}
+    <p>
+      {{{p}}}
+    </p>
+{{/details}}
+  </body>
+</html>

static/templates/index.html

@@ -24,7 +24,7 @@
     />
     <link
       type="text/css"
-      href="{{root}}/skin/home.css"
+      href="{{root}}/skin/index.css"
       rel="Stylesheet"
     />
     <style>

static/templates/no_search_result.html

@@ -1,103 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta content="text/html; charset=utf-8" http-equiv="content-type" />
-    <style type="text/css">
-      body{
-      color: #000000;
-      font: small/normal Arial,Helvetica,Sans-Serif;
-      margin-top: 0.5em;
-      font-size: 90%;
-      }
-
-      a{
-      color: #04c;
-      }
-
-      a:visited {
-      color: #639
-      }
-
-      a:hover {
-      text-decoration: underline
-      }
-
-      .header {
-      font-size: 120%;
-      }
-
-      ul {
-      margin:0;
-      padding:0
-      }
-
-      .results {
-      font-size: 110%;
-      }
-
-      .results li {
-      list-style-type:none;
-      margin-top: 0.5em;
-      }
-
-      .results a {
-      font-size: 110%;
-      text-decoration: underline
-      }
-
-      cite {
-      font-style:normal;
-      word-wrap:break-word;
-      display: block;
-      font-size: 100%;
-      }
-
-      .informations {
-      color: #388222;
-      font-size: 100%;
-      }
-
-      .footer {
-      padding: 0;
-      margin-top: 1em;
-      width: 100%;
-      float: left
-      }
-
-      .footer a, .footer span {
-      display: block;
-      padding: .3em .7em;
-      margin: 0 .38em 0 0;
-      text-align:center;
-      text-decoration: none;
-      }
-
-      .footer a:hover {
-      background: #ededed;
-      }
-
-      .footer ul, .footer li {
-      list-style:none;
-      margin: 0;
-      padding: 0;
-      }
-
-      .footer li {
-      float: left;
-      }
-
-      .selected {
-      background: #ededed;
-      }
-
-    </style>
-    <title>Fulltext search unavailable</title>
-  </head>
-  <body bgcolor="white">
-    <div class="header">Not found</div>
-    <p>
-      There is no article with the title <b> "{{pattern}}"</b>
-      and the fulltext search engine is not available for this content.
-    </p>
-  </body>
-</html>

static/templates/search_result.html

@@ -57,6 +57,11 @@
       font-size: 100%;
       }
 
+      .book-title {
+      color: #662200;
+      font-size: 100%;
+      }
+
       .footer {
       padding: 0;
       margin-top: 1em;
@@ -102,11 +107,11 @@
         </b> of <b>
           {{count}}
         </b> for <b>
-          {{searchPattern}}
+          "{{{searchPattern}}}"
         </b>
       {{/hasResults}}
       {{^hasResults}}
-        No results were found for <b>{{searchPattern}}</b>
+        No results were found for <b>"{{{searchPattern}}}"</b>
       {{/hasResults}}
     </div>
 
@@ -120,6 +125,9 @@
             {{#snippet}}
               <cite>{{>snippet}}...</cite>
             {{/snippet}}
+            {{#bookTitle}}
+              <div class="book-title">from {{bookTitle}}</div>
+            {{/bookTitle}}
             {{#wordCount}}
               <div class="informations">{{wordCount}} words</div>
             {{/wordCount}}

static/templates/taskbar_part.html

@@ -5,18 +5,18 @@
         <form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
           {{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
           <label for="kiwixsearchbox">&#x1f50d;</label>
-          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
+          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search '{{title}}'" aria-label="Search '{{title}}'">
         </form>
       </div>
         <input type="checkbox" id="kiwix_button_show_toggle">
         <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
         <div class="kiwix_button_cont">
           {{#withlibrarybutton}}
-            <a id="kiwix_serve_taskbar_library_button" href="{{root}}/"><button>&#x1f3e0;</button></a>
+            <a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="{{root}}/"><button>&#x1f3e0;</button></a>
           {{/withlibrarybutton}}
           {{#hascontent}}
-          <a id="kiwix_serve_taskbar_home_button" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
-          <a id="kiwix_serve_taskbar_random_button"
+          <a id="kiwix_serve_taskbar_home_button" title="Go to the main page of '{{title}}'" aria-label="Go to the main page of '{{title}}'" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
+          <a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
             href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
           {{/hascontent}}
         </div>

test/book.cpp

@@ -2,42 +2,6 @@
 #include "../include/book.h"
 #include <pugixml.hpp>
 
-TEST(BookTest, updateTest)
-{
-    kiwix::Book book;
-
-    book.setId("xyz");
-    book.setReadOnly(false);
-    book.setPath("/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim");
-    book.setPathValid(true);
-    book.setUrl("book-url");
-    book.setTags("youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes");
-    book.setName("skin-of-color-society_en_all");
-    book.setFavicon("book-favicon");
-    book.setFaviconMimeType("book-favicon-mimetype");
-
-    kiwix::Book newBook;
-
-    newBook.setReadOnly(true);
-    EXPECT_FALSE(newBook.update(book));
-
-    newBook.setReadOnly(false);
-    EXPECT_FALSE(newBook.update(book));
-
-    newBook.setId("xyz");
-    EXPECT_TRUE(newBook.update(book));
-
-    EXPECT_EQ(newBook.readOnly(), book.readOnly());
-    EXPECT_EQ(newBook.getPath(), book.getPath());
-    EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
-    EXPECT_EQ(newBook.getUrl(), book.getUrl());
-    EXPECT_EQ(newBook.getTags(), book.getTags());
-    EXPECT_EQ(newBook.getCategory(), book.getCategory());
-    EXPECT_EQ(newBook.getName(), book.getName());
-    EXPECT_EQ(newBook.getFavicon(), book.getFavicon());
-    EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType());
-}
-
 namespace
 {
 
@@ -68,6 +32,9 @@ TEST(BookTest, updateFromXMLTest)
             articleCount="123456"
             mediaCount="234567"
             size="345678"
+            favicon="ZmFrZS1ib29rLWZhdmljb24tZGF0YQ=="
+            faviconMimeType="text/plain"
+            faviconUrl="http://who.org/zara.fav"
           >
       </book>
     )");
@@ -85,6 +52,10 @@ TEST(BookTest, updateFromXMLTest)
     EXPECT_EQ(book.getArticleCount(), 123456U);
     EXPECT_EQ(book.getMediaCount(), 234567U);
     EXPECT_EQ(book.getSize(), 345678U*1024U);
+    auto defaultIllustration = book.getIllustration(48);
+    EXPECT_EQ(defaultIllustration->getData(), "fake-book-favicon-data");
+    EXPECT_EQ(defaultIllustration->mimeType, "text/plain");
+    EXPECT_EQ(defaultIllustration->url, "http://who.org/zara.fav");
 }
 
 TEST(BookTest, updateFromXMLCategoryHandlingTest)
@@ -166,3 +137,76 @@ TEST(BookTest, updateCopiesCategory)
     newBook.update(book);
     EXPECT_EQ(newBook.getCategory(), "ted");
 }
+
+TEST(BookTest, updateTest)
+{
+    const XMLDoc xml(R"(
+      <book id="xyz"
+            path="/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim"
+            url="book-url"
+            name="skin-of-color-society_en_all"
+            tags="youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes"
+            favicon="Ym9vay1mYXZpY29u"
+            faviconMimeType="book-favicon-mimetype"
+          >
+      </book>
+    )");
+
+    kiwix::Book book;
+    book.updateFromXml(xml.child("book"), "/data/zim");
+
+    book.setReadOnly(false);
+    book.setPathValid(true);
+
+    kiwix::Book newBook;
+
+    newBook.setReadOnly(true);
+    EXPECT_FALSE(newBook.update(book));
+
+    newBook.setReadOnly(false);
+    EXPECT_FALSE(newBook.update(book));
+
+    newBook.setId("xyz");
+    EXPECT_TRUE(newBook.update(book));
+
+    EXPECT_EQ(newBook.readOnly(), book.readOnly());
+    EXPECT_EQ(newBook.getPath(), book.getPath());
+    EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
+    EXPECT_EQ(newBook.getUrl(), book.getUrl());
+    EXPECT_EQ(newBook.getTags(), book.getTags());
+    EXPECT_EQ(newBook.getCategory(), book.getCategory());
+    EXPECT_EQ(newBook.getName(), book.getName());
+    auto defaultIllustration = book.getIllustration(48);
+    auto newDefaultIllustration = newBook.getIllustration(48);
+    EXPECT_EQ(newDefaultIllustration->getData(), defaultIllustration->getData());
+    EXPECT_EQ(newDefaultIllustration->mimeType, defaultIllustration->mimeType);
+}
+
+namespace
+{
+
+std::string path2HumanReadableId(const std::string& path)
+{
+    const XMLDoc xml("<book id=\"xyz\" path=\"" + path + "\"></book>");
+
+    kiwix::Book book;
+    book.updateFromXml(xml.child("book"), "/data/zim");
+    return book.getHumanReadableIdFromPath();
+}
+
+} // unnamed namespace
+
+TEST(BookTest, getHumanReadableIdFromPath)
+{
+  EXPECT_EQ("abc",     path2HumanReadableId("abc.zim"));
+  EXPECT_EQ("abc",     path2HumanReadableId("ABC.zim"));
+  EXPECT_EQ("abc",     path2HumanReadableId("âbç.zim"));
+  EXPECT_EQ("ancient", path2HumanReadableId("ancient.zimbabwe"));
+  EXPECT_EQ("ab_cd",   path2HumanReadableId("ab cd.zim"));
+#ifdef _WIN32
+  EXPECT_EQ("abc",     path2HumanReadableId("C:\\Data\\ZIM\\abc.zim"));
+#else
+  EXPECT_EQ("abc",     path2HumanReadableId("/Data/ZIM/abc.zim"));
+#endif
+  EXPECT_EQ("3plus2",  path2HumanReadableId("3+2.zim"));
+}

test/data/library.xml

@@ -14,6 +14,8 @@
         articleCount="284"
         mediaCount="2"
         size="556"
+        faviconMimeType="image/png"
+        favicon="SOME DATA"
       ></book>
   <book
         id="raycharles_uncategorized"
@@ -33,8 +35,8 @@
       ></book>
   <book
         id="charlesray"
-        path="./zimfile.zim"
-        url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
+        path="./zimfile&amp;other.zim"
+        url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim"
         title="Charles, Ray"
         description="Wikipedia articles about Ray Charles"
         language="fra"

test/data/zimfile&other.zim

@@ -0,0 +1 @@
+zimfile.zim

test/httplib.h

@@ -2889,7 +2889,7 @@ inline ssize_t Stream::write(const std::string &s) {
 
 template <typename... Args>
 inline ssize_t Stream::write_format(const char *fmt, const Args &... args) {
-  std::array<char, 2048> buf;
+  std::array<char, 2048> buf{};
 
 #if defined(_MSC_VER) && _MSC_VER < 1900
   auto sn = _snprintf_s(buf, bufsiz, buf.size() - 1, fmt, args...);
@@ -3756,7 +3756,7 @@ inline socket_t Client::create_client_socket() const {
 }
 
 inline bool Client::read_response_line(Stream &strm, Response &res) {
-  std::array<char, 2048> buf;
+  std::array<char, 2048> buf{};
 
   detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
 

test/library.cpp

@@ -22,13 +22,18 @@
 
 
 const char * sampleOpdsStream = R"(
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:dc="http://purl.org/dc/terms/"
+      xmlns:opds="http://opds-spec.org/2010/catalog">
   <id>00000000-0000-0000-0000-000000000000</id>
   <entry>
     <title>Encyclopédie de la Tunisie</title>
+    <name>wikipedia_fr_tunisie_novid_2018-10</name>
+    <flavour>unforgettable</flavour>
     <id>urn:uuid:0c45160e-f917-760a-9159-dfe3c53cdcdd</id>
     <icon>/meta?name=favicon&amp;content=wikipedia_fr_tunisie_novid_2018-10</icon>
     <updated>2018-10-08T00:00::00:Z</updated>
+    <dc:issued>8 Oct 2018</dc:issued>
     <language>fra</language>
     <summary>Le meilleur de Wikipédia sur la Tunisie</summary>
     <tags>wikipedia;novid;_ftindex</tags>
@@ -36,8 +41,13 @@ const char * sampleOpdsStream = R"(
     <author>
       <name>Wikipedia</name>
     </author>
+    <publisher>
+      <name>Wikipedia Publishing House</name>
+    </publisher>
     <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="http://download.kiwix.org/zim/wikipedia/wikipedia_fr_tunisie_novid_2018-10.zim.meta4" length="90030080" />
     <link rel="http://opds-spec.org/image/thumbnail" type="image/png" href="/meta?name=favicon&amp;content=wikipedia_fr_tunisie_novid_2018-10" />
+    <mediaCount>1100</mediaCount>
+    <articleCount>172</articleCount>
   </entry>
   <entry>
     <title>Tania Louis</title>
@@ -224,6 +234,64 @@ const char sampleLibraryXML[] = R"(
 namespace
 {
 
+TEST(LibraryOpdsImportTest, allInOne)
+{
+  kiwix::Library lib;
+  kiwix::Manager manager(&lib);
+  manager.readOpds(sampleOpdsStream, "library-opds-import.unittests.dev");
+
+  EXPECT_EQ(10U, lib.getBookCount(true, true));
+
+  {
+  const kiwix::Book& book1 = lib.getBookById("0c45160e-f917-760a-9159-dfe3c53cdcdd");
+
+  EXPECT_EQ(book1.getTitle(), "Encyclopédie de la Tunisie");
+  EXPECT_EQ(book1.getName(), "wikipedia_fr_tunisie_novid_2018-10");
+  EXPECT_EQ(book1.getFlavour(), "unforgettable");
+  EXPECT_EQ(book1.getLanguage(), "fra");
+  EXPECT_EQ(book1.getDate(), "8 Oct 2018");
+  EXPECT_EQ(book1.getDescription(), "Le meilleur de Wikipédia sur la Tunisie");
+  EXPECT_EQ(book1.getCreator(), "Wikipedia");
+  EXPECT_EQ(book1.getPublisher(), "Wikipedia Publishing House");
+  EXPECT_EQ(book1.getTags(), "wikipedia;novid;_ftindex");
+  EXPECT_EQ(book1.getCategory(), "");
+  EXPECT_EQ(book1.getUrl(), "http://download.kiwix.org/zim/wikipedia/wikipedia_fr_tunisie_novid_2018-10.zim.meta4");
+  EXPECT_EQ(book1.getSize(), 90030080UL);
+  EXPECT_EQ(book1.getMediaCount(), 1100U); // Roman MC (MediaCount) is 1100
+  EXPECT_EQ(book1.getArticleCount(), 172U); // Hex AC (ArticleCount) is 172
+
+  const auto illustration = book1.getIllustration(48);
+  EXPECT_EQ(illustration->width, 48U);
+  EXPECT_EQ(illustration->height, 48U);
+  EXPECT_EQ(illustration->mimeType, "image/png");
+  EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=wikipedia_fr_tunisie_novid_2018-10");
+  }
+
+  {
+  const kiwix::Book& book2 = lib.getBookById("0189d9be-2fd0-b4b6-7300-20fab0b5cdc8");
+  EXPECT_EQ(book2.getTitle(), "TED talks - Business");
+  EXPECT_EQ(book2.getName(), "");
+  EXPECT_EQ(book2.getFlavour(), "");
+  EXPECT_EQ(book2.getLanguage(), "eng");
+  EXPECT_EQ(book2.getDate(), "2018-07-23");
+  EXPECT_EQ(book2.getDescription(), "Ideas worth spreading");
+  EXPECT_EQ(book2.getCreator(), "TED");
+  EXPECT_EQ(book2.getPublisher(), "");
+  EXPECT_EQ(book2.getTags(), "");
+  EXPECT_EQ(book2.getCategory(), "");
+  EXPECT_EQ(book2.getUrl(), "http://download.kiwix.org/zim/ted/ted_en_business_2018-07.zim.meta4");
+  EXPECT_EQ(book2.getSize(), 8855827456UL);
+  EXPECT_EQ(book2.getMediaCount(), 0U);
+  EXPECT_EQ(book2.getArticleCount(), 0U);
+
+  const auto illustration = book2.getIllustration(48);
+  EXPECT_EQ(illustration->width, 48U);
+  EXPECT_EQ(illustration->height, 48U);
+  EXPECT_EQ(illustration->mimeType, "image/png");
+  EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=ted_en_business_2018-07");
+  }
+}
+
 class LibraryTest : public ::testing::Test {
  protected:
   typedef kiwix::Library::BookIdCollection BookIdCollection;
@@ -232,7 +300,7 @@ class LibraryTest : public ::testing::Test {
   void SetUp() override {
      kiwix::Manager manager(&lib);
      manager.readOpds(sampleOpdsStream, "foo.urlHost");
-     manager.readXml(sampleLibraryXML, true, "./test/library.xml", true);
+     manager.readXml(sampleLibraryXML, false, "./test/library.xml", true);
   }
 
     kiwix::Bookmark createBookmark(const std::string &id) {
@@ -292,7 +360,8 @@ TEST_F(LibraryTest, sanityCheck)
   EXPECT_EQ(lib.getBooksPublishers(), std::vector<std::string>({
       "",
       "Kiwix",
-      "Kiwix & Some Enthusiasts"
+      "Kiwix & Some Enthusiasts",
+      "Wikipedia Publishing House"
   }));
 }
 
@@ -660,13 +729,14 @@ TEST_F(LibraryTest, filterByMultipleCriteria)
 
 TEST_F(LibraryTest, getBookByPath)
 {
-  auto& book = lib.getBookById(lib.getBooksIds()[0]);
+  kiwix::Book book = lib.getBookById(lib.getBooksIds()[0]);
 #ifdef _WIN32
   auto path = "C:\\some\\abs\\path.zim";
 #else
   auto path = "/some/abs/path.zim";
 #endif
   book.setPath(path);
+  lib.addBook(book);
   EXPECT_EQ(lib.getBookByPath(path).getId(), book.getId());
   EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
 }
@@ -683,9 +753,9 @@ TEST_F(LibraryTest, removeBookByIdRemovesTheBook)
 
 TEST_F(LibraryTest, removeBookByIdDropsTheReader)
 {
-  EXPECT_NE(nullptr, lib.getReaderById("raycharles"));
+  EXPECT_NE(nullptr, lib.getArchiveById("raycharles"));
   lib.removeBookById("raycharles");
-  EXPECT_THROW(lib.getReaderById("raycharles"), std::out_of_range);
+  EXPECT_THROW(lib.getArchiveById("raycharles"), std::out_of_range);
 };
 
 TEST_F(LibraryTest, removeBookByIdUpdatesTheSearchDB)
@@ -706,4 +776,35 @@ TEST_F(LibraryTest, removeBookByIdUpdatesTheSearchDB)
   EXPECT_THROW(lib.getBookById("raycharles"), std::out_of_range);
 };
 
+TEST_F(LibraryTest, removeBooksNotUpdatedSince)
+{
+  EXPECT_FILTER_RESULTS(kiwix::Filter(),
+    "An example ZIM archive",
+    "Encyclopédie de la Tunisie",
+    "Granblue Fantasy Wiki",
+    "Géographie par Wikipédia",
+    "Islam Stack Exchange",
+    "Mathématiques",
+    "Movies & TV Stack Exchange",
+    "Mythology & Folklore Stack Exchange",
+    "Ray Charles",
+    "TED talks - Business",
+    "Tania Louis",
+    "Wikiquote"
+  );
+
+  const uint64_t rev = lib.getRevision();
+  for ( const auto& id : lib.filter(kiwix::Filter().query("exchange")) ) {
+    lib.addBook(lib.getBookByIdThreadSafe(id));
+  }
+
+  EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev));
+
+  EXPECT_FILTER_RESULTS(kiwix::Filter(),
+    "Islam Stack Exchange",
+    "Movies & TV Stack Exchange",
+    "Mythology & Folklore Stack Exchange",
+  );
+};
+
 };

test/manager.cpp

@@ -67,3 +67,29 @@ TEST(ManagerTest, readXml)
     EXPECT_EQ(45U, book.getMediaCount());
     EXPECT_EQ(678U*1024, book.getSize());
 }
+
+TEST(Manager, reload)
+{
+  kiwix::Library lib;
+  kiwix::Manager manager(&lib);
+
+  manager.reload({ "./test/library.xml" });
+  EXPECT_EQ(lib.getBooksIds(), (kiwix::Library::BookIdCollection{
+        "charlesray",
+        "raycharles",
+        "raycharles_uncategorized"
+  }));
+
+  lib.removeBookById("raycharles");
+  EXPECT_EQ(lib.getBooksIds(), (kiwix::Library::BookIdCollection{
+        "charlesray",
+        "raycharles_uncategorized"
+  }));
+
+  manager.reload({ "./test/library.xml" });
+  EXPECT_EQ(lib.getBooksIds(), kiwix::Library::BookIdCollection({
+        "charlesray",
+        "raycharles",
+        "raycharles_uncategorized"
+  }));
+}

test/meson.build

@@ -8,9 +8,8 @@ tests = [
     'kiwixserve',
     'book',
     'manager',
-    'opds_catalog',
-    'reader',
-    'searcher'
+    'name_mapper',
+    'opds_catalog'
 ]
 
 if build_machine.system() != 'windows'
@@ -28,7 +27,9 @@ if gtest_dep.found() and not meson.is_cross_build()
     data_files = [
       'example.zim',
       'zimfile.zim',
+      'zimfile&other.zim',
       'corner_cases.zim',
+      'poor.zim',
       'library.xml'
     ]
     foreach file : data_files

test/name_mapper.cpp

@@ -0,0 +1,147 @@
+#include "../include/name_mapper.h"
+
+#include "../include/library.h"
+#include "../include/manager.h"
+#include "gtest/gtest.h"
+
+namespace
+{
+
+const char libraryXML[] = R"(
+<library version="1.0">
+  <book id="01"         path="/data/zero_one.zim">         </book>
+  <book id="02"         path="/data/zero two.zim">         </book>
+  <book id="03"         path="/data/ZERO thrêë.zim">       </book>
+  <book id="04-2021-10" path="/data/zero_four_2021-10.zim"></book>
+  <book id="04-2021-11" path="/data/zero_four_2021-11.zim"></book>
+</library>
+)";
+
+class NameMapperTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+     kiwix::Manager manager(&lib);
+     manager.readXml(libraryXML, false, "./library.xml", true);
+     for ( const std::string& id : lib.getBooksIds() ) {
+       kiwix::Book bookCopy = lib.getBookById(id);
+       bookCopy.setPathValid(true);
+       lib.addBook(bookCopy);
+     }
+  }
+
+  kiwix::Library lib;
+};
+
+class CapturedStderr
+{
+  std::ostringstream buffer;
+  std::streambuf* const sbuf;
+public:
+  CapturedStderr()
+    : sbuf(std::cerr.rdbuf())
+  {
+    std::cerr.rdbuf(buffer.rdbuf());
+  }
+
+  CapturedStderr(const CapturedStderr&) = delete;
+
+  ~CapturedStderr()
+  {
+    std::cerr.rdbuf(sbuf);
+  }
+
+  operator std::string() const { return buffer.str(); }
+};
+
+} // unnamed namespace
+
+void checkUnaliasedEntriesInNameMapper(const kiwix::NameMapper& nm)
+{
+  EXPECT_EQ("zero_one",           nm.getNameForId("01"));
+  EXPECT_EQ("zero_two",           nm.getNameForId("02"));
+  EXPECT_EQ("zero_three",         nm.getNameForId("03"));
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("zero_four_2021-11",  nm.getNameForId("04-2021-11"));
+
+  EXPECT_EQ("01",         nm.getIdForName("zero_one"));
+  EXPECT_EQ("02",         nm.getIdForName("zero_two"));
+  EXPECT_EQ("03",         nm.getIdForName("zero_three"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_EQ("04-2021-11", nm.getIdForName("zero_four_2021-11"));
+}
+
+TEST_F(NameMapperTest, HumanReadableNameMapperWithoutAliases)
+{
+  CapturedStderr stderror;
+  kiwix::HumanReadableNameMapper nm(lib, false);
+  EXPECT_EQ("", std::string(stderror));
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+
+  lib.removeBookById("04-2021-10");
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+}
+
+TEST_F(NameMapperTest, HumanReadableNameMapperWithAliases)
+{
+  CapturedStderr stderror;
+  kiwix::HumanReadableNameMapper nm(lib, true);
+  EXPECT_EQ(
+      "Path collision: /data/zero_four_2021-10.zim and"
+      " /data/zero_four_2021-11.zim can't share the same URL path 'zero_four'."
+      " Therefore, only /data/zero_four_2021-10.zim will be served.\n"
+      , std::string(stderror)
+  );
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+
+  lib.removeBookById("04-2021-10");
+  EXPECT_EQ("zero_four_2021-10",  nm.getNameForId("04-2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four_2021-10"));
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+}
+
+TEST_F(NameMapperTest, UpdatableNameMapperWithoutAliases)
+{
+  CapturedStderr stderror;
+  kiwix::UpdatableNameMapper nm(lib, false);
+  EXPECT_EQ("", std::string(stderror));
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+
+  lib.removeBookById("04-2021-10");
+  nm.update();
+  EXPECT_THROW(nm.getNameForId("04-2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four_2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four"), std::out_of_range);
+}
+
+TEST_F(NameMapperTest, UpdatableNameMapperWithAliases)
+{
+  CapturedStderr stderror;
+  kiwix::UpdatableNameMapper nm(lib, true);
+  EXPECT_EQ(
+      "Path collision: /data/zero_four_2021-10.zim and"
+      " /data/zero_four_2021-11.zim can't share the same URL path 'zero_four'."
+      " Therefore, only /data/zero_four_2021-10.zim will be served.\n"
+      , std::string(stderror)
+  );
+
+  checkUnaliasedEntriesInNameMapper(nm);
+  EXPECT_EQ("04-2021-10", nm.getIdForName("zero_four"));
+
+  {
+    CapturedStderr nmUpdateStderror;
+    lib.removeBookById("04-2021-10");
+    nm.update();
+    EXPECT_EQ("", std::string(nmUpdateStderror));
+  }
+  EXPECT_EQ("04-2021-11", nm.getIdForName("zero_four"));
+  EXPECT_THROW(nm.getNameForId("04-2021-10"), std::out_of_range);
+  EXPECT_THROW(nm.getIdForName("zero_four_2021-10"), std::out_of_range);
+}

test/pathTools.cpp

@@ -20,6 +20,10 @@
 #include "gtest/gtest.h"
 #include <string>
 #include <vector>
+#ifndef _WIN32
+# include <unistd.h>
+# include <fcntl.h>
+#endif
 #include "../include/tools.h"
 #include "../src/tools/pathTools.h"
 
@@ -190,6 +194,34 @@ TEST(pathTools, appendToDirectory)
             P5("a","b","c",".","foo.xml"));
 }
 
+TEST(pathTools, fileExists)
+{
+  ASSERT_TRUE(kiwix::fileExists(P3(".","test","example.zim")));
+  ASSERT_FALSE(kiwix::fileExists(P3(".","test","noFile.zim")));
+}
+
+TEST(pathTools, fileReadable)
+{
+  ASSERT_TRUE(kiwix::fileReadable(P3(".","test","example.zim")));
+  ASSERT_FALSE(kiwix::fileReadable(P3(".","test","noFile.zim")));
+#ifdef _POSIX_SOURCE
+  std::string path = P3(".","test","example.zim");
+  auto myprivs = geteuid();
+  if (myprivs != 0) {
+    int fd = open(path.c_str(), O_RDONLY);
+    if (fd != -1) {
+      int wResp = fchmod(fd, ~(S_IRWXU | S_IRWXG | S_IRWXO)); // remove all permissions
+      if (wResp == 0) {
+        EXPECT_FALSE(kiwix::fileReadable(P3(".","test","example.zim")));
+      }
+      int resetResp = fchmod(fd, S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR | S_IWGRP); // reset back permissions to -rw-rw-r--
+      if (resetResp == 0) {
+        EXPECT_TRUE(kiwix::fileReadable(P3(".","test","example.zim")));
+      }
+    }
+  }
+#endif
+}
 
 TEST(pathTools, goUp)
 {

test/reader.cpp

@@ -1,62 +0,0 @@
-
-#include "gtest/gtest.h"
-#include "../include/reader.h"
-#include "zim/archive.h"
-
-namespace kiwix
-{
-  /**
-   * This test file is written primarily to demonstrate how Reader is simply a
-   * wrapper over an archive. We will be dropping this wrapper soon.
-   **/
-  TEST (Reader, archiveWrapper) {
-    Reader reader("./test/zimfile.zim");
-    zim::Archive archive = *reader.getZimArchive();
-
-    std::ostringstream s;
-    s << archive.getUuid();
-
-    ASSERT_EQ(reader.getId(), s.str());
-    ASSERT_EQ(reader.getGlobalCount(), archive.getEntryCount());
-    ASSERT_EQ(reader.getMainPage().getTitle(), archive.getMainEntry().getTitle());
-    ASSERT_EQ(reader.hasFulltextIndex(), archive.hasFulltextIndex());
-    ASSERT_NO_THROW(reader.getRandomPage());
-  }
-
-  TEST (Reader, getFunctions) {
-    zim::Archive archive("./test/zimfile.zim");
-    Reader reader("./test/zimfile.zim");
-
-    auto archiveEntry = archive.getRandomEntry();
-    ASSERT_TRUE(reader.pathExists(archiveEntry.getPath()));
-    auto readerEntry = reader.getEntryFromPath(archiveEntry.getPath());
-    ASSERT_EQ(readerEntry.getTitle(), archiveEntry.getTitle());
-
-    ASSERT_FALSE(reader.pathExists("invalidEntryPath"));
-    ASSERT_THROW(reader.getEntryFromPath("invalidEntryPath"), NoEntry);
-
-    readerEntry = reader.getEntryFromTitle(archiveEntry.getTitle());
-    ASSERT_EQ(readerEntry.getTitle(), archiveEntry.getTitle());
-  }
-
-  TEST (Reader, suggestions) {
-    Reader reader("./test/zimfile.zim");
-    SuggestionsList_t suggestions;
-    reader.searchSuggestionsSmart("The Genius", 4, suggestions);
-
-    std::vector<std::string> suggestionResult, expectedResult;
-    std::string suggestionTitle;
-    for (auto it = suggestions.begin(); it != suggestions.end(); it++) {
-      suggestionResult.push_back(it->getTitle());
-    }
-
-    expectedResult = {
-      "The Genius After Hours",
-      "The Genius Hits the Road",
-      "The Genius Sings the Blues",
-      "The Genius of Ray Charles"
-    };
-
-    ASSERT_EQ(suggestionResult, expectedResult);
-  }
-}

test/searcher.cpp

@@ -1,67 +0,0 @@
-#include "gtest/gtest.h"
-#include "../include/searcher.h"
-#include "../include/reader.h"
-
-namespace kiwix
-{
-
-TEST(Searcher, search) {
-  Reader reader("./test/example.zim");
-
-  Searcher searcher;
-  searcher.add_reader(&reader);
-  ASSERT_EQ(searcher.get_reader(0)->getTitle(), reader.getTitle());
-
-  searcher.search("wiki", 0, 2);
-  searcher.restart_search();
-  ASSERT_EQ(searcher.getEstimatedResultCount(), (unsigned int)2);
-
-  auto result = searcher.getNextResult();
-  ASSERT_EQ(result->get_title(), "FreedomBox for Communities/Offline Wikipedia - Wikibooks, open books for an open world");
-  result = searcher.getNextResult();
-  ASSERT_EQ(result->get_title(), "Wikibooks");
-}
-
-TEST(Searcher, suggestion) {
-  Reader reader("./test/zimfile.zim");
-
-  Searcher searcher;
-  searcher.add_reader(&reader);
-  ASSERT_EQ(searcher.get_reader(0)->getTitle(), reader.getTitle());
-
-  std::string query = "ray";
-  searcher.suggestions(query, true);
-  searcher.restart_search();
-
-  auto result = searcher.getNextResult();
-  ASSERT_EQ(result->get_title(), "Charles, Ray");
-  ASSERT_EQ(result->get_url(), "A/Charles,_Ray");
-  ASSERT_EQ(result->get_snippet(), "Charles, <b>Ray</b>");
-  ASSERT_EQ(result->get_score(), 0);
-  ASSERT_EQ(result->get_content(), "");
-  ASSERT_EQ(result->get_size(), 0);
-  ASSERT_EQ(result->get_wordCount(), 0);
-  ASSERT_EQ(result->get_zimId(), "");
-
-  result = searcher.getNextResult();
-  ASSERT_EQ(result->get_title(), "Ray (film)");
-}
-
-TEST(Searcher, incrementalRange) {
-  // Attempt to get 50 results in steps of 5
-  zim::Archive archive("./test/zimfile.zim");
-  zim::Searcher ftsearcher(archive);
-  zim::Query query;
-  query.setQuery("ray");
-  auto search = ftsearcher.search(query);
-
-  int suggCount = 0;
-  for (int i = 0; i < 10; i++) {
-    auto srs = search.getResults(i*5, 5); // get 5 results
-    ASSERT_EQ(srs.size(), 5);
-    suggCount += srs.size();
-  }
-  ASSERT_EQ(suggCount, 50);
-}
-
-}