Patch httplib.h for gcc maybe-uninitialized warning.

Recent version of gcc warn about the use `buf` where it may be
uninitialized.
But here, buf is used as a output buffer to initialize it.

I'm not sure we can fix this differently than ignoring the warning.

.github/workflows/ci.yml

@@ -13,9 +13,9 @@ jobs:
         with:
           python-version: '3.5'
       - name: Install packages
-        uses: mstksg/get-package@v1
-        with:
-          brew: gcovr pkg-config ninja
+        run: |
+          brew update
+          brew install gcovr pkg-config ninja
       - name: Install python modules
         run: pip3 install meson==0.49.2 pytest
       - name: Install deps
@@ -53,30 +53,41 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        name:
           - native_static
           - native_dyn
+          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
           - win32_dyn
         include:
-          - target: native_static
+          - name: native_static
+            target: native_static
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: native_dyn
+          - name: native_dyn
+            target: native_dyn
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm
+          - name: native_dyn_bionic
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm64
+          - name: android_arm64
+            target: android_arm64
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: win32_static
+          - name: win32_static
+            target: win32_static
             image_variant: f31
             lib_postfix: '64'
-          - target: win32_dyn
+          - name: win32_dyn
+            target: win32_dyn
             image_variant: f31
             lib_postfix: '64'
     env:
@@ -123,7 +134,7 @@ jobs:
         if [[ "${{matrix.target}}" =~ android_.* ]]; then
           MESON_OPTION="$MESON_OPTION -Dandroid=true"
         fi
-        cd $HOME/kiwix-lib
+        cd $HOME/libkiwix
         meson . build ${MESON_OPTION}
         cd build
         ninja
@@ -134,7 +145,7 @@ jobs:
       if: startsWith(matrix.target, 'native_')
       shell: bash
       run: |
-        cd $HOME/kiwix-lib/build
+        cd $HOME/libkiwix/build
         meson test --verbose
         ninja coverage
       env:
@@ -143,10 +154,10 @@ jobs:
     - name: Publish coverage
       shell: bash
       run: |
-        cd $HOME/kiwix-lib
+        cd $HOME/libkiwix
         curl https://codecov.io/bash -o codecov.sh
         bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
         rm codecov.sh
-      if: startsWith(matrix.target, 'native_')
+      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -0,0 +1,87 @@
+name: Packages
+on: [push, pull_request]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distro: [ubuntu-hirsute, ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
+    steps:
+      - uses: actions/checkout@v2
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
+        if: matrix.distro == 'ubuntu-hirsute'
+        name: Build package for ubuntu-hirsute
+        id: build-ubuntu-hirsute
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-groovy
+        if: matrix.distro == 'ubuntu-groovy'
+        name: Build package for ubuntu-groovy
+        id: build-ubuntu-groovy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to master
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.gitignore

@@ -2,3 +2,6 @@
 *.swp
 subprojects/googletest-release*
 *.class
+build/
+.vscode/
+builddir/

ChangeLog

@@ -1,3 +1,69 @@
+libkiwix 10.0.0
+===============
+
+ * ...
+
+kiwix-lib 9.4.1
+===============
+
+ * Fix `M/Counter` parsing.
+ * [SERVER] Adjust body padding-top for taskbar
+ * Fix potential crash when stoping a server not started.
+ * Various fix in build system and the CI.
+
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
 kiwix-lib 9.2
 =============
 

README.md

@@ -1,15 +1,14 @@
-Kiwix library
-=============
+Libkiwix
+========
 
-The Kiwix library provides the [Kiwix](https://kiwix.org) software
-suite core. It contains the code shared by all Kiwix ports (Windows,
+The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
+core. It contains the code shared by all Kiwix ports (Windows,
 GNU/Linux, macOS, Android, iOS, ...).
 
-[![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
-[![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
-[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
-[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
-[![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
+[![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -17,31 +16,38 @@ Disclaimer
 
 This document assumes you have a little knowledge about software
 compilation. If you experience difficulties with the dependencies or
-with the Kiwix libary compilation itself, we recommend to have a look
-to [kiwix-build](https://github.com/kiwix/kiwix-build).
+with the Libkiwix compilation itself, we recommend to have a look to
+[kiwix-build](https://github.com/kiwix/kiwix-build).
 
 Preamble
 --------
 
-Although the Kiwix library can be (cross-)compiled on/for many
-sytems, the following documentation explains how to do it on POSIX
-ones. It is primarly thought for GNU/Linux systems and has been tested
-on recent releases of Ubuntu and Fedora.
+Although the Libkiwix can be (cross-)compiled on/for many sytems, the
+following documentation explains how to do it on POSIX ones. It is
+primarly thought for GNU/Linux systems and has been tested on recent
+releases of Ubuntu and Fedora.
 
 Dependencies
 ------------
 
-The Kiwix library relies on many third parts software libraries. They
-are prerequisites to the Kiwix library compilation. Following
-libraries need to be available:
-
+The Libkiwix relies on many third party software libraries. They are
+prerequisites to the Libkiwix compilation. Following libraries need to
+be available:
 * [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
 * [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
 * [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
-* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
-set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
+set CPPFLAGS with correct `-I` option). Use Mustache version 4.1 or above.
+* [Libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
+* [Microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
+* [Zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
+
+To test the code:
+* [Google Test](https://github.com/google/googletest) (package `googletest` on Ubuntu)
+
+The following dependency needs to be available at runtime:
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -50,13 +56,13 @@ In the worse case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
-`kiwix-lib` directory as install prefix.
+`libkiwix` directory as install prefix.
 
 Environment
 -------------
 
-The Kiwix library builds using [Meson](https://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+The Libkiwix builds using [Meson](https://mesonbuild.com/) version
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
@@ -71,7 +77,7 @@ section.
 Compilation
 -----------
 
-Once all dependencies are installed, you can compile the Kiwix library
+Once all dependencies are installed, you can compile the Libkiwix
 with:
 ```bash
 meson . build
@@ -79,7 +85,7 @@ ninja -C build
 ```
 
 By default, it will compile dynamic linked libraries. All binary files
-will be created in the "build" directory created automatically by
+will be created in the `build` directory created automatically by
 Meson. If you want statically linked libraries, you can add
 `--default-library=static` option to the Meson command.
 
@@ -97,7 +103,7 @@ meson test
 Installation
 ------------
 
-If you want to install the Kiwix library and the headers you just have
+If you want to install the Libkiwix and the headers you just have
 compiled on your system, here we go:
 ```bash
 ninja -C build install
@@ -140,6 +146,10 @@ cp ninja ../bin
 cd ..
 ```
 
+If you compile manually Libmicrohttpd, you might need to compile it
+without GNU TLS, a bug here will empeach further compilation
+otherwise.
+
 If the compilation still fails, you might need to get a more recent
 version of a dependency than the one packaged by your Linux
 distribution. Try then with a source tarball distributed by the

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,10 +26,10 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '9.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
-        url 'https://github.com/kiwix/kiwix-lib'
+        url 'https://github.com/kiwix/libkiwix'
         licenses {
             license {
                 name 'GPLv3'
@@ -44,9 +44,9 @@ task writePom {
           }
       }
       scm {
-          connection 'https://github.com/kiwix/kiwix-lib.git'
-          developerConnection 'https://github.com/kiwix/kiwix-lib.git'
-          url 'https://github.com/kiwix/kiwix-lib'
+          connection 'https://github.com/kiwix/libkiwix.git'
+          developerConnection 'https://github.com/kiwix/libkiwix.git'
+          url 'https://github.com/kiwix/libkiwix'
       }
     }
   }.withXml {

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,46 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+ meson,
+ pkg-config,
+ libzim-dev (>= 6.1.8),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/libkiwix
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 6.0.0),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev,
+ libmicrohttpd-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix10
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
+Conflicts: libkiwix0, libkiwix3, libkiwix9
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix-dev.manpages

@@ -0,0 +1 @@
+usr/share/man/man1/kiwix-compile-resources.1*

debian/libkiwix10.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

include/book.h

@@ -59,6 +59,7 @@ class Book
   const std::string& getDate() const { return m_date; }
   const std::string& getUrl() const { return m_url; }
   const std::string& getName() const { return m_name; }
+  std::string getCategory() const;
   const std::string& getTags() const { return m_tags; }
   std::string getTagStr(const std::string& tagName) const;
   bool getTagBool(const std::string& tagName) const;
@@ -94,6 +95,9 @@ class Book
   void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
   void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
 
+ private:
+  std::string getCategoryFromTags() const;
+
  protected:
   std::string m_id;
   std::string m_downloadId;
@@ -101,6 +105,7 @@ class Book
   bool m_pathValid = false;
   std::string m_title;
   std::string m_description;
+  std::string m_category;
   std::string m_language;
   std::string m_creator;
   std::string m_publisher;

include/downloader.h

@@ -23,7 +23,6 @@
 #include <string>
 #include <vector>
 #include <map>
-#include <pthread.h>
 #include <memory>
 #include <stdexcept>
 

include/entry.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -21,7 +21,8 @@
 #define KIWIX_ENTRY_H
 
 #include <stdio.h>
-#include <zim/article.h>
+#include <zim/entry.h>
+#include <zim/item.h>
 #include <exception>
 #include <string>
 
@@ -41,19 +42,12 @@ class NoEntry : public std::exception {};
 class Entry
 {
   public:
-    /**
-     * Default constructor.
-     *
-     * Construct an invalid entry.
-     */
-    Entry() = default;
-
     /**
      * Construct an entry making reference to an zim article.
      *
      * @param article a zim::Article object
      */
-    Entry(zim::Article article);
+    Entry(zim::Entry entry);
     virtual ~Entry() = default;
 
     /**
@@ -63,14 +57,14 @@ class Entry
      *
      * @return the path of the entry.
      */
-    std::string getPath() const;
+    std::string getPath() const { return entry.getPath(); }
 
     /**
      * Get the title of the entry.
      *
      * @return the title of the entry.
      */
-    std::string getTitle() const;
+    std::string getTitle() const { return entry.getTitle(); }
 
     /**
      * Get the content of the entry.
@@ -80,7 +74,7 @@ class Entry
      *
      * @return the content of the entry.
      */
-    std::string getContent() const;
+    std::string getContent() const { return entry.getItem().getData(); }
 
     /**
      * Get the blob of the entry.
@@ -90,7 +84,7 @@ class Entry
      * @param offset The starting offset of the blob.
      * @return the blob of the entry.
      */
-    zim::Blob   getBlob(offset_type offset = 0) const;
+    zim::Blob   getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
 
     /**
      * Get the blob of the entry.
@@ -101,7 +95,7 @@ class Entry
      * @param size The size of the blob.
      * @return the blob of the entry.
      */
-    zim::Blob   getBlob(offset_type offset, size_type size) const;
+    zim::Blob   getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
 
     /**
      * Get the info for direct access to the content of the entry.
@@ -109,7 +103,7 @@ class Entry
      * Some entry (ie binary ones) have their content plain stored
      * in the zim file. Knowing the offset where the content is stored
      * an user can directly read the content in the zim file bypassing the
-     * kiwix-lib/libzim.
+     * libkiwix/libzim.
      *
      * @return A pair specifying where to read the content.
      *         The string is the real file to read (may be different that .zim
@@ -117,7 +111,7 @@ class Entry
      *         The offset is the offset to read in the file.
      *         Return <"",0> if is not possible to read directly.
      */
-    std::pair<std::string, offset_type> getDirectAccessInfo() const;
+    zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
 
     /**
      * Get the size of the entry.
@@ -174,17 +168,14 @@ class Entry
     Entry getFinalEntry() const;
 
     /**
-     * Convert the entry to a boolean value.
+     * Get the zim entry wrapped by this (kiwix) entry
      *
-     * @return True if the entry is valid.
+     * @return the zim entry
      */
-    explicit operator bool() const { return good(); }
+    const zim::Entry& getZimEntry() const { return entry; }
 
   private:
-    zim::Article article;
-    mutable zim::Article final_article;
-
-    bool good() const { return article.good(); }
+    zim::Entry entry;
 };
 
 }

include/library.h

@@ -35,6 +35,7 @@ namespace kiwix
 {
 
 class OPDSDumper;
+class Library;
 
 enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
 enum supportedListMode {
@@ -48,18 +49,23 @@ enum supportedListMode {
 };
 
 class Filter {
-  private:
+  public: // types
+    using Tags = std::vector<std::string>;
+
+  private: // data
     uint64_t activeFilters;
-    std::vector<std::string> _acceptTags;
-    std::vector<std::string> _rejectTags;
+    Tags _acceptTags;
+    Tags _rejectTags;
+    std::string _category;
     std::string _lang;
     std::string _publisher;
     std::string _creator;
     size_t _maxSize;
     std::string _query;
+    bool _queryIsPartial;
     std::string _name;
 
-  public:
+  public: // functions
     Filter();
     ~Filter() = default;
 
@@ -93,16 +99,42 @@ class Filter {
     /**
      * Set the filter to only accept book with corresponding tag.
      */
-    Filter& acceptTags(std::vector<std::string> tags);
-    Filter& rejectTags(std::vector<std::string> tags);
+    Filter& acceptTags(const Tags& tags);
+    Filter& rejectTags(const Tags& tags);
 
+    Filter& category(std::string category);
     Filter& lang(std::string lang);
     Filter& publisher(std::string publisher);
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
-    Filter& query(std::string query);
+    Filter& query(std::string query, bool partial=true);
     Filter& name(std::string name);
 
+    bool hasQuery() const;
+    const std::string& getQuery() const { return _query; }
+    bool queryIsPartial() const { return _queryIsPartial; }
+
+    bool hasName() const;
+    const std::string& getName() const { return _name; }
+
+    bool hasCategory() const;
+    const std::string& getCategory() const { return _category; }
+
+    bool hasLang() const;
+    const std::string& getLang() const { return _lang; }
+
+    bool hasPublisher() const;
+    const std::string& getPublisher() const { return _publisher; }
+
+    bool hasCreator() const;
+    const std::string& getCreator() const { return _creator; }
+
+    const Tags& getAcceptTags() const { return _acceptTags; }
+    const Tags& getRejectTags() const { return _rejectTags; }
+
+private: // functions
+    friend class Library;
+
     bool accept(const Book& book) const;
 };
 
@@ -115,11 +147,24 @@ class Library
   std::map<std::string, kiwix::Book> m_books;
   std::map<std::string, std::shared_ptr<Reader>> m_readers;
   std::vector<kiwix::Bookmark> m_bookmarks;
+  class BookDB;
+  std::unique_ptr<BookDB> m_bookDB;
+
+ public:
+  typedef std::vector<std::string> BookIdCollection;
 
  public:
   Library();
   ~Library();
 
+  /**
+   * Library is not a copiable object. However it can be moved.
+   */
+  Library(const Library& ) = delete;
+  Library(Library&& );
+  void operator=(const Library& ) = delete;
+  Library& operator=(Library&& );
+
   /**
    * Add a book to the library.
    *
@@ -148,7 +193,9 @@ class Library
    */
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
+  const Book& getBookById(const std::string& id) const;
   Book& getBookById(const std::string& id);
+  const Book& getBookByPath(const std::string& path) const;
   Book& getBookByPath(const std::string& path);
   std::shared_ptr<Reader> getReaderById(const std::string& id);
 
@@ -166,7 +213,7 @@ class Library
    * @param path the path of the file to write to.
    * @return True if the library has been correctly saved.
    */
-  bool writeToFile(const std::string& path);
+  bool writeToFile(const std::string& path) const;
 
   /**
    * Write the library bookmarks to a file.
@@ -174,7 +221,7 @@ class Library
    * @param path the path of the file to write to.
    * @return True if the library has been correctly saved.
    */
-  bool writeBookmarksToFile(const std::string& path);
+  bool writeBookmarksToFile(const std::string& path) const;
 
   /**
    * Get the number of book in the library.
@@ -183,42 +230,49 @@ class Library
    * @param remoteBooks If we must count remote books (books with an url)
    * @return The number of books.
    */
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
 
   /**
-   * Get all langagues of the books in the library.
+   * Get all languagues of the books in the library.
    *
    * @return A list of languages.
    */
-  std::vector<std::string> getBooksLanguages();
+  std::vector<std::string> getBooksLanguages() const;
+
+  /**
+   * Get all categories of the books in the library.
+   *
+   * @return A list of categories.
+   */
+  std::vector<std::string> getBooksCategories() const;
 
   /**
    * Get all book creators of the books in the library.
    *
    * @return A list of book creators.
    */
-  std::vector<std::string> getBooksCreators();
+  std::vector<std::string> getBooksCreators() const;
 
   /**
    * Get all book publishers of the books in the library.
    *
    * @return A list of book publishers.
    */
-  std::vector<std::string> getBooksPublishers();
+  std::vector<std::string> getBooksPublishers() const;
 
   /**
    * Get all bookmarks.
    *
    * @return A list of bookmarks
    */
-  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true) const;
 
   /**
    * Get all book ids of the books in the library.
    *
    * @return A list of book ids.
    */
-  std::vector<std::string> getBooksIds();
+  BookIdCollection getBooksIds() const;
 
   /**
    * Filter the library and generate a new one with the keep elements.
@@ -228,7 +282,7 @@ class Library
    * @param search List only books with search in the title or description.
    * @return The list of bookIds corresponding to the query.
    */
-  DEPRECATED std::vector<std::string> filter(const std::string& search);
+  DEPRECATED BookIdCollection filter(const std::string& search) const;
 
 
   /**
@@ -237,7 +291,7 @@ class Library
    * @param filter The filter to use.
    * @return The list of bookIds corresponding to the filter.
    */
-  std::vector<std::string> filter(const Filter& filter);
+  BookIdCollection filter(const Filter& filter) const;
 
 
   /**
@@ -247,7 +301,7 @@ class Library
    * @param comparator how to sort the books
    * @return The sorted list of books
    */
-  void sort(std::vector<std::string>& bookIds, supportedListSortBy sortBy, bool ascending);
+  void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
 
   /**
    * List books in the library.
@@ -271,7 +325,7 @@ class Library
    *                Set to 0 to cancel this filter.
    * @return The list of bookIds corresponding to the query.
    */
-  DEPRECATED std::vector<std::string> listBooksIds(
+  DEPRECATED BookIdCollection listBooksIds(
     int supportedListMode = ALL,
     supportedListSortBy sortBy = UNSORTED,
     const std::string& search = "",
@@ -279,11 +333,16 @@ class Library
     const std::string& creator = "",
     const std::string& publisher = "",
     const std::vector<std::string>& tags = {},
-    size_t maxSize = 0);
+    size_t maxSize = 0) const;
 
   friend class OPDSDumper;
   friend class libXMLDumper;
+
+private: // functions
+  BookIdCollection filterViaBookDB(const Filter& filter) const;
+  void updateBookDB(const Book& book);
 };
+
 }
 
 #endif

include/libxml_dumper.h

@@ -38,7 +38,7 @@ class LibXMLDumper
 {
  public:
   LibXMLDumper() = default;
-  LibXMLDumper(Library* library);
+  LibXMLDumper(const Library* library);
   ~LibXMLDumper();
 
   /**
@@ -69,10 +69,10 @@ class LibXMLDumper
    *
    * @param library The library to dump.
    */
-  void setLibrary(Library* library) { this->library = library; }
+  void setLibrary(const Library* library) { this->library = library; }
 
  protected:
-  kiwix::Library* library;
+  const kiwix::Library* library;
   std::string baseDir;
  private:
   void handleBook(Book book, pugi::xml_node root_node);

include/meson.build

@@ -25,7 +25,6 @@ install_headers(
   'tools/pathTools.h',
   'tools/regexTools.h',
   'tools/stringTools.h',
-  'tools/lock.h',
   subdir:'kiwix/tools'
 )
 

include/opds_dumper.h

@@ -51,24 +51,35 @@ class OPDSDumper
   /**
    * Dump the OPDS feed.
    *
-   * @param id The id of the library.
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
    * @return The OPDS feed.
    */
-  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds);
+  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const;
 
   /**
-   * Set the id of the opds stream.
+   * Dump the OPDS feed.
+   *
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
+   * @return The OPDS feed.
+   */
+  std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const;
+
+  /**
+   * Dump the categories OPDS feed.
+   *
+   * @param categories list of category names
+   * @return The OPDS feed.
+   */
+  std::string categoriesOPDSFeed(const std::vector<std::string>& categories) const;
+
+  /**
+   * Set the id of the library.
    *
    * @param id the id to use.
    */
-  void setId(const std::string& id) { this->id = id;}
-
-  /**
-   * Set the title oft the opds stream.
-   *
-   * @param title the title to use.
-   */
-  void setTitle(const std::string& title) { this->title = title; }
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
 
   /**
    * Set the root location used when generating url.
@@ -77,13 +88,6 @@ class OPDSDumper
    */
   void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
 
-  /**
-   * Set the search url.
-   *
-   * @param searchUrl the search url to use.
-   */
-  void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; }
-
   /**
    * Set some informations about the search results.
    *
@@ -93,27 +97,13 @@ class OPDSDumper
    */
   void setOpenSearchInfo(int totalResult, int startIndex, int count);
 
-  /**
-   * Set the library to dump.
-   *
-   * @param library The library to dump.
-   */
-  void setLibrary(Library* library) { this->library = library; }
-
  protected:
   kiwix::Library* library;
-  std::string id;
-  std::string title;
-  std::string date;
+  std::string libraryId;
   std::string rootLocation;
-  std::string searchDescriptionUrl;
   int m_totalResults;
   int m_startIndex;
   int m_count;
-  bool m_isSearchResult = false;
-
- private:
-  pugi::xml_node handleBook(Book book, pugi::xml_node root_node);
 };
 }
 

include/reader.h

@@ -21,10 +21,8 @@
 #define KIWIX_READER_H
 
 #include <stdio.h>
-#include <zim/article.h>
-#include <zim/file.h>
-#include <zim/fileiterator.h>
 #include <zim/zim.h>
+#include <zim/archive.h>
 #include <exception>
 #include <map>
 #include <sstream>
@@ -39,10 +37,47 @@ using namespace std;
 namespace kiwix
 {
 
+/**
+ * The SuggestionItem is a helper class that contains the info about a single
+ * suggestion item.
+ */
+
+class SuggestionItem
+{
+  // Functions
+  private:
+    // Create a sugggestion item.
+    explicit SuggestionItem(std::string title, std::string normalizedTitle,
+                            std::string path, std::string snippet = "") :
+      title(title),
+      normalizedTitle(normalizedTitle),
+      path(path),
+      snippet(snippet) {}
+
+  public:
+    const std::string getTitle()           {return title;}
+    const std::string getNormalizedTitle() {return normalizedTitle;}
+    const std::string getPath()            {return path;}
+    const std::string getSnippet()         {return snippet;}
+
+    const bool hasSnippet()                {return !snippet.empty();}
+
+  // Data
+  private:
+    std::string title;
+    std::string normalizedTitle;
+    std::string path;
+    std::string snippet;
+
+  friend class Reader;
+};
+
 /**
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
  */
+
+using SuggestionsList_t = std::vector<SuggestionItem>;
 class Reader
 {
  public:
@@ -55,8 +90,12 @@ class Reader
    *                    unsplitted path as if the file were not splitted
    *                    (.zim extesion).
    */
-  Reader(const string zimFilePath);
-  ~Reader();
+  explicit Reader(const string zimFilePath);
+#ifndef _WIN32
+  explicit Reader(int fd);
+  Reader(int fd, zim::offset_type offset, zim::size_type size);
+#endif
+  ~Reader() = default;
 
   /**
    * Get the number of "displayable" entries in the zim file.
@@ -99,17 +138,6 @@ class Reader
    */
   string getId() const;
 
-  /**
-   * Get the url of a random page.
-   *
-   * Deprecated : Use `getRandomPage` instead.
-   *
-   * @return Url of a random page. The page is picked from all entries in
-   *         the 'A' namespace.
-   *         The main page is excluded from the potential results.
-   */
-  DEPRECATED string getRandomPageUrl() const;
-
   /**
    * Get a random page.
    *
@@ -119,31 +147,6 @@ class Reader
    */
   Entry getRandomPage() const;
 
-  /**
-   * Get the url of the first page.
-   *
-   * Deprecated : Use `getFirstPage` instead.
-   *
-   * @return Url of the first entry in the 'A' namespace.
-   */
-  DEPRECATED string getFirstPageUrl() const;
-
-  /**
-   * Get the entry of the first page.
-   *
-   * @return The first entry in the 'A' namespace.
-   */
-  Entry getFirstPage() const;
-
-  /**
-   * Get the url of the main page.
-   *
-   * Deprecated : Use `getMainPage` instead.
-   *
-   * @return Url of the main page as specified in the zim file.
-   */
-  DEPRECATED string getMainPageUrl() const;
-
   /**
    * Get the entry of the main page.
    *
@@ -332,93 +335,15 @@ class Reader
    */
   Entry getEntryFromTitle(const std::string& title) const;
 
-  /**
-   * Get the url of a page specified by a title.
-   *
-   * @param[in] title the title of the page.
-   * @param[out] url the url of the page.
-   * @return True if the page can be found.
-   */
-  DEPRECATED bool getPageUrlFromTitle(const string& title, string& url) const;
-
-  /**
-   * Get the mimetype of a entry specified by a url.
-   *
-   * @param[in] url the url of the entry.
-   * @param[out] mimeType the mimeType of the entry.
-   * @return True if the mimeType has been found.
-   */
-  DEPRECATED bool getMimeTypeByUrl(const string& url, string& mimeType) const;
-
-  /**
-   * Get the content of an entry specifed by a url.
-   *
-   * Alias to `getContentByEncodedUrl`
-   */
-  DEPRECATED bool getContentByUrl(const string& url,
-                       string& content,
-                       string& title,
-                       unsigned int& contentLength,
-                       string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url encoded url.
-   *
-   * Equivalent to getContentByDecodedUrl(urlDecode(url), ...).
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-
-  /**
-   * Get the content of an entry specified by an url encoded url.
-   *
-   * Equivalent to getContentByEncodedUrl but without baseUrl.
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * @param[in] url The url of the entry.
-   * @param[out] content The content of the entry.
-   * @param[out] title the title of the entry.
-   * @param[out] contentLength The size of the entry (size of content).
-   * @param[out] contentType The mimeType of the entry.
-   * @param[out] baseUrl Return the true url of the entry.
-   *                     If the specified entry is a redirection, contains
-   *                     the url of the targeted entry.
-   * @return True if the entry has been found.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * Equivalent to getContentByDecodedUrl but withou the baseUrl.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-
   /**
    * Search for entries with title starting with prefix (case sensitive).
    *
    * Suggestions are stored in an internal vector and can be retrieved using
    * `getNextSuggestion` method.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestions(const string& prefix,
+   *                        unsigned int suggestionsCount,
+   *                        SuggestionsList_t& results);
    *
    * @param prefix The prefix to search.
    * @param suggestionsCount How many suggestions to search for.
@@ -426,12 +351,49 @@ class Reader
    *              If false, add suggestions to the internal vector
    *              (until internal vector size is suggestionCount (or no more
    *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
+   * @return True if some suggestions have been added to the internal vector.
    */
-  bool searchSuggestions(const string& prefix,
+  DEPRECATED bool searchSuggestions(const string& prefix,
                          unsigned int suggestionsCount,
                          const bool reset = true);
 
+  /**
+   * Search for entries with title starting with prefix (case sensitive).
+   *
+   * Suggestions are added to the `result` vector.
+   *
+   * @param prefix The prefix to search.
+   * @param suggestionsCount How many suggestions to search for.
+   * @param result The vector where to store the suggestions.
+   * @return True if some suggestions have been added to the vector.
+   */
+
+  bool searchSuggestions(const string& prefix,
+                         unsigned int suggestionsCount,
+                         SuggestionsList_t& resuls);
+
+  /**
+   * Search for entries for the given prefix.
+   *
+   * If the zim file has a internal fulltext index, the suggestions will be
+   * searched using it.
+   * Else the suggestions will be search using `searchSuggestions` while trying
+   * to be smart about case sensitivity (using `getTitleVariants`).
+   *
+   * In any case, suggestions are stored in an internal vector and can be
+   * retrieved using `getNextSuggestion` method.
+   * The internal vector will be reset.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestionsSmart(const string& prefix,
+   *                             unsigned int suggestionsCount,
+   *                             SuggestionsList_t& results);
+   *
+   * @param prefix The prefix to search for.
+   * @param suggestionsCount How many suggestions to search for.
+   */
+  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount);
+
   /**
    * Search for entries for the given prefix.
    *
@@ -446,19 +408,13 @@ class Reader
    *
    * @param prefix The prefix to search for.
    * @param suggestionsCount How many suggestions to search for.
+   * @param results The vector where to store the suggestions
+   * @return True if some suggestions have been added to the results.
    */
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
+   bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount,
+                              SuggestionsList_t& results);
 
-  /**
-   * Check if the url exists in the zim file.
-   *
-   * Deprecated : Use `pathExists` instead.
-   *
-   * @param url the url to check.
-   * @return True if the url exits in the zim file.
-   */
-  DEPRECATED bool urlExists(const string& url) const;
 
   /**
    * Check if the path exists in the zim file.
@@ -490,7 +446,7 @@ class Reader
    * @param[out] title the title of the suggestion.
    * @return True if title has been set.
    */
-  bool getNextSuggestion(string& title);
+  DEPRECATED bool getNextSuggestion(string& title);
 
   /**
    * Get the next suggestion title and url.
@@ -499,7 +455,7 @@ class Reader
    * @param[out] url the url of the suggestion.
    * @return True if title and url have been set.
    */
-  bool getNextSuggestion(string& title, string& url);
+  DEPRECATED bool getNextSuggestion(string& title, string& url);
 
   /**
    * Get if we can check zim file integrity (has a checksum).
@@ -515,16 +471,6 @@ class Reader
    */
   bool isCorrupted() const;
 
-  /**
-   * Parse a full url into a namespace and url.
-   *
-   * @param[in] url The full url ("/N/url").
-   * @param[out] ns The namespace (N).
-   * @param[out] title The url (url).
-   * @return True
-   */
-  DEPRECATED bool parseUrl(const string& url, char* ns, string& title) const;
-
   /**
    * Return the total size of the zim file.
    *
@@ -539,28 +485,14 @@ class Reader
    *
    * @return The libzim file handler.
    */
-  zim::File* getZimFileHandler() const;
-
-  /**
-   * Get the zim article object associated to a url.
-   *
-   * @param[in] url The url of the article.
-   * @param[out] article The libzim article object.
-   * @return True if the url is good (article.good()).
-   */
-  DEPRECATED bool getArticleObjectByDecodedUrl(const string& url,
-                                    zim::Article& article) const;
+  zim::Archive* getZimArchive() const;
 
  protected:
-  zim::File* zimFileHandler;
-  zim::size_type firstArticleOffset;
-  zim::size_type lastArticleOffset;
-  zim::size_type nsACount;
-  zim::size_type nsICount;
+  std::unique_ptr<zim::Archive> zimArchive;
   std::string zimFilePath;
 
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
+  SuggestionsList_t suggestions;
+  SuggestionsList_t::iterator suggestionsOffset;
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;

include/search_renderer.h

@@ -60,6 +60,13 @@ class SearchRenderer
    */
   void setSearchProtocolPrefix(const std::string& prefix);
 
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
   /**
    * Generate the html page with the resutls of the search.
    */
@@ -73,10 +80,9 @@ class SearchRenderer
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
+  unsigned int pageLength;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
-  unsigned int resultEnd;
 };
 
 

include/searcher.h

@@ -27,11 +27,10 @@
 #include <cctype>
 #include <locale>
 #include <string>
-#include <vector>
+#include <memory>
 #include <vector>
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
-#include "kiwix_config.h"
 
 using namespace std;
 
@@ -49,7 +48,7 @@ class Result
   virtual std::string get_content() = 0;
   virtual int get_wordCount() = 0;
   virtual int get_size() = 0;
-  virtual int get_readerIndex() = 0;
+  virtual std::string get_zimId() = 0;
 };
 
 struct SearcherInternal;
@@ -155,7 +154,7 @@ class Searcher
                      const bool verbose = false);
 
   std::vector<Reader*> readers;
-  SearcherInternal* internal;
+  std::unique_ptr<SearcherInternal> internal;
   std::string searchPattern;
   unsigned int estimatedResultCount;
   unsigned int resultStart;

include/tools/lock.h

@@ -1,46 +0,0 @@
-
-
-#ifndef KIWIXLIB_TOOL_LOCK_H
-#define KIWIXLIB_TOOL_LOCK_H
-
-#include <pthread.h>
-
-namespace kiwix {
-
-class Lock
-{
-  public:
-    explicit Lock(pthread_mutex_t* mutex) :
-      mp_mutex(mutex)
-    {
-      pthread_mutex_lock(mp_mutex);
-    }
-    ~Lock() {
-      if (mp_mutex != nullptr) {
-        pthread_mutex_unlock(mp_mutex);
-      }
-    }
-    Lock(Lock && other) :
-      mp_mutex(other.mp_mutex)
-    {
-      other.mp_mutex = nullptr;
-    }
-    Lock & operator=(Lock && other)
-    {
-      mp_mutex = other.mp_mutex;
-      other.mp_mutex = nullptr;
-      return *this;
-    }
-
-
-  private:
-    pthread_mutex_t* mp_mutex;
-
-    Lock(Lock const &) = delete;
-    Lock & operator=(Lock const &) = delete;
-};
-
-
-}
-
-#endif //KIWIXLIB_TOOL_LOCK_H

include/tools/otherTools.h

@@ -22,6 +22,9 @@
 
 #include <string>
 #include <vector>
+#include <map>
+#include <zim/zim.h>
+#include <mustache.hpp>
 
 namespace pugi {
   class xml_node;
@@ -41,6 +44,13 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
+  using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
+  MimeCounterType parseMimetypeCounter(const std::string& counterData);
+
+  std::string gen_date_str();
+  std::string gen_uuid(const std::string& s);
+
+  std::string render_template(const std::string& template_str, kainjow::mustache::data data);
 }
 
 #endif

include/tools/regexTools.h

@@ -29,5 +29,8 @@ std::string replaceRegex(const std::string& content,
 std::string appendToFirstOccurence(const std::string& content,
                                    const std::string& regex,
                                    const std::string& replacement);
+std::string prependToFirstOccurence(const std::string& content,
+                                    const std::string& regex,
+                                    const std::string& replacement);
 
 #endif

include/tools/stringTools.h

@@ -43,7 +43,7 @@ void loadICUExternalTables();
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
 std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);

meson.build

@@ -1,43 +1,68 @@
-project('kiwix-lib', 'cpp',
-  version : '9.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
-  license : 'GPL',
+project('libkiwix', 'cpp',
+  version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
 wrapper = get_option('wrapper')
 
-static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
-if 'android' in wrapper
+static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
+if wrapper.contains('android')
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
-if 'java' in wrapper
+if wrapper.contains('java')
   add_languages('java')
 endif
 
-thread_dep = dependency('threads')
+# See https://github.com/kiwix/libkiwix/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
+  extra_libs += '-latomic'
+endif
+
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
+  # C++ std::thread is implemented using pthread on linux by gcc
+  thread_dep = dependency('threads')
+else
+  thread_dep = dependency('', required:false)
+endif
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+zlib_dep = dependency('zlib', static:static_deps)
+xapian_dep = dependency('xapian-core', static:static_deps)
 
-if not compiler.has_header('mustache.hpp')
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
   error('Cannot found header mustache.hpp')
 endif
 
+libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
+if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
+  error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
+endif
+
+
 extra_cflags = ''
 if target_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
+if target_machine.system() == 'windows'
+  add_project_arguments('-DNOMINMAX', language: 'cpp')
+endif
 
-inc = include_directories('include')
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
+
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
@@ -54,7 +79,7 @@ subdir('static')
 subdir('src')
 subdir('test')
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/meson.build

@@ -2,3 +2,5 @@
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')

src/aria2.cpp

@@ -19,6 +19,11 @@
 #endif
 
 
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+  }
 
 namespace kiwix {
 
@@ -26,8 +31,8 @@ Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
   m_secret("kiwixariarpc"),
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
+  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
+  mp_curl(nullptr)
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -58,6 +63,7 @@ Aria2::Aria2():
     // Try to use a potential installed aria2c.
     callCmd.push_back(ARIA2_CMD);
   }
+  callCmd.push_back("--follow-metalink=mem");
   callCmd.push_back("--enable-rpc");
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
@@ -84,27 +90,21 @@ Aria2::Aria2():
   }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
-  char errbuf[CURL_ERROR_SIZE];
+
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
-    errbuf[0] = 0;
+    m_curlErrorBuffer[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
     } else if (watchdog == 1) {
-      std::cerr <<" curl_easy_perform() failed." << std::endl;
-      fprintf(stderr, "\nlibcurl: (%d) ", res);
-      if (errbuf[0] != 0) {
-        std::cerr << errbuf << std::endl;
-      } else {
-        std::cerr << curl_easy_strerror(res) << std::endl;
-      }
+      LOG_ARIA_ERROR();
     }
   }
   if (!watchdog) {
@@ -115,6 +115,7 @@ Aria2::Aria2():
 
 Aria2::~Aria2()
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   curl_easy_cleanup(mp_curl);
 }
 
@@ -126,38 +127,44 @@ void Aria2::close()
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
+  auto outStream = static_cast<std::stringstream*>(userdata);
+  outStream->write(ptr, nmemb);
   return nmemb;
 }
 
 std::string Aria2::doRequest(const MethodCall& methodCall)
 {
-  pthread_mutex_lock(&m_lock);
   auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
+  std::stringstream outStream;
   CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
-  res = curl_easy_perform(mp_curl);
-  if (res == CURLE_OK) {
-    long response_code;
+  long response_code;
+  {
+    std::unique_lock<std::mutex> lock(m_lock);
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
+    m_curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(mp_curl);
+    if (res != CURLE_OK) {
+      LOG_ARIA_ERROR();
+      throw std::runtime_error("Cannot perform request");
+    }
     curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
-    pthread_mutex_unlock(&m_lock);
-    if (response_code != 200) {
-      throw std::runtime_error("Invalid return code from aria");
-    }
-    auto responseContent = stringstream.str();
-    MethodResponse response(responseContent);
-    if (response.isFault()) {
-      throw AriaError(response.getFault().getFaultString());
-    }
-    return responseContent;
   }
-  pthread_mutex_unlock(&m_lock);
-  throw std::runtime_error("Cannot perform request");
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
 }
 
 std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)

src/aria2.h

@@ -12,8 +12,8 @@
 #include "xmlrpc.h"
 
 #include <memory>
+#include <mutex>
 #include <curl/curl.h>
-#include <pthread.h>
 
 namespace kiwix {
 
@@ -24,8 +24,9 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
+    std::unique_ptr<char[]> m_curlErrorBuffer;
     CURL* mp_curl;
-    pthread_mutex_t m_lock;
+    std::mutex m_lock;
 
     std::string doRequest(const MethodCall& methodCall);
 

src/book.cpp

@@ -61,6 +61,7 @@ bool Book::update(const kiwix::Book& other)
   m_name = other.m_name;
   m_flavour = other.m_flavour;
   m_tags = other.m_tags;
+  m_category = other.m_category;
   m_origId = other.m_origId;
   m_articleCount = other.m_articleCount;
   m_mediaCount = other.m_mediaCount;
@@ -88,6 +89,7 @@ void Book::update(const kiwix::Reader& reader)
   m_name = reader.getName();
   m_flavour = reader.getFlavour();
   m_tags = reader.getTags();
+  m_category = getCategoryFromTags();
   m_origId = reader.getOrigId();
   m_articleCount = reader.getArticleCount();
   m_mediaCount = reader.getMediaCount();
@@ -127,6 +129,8 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
+  const auto catattr = node.attribute("category");
+  m_category = catattr.empty() ? getCategoryFromTags() : catattr.value();
 }
 #undef ATTR
 
@@ -156,6 +160,8 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_name = VALUE("name");
   m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
+  const auto catnode = node.child("category");
+  m_category = catnode.empty() ? getCategoryFromTags() : catnode.child_value();
   m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
   m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
@@ -220,4 +226,21 @@ bool Book::getTagBool(const std::string& tagName) const {
   return convertStrToBool(getTagStr(tagName));
 }
 
+std::string Book::getCategory() const
+{
+  return m_category;
+}
+
+std::string Book::getCategoryFromTags() const
+{
+  try
+  {
+    return getTagStr("category");
+  }
+  catch ( const std::out_of_range& )
+  {
+    return "";
+  }
+}
+
 }

src/downloader.cpp

@@ -133,7 +133,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what();
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
   }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
@@ -141,7 +141,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellWaiting failed : " << e.what();
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
 }
 

src/entry.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -20,121 +20,54 @@
 #include "reader.h"
 #include <time.h>
 
-#include <zim/search.h>
-
 namespace kiwix
 {
 
-Entry::Entry(zim::Article article)
-  : article(article)
+Entry::Entry(zim::Entry entry)
+  : entry(entry)
 {
 }
 
-#define RETURN_IF_INVALID(WHAT) if(!good()) { return (WHAT); }
-
-std::string Entry::getPath() const
-{
-  RETURN_IF_INVALID("");
-  return article.getLongUrl();
-}
-
-std::string Entry::getTitle() const
-{
-  RETURN_IF_INVALID("");
-  return article.getTitle();
-}
-
-std::string Entry::getContent() const
-{
-  RETURN_IF_INVALID("");
-  return article.getData();
-}
-
-zim::Blob Entry::getBlob(offset_type offset) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset);
-}
-
-zim::Blob Entry::getBlob(offset_type offset, size_type size) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset, size);
-}
-
-std::pair<std::string, offset_type> Entry::getDirectAccessInfo() const
-{
-  RETURN_IF_INVALID(std::make_pair("", 0));
-  return article.getDirectAccessInformation();
-}
-
 size_type Entry::getSize() const
 {
-  RETURN_IF_INVALID(0);
-  return article.getArticleSize();
+  if (entry.isRedirect()) {
+    return 0;
+  } else {
+    return entry.getItem().getSize();
+  }
 }
 
 std::string Entry::getMimetype() const
 {
-  RETURN_IF_INVALID("");
-  try {
-    return article.getMimeType();
-  } catch (exception& e) {
-    return "application/octet-stream";
-  }
+  return entry.getItem(true).getMimetype();
 }
 
 bool Entry::isRedirect() const
 {
-  RETURN_IF_INVALID(false);
-  return article.isRedirect();
-}
-
-bool Entry::isLinkTarget() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isLinktarget();
-}
-
-bool Entry::isDeleted() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isDeleted();
+  return entry.isRedirect();
 }
 
 Entry Entry::getRedirectEntry() const
 {
-  RETURN_IF_INVALID(Entry());
-  if ( !article.isRedirect() ) {
+  if ( !entry.isRedirect() ) {
     throw NoEntry();
   }
 
-  auto targeted_article = article.getRedirectArticle();
-  if ( !targeted_article.good()) {
-    throw NoEntry();
-  }
-  return targeted_article;
+  return entry.getRedirectEntry();
 }
 
 Entry Entry::getFinalEntry() const
 {
-  RETURN_IF_INVALID(Entry());
-  if (final_article.good()) {
-    return final_article;
-  }
   int loopCounter = 42;
-  final_article = article;
-  while (final_article.isRedirect() && loopCounter--) {
-    final_article = final_article.getRedirectArticle();
-    if ( !final_article.good()) {
-      throw NoEntry();
-    }
+  auto final_entry = entry;
+  while (final_entry.isRedirect() && loopCounter--) {
+    final_entry = final_entry.getRedirectEntry();
   }
   // Prevent infinite loops.
-  if (final_article.isRedirect()) {
+  if (final_entry.isRedirect()) {
     throw NoEntry();
   }
-  return final_article;
+  return final_entry;
 }
 
 }

src/library.cpp

@@ -30,14 +30,42 @@
 #include <pugixml.hpp>
 #include <algorithm>
 #include <set>
+#include <unicode/locid.h>
+#include <xapian.h>
 
 namespace kiwix
 {
 
+namespace
+{
+
+std::string iso639_3ToXapian(const std::string& lang) {
+  return icu::Locale(lang.c_str()).getLanguage();
+};
+
+std::string normalizeText(const std::string& text)
+{
+  return removeAccents(text);
+}
+
+} // unnamed namespace
+
+class Library::BookDB : public Xapian::WritableDatabase
+{
+public:
+  BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
+};
+
 /* Constructor */
 Library::Library()
+  : m_bookDB(new BookDB)
 {
 }
+
+Library::Library(Library&& ) = default;
+
+Library& Library::operator=(Library&& ) = default;
+
 /* Destructor */
 Library::~Library()
 {
@@ -47,6 +75,7 @@ Library::~Library()
 bool Library::addBook(const Book& book)
 {
   /* Try to find it */
+  updateBookDB(book);
   try {
     auto& oldbook = m_books.at(book.getId());
     oldbook.update(book);
@@ -77,15 +106,23 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 
 bool Library::removeBookById(const std::string& id)
 {
+  m_bookDB->delete_document("Q" + id);
+  m_readers.erase(id);
   return m_books.erase(id) == 1;
 }
 
-Book& Library::getBookById(const std::string& id)
+const Book& Library::getBookById(const std::string& id) const
 {
   return m_books.at(id);
 }
 
-Book& Library::getBookByPath(const std::string& path)
+Book& Library::getBookById(const std::string& id)
+{
+  const Library& const_self = *this;
+  return const_cast<Book&>(const_self.getBookById(id));
+}
+
+const Book& Library::getBookByPath(const std::string& path) const
 {
   for(auto& it: m_books) {
     auto& book = it.second;
@@ -97,6 +134,12 @@ Book& Library::getBookByPath(const std::string& path)
   throw std::out_of_range(ss.str());
 }
 
+Book& Library::getBookByPath(const std::string& path)
+{
+  const Library& const_self = *this;
+  return const_cast<Book&>(const_self.getBookByPath(path));
+}
+
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {
@@ -112,7 +155,7 @@ std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
-                                   const bool remoteBooks)
+                                   const bool remoteBooks) const
 {
   unsigned int result = 0;
   for (auto& pair: m_books) {
@@ -125,7 +168,7 @@ unsigned int Library::getBookCount(const bool localBooks,
   return result;
 }
 
-bool Library::writeToFile(const std::string& path)
+bool Library::writeToFile(const std::string& path) const
 {
   auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
@@ -133,13 +176,13 @@ bool Library::writeToFile(const std::string& path)
   return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
 }
 
-bool Library::writeBookmarksToFile(const std::string& path)
+bool Library::writeBookmarksToFile(const std::string& path) const
 {
   LibXMLDumper dumper(this);
   return writeTextFile(path, dumper.dumpLibXMLBookmark());
 }
 
-std::vector<std::string> Library::getBooksLanguages()
+std::vector<std::string> Library::getBooksLanguages() const
 {
   std::vector<std::string> booksLanguages;
   std::map<std::string, bool> booksLanguagesMap;
@@ -158,7 +201,22 @@ std::vector<std::string> Library::getBooksLanguages()
   return booksLanguages;
 }
 
-std::vector<std::string> Library::getBooksCreators()
+std::vector<std::string> Library::getBooksCategories() const
+{
+  std::set<std::string> categories;
+
+  for (const auto& pair: m_books) {
+    const auto& book = pair.second;
+    const auto& c = book.getCategory();
+    if ( !c.empty() ) {
+      categories.insert(c);
+    }
+  }
+
+  return std::vector<std::string>(categories.begin(), categories.end());
+}
+
+std::vector<std::string> Library::getBooksCreators() const
 {
   std::vector<std::string> booksCreators;
   std::map<std::string, bool> booksCreatorsMap;
@@ -177,7 +235,7 @@ std::vector<std::string> Library::getBooksCreators()
   return booksCreators;
 }
 
-std::vector<std::string> Library::getBooksPublishers()
+std::vector<std::string> Library::getBooksPublishers() const
 {
   std::vector<std::string> booksPublishers;
   std::map<std::string, bool> booksPublishersMap;
@@ -196,7 +254,7 @@ std::vector<std::string> Library::getBooksPublishers()
   return booksPublishers;
 }
 
-const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
 {
   if (!onlyValidBookmarks) {
     return m_bookmarks;
@@ -211,9 +269,9 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
   return validBookmarks;
 }
 
-std::vector<std::string> Library::getBooksIds()
+Library::BookIdCollection Library::getBooksIds() const
 {
-  std::vector<std::string> bookIds;
+  BookIdCollection bookIds;
 
   for (auto& pair: m_books) {
     bookIds.push_back(pair.first);
@@ -222,7 +280,7 @@ std::vector<std::string> Library::getBooksIds()
   return bookIds;
 }
 
-std::vector<std::string> Library::filter(const std::string& search)
+Library::BookIdCollection Library::filter(const std::string& search) const
 {
   if (search.empty()) {
     return getBooksIds();
@@ -232,16 +290,195 @@ std::vector<std::string> Library::filter(const std::string& search)
 }
 
 
-std::vector<std::string> Library::filter(const Filter& filter)
+void Library::updateBookDB(const Book& book)
 {
-  std::vector<std::string> bookIds;
-  for(auto& pair:m_books) {
-    auto book = pair.second;
-    if(filter.accept(book)) {
-      bookIds.push_back(pair.first);
+  Xapian::Stem stemmer;
+  Xapian::TermGenerator indexer;
+  const std::string lang = book.getLanguage();
+  try {
+    stemmer = Xapian::Stem(iso639_3ToXapian(lang));
+    indexer.set_stemmer(stemmer);
+    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
+  } catch (...) {}
+  Xapian::Document doc;
+  indexer.set_document(doc);
+
+  const std::string title = normalizeText(book.getTitle());
+  const std::string desc = normalizeText(book.getDescription());
+
+  // Index title and description without prefixes for general search
+  indexer.index_text(title);
+  indexer.increase_termpos();
+  indexer.index_text(desc);
+
+  // Index all fields for field-based search
+  indexer.index_text(title, 1, "S");
+  indexer.index_text(desc,  1, "XD");
+  indexer.index_text(lang,  1, "L");
+  indexer.index_text(normalizeText(book.getCreator()),   1, "A");
+  indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
+  indexer.index_text(normalizeText(book.getName()),      1, "XN");
+  indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
+
+  for ( const auto& tag : split(normalizeText(book.getTags()), ";") )
+    doc.add_boolean_term("XT" + tag);
+
+  const std::string idterm = "Q" + book.getId();
+  doc.add_boolean_term(idterm);
+
+  doc.set_data(book.getId());
+
+  m_bookDB->replace_document(idterm, doc);
+}
+
+namespace
+{
+
+bool willSelectEverything(const Xapian::Query& query)
+{
+  return query.get_type() == Xapian::Query::LEAF_MATCH_ALL;
+}
+
+
+Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
+{
+  if ( !filter.hasQuery() || filter.getQuery().empty() ) {
+    // This is a thread-safe way to construct an equivalent of
+    // a Xapian::Query::MatchAll query
+    return Xapian::Query(std::string());
+  }
+
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_AND);
+  queryParser.add_prefix("title", "S");
+  queryParser.add_prefix("description", "XD");
+  queryParser.add_prefix("name", "XN");
+  queryParser.add_prefix("category", "XC");
+  queryParser.add_prefix("lang", "L");
+  queryParser.add_prefix("publisher", "XP");
+  queryParser.add_prefix("creator", "A");
+  queryParser.add_prefix("tag", "XT");
+  const auto partialQueryFlag = filter.queryIsPartial()
+                              ? Xapian::QueryParser::FLAG_PARTIAL
+                              : 0;
+  // Language assumed for the query is not known for sure so stemming
+  // is not applied
+  //queryParser.set_stemmer(Xapian::Stem(iso639_3ToXapian(???)));
+  //queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_SOME);
+  const auto flags = Xapian::QueryParser::FLAG_PHRASE
+                   | Xapian::QueryParser::FLAG_BOOLEAN
+                   | Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE
+                   | Xapian::QueryParser::FLAG_LOVEHATE
+                   | Xapian::QueryParser::FLAG_WILDCARD
+                   | partialQueryFlag;
+  return queryParser.parse_query(normalizeText(filter.getQuery()), flags);
+}
+
+Xapian::Query nameQuery(const std::string& name)
+{
+  return Xapian::Query("XN" + normalizeText(name));
+}
+
+Xapian::Query categoryQuery(const std::string& category)
+{
+  return Xapian::Query("XC" + normalizeText(category));
+}
+
+Xapian::Query langQuery(const std::string& lang)
+{
+  return Xapian::Query("L" + normalizeText(lang));
+}
+
+Xapian::Query publisherQuery(const std::string& publisher)
+{
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_OR);
+  queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_NONE);
+  const auto flags = 0;
+  const auto q = queryParser.parse_query(normalizeText(publisher), flags, "XP");
+  return Xapian::Query(Xapian::Query::OP_PHRASE, q.get_terms_begin(), q.get_terms_end(), q.get_length());
+}
+
+Xapian::Query creatorQuery(const std::string& creator)
+{
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_OR);
+  queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_NONE);
+  const auto flags = 0;
+  const auto q = queryParser.parse_query(normalizeText(creator), flags, "A");
+  return Xapian::Query(Xapian::Query::OP_PHRASE, q.get_terms_begin(), q.get_terms_end(), q.get_length());
+}
+
+Xapian::Query tagsQuery(const Filter::Tags& acceptTags, const Filter::Tags& rejectTags)
+{
+  Xapian::Query q = Xapian::Query(std::string());
+  if (!acceptTags.empty()) {
+    for ( const auto& tag : acceptTags )
+      q &= Xapian::Query("XT" + normalizeText(tag));
+  }
+
+  if (!rejectTags.empty()) {
+    for ( const auto& tag : rejectTags )
+      q = Xapian::Query(Xapian::Query::OP_AND_NOT, q, "XT" + normalizeText(tag));
+  }
+  return q;
+}
+
+Xapian::Query buildXapianQuery(const Filter& filter)
+{
+  auto q = buildXapianQueryFromFilterQuery(filter);
+  if ( filter.hasName() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, nameQuery(filter.getName()));
+  }
+  if ( filter.hasCategory() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, categoryQuery(filter.getCategory()));
+  }
+  if ( filter.hasLang() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, langQuery(filter.getLang()));
+  }
+  if ( filter.hasPublisher() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, publisherQuery(filter.getPublisher()));
+  }
+  if ( filter.hasCreator() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, creatorQuery(filter.getCreator()));
+  }
+  if ( !filter.getAcceptTags().empty() || !filter.getRejectTags().empty() ) {
+    const auto tq = tagsQuery(filter.getAcceptTags(), filter.getRejectTags());
+    q = Xapian::Query(Xapian::Query::OP_AND, q, tq);;
+  }
+  return q;
+}
+
+} // unnamed namespace
+
+Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
+{
+  const auto query = buildXapianQuery(filter);
+
+  if ( willSelectEverything(query) )
+    return getBooksIds();
+
+  BookIdCollection bookIds;
+
+  Xapian::Enquire enquire(*m_bookDB);
+  enquire.set_query(query);
+  const auto results = enquire.get_mset(0, m_books.size());
+  for ( auto it = results.begin(); it != results.end(); ++it  ) {
+    bookIds.push_back(it.get_document().get_data());
+  }
+
+  return bookIds;
+}
+
+Library::BookIdCollection Library::filter(const Filter& filter) const
+{
+  BookIdCollection result;
+  for(auto id : filterViaBookDB(filter)) {
+    if(filter.accept(m_books.at(id))) {
+      result.push_back(id);
     }
   }
-  return bookIds;
+  return result;
 }
 
 template<supportedListSortBy SORT>
@@ -257,13 +494,13 @@ struct KEY_TYPE<SIZE> {
 template<supportedListSortBy sort>
 class Comparator {
   private:
-    Library* lib;
-    bool     ascending;
+    const Library* const lib;
+    const bool     ascending;
 
     inline typename KEY_TYPE<sort>::TYPE get_key(const std::string& id);
 
   public:
-    Comparator(Library* lib, bool ascending) : lib(lib), ascending(ascending) {}
+    Comparator(const Library* lib, bool ascending) : lib(lib), ascending(ascending) {}
     inline bool operator() (const std::string& id1, const std::string& id2) {
       if (ascending) {
         return get_key(id1) < get_key(id2);
@@ -303,7 +540,7 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
   return lib->getBookById(id).getPublisher();
 }
 
-void Library::sort(std::vector<std::string>& bookIds, supportedListSortBy sort, bool ascending)
+void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
 {
   switch(sort) {
     case TITLE:
@@ -327,7 +564,7 @@ void Library::sort(std::vector<std::string>& bookIds, supportedListSortBy sort,
 }
 
 
-std::vector<std::string> Library::listBooksIds(
+Library::BookIdCollection Library::listBooksIds(
     int mode,
     supportedListSortBy sortBy,
     const std::string& search,
@@ -335,7 +572,7 @@ std::vector<std::string> Library::listBooksIds(
     const std::string& creator,
     const std::string& publisher,
     const std::vector<std::string>& tags,
-    size_t maxSize) {
+    size_t maxSize) const {
 
   Filter _filter;
   if (mode & LOCAL)
@@ -391,6 +628,7 @@ enum filterTypes {
   MAXSIZE = FLAG(11),
   QUERY = FLAG(12),
   NAME = FLAG(13),
+  CATEGORY = FLAG(14),
 };
 
 Filter& Filter::local(bool accept)
@@ -429,20 +667,27 @@ Filter& Filter::valid(bool accept)
   return *this;
 }
 
-Filter& Filter::acceptTags(std::vector<std::string> tags)
+Filter& Filter::acceptTags(const Tags& tags)
 {
   _acceptTags = tags;
   activeFilters |= ACCEPTTAGS;
   return *this;
 }
 
-Filter& Filter::rejectTags(std::vector<std::string> tags)
+Filter& Filter::rejectTags(const Tags& tags)
 {
   _rejectTags = tags;
   activeFilters |= REJECTTAGS;
   return *this;
 }
 
+Filter& Filter::category(std::string category)
+{
+  _category = category;
+  activeFilters |= CATEGORY;
+  return *this;
+}
+
 Filter& Filter::lang(std::string lang)
 {
   _lang = lang;
@@ -471,9 +716,10 @@ Filter& Filter::maxSize(size_t maxSize)
   return *this;
 }
 
-Filter& Filter::query(std::string query)
+Filter& Filter::query(std::string query, bool partial)
 {
   _query = query;
+  _queryIsPartial = partial;
   activeFilters |= QUERY;
   return *this;
 }
@@ -487,6 +733,36 @@ Filter& Filter::name(std::string name)
 
 #define ACTIVE(X) (activeFilters & (X))
 #define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
+bool Filter::hasQuery() const
+{
+  return ACTIVE(QUERY);
+}
+
+bool Filter::hasName() const
+{
+  return ACTIVE(NAME);
+}
+
+bool Filter::hasCategory() const
+{
+  return ACTIVE(CATEGORY);
+}
+
+bool Filter::hasLang() const
+{
+  return ACTIVE(LANG);
+}
+
+bool Filter::hasPublisher() const
+{
+  return ACTIVE(_PUBLISHER);
+}
+
+bool Filter::hasCreator() const
+{
+  return ACTIVE(_CREATOR);
+}
+
 bool Filter::accept(const Book& book) const
 {
   auto local = !book.getPath().empty();
@@ -502,40 +778,8 @@ bool Filter::accept(const Book& book) const
   FILTER(_NOREMOTE, !remote)
 
   FILTER(MAXSIZE, book.getSize() <= _maxSize)
-  FILTER(LANG, book.getLanguage() == _lang)
-  FILTER(_PUBLISHER, book.getPublisher() == _publisher)
-  FILTER(_CREATOR, book.getCreator() == _creator)
-  FILTER(NAME, book.getName() == _name)
-
-  if (ACTIVE(ACCEPTTAGS)) {
-    if (!_acceptTags.empty()) {
-      auto vBookTags = split(book.getTags(), ";");
-      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
-      for (auto& t: _acceptTags) {
-        if (sBookTags.find(t) == sBookTags.end()) {
-          return false;
-        }
-      }
-    }
-  }
-  if (ACTIVE(REJECTTAGS)) {
-    if (!_rejectTags.empty()) {
-      auto vBookTags = split(book.getTags(), ";");
-      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
-      for (auto& t: _rejectTags) {
-        if (sBookTags.find(t) != sBookTags.end()) {
-          return false;
-        }
-      }
-    }
-  }
-  if ( ACTIVE(QUERY)
-    && !(matchRegex(book.getTitle(), "\\Q" + _query + "\\E")
-        || matchRegex(book.getDescription(), "\\Q" + _query + "\\E")))
-    return false;
 
   return true;
-
 }
 
 }

src/libxml_dumper.cpp

@@ -28,7 +28,7 @@
 namespace kiwix
 {
 /* Constructor */
-LibXMLDumper::LibXMLDumper(Library* library)
+LibXMLDumper::LibXMLDumper(const Library* library)
   : library(library)
 {
 }

src/manager.cpp

@@ -80,7 +80,7 @@ bool Manager::readXml(const std::string& xml,
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)xml.data(), xml.size());
+      = doc.load_buffer((void*)xml.data(), xml.size());
 
   if (result) {
     this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
@@ -124,7 +124,7 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)content.data(), content.size());
+      = doc.load_buffer((void*)content.data(), content.size());
 
   if (result) {
     this->parseOpdsDom(doc, urlHost);

src/meson.build

@@ -21,9 +21,12 @@ kiwix_sources = [
   'tools/otherTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
+  'server/byte_range.cpp',
   'server/etag.cpp',
   'server/request_context.cpp',
-  'server/response.cpp'
+  'server/response.cpp',
+  'server/internalServer.cpp',
+  'server/internalServer_catalog_v2.cpp'
 ]
 kiwix_sources += lib_resources
 
@@ -33,13 +36,13 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if 'android' in wrapper
+if wrapper.contains('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
-if 'android' in wrapper or 'java' in wrapper
+if wrapper.contains('android') or wrapper.contains('java')
   subdir('wrapper/java')
 endif
 
@@ -52,6 +55,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
                    install_dir: install_dir)

src/microhttpd_wrapper.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <microhttpd.h>
+
+#if MHD_VERSION < 0x00097002
+typedef int MHD_Result;
+#endif

src/opds_dumper.cpp

@@ -21,10 +21,13 @@
 #include "book.h"
 
 #include "tools/otherTools.h"
-#include <iomanip>
+
+#include "kiwixlib-resources.h"
+#include <mustache.hpp>
 
 namespace kiwix
 {
+
 /* Constructor */
 OPDSDumper::OPDSDumper(Library* library)
   : library(library)
@@ -35,120 +38,111 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-std::string gen_date_str()
-{
-  auto now = time(0);
-  auto tm = localtime(&now);
-
-  std::stringstream is;
-  is << std::setw(2) << std::setfill('0')
-     << 1900+tm->tm_year << "-"
-     << std::setw(2) << std::setfill('0') << tm->tm_mon << "-"
-     << std::setw(2) << std::setfill('0') << tm->tm_mday << "T"
-     << std::setw(2) << std::setfill('0') << tm->tm_hour << ":"
-     << std::setw(2) << std::setfill('0') << tm->tm_min << ":"
-     << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z";
-  return is.str();
-}
-
-static std::string gen_date_from_yyyy_mm_dd(const std::string& date)
-{
-  std::stringstream is;
-  is << date << "T00:00::00:Z";
-  return is.str();
-}
-
 void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 {
   m_totalResults = totalResults;
   m_startIndex = startIndex,
   m_count = count;
-  m_isSearchResult = true;
 }
 
-#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
-
-pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
-  auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
-  ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
-  ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage());
-  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
-  ADD_TEXT_ENTRY(entry_node, "name", book.getName());
-  ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
-  ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
-  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
-  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-
-  auto content_node = entry_node.append_child("link");
-  content_node.append_attribute("type") = "text/html";
-  content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str();
-
-  auto author_node = entry_node.append_child("author");
-  ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
-
-  auto publisher_node = entry_node.append_child("publisher");
-  ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher());
-
-  if (! book.getUrl().empty()) {
-    auto acquisition_link = entry_node.append_child("link");
-    acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";
-    acquisition_link.append_attribute("type") = "application/x-zim";
-    acquisition_link.append_attribute("href") = book.getUrl().c_str();
-    acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str();
-  }
-
-  if (! book.getFaviconMimeType().empty() ) {
-    auto image_link = entry_node.append_child("link");
-    image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail";
-    image_link.append_attribute("type") = book.getFaviconMimeType().c_str();
-    image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str();
-  }
-  return entry_node;
-}
-
-string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds)
+namespace
 {
-  date = gen_date_str();
-  pugi::xml_document doc;
 
-  auto root_node = doc.append_child("feed");
-  root_node.append_attribute("xmlns") = "http://www.w3.org/2005/Atom";
-  root_node.append_attribute("xmlns:opds") = "http://opds-spec.org/2010/catalog";
+typedef kainjow::mustache::data MustacheData;
+typedef kainjow::mustache::list BookData;
 
-  ADD_TEXT_ENTRY(root_node, "id", id);
-
-  ADD_TEXT_ENTRY(root_node, "title", title);
-  ADD_TEXT_ENTRY(root_node, "updated", date);
-
-  if (m_isSearchResult) {
-    ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults));
-    ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex));
-    ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count));
+BookData getBookData(const Library* library, const std::vector<std::string>& bookIds)
+{
+  BookData bookData;
+  for ( const auto& bookId : bookIds ) {
+    const Book& book = library->getBookById(bookId);
+    const MustacheData bookUrl = book.getUrl().empty()
+                               ? MustacheData(false)
+                               : MustacheData(book.getUrl());
+    bookData.push_back(kainjow::mustache::object{
+      {"id", "urn:uuid:"+book.getId()},
+      {"name", book.getName()},
+      {"title", book.getTitle()},
+      {"description", book.getDescription()},
+      {"language", book.getLanguage()},
+      {"content_id",  book.getHumanReadableIdFromPath()},
+      {"updated", book.getDate() + "T00:00:00Z"},
+      {"category", book.getCategory()},
+      {"flavour", book.getFlavour()},
+      {"tags", book.getTags()},
+      {"article_count", to_string(book.getArticleCount())},
+      {"media_count", to_string(book.getMediaCount())},
+      {"author_name", book.getCreator()},
+      {"publisher_name", book.getPublisher()},
+      {"url", bookUrl},
+      {"size", to_string(book.getSize())},
+    });
   }
 
-  auto self_link_node = root_node.append_child("link");
-  self_link_node.append_attribute("rel") = "self";
-  self_link_node.append_attribute("href") = "";
-  self_link_node.append_attribute("type") = "application/atom+xml";
+  return bookData;
+}
 
+} // unnamed namespace
 
-  if (!searchDescriptionUrl.empty() ) {
-    auto search_link = root_node.append_child("link");
-    search_link.append_attribute("rel") = "search";
-    search_link.append_attribute("type") = "application/opensearchdescription+xml";
-    search_link.append_attribute("href") = searchDescriptionUrl.c_str();
+string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
+{
+  const auto bookData = getBookData(library, bookIds);
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"root", rootLocation},
+     {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
+     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", bookData }
+  };
+
+  return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
+}
+
+string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const
+{
+  const auto bookData = getBookData(library, bookIds);
+
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"endpoint_root", rootLocation + "/catalog/v2"},
+     {"feed_id", gen_uuid(libraryId + "/entries?"+query)},
+     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", bookData }
+  };
+
+  return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
+}
+
+std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categories) const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : categories ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
   }
 
-  if (library) {
-    for (auto& bookId: bookIds) {
-      handleBook(library->getBookById(bookId), root_node);
-    }
-  }
-
-  return nodeToString(root_node);
+  return render_template(
+             RESOURCE::templates::catalog_v2_categories_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/categories")},
+               {"categories", categoryData }
+             }
+  );
 }
 
 }

src/reader.cpp

@@ -21,6 +21,8 @@
 #include <time.h>
 
 #include <zim/search.h>
+#include <zim/item.h>
+#include <zim/error.h>
 
 #include "tools/otherTools.h"
 
@@ -65,7 +67,9 @@ std::string hexUUID(std::string in)
 namespace kiwix
 {
 /* Constructor */
-Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
+Reader::Reader(const string zimFilePath)
+  :  zimArchive(nullptr),
+     zimFilePath(zimFilePath)
 {
   string tmpZimFilePath = zimFilePath;
 
@@ -76,58 +80,45 @@ Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
     tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
   }
 
-  this->zimFileHandler = new zim::File(tmpZimFilePath);
-
-  if (this->zimFileHandler != NULL) {
-    this->firstArticleOffset
-        = this->zimFileHandler->getNamespaceBeginOffset('A');
-    this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
-    this->nsACount = this->zimFileHandler->getNamespaceCount('A');
-    this->nsICount = this->zimFileHandler->getNamespaceCount('I');
-    this->zimFilePath = zimFilePath;
-  }
+  zimArchive.reset(new zim::Archive(tmpZimFilePath));
 
   /* initialize random seed: */
-  srand(time(NULL));
+  srand(time(nullptr));
 }
 
-/* Destructor */
-Reader::~Reader()
+#ifndef _WIN32
+Reader::Reader(int fd)
+  :  zimArchive(new zim::Archive(fd)),
+     zimFilePath("")
 {
-  if (this->zimFileHandler != NULL) {
-    delete this->zimFileHandler;
+  /* initialize random seed: */
+  srand(time(nullptr));
+}
+
+Reader::Reader(int fd, zim::offset_type offset, zim::size_type size)
+  :  zimArchive(new zim::Archive(fd, offset, size)),
+     zimFilePath("")
+{
+  /* initialize random seed: */
+  srand(time(nullptr));
+}
+#endif // #ifndef _WIN32
+
+zim::Archive* Reader::getZimArchive() const
+{
+  return zimArchive.get();
+}
+
+MimeCounterType Reader::parseCounterMetadata() const
+{
+  try {
+    auto counterContent = zimArchive->getMetadata("Counter");
+    return parseMimetypeCounter(counterContent);
+  } catch (zim::EntryNotFound& e) {
+    return {};
   }
 }
 
-zim::File* Reader::getZimFileHandler() const
-{
-  return this->zimFileHandler;
-}
-std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
-{
-  std::map<const std::string, unsigned int> counters;
-  string mimeType, item, counterString;
-  unsigned int counter;
-
-  zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
-
-  if (article.good()) {
-    stringstream ssContent(article.getData());
-
-    while (getline(ssContent, item, ';')) {
-      stringstream ssItem(item);
-      getline(ssItem, mimeType, '=');
-      getline(ssItem, counterString, '=');
-      if (!counterString.empty() && !mimeType.empty()) {
-        sscanf(counterString.c_str(), "%u", &counter);
-        counters.insert(pair<string, int>(mimeType, counter));
-      }
-    }
-  }
-
-  return counters;
-}
-
 /* Get the count of articles which can be indexed/displayed */
 unsigned int Reader::getArticleCount() const
 {
@@ -135,12 +126,9 @@ unsigned int Reader::getArticleCount() const
       = this->parseCounterMetadata();
   unsigned int counter = 0;
 
-  if (counterMap.empty()) {
-    counter = this->nsACount;
-  } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
+  for(auto &pair:counterMap) {
+    if (startsWith(pair.first, "text/html")) {
+      counter += pair.second;
     }
   }
 
@@ -154,139 +142,68 @@ unsigned int Reader::getMediaCount() const
       = this->parseCounterMetadata();
   unsigned int counter = 0;
 
-  if (counterMap.empty()) {
-    counter = this->nsICount;
-  } else {
-    auto it = counterMap.find("image/jpeg");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/gif");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/png");
-    if (it != counterMap.end()) {
-      counter += it->second;
+  for (auto &pair:counterMap) {
+    if (startsWith(pair.first, "image/") ||
+        startsWith(pair.first, "video/") ||
+        startsWith(pair.first, "audio/")) {
+      counter += pair.second;
     }
   }
+
   return counter;
 }
 
 /* Get the total of all items of a ZIM file, redirects included */
 unsigned int Reader::getGlobalCount() const
 {
-  return this->zimFileHandler->getCountArticles();
+  return zimArchive->getEntryCount();
 }
 
 /* Return the UID of the ZIM file */
 string Reader::getId() const
 {
   std::ostringstream s;
-  s << this->zimFileHandler->getFileheader().getUuid();
+  s << zimArchive->getUuid();
   return s.str();
 }
 
-/* Return a page url from a title */
-bool Reader::getPageUrlFromTitle(const string& title, string& url) const
-{
-  try {
-    auto entry = getEntryFromTitle(title);
-    entry = entry.getFinalEntry();
-    url = entry.getPath();
-    return true;
-  } catch (NoEntry& e) {
-    return false;
-  }
-}
-
-/* Return an URL from a title */
-string Reader::getRandomPageUrl() const
-{
-  return getRandomPage().getPath();
-}
-
 Entry Reader::getRandomPage() const
 {
-  if (!this->zimFileHandler) {
+  try {
+    return zimArchive->getRandomEntry();
+  } catch(...) {
     throw NoEntry();
   }
-
-  zim::Article article;
-  std::string mainPagePath = this->getMainPage().getPath();
-  int watchdog = 42;
-
-  do {
-    auto idx = this->firstArticleOffset
-          + (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
-                             * this->nsACount);
-    article = zimFileHandler->getArticle(idx);
-    if (!watchdog--) {
-      throw NoEntry();
-    }
-  } while (!article.good() && article.getLongUrl() == mainPagePath);
-
-  return article;
-}
-
-/* Return the welcome page URL */
-string Reader::getMainPageUrl() const
-{
-  return getMainPage().getPath();
 }
 
 Entry Reader::getMainPage() const
 {
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  zim::Article article;
-  if (this->zimFileHandler->getFileheader().hasMainPage())
-  {
-    article = zimFileHandler->getArticle(
-        this->zimFileHandler->getFileheader().getMainPage());
-  }
-
-  if (!article.good())
-  {
-    return getFirstPage();
-  }
-
-  return article;
+  return zimArchive->getMainEntry();
 }
 
 bool Reader::getFavicon(string& content, string& mimeType) const
 {
-  static const char* const paths[] = {"-/favicon", "-/favicon.png", "I/favicon.png", "I/favicon"};
-
-  for (auto &path: paths) {
-    try {
-      auto entry = getEntryFromPath(path);
-      entry = entry.getFinalEntry();
-      content = entry.getContent();
-      mimeType = entry.getMimetype();
-      return true;
-    } catch(NoEntry& e) {};
-  }
+  try {
+    auto item = zimArchive->getIllustrationItem();
+    content = item.getData();
+    mimeType = item.getMimetype();
+    return true;
+  } catch(zim::EntryNotFound& e) {};
 
   return false;
 }
 
 string Reader::getZimFilePath() const
 {
-  return this->zimFilePath;
+  return zimFilePath;
 }
 /* Return a metatag value */
 bool Reader::getMetadata(const string& name, string& value) const
 {
   try {
-    auto entry = getEntryFromPath("M/"+name);
-    value = entry.getContent();
+    value = zimArchive->getMetadata(name);
     return true;
-  } catch(NoEntry& e) {
+  } catch(zim::EntryNotFound& e) {
     return false;
   }
 }
@@ -300,10 +217,9 @@ string Reader::getName() const
 
 string Reader::getTitle() const
 {
-  string value;
-  this->getMetadata("Title", value);
+  string value = zimArchive->getMetadata("Title");
   if (value.empty()) {
-    value = getLastPathElement(zimFileHandler->getFilename());
+    value = getLastPathElement(zimFilePath);
     std::replace(value.begin(), value.end(), '_', ' ');
     size_t pos = value.find(".zim");
     value = value.substr(0, pos);
@@ -424,83 +340,17 @@ string Reader::getOrigId() const
   return origID;
 }
 
-/* Return the first page URL */
-string Reader::getFirstPageUrl() const
-{
-  return getFirstPage().getPath();
-}
-
-Entry Reader::getFirstPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  auto firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
-  auto article = zimFileHandler->getArticle(firstPageOffset);
-
-  if (! article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-bool _parseUrl(const string& url, char* ns, string& title)
-{
-  /* Offset to visit the url */
-  unsigned int urlLength = url.size();
-  unsigned int offset = 0;
-
-  /* Ignore the first '/' */
-  if (url[offset] == '/')
-    offset++;
-
-  if (url[offset] == '/' || offset >= urlLength)
-    return false;
-
-  /* Get namespace */
-  *ns = url[offset++];
-
-  if (url[offset] != '/' || offset >= urlLength)
-    return false;
-
-  offset++;
-
-  if ( offset >= urlLength)
-    return false;
-
-  /* Get content title */
-  title = url.substr(offset, urlLength - offset);
-
-  return true;
-}
-
-bool Reader::parseUrl(const string& url, char* ns, string& title) const
-{
-  return _parseUrl(url, ns, title);
-}
-
 Entry Reader::getEntryFromPath(const std::string& path) const
 {
-  char ns = 0;
-  std::string short_url;
-
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-  _parseUrl(path, &ns, short_url);
-
-  if (short_url.empty() && ns == 0) {
+  if (path.empty() || path == "/") {
     return getMainPage();
   }
 
-  auto article = zimFileHandler->getArticle(ns, short_url);
-  if (!article.good()) {
+  try {
+    return zimArchive->getEntryByPath(path);
+  } catch (zim::EntryNotFound& e) {
     throw NoEntry();
   }
-
-  return article;
 }
 
 Entry Reader::getEntryFromEncodedPath(const std::string& path) const
@@ -510,211 +360,30 @@ Entry Reader::getEntryFromEncodedPath(const std::string& path) const
 
 Entry Reader::getEntryFromTitle(const std::string& title) const
 {
-  if (!this->zimFileHandler) {
+  try {
+    return zimArchive->getEntryByTitle(title);
+  } catch(zim::EntryNotFound& e) {
     throw NoEntry();
   }
-
-  auto article = this->zimFileHandler->getArticleByTitle('A', title);
-  if (!article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-/* Return article by url */
-bool Reader::getArticleObjectByDecodedUrl(const string& url,
-                                          zim::Article& article) const
-{
-  if (this->zimFileHandler == NULL) {
-    return false;
-  }
-
-  /* Parse the url */
-  char ns = 0;
-  string urlStr;
-  _parseUrl(url, &ns, urlStr);
-
-  /* Main page */
-  if (urlStr.empty() && ns == 0) {
-    _parseUrl(this->getMainPage().getPath(), &ns, urlStr);
-  }
-
-  /* Extract the content from the zim file */
-  article = zimFileHandler->getArticle(ns, urlStr);
-  return article.good();
-}
-
-/* Return the mimeType without the content */
-bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
-{
-  try {
-    auto entry = getEntryFromPath(url);
-    mimeType = entry.getMimetype();
-    return true;
-  } catch (NoEntry& e) {
-    mimeType = "";
-    return false;
-  }
-}
-
-bool get_content_by_decoded_url(const Reader& reader,
-                                const string& url,
-                                string& content,
-                                string& title,
-                                unsigned int& contentLength,
-                                string& contentType,
-                                string& baseUrl)
-{
-  content = "";
-  contentType = "";
-  contentLength = 0;
-
-  try {
-    auto entry = reader.getEntryFromPath(url);
-    entry = entry.getFinalEntry();
-    baseUrl = entry.getPath();
-    contentType = entry.getMimetype();
-    content = entry.getContent();
-    contentLength = entry.getSize();
-    title = entry.getTitle();
-
-    /* Try to set a stub HTML header/footer if necesssary */
-    if (contentType.find("text/html") != string::npos
-      && content.find("<body") == std::string::npos
-      && content.find("<BODY") == std::string::npos) {
-      content = "<html><head><title>" + title +
-              "</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
-              "charset=utf-8\" /></head><body>" +
-              content + "</body></html>";
-    }
-    return true;
-  } catch (NoEntry& e) {
-    return false;
-  }
-}
-
-
-/* Get a content from a zim file */
-bool Reader::getContentByUrl(const string& url,
-                             string& content,
-                             string& title,
-                             unsigned int& contentLength,
-                             string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
-}
-
-/* Check if an article exists */
-bool Reader::urlExists(const string& url) const
-{
-  return pathExists(url);
 }
 
 bool Reader::pathExists(const string& path) const
 {
-  if (!zimFileHandler)
-  {
-    return false;
-  }
-
-  char ns = 0;
-  string titleStr;
-  _parseUrl(path, &ns, titleStr);
-  zim::File::const_iterator findItr = zimFileHandler->find(ns, titleStr);
-  return findItr != zimFileHandler->end() && findItr->getUrl() == titleStr;
+  return zimArchive->hasEntryByPath(path);
 }
 
 /* Does the ZIM file has a fulltext index */
 bool Reader::hasFulltextIndex() const
 {
-  if (!zimFileHandler || zimFileHandler->is_multiPart() )
-  {
-    return false;
-  }
-
-  return ( pathExists("Z//fulltextIndex/xapian")
-        || pathExists("X/fulltext/xapian"));
+  return zimArchive->hasFulltextIndex();
 }
 
 /* Search titles by prefix */
+
 bool Reader::searchSuggestions(const string& prefix,
                                unsigned int suggestionsCount,
                                const bool reset)
 {
-  bool retVal = false;
-
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
   if (reset) {
@@ -726,39 +395,48 @@ bool Reader::searchSuggestions(const string& prefix,
     }
   }
 
+  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
+
+  /* Set the cursor to the begining */
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+
+bool Reader::searchSuggestions(const string& prefix,
+                               unsigned int suggestionsCount,
+                               SuggestionsList_t& results)
+{
+  bool retVal = false;
+
   /* Return if no prefix */
   if (prefix.size() == 0) {
     return false;
   }
 
-  for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
-       articleItr != zimFileHandler->end()
-       && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
-       ++articleItr) {
+  for (auto& entry: zimArchive->findByTitle(prefix)) {
+    if (results.size() >= suggestionsCount) {
+      break;
+    }
     /* Extract the interesting part of article title & url */
     std::string normalizedArticleTitle
-        = kiwix::normalize(articleItr->getTitle());
-    std::string articleFinalUrl = "/A/" + articleItr->getUrl();
-    if (articleItr->isRedirect()) {
-      zim::Article article = *articleItr;
-      unsigned int loopCounter = 0;
-      while (article.isRedirect() && loopCounter++ < 42) {
-        article = article.getRedirectArticle();
-      }
-      articleFinalUrl = "/A/" + article.getUrl();
-    }
+        = kiwix::normalize(entry.getTitle());
+
+    // Get the final path.
+    auto item = entry.getItem(true);
+    std::string articleFinalUrl = item.getPath();
 
     /* Go through all already found suggestions and skip if this
        article is already in the suggestions list (with an other
        title) */
     bool insert = true;
-    std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
+    std::vector<SuggestionItem>::iterator suggestionItr;
+    for (suggestionItr = results.begin();
+         suggestionItr != results.end();
          suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
+      int result = normalizedArticleTitle.compare((*suggestionItr).getNormalizedTitle());
+      if (result == 0 && articleFinalUrl.compare((*suggestionItr).getPath()) == 0) {
         insert = false;
         break;
       } else if (result < 0) {
@@ -768,20 +446,14 @@ bool Reader::searchSuggestions(const string& prefix,
 
     /* Insert if possible */
     if (insert) {
-      std::vector<std::string> suggestion;
-      suggestion.push_back(articleItr->getTitle());
-      suggestion.push_back(articleFinalUrl);
-      suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
+      SuggestionItem suggestion(entry.getTitle(), normalizedArticleTitle, articleFinalUrl);
+      results.insert(suggestionItr, suggestion);
     }
 
     /* Suggestions where found */
     retVal = true;
   }
 
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
   return retVal;
 }
 
@@ -796,39 +468,50 @@ std::vector<std::string> Reader::getTitleVariants(
   return variants;
 }
 
-/* Try also a few variations of the prefix to have better results */
+
 bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
+{
+  this->suggestions.clear();
+  this->suggestionsOffset = this->suggestions.begin();
+
+  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
+
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+/* Try also a few variations of the prefix to have better results */
+bool Reader::searchSuggestionsSmart(const string& prefix,
+                                    unsigned int suggestionsCount,
+                                    SuggestionsList_t& results)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
   bool retVal = false;
 
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
-  const auto suggestionSearch
-      = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
 
-  if (suggestionSearch->get_matches_estimated()) {
-    for (auto current = suggestionSearch->begin();
-         current != suggestionSearch->end();
+  auto suggestionSearcher = zim::Searcher(*zimArchive);
+  zim::Query suggestionQuery;
+  suggestionQuery.setQuery(prefix, true);
+  auto suggestionSearch = suggestionSearcher.search(suggestionQuery);
+
+  if (suggestionSearch.getEstimatedMatches()) {
+    const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
+    for (auto current = suggestions.begin();
+         current != suggestions.end();
          current++) {
-      if (!current->good()) {
-          continue;
-      }
-      std::vector<std::string> suggestion;
-      suggestion.push_back(current->getTitle());
-      suggestion.push_back("/A/" + current->getUrl());
-      suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
+      SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
+                                current.getPath(), current.getSnippet());
+      results.push_back(suggestion);
     }
-    this->suggestionsOffset = this->suggestions.begin();
     retVal = true;
   } else {
     for (std::vector<std::string>::iterator variantsItr = variants.begin();
          variantsItr != variants.end();
          variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
+      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
                || retVal;
     }
   }
@@ -841,7 +524,7 @@ bool Reader::getNextSuggestion(string& title)
 {
   if (this->suggestionsOffset != this->suggestions.end()) {
     /* title */
-    title = (*(this->suggestionsOffset))[0];
+    title = (*(this->suggestionsOffset)).getTitle();
 
     /* increment the cursor for the next call */
     this->suggestionsOffset++;
@@ -856,8 +539,8 @@ bool Reader::getNextSuggestion(string& title, string& url)
 {
   if (this->suggestionsOffset != this->suggestions.end()) {
     /* title */
-    title = (*(this->suggestionsOffset))[0];
-    url = (*(this->suggestionsOffset))[1];
+    title = (*(this->suggestionsOffset)).getTitle();
+    url = (*(this->suggestionsOffset)).getPath();
 
     /* increment the cursor for the next call */
     this->suggestionsOffset++;
@@ -871,14 +554,14 @@ bool Reader::getNextSuggestion(string& title, string& url)
 /* Check if the file has as checksum */
 bool Reader::canCheckIntegrity() const
 {
-  return this->zimFileHandler->getChecksum() != "";
+  return zimArchive->hasChecksum();
 }
 
 /* Return true if corrupted, false otherwise */
 bool Reader::isCorrupted() const
 {
   try {
-    if (this->zimFileHandler->verify() == true) {
+    if (zimArchive->check() == true) {
       return false;
     }
   } catch (exception& e) {
@@ -892,13 +575,7 @@ bool Reader::isCorrupted() const
 /* Return the file size, works also for splitted files */
 unsigned int Reader::getFileSize() const
 {
-  zim::File* file = this->getZimFileHandler();
-  zim::size_type size = 0;
-
-  if (file != NULL) {
-    size = file->getFilesize();
-  }
-
-  return (size / 1024);
+  return zimArchive->getFilesize() / 1024;
 }
+
 }

src/search_renderer.cpp

@@ -77,9 +77,7 @@ std::string SearchRenderer::getHtml()
     result.set("title", p_result->get_title());
     result.set("url", p_result->get_url());
     result.set("snippet", p_result->get_snippet());
-    auto readerIndex = p_result->get_readerIndex();
-    auto reader = mp_searcher->get_reader(readerIndex);
-    result.set("resultContentId", mp_nameMapper->getNameForId(reader->getId()));
+    result.set("resultContentId", mp_nameMapper->getNameForId(p_result->get_zimId()));
 
     if (p_result->get_wordCount() >= 0) {
       result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
@@ -93,30 +91,30 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
   auto resultStart = mp_searcher->getResultStart();
-  auto resultEnd = mp_searcher->getResultEnd();
-  auto resultCountPerPage = resultEnd - resultStart;
+  auto resultEnd = 0U;
   auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
   auto currentPage = 0U;
   auto pageStart = 0U;
   auto pageEnd = 0U;
   auto lastPageStart = 0U;
-  if (resultCountPerPage) {
-    currentPage = resultStart/resultCountPerPage;
+  if (pageLength) {
+    currentPage = resultStart/pageLength;
     pageStart = currentPage > 4 ? currentPage-4 : 0;
     pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / resultCountPerPage) {
-      pageEnd = estimatedResultCount / resultCountPerPage;
+    if (pageEnd > estimatedResultCount / pageLength) {
+      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
     }
-    if (estimatedResultCount > resultCountPerPage) {
-      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
+    if (estimatedResultCount > pageLength) {
+      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
     }
   }
 
+  resultEnd = resultStart+pageLength; //setting result end
+
   for (unsigned int i = pageStart; i < pageEnd; i++) {
     kainjow::mustache::data page;
     page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * resultCountPerPage));
-    page.set("end", to_string((i + 1) * resultCountPerPage));
+    page.set("start", to_string(i * pageLength));
 
     if (i == currentPage) {
       page.set("selected", true);
@@ -137,9 +135,8 @@ std::string SearchRenderer::getHtml()
   allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
   allData.set("resultStart", to_string(resultStart + 1));
   allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("resultRange", to_string(resultCountPerPage));
+  allData.set("pageLength", to_string(pageLength));
   allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("lastResult", to_string(estimatedResultCount));
   allData.set("protocolPrefix", this->protocolPrefix);
   allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
   allData.set("contentId", this->searchContent);
@@ -149,4 +146,4 @@ std::string SearchRenderer::getHtml()
   return ss.str();
 }
 
-}
+}

src/searcher.cpp

@@ -35,7 +35,7 @@ namespace kiwix
 class _Result : public Result
 {
  public:
-  _Result(zim::Search::iterator& iterator);
+  _Result(zim::SearchResultSet::iterator iterator);
   virtual ~_Result(){};
 
   virtual std::string get_url();
@@ -45,29 +45,25 @@ class _Result : public Result
   virtual std::string get_content();
   virtual int get_wordCount();
   virtual int get_size();
-  virtual int get_readerIndex();
+  virtual std::string get_zimId();
 
  private:
-  zim::Search::iterator iterator;
+  zim::SearchResultSet::iterator iterator;
 };
 
-struct SearcherInternal {
-  const zim::Search* _search;
-  zim::Search::iterator current_iterator;
-
-  SearcherInternal() : _search(NULL) {}
-  ~SearcherInternal()
+struct SearcherInternal : zim::SearchResultSet {
+  explicit SearcherInternal(const zim::SearchResultSet& srs)
+    : zim::SearchResultSet(srs)
+    , current_iterator(srs.begin())
   {
-    if (_search != NULL) {
-      delete _search;
-    }
   }
+
+  zim::SearchResultSet::iterator current_iterator;
 };
 
 /* Constructor */
 Searcher::Searcher()
-    : internal(new SearcherInternal()),
-      searchPattern(""),
+    : searchPattern(""),
       estimatedResultCount(0),
       resultStart(0),
       resultEnd(0)
@@ -78,7 +74,6 @@ Searcher::Searcher()
 /* Destructor */
 Searcher::~Searcher()
 {
-  delete internal;
 }
 
 bool Searcher::add_reader(Reader* reader)
@@ -115,20 +110,20 @@ void Searcher::search(const std::string& search,
   if (resultStart != resultEnd) {
     /* Perform the search */
     string unaccentedSearch = removeAccents(search);
-    std::vector<const zim::File*> zims;
+    std::vector<zim::Archive> archives;
     for (auto current = this->readers.begin(); current != this->readers.end();
          current++) {
       if ( (*current)->hasFulltextIndex() ) {
-          zims.push_back((*current)->getZimFileHandler());
+          archives.push_back(*(*current)->getZimArchive());
       }
     }
-    zim::Search* search = new zim::Search(zims);
-    search->set_verbose(verbose);
-    search->set_query(unaccentedSearch);
-    search->set_range(resultStart, resultEnd);
-    internal->_search = search;
-    internal->current_iterator = internal->_search->begin();
-    this->estimatedResultCount = internal->_search->get_matches_estimated();
+    zim::Searcher searcher(archives);
+    zim::Query query;
+    query.setQuery(unaccentedSearch, false);
+    query.setVerbose(verbose);
+    zim::Search search = searcher.search(query);
+    internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+    this->estimatedResultCount = search.getEstimatedMatches();
   }
 
   return;
@@ -158,33 +153,33 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     return;
   }
 
-  std::vector<const zim::File*> zims;
+  std::vector<zim::Archive> archives;
   for (auto current = this->readers.begin(); current != this->readers.end();
        current++) {
-    zims.push_back((*current)->getZimFileHandler());
+    archives.push_back(*(*current)->getZimArchive());
   }
-  zim::Search* search = new zim::Search(zims);
-  search->set_verbose(verbose);
-  search->set_query("");
-  search->set_georange(latitude, longitude, distance);
-  search->set_range(resultStart, resultEnd);
-  internal->_search = search;
-  internal->current_iterator = internal->_search->begin();
-  this->estimatedResultCount = internal->_search->get_matches_estimated();
+  zim::Searcher searcher(archives);
+  zim::Query query;
+  query.setVerbose(verbose);
+  query.setQuery("", false);
+  query.setGeorange(latitude, longitude, distance);
+  zim::Search search = searcher.search(query);
+  internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+  this->estimatedResultCount = search.getEstimatedMatches();
 }
 
 
 void Searcher::restart_search()
 {
-  if (internal->_search) {
-    internal->current_iterator = internal->_search->begin();
+  if (internal.get()) {
+    internal->current_iterator = internal->begin();
   }
 }
 
 Result* Searcher::getNextResult()
 {
-  if (internal->_search &&
-             internal->current_iterator != internal->_search->end()) {
+  if (internal.get() &&
+             internal->current_iterator != internal->end()) {
     Result* result = new _Result(internal->current_iterator);
     internal->current_iterator++;
     return result;
@@ -213,19 +208,18 @@ void Searcher::suggestions(std::string& searchPattern, const bool verbose)
   this->resultEnd = 10;
   string unaccentedSearch = removeAccents(searchPattern);
 
-  std::vector<const zim::File*> zims;
+  std::vector<zim::Archive> archives;
   for (auto current = this->readers.begin(); current != this->readers.end();
        current++) {
-    zims.push_back((*current)->getZimFileHandler());
+    archives.push_back(*(*current)->getZimArchive());
   }
-  zim::Search* search = new zim::Search(zims);
-  search->set_verbose(verbose);
-  search->set_query(unaccentedSearch);
-  search->set_range(resultStart, resultEnd);
-  search->set_suggestion_mode(true);
-  internal->_search = search;
-  internal->current_iterator = internal->_search->begin();
-  this->estimatedResultCount = internal->_search->get_matches_estimated();
+  zim::Searcher searcher(archives);
+  zim::Query query;
+  query.setVerbose(verbose);
+  query.setQuery(unaccentedSearch, true);
+  zim::Search search = searcher.search(query);
+  internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
+  this->estimatedResultCount = search.getEstimatedMatches();
 }
 
 /* Return the result count estimation */
@@ -234,45 +228,44 @@ unsigned int Searcher::getEstimatedResultCount()
   return this->estimatedResultCount;
 }
 
-_Result::_Result(zim::Search::iterator& iterator)
+_Result::_Result(zim::SearchResultSet::iterator iterator)
     : iterator(iterator)
 {
 }
 
 std::string _Result::get_url()
 {
-  return iterator.get_url();
+  return iterator.getPath();
 }
 std::string _Result::get_title()
 {
-  return iterator.get_title();
+  return iterator.getTitle();
 }
 int _Result::get_score()
 {
-  return iterator.get_score();
+  return iterator.getScore();
 }
 std::string _Result::get_snippet()
 {
-  return iterator.get_snippet();
+  return iterator.getSnippet();
 }
 std::string _Result::get_content()
 {
-  if (iterator->good()) {
-    return iterator->getData();
-  }
-  return "";
+  return iterator->getItem(true).getData();
 }
 int _Result::get_size()
 {
-  return iterator.get_size();
+  return iterator.getSize();
 }
 int _Result::get_wordCount()
 {
-  return iterator.get_wordCount();
+  return iterator.getWordCount();
 }
-int _Result::get_readerIndex()
+std::string _Result::get_zimId()
 {
-  return iterator.get_fileIndex();
+  std::ostringstream s;
+  s << iterator.getZimId();
+  return s.str();
 }
 
 

src/server.cpp

@@ -19,140 +19,15 @@
 
 #include "server.h"
 
-#ifdef _WIN32
-# if !defined(__MINGW32__) && (_MSC_VER < 1600)
-#   include "stdint4win.h"
-# endif
-# include <winsock2.h>
-# include <ws2tcpip.h>
-# ifdef __GNUC__
-  // inet_pton is not declared in mingw, even if the function exists.
-  extern "C" {
-    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
-  }
-# endif
-  typedef UINT64 uint64_t;
-  typedef UINT16 uint16_t;
-#endif
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
 #include "library.h"
 #include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
-#include "search_renderer.h"
-#include "opds_dumper.h"
 
-#include <zim/uuid.h>
-
-#include <mustache.hpp>
-
-#include <pthread.h>
-#include <atomic>
 #include <string>
-#include <vector>
-#include <chrono>
-#include "kiwixlib-resources.h"
 
-#ifndef _WIN32
-# include <arpa/inet.h>
-#endif
-
-#include "server/request_context.h"
-#include "server/response.h"
-
-#define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#include "server/internalServer.h"
 
 namespace kiwix {
 
-static IdNameMapper defaultNameMapper;
-
-typedef kainjow::mustache::data MustacheData;
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls);
-
-
-class InternalServer {
-  public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
-                   int port,
-                   std::string root,
-                   int nbThreads,
-                   bool verbose,
-                   bool withTaskbar,
-                   bool withLibraryButton,
-                   bool blockExternalLinks);
-    virtual ~InternalServer() = default;
-
-    int handlerCallback(struct MHD_Connection* connection,
-                        const char* url,
-                        const char* method,
-                        const char* version,
-                        const char* upload_data,
-                        size_t* upload_data_size,
-                        void** cont_cls);
-    bool start();
-    void stop();
-
-  private: // functions
-    Response handle_request(const RequestContext& request);
-    Response build_500(const std::string& msg);
-    Response build_404(const RequestContext& request, const std::string& zimName);
-    Response build_304(const RequestContext& request, const ETag& etag) const;
-    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
-    Response build_homepage(const RequestContext& request);
-    Response handle_skin(const RequestContext& request);
-    Response handle_catalog(const RequestContext& request);
-    Response handle_meta(const RequestContext& request);
-    Response handle_search(const RequestContext& request);
-    Response handle_suggest(const RequestContext& request);
-    Response handle_random(const RequestContext& request);
-    Response handle_captured_external(const RequestContext& request);
-    Response handle_content(const RequestContext& request);
-
-    MustacheData get_default_data() const;
-    MustacheData homepage_data() const;
-    Response get_default_response() const;
-
-    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
-
-  private: // data
-    std::string m_addr;
-    int m_port;
-    std::string m_root;
-    int m_nbThreads;
-    std::atomic_bool m_verbose;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    struct MHD_Daemon* mp_daemon;
-
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
-
-    std::string m_server_id;
-};
-
-
 Server::Server(Library* library, NameMapper* nameMapper) :
   mp_library(library),
   mp_nameMapper(nameMapper),
@@ -178,8 +53,10 @@ bool Server::start() {
 }
 
 void Server::stop() {
-  mp_server->stop();
-  mp_server.reset(nullptr);
+  if (mp_server) {
+    mp_server->stop();
+    mp_server.reset(nullptr);
+  }
 }
 
 void Server::setRoot(const std::string& root)
@@ -193,773 +70,4 @@ void Server::setRoot(const std::string& root)
   }
 }
 
-
-InternalServer::InternalServer(Library* library,
-                               NameMapper* nameMapper,
-                               std::string addr,
-                               int port,
-                               std::string root,
-                               int nbThreads,
-                               bool verbose,
-                               bool withTaskbar,
-                               bool withLibraryButton,
-                               bool blockExternalLinks) :
-  m_addr(addr),
-  m_port(port),
-  m_root(root),
-  m_nbThreads(nbThreads),
-  m_verbose(verbose),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  mp_daemon(nullptr),
-  mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
-{}
-
-bool InternalServer::start() {
-#ifdef _WIN32
-  int flags = MHD_USE_SELECT_INTERNALLY;
-#else
-  int flags = MHD_USE_POLL_INTERNALLY;
-#endif
-  if (m_verbose.load())
-    flags |= MHD_USE_DEBUG;
-
-
-  struct sockaddr_in sockAddr;
-  memset(&sockAddr, 0, sizeof(sockAddr));
-  sockAddr.sin_family = AF_INET;
-  sockAddr.sin_port = htons(m_port);
-  if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
-      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-  } else {
-    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
-      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
-      return false;
-    }
-  }
-
-  mp_daemon = MHD_start_daemon(flags,
-                            m_port,
-                            NULL,
-                            NULL,
-                            &staticHandlerCallback,
-                            this,
-                            MHD_OPTION_SOCK_ADDR, &sockAddr,
-                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
-                            MHD_OPTION_END);
-  if (mp_daemon == nullptr) {
-    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
-              << " is maybe already occupied or need more permissions to be open. "
-                 "Please try as root or with a port number higher or equal to 1024."
-              << std::endl;
-    return false;
-  }
-  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
-  m_server_id = kiwix::to_string(server_start_time.count());
-  return true;
-}
-
-void InternalServer::stop()
-{
-  MHD_stop_daemon(mp_daemon);
-}
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls)
-{
-  InternalServer* _this = static_cast<InternalServer*>(cls);
-
-  return _this->handlerCallback(connection,
-                                url,
-                                method,
-                                version,
-                                upload_data,
-                                upload_data_size,
-                                cont_cls);
-}
-
-int InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                    const char* url,
-                                    const char* method,
-                                    const char* version,
-                                    const char* upload_data,
-                                    size_t* upload_data_size,
-                                    void** cont_cls)
-{
-  auto start_time = std::chrono::steady_clock::now();
-  if (m_verbose.load() ) {
-    printf("======================\n");
-    printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
-  }
-  RequestContext request(connection, m_root, url, method, version);
-
-  if (m_verbose.load() ) {
-    request.print_debug_info();
-  }
-  /* Unexpected method */
-  if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST
-   && request.get_method() != RequestMethod::HEAD) {
-    printf("Reject request because of unhandled request method.\n");
-    printf("----------------------\n");
-    return MHD_NO;
-  }
-
-  auto response = handle_request(request);
-
-  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
-    printf("========== INTERNAL ERROR !! ============\n");
-    if (!m_verbose.load()) {
-      printf("Requesting : \n");
-      printf("full_url : %s\n", url);
-      request.print_debug_info();
-    }
-  }
-
-  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response.set_server_id(m_server_id);
-
-  auto ret = response.send(request, connection);
-  auto end_time = std::chrono::steady_clock::now();
-  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
-  if (m_verbose.load()) {
-    printf("Request time : %fs\n", time_span.count());
-    printf("----------------------\n");
-  }
-  return ret;
-}
-
-Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
-{
-  auto response = get_default_response();
-  response.set_code(MHD_HTTP_NOT_MODIFIED);
-  response.set_etag(etag);
-  response.set_content("");
-  return response;
-}
-
-Response InternalServer::handle_request(const RequestContext& request)
-{
-  try {
-    if (! request.is_valid_url())
-      return build_404(request, "");
-
-    const ETag etag = get_matching_if_none_match_etag(request);
-    if ( etag )
-      return build_304(request, etag);
-
-    if (kiwix::startsWith(request.get_url(), "/skin/"))
-      return handle_skin(request);
-
-    if (startsWith(request.get_url(), "/catalog"))
-      return handle_catalog(request);
-
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
-
-    if (request.get_url() == "/search")
-      return handle_search(request);
-
-    if (request.get_url() == "/suggest")
-     return handle_suggest(request);
-
-    if (request.get_url() == "/random")
-      return handle_random(request);
-
-    if (request.get_url() == "/catch/external")
-      return handle_captured_external(request);
-
-    return handle_content(request);
-  } catch (std::exception& e) {
-    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return build_500(e.what());
-  } catch (...) {
-    fprintf(stderr, "===== Unhandled unknown error\n");
-    return build_500("Unknown error");
-  }
-}
-
-MustacheData InternalServer::get_default_data() const
-{
-  MustacheData data;
-  data.set("root", m_root);
-  return data;
-}
-
-Response InternalServer::get_default_response() const
-{
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
-}
-
-
-Response InternalServer::build_404(const RequestContext& request,
-                                   const std::string& bookName)
-{
-  MustacheData results;
-  results.set("url", request.get_full_url());
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::_404_html, results);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_NOT_FOUND);
-  response.set_compress(true);
-  response.set_taskbar(bookName, "");
-
-  return response;
-}
-
-Response InternalServer::build_500(const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  Response response(m_root, true, false, false, false);
-  response.set_template(RESOURCE::templates::_500_html, data);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
-MustacheData InternalServer::homepage_data() const
-{
-  auto data = get_default_data();
-
-  MustacheData books{MustacheData::type::list};
-  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = mp_library->getBookById(bookId);
-
-    MustacheData book;
-    book.set("name", mp_nameMapper->getNameForId(bookId));
-    book.set("title", currentBook.getTitle());
-    book.set("description", currentBook.getDescription());
-    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
-    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
-    books.push_back(book);
-  }
-
-  data.set("books", books);
-  return data;
-}
-
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
-Response InternalServer::build_homepage(const RequestContext& request)
-{
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, homepage_data());
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_meta(const RequestContext& request)
-{
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
-  } else {
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-  response.set_content(content);
-  response.set_mimeType(mimeType);
-  response.set_compress(false);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_suggest(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_suggest\n");
-  }
-
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-  std::string suggestion;
-
-  std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
-  }
-
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    reader->searchSuggestionsSmart(term, maxSuggestionCount);
-    while (reader->getNextSuggestion(suggestion)) {
-      MustacheData result;
-      result.set("label", suggestion);
-      result.set("value", suggestion);
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
-    }
-  }
-
-  /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
-    MustacheData result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term + " ");
-    result.set("first", first);
-    results.push_back(result);
-  }
-
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::suggestion_json, data);
-  response.set_mimeType("application/json; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_skin(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_skin\n");
-  }
-
-  auto response = get_default_response();
-  auto resourceName = request.get_url().substr(1);
-  try {
-    response.set_content(getResource(resourceName));
-  } catch (const ResourceNotFound& e) {
-    return build_404(request, "");
-  }
-  response.set_mimeType(getMimeTypeForFile(resourceName));
-  response.set_compress(true);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_search(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_search\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
-
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
-
-  std::shared_ptr<Reader> reader(nullptr);
-  try {
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Try first to load directly the article */
-  if (reader != nullptr && !patternString.empty()) {
-    std::string patternCorrespondingUrl;
-    auto variants = reader->getTitleVariants(patternString);
-    auto variantsItr = variants.begin();
-
-    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
-      try {
-        auto entry = reader->getEntryFromTitle(*variantsItr);
-        entry = entry.getFinalEntry();
-        patternCorrespondingUrl = entry.getPath();
-        break;
-      } catch(kiwix::NoEntry& e) {
-        variantsItr++;
-      }
-    }
-
-    /* If article found then redirect directly to it */
-    if (!patternCorrespondingUrl.empty()) {
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
-      return response;
-    }
-  }
-
-  /* Make the search */
-  auto response = get_default_response();
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
-  response.set_compress(true);
-
-  if ( (!reader && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    response.set_template(RESOURCE::templates::no_search_result_html, data);
-    response.set_code(MHD_HTTP_NOT_FOUND);
-    return response;
-  }
-
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-  auto end = 25;
-  try {
-    end = request.get_argument<unsigned int>("end");
-  } catch (const std::exception&) {}
-  if (start>end) {
-    auto tmp = start;
-    start = end;
-    end = tmp;
-  }
-  if (end > start + MAX_SEARCH_LEN) {
-    end = start + MAX_SEARCH_LEN;
-  }
-
-  /* Get the results */
-  try {
-    if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
-    } else {
-      searcher.search(patternString,
-                       start, end, m_verbose.load());
-    }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
-    response.set_content(renderer.getHtml());
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-  return response;
-}
-
-Response InternalServer::handle_random(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_random\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  try {
-    auto entry = reader->getRandomPage();
-    return build_redirect(bookName, entry.getFinalEntry());
-  } catch(kiwix::NoEntry& e) {
-    return build_404(request, bookName);
-  }
-}
-
-Response InternalServer::handle_captured_external(const RequestContext& request)
-{
-  std::string source = "";
-  try {
-    source = kiwix::urlDecode(request.get_argument("source"));
-  } catch (const std::out_of_range& e) {}
-
-  if (source.empty())
-    return build_404(request, "");
-
-  auto data = get_default_data();
-  data.set("source", source);
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::captured_external_html, data);
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return build_404(request, "");
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return build_404(request, "");
-  }
-
-  auto response = get_default_response();
-  response.set_compress(true);
-  if (url == "searchdescription.xml") {
-    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
-    response.set_mimeType("application/opensearchdescription+xml");
-    return response;
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper;
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    opdsDumper.setTitle("All zims");
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
-    string query("<Empty query>");
-    size_t count(10);
-    size_t startIndex(0);
-    try {
-      query = request.get_argument("q");
-      filter.query(query);
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      count = extractFromString<unsigned long>(request.get_argument("count"));
-    } catch (...) {}
-    try {
-      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
-    } catch (...) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    opdsDumper.setTitle("Search result for " + query);
-    uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(filter);
-    auto totalResults = bookIdsToDump.size();
-    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
-    if (count>0 && bookIdsToDump.size() > count) {
-      bookIdsToDump.resize(count);
-    }
-    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
-  }
-
-  opdsDumper.setId(kiwix::to_string(uuid));
-  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
-  return response;
-}
-
-namespace
-{
-
-std::string get_book_name(const RequestContext& request)
-{
-  try {
-    return request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return std::string();
-  }
-}
-
-} // unnamed namespace
-
-std::shared_ptr<Reader>
-InternalServer::get_reader(const std::string& bookName) const
-{
-  std::shared_ptr<Reader> reader;
-  try {
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-  }
-  return reader;
-}
-
-Response
-InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
-{
-  auto response = get_default_response();
-  response.set_redirection(m_root + "/" + bookName + "/" +
-                           kiwix::urlEncode(entry.getPath()));
-  return response;
-}
-
-Response InternalServer::handle_content(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_content\n");
-  }
-
-  const std::string bookName = get_book_name(request);
-  if (bookName.empty())
-    return build_homepage(request);
-
-  const std::shared_ptr<Reader> reader = get_reader(bookName);
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  auto urlStr = request.get_url().substr(bookName.size()+1);
-  if (urlStr[0] == '/') {
-    urlStr = urlStr.substr(1);
-  }
-
-  kiwix::Entry entry;
-
-  try {
-    entry = reader->getEntryFromPath(urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
-      return build_redirect(bookName, entry.getFinalEntry());
-    }
-  } catch(kiwix::NoEntry& e) {
-    if (m_verbose.load())
-      printf("Failed to find %s\n", urlStr.c_str());
-
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-
-  response.set_entry(entry, request);
-
-  if (m_verbose.load()) {
-    printf("Found %s\n", entry.getPath().c_str());
-    printf("mimeType: %s\n", response.get_mimeType().c_str());
-  }
-
-  if (response.get_mimeType().find("text/html") != string::npos)
-    response.set_taskbar(bookName, reader->getTitle());
-
-  return response;
-}
-
 }

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/internalServer.cpp

@@ -0,0 +1,810 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "internalServer.h"
+
+#ifdef __FreeBSD__
+#include <netinet/in.h>
+#endif
+
+#ifdef _WIN32
+# if !defined(__MINGW32__) && (_MSC_VER < 1600)
+#   include "stdint4win.h"
+# endif
+# include <winsock2.h>
+# include <ws2tcpip.h>
+# ifdef __GNUC__
+  // inet_pton is not declared in mingw, even if the function exists.
+  extern "C" {
+    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
+  }
+# endif
+  typedef UINT64 uint64_t;
+  typedef UINT16 uint16_t;
+#endif
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+#include "library.h"
+#include "name_mapper.h"
+#include "entry.h"
+#include "searcher.h"
+#include "search_renderer.h"
+#include "opds_dumper.h"
+
+#include <zim/uuid.h>
+
+#include <mustache.hpp>
+
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+#include "kiwixlib-resources.h"
+
+#ifndef _WIN32
+# include <arpa/inet.h>
+#endif
+
+#include "request_context.h"
+#include "response.h"
+
+#define MAX_SEARCH_LEN 140
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+namespace
+{
+
+inline std::string normalizeRootUrl(std::string rootUrl)
+{
+  while ( !rootUrl.empty() && rootUrl.back() == '/' )
+    rootUrl.pop_back();
+
+  while ( !rootUrl.empty() && rootUrl.front() == '/' )
+    rootUrl = rootUrl.substr(1);
+  return rootUrl.empty() ? rootUrl : "/" + rootUrl;
+}
+
+} // unnamed namespace
+
+static IdNameMapper defaultNameMapper;
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls);
+
+
+InternalServer::InternalServer(Library* library,
+                               NameMapper* nameMapper,
+                               std::string addr,
+                               int port,
+                               std::string root,
+                               int nbThreads,
+                               bool verbose,
+                               bool withTaskbar,
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
+  m_addr(addr),
+  m_port(port),
+  m_root(normalizeRootUrl(root)),
+  m_nbThreads(nbThreads),
+  m_verbose(verbose),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  mp_daemon(nullptr),
+  mp_library(library),
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+{}
+
+bool InternalServer::start() {
+#ifdef _WIN32
+  int flags = MHD_USE_SELECT_INTERNALLY;
+#else
+  int flags = MHD_USE_POLL_INTERNALLY;
+#endif
+  if (m_verbose.load())
+    flags |= MHD_USE_DEBUG;
+
+
+  struct sockaddr_in sockAddr;
+  memset(&sockAddr, 0, sizeof(sockAddr));
+  sockAddr.sin_family = AF_INET;
+  sockAddr.sin_port = htons(m_port);
+  if (m_addr.empty()) {
+    if (0 != INADDR_ANY)
+      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+  } else {
+    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
+      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
+      return false;
+    }
+  }
+
+  mp_daemon = MHD_start_daemon(flags,
+                            m_port,
+                            NULL,
+                            NULL,
+                            &staticHandlerCallback,
+                            this,
+                            MHD_OPTION_SOCK_ADDR, &sockAddr,
+                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_END);
+  if (mp_daemon == nullptr) {
+    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
+              << " is maybe already occupied or need more permissions to be open. "
+                 "Please try as root or with a port number higher or equal to 1024."
+              << std::endl;
+    return false;
+  }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
+  m_library_id = m_server_id;
+  return true;
+}
+
+void InternalServer::stop()
+{
+  MHD_stop_daemon(mp_daemon);
+}
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls)
+{
+  InternalServer* _this = static_cast<InternalServer*>(cls);
+
+  return _this->handlerCallback(connection,
+                                url,
+                                method,
+                                version,
+                                upload_data,
+                                upload_data_size,
+                                cont_cls);
+}
+
+MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                           const char* url,
+                                           const char* method,
+                                           const char* version,
+                                           const char* upload_data,
+                                           size_t* upload_data_size,
+                                           void** cont_cls)
+{
+  auto start_time = std::chrono::steady_clock::now();
+  if (m_verbose.load() ) {
+    printf("======================\n");
+    printf("Requesting : \n");
+    printf("full_url  : %s\n", url);
+  }
+  RequestContext request(connection, m_root, url, method, version);
+
+  if (m_verbose.load() ) {
+    request.print_debug_info();
+  }
+  /* Unexpected method */
+  if (request.get_method() != RequestMethod::GET
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
+    printf("Reject request because of unhandled request method.\n");
+    printf("----------------------\n");
+    return MHD_NO;
+  }
+
+  auto response = handle_request(request);
+
+  if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
+    printf("========== INTERNAL ERROR !! ============\n");
+    if (!m_verbose.load()) {
+      printf("Requesting : \n");
+      printf("full_url : %s\n", url);
+      request.print_debug_info();
+    }
+  }
+
+  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response->set_server_id(m_server_id);
+
+  auto ret = response->send(request, connection);
+  auto end_time = std::chrono::steady_clock::now();
+  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
+  if (m_verbose.load()) {
+    printf("Request time : %fs\n", time_span.count());
+    printf("----------------------\n");
+  }
+  return ret;
+}
+
+std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
+{
+  try {
+    if (! request.is_valid_url())
+      return Response::build_404(*this, request, "", "");
+
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return Response::build_304(*this, etag);
+
+    if (startsWith(request.get_url(), "/skin/"))
+      return handle_skin(request);
+
+    if (startsWith(request.get_url(), "/catalog/"))
+      return handle_catalog(request);
+
+    if (request.get_url() == "/meta")
+      return handle_meta(request);
+
+    if (request.get_url() == "/search")
+      return handle_search(request);
+
+    if (request.get_url() == "/suggest")
+     return handle_suggest(request);
+
+    if (request.get_url() == "/random")
+      return handle_random(request);
+
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
+    return handle_content(request);
+  } catch (std::exception& e) {
+    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
+    return Response::build_500(*this, e.what());
+  } catch (...) {
+    fprintf(stderr, "===== Unhandled unknown error\n");
+    return Response::build_500(*this, "Unknown error");
+  }
+}
+
+MustacheData InternalServer::get_default_data() const
+{
+  MustacheData data;
+  data.set("root", m_root);
+  return data;
+}
+
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
+{
+  return ContentResponse::build(*this, RESOURCE::templates::index_html, get_default_data(), "text/html; charset=utf-8", true);
+}
+
+std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+{
+  std::string bookName;
+  std::string bookId;
+  std::string meta_name;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    meta_name = request.get_argument("name");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  auto response = ContentResponse::build(*this, content, mimeType);
+  response->set_cacheable();
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_suggest\n");
+  }
+
+  std::string content;
+  std::string mimeType;
+  unsigned int maxSuggestionCount = 10;
+  unsigned int suggestionCount = 0;
+
+  std::string bookName;
+  std::string bookId;
+  std::string term;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    term = request.get_argument("term");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  if (m_verbose.load()) {
+    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+  }
+
+  MustacheData results{MustacheData::type::list};
+
+  bool first = true;
+  if (reader != nullptr) {
+    /* Get the suggestions */
+    SuggestionsList_t suggestions;
+    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
+    for(auto& suggestion:suggestions) {
+      MustacheData result;
+      result.set("label", suggestion.getTitle());
+
+      if (suggestion.hasSnippet()) {
+        result.set("label", suggestion.getSnippet());
+      }
+
+      result.set("value", suggestion.getTitle());
+      result.set("kind", "path");
+      result.set("path", suggestion.getPath());
+      result.set("first", first);
+      first = false;
+      results.push_back(result);
+      suggestionCount++;
+    }
+  }
+
+  /* Propose the fulltext search if possible */
+  if (reader->hasFulltextIndex()) {
+    MustacheData result;
+    result.set("label", "containing '" + term + "'...");
+    result.set("value", term + " ");
+    result.set("kind", "pattern");
+    result.set("first", first);
+    results.push_back(result);
+  }
+
+  auto data = get_default_data();
+  data.set("suggestions", results);
+
+  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_skin\n");
+  }
+
+  auto resourceName = request.get_url().substr(1);
+  try {
+    auto response = ContentResponse::build(
+        *this,
+        getResource(resourceName),
+        getMimeTypeForFile(resourceName));
+    response->set_cacheable();
+    return std::move(response);
+  } catch (const ResourceNotFound& e) {
+    return Response::build_404(*this, request, "", "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_search\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+  } catch (const std::out_of_range&) {}
+
+  std::string patternString;
+  try {
+    patternString = request.get_argument("pattern");
+  } catch (const std::out_of_range&) {}
+
+  /* Retrive geo search */
+  bool has_geo_query = false;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
+  try {
+    latitude = request.get_argument<float>("latitude");
+    longitude = request.get_argument<float>("longitude");
+    distance = request.get_argument<float>("distance");
+    has_geo_query = true;
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  std::shared_ptr<Reader> reader(nullptr);
+  try {
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {}
+
+  /* Make the search */
+  if ( (!reader && !bookName.empty())
+    || (patternString.empty() && ! has_geo_query) ) {
+    auto data = get_default_data();
+    data.set("pattern", encodeDiples(patternString));
+    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_code(MHD_HTTP_NOT_FOUND);
+    return std::move(response);
+  }
+
+  Searcher searcher;
+  if (reader) {
+    searcher.add_reader(reader.get());
+  } else {
+    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+      auto currentReader = mp_library->getReaderById(bookId);
+      if (currentReader) {
+        searcher.add_reader(currentReader.get());
+      }
+    }
+  }
+
+  auto start = 0;
+  try {
+    start = request.get_argument<unsigned int>("start");
+  } catch (const std::exception&) {}
+
+  auto pageLength = 25;
+  try {
+    pageLength = request.get_argument<unsigned int>("pageLength");
+  } catch (const std::exception&) {}
+  if (pageLength > MAX_SEARCH_LEN) {
+    pageLength = MAX_SEARCH_LEN;
+  }
+  if (pageLength == 0) {
+    pageLength = 25;
+  }
+
+  auto end = start + pageLength;
+
+  /* Get the results */
+  try {
+    if (patternString.empty()) {
+      searcher.geo_search(latitude, longitude, distance,
+                           start, end, m_verbose.load());
+    } else {
+      searcher.search(patternString,
+                       start, end, m_verbose.load());
+    }
+    SearchRenderer renderer(&searcher, mp_nameMapper);
+    renderer.setSearchPattern(patternString);
+    renderer.setSearchContent(bookName);
+    renderer.setProtocolPrefix(m_root + "/");
+    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setPageLength(pageLength);
+    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+
+    return std::move(response);
+  } catch (const std::exception& e) {
+    std::cerr << e.what() << std::endl;
+    return Response::build_500(*this, e.what());
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_random\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+
+  try {
+    auto entry = reader->getRandomPage();
+    return build_redirect(bookName, entry.getFinalEntry());
+  } catch(kiwix::NoEntry& e) {
+    return Response::build_404(*this, request, bookName, "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return Response::build_404(*this, request, "", "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, "", "");
+  }
+
+  if (url == "v2") {
+    return handle_catalog_v2(request);
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return Response::build_404(*this, request, "", "");
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper(mp_library);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(m_library_id);
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    bookIdsToDump = search_catalog(request, opdsDumper);
+    uuid = zim::Uuid::generate();
+  }
+
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()),
+      "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
+  return std::move(response);
+}
+
+namespace
+{
+
+Filter get_search_filter(const RequestContext& request)
+{
+    auto filter = kiwix::Filter().valid(true).local(true);
+    try {
+      filter.query(request.get_argument("q"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(request.get_argument<unsigned long>("maxsize"));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.category(request.get_argument("category"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
+    } catch (...) {}
+    return filter;
+}
+
+template<class T>
+std::vector<T> subrange(const std::vector<T>& v, size_t s, size_t n)
+{
+  const size_t e = std::min(v.size(), s+n);
+  return std::vector<T>(v.begin()+std::min(v.size(), s), v.begin()+e);
+}
+
+} // unnamed namespace
+
+std::vector<std::string>
+InternalServer::search_catalog(const RequestContext& request,
+                               kiwix::OPDSDumper& opdsDumper)
+{
+    const auto filter = get_search_filter(request);
+    const std::string q = filter.hasQuery()
+                        ? filter.getQuery()
+                        : "<Empty query>";
+    std::vector<std::string> bookIdsToDump = mp_library->filter(filter);
+    const auto totalResults = bookIdsToDump.size();
+    const size_t count = request.get_optional_param("count", 10UL);
+    const size_t startIndex = request.get_optional_param("start", 0UL);
+    bookIdsToDump = subrange(bookIdsToDump, startIndex, count);
+    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
+    return bookIdsToDump;
+}
+
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+std::string searchSuggestionHTML(const std::string& searchURL, const std::string& pattern)
+{
+  kainjow::mustache::mustache tmpl("Make a full text search for <a href=\"{{{searchURL}}}\">{{pattern}}</a>");
+  MustacheData data;
+  data.set("pattern", pattern);
+  data.set("searchURL", searchURL);
+  return (tmpl.render(data));
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+std::unique_ptr<Response>
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
+  return Response::build_redirect(*this, redirectUrl);
+}
+
+std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
+{
+  const std::string url = request.get_url();
+  const std::string pattern = url.substr((url.find_last_of('/'))+1);
+  if (m_verbose.load()) {
+    printf("** running handle_content\n");
+  }
+
+  const std::string bookName = get_book_name(request);
+  if (bookName.empty())
+    return build_homepage(request);
+
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
+  if (reader == nullptr) {
+    std::string searchURL = m_root+"/search?pattern="+pattern; // Make a full search on the entire library.
+    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
+
+    return Response::build_404(*this, request, bookName, "", details);
+  }
+
+  auto urlStr = request.get_url().substr(bookName.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
+
+  try {
+    auto entry = reader->getEntryFromPath(urlStr);
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      return build_redirect(bookName, entry.getFinalEntry());
+    }
+    auto response = ItemResponse::build(*this, request, entry.getZimEntry().getItem());
+    try {
+      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
+    } catch (std::bad_cast& e) {}
+
+    if (m_verbose.load()) {
+      printf("Found %s\n", entry.getPath().c_str());
+      printf("mimeType: %s\n", entry.getMimetype().c_str());
+    }
+
+    return response;
+  } catch(kiwix::NoEntry& e) {
+    if (m_verbose.load())
+      printf("Failed to find %s\n", urlStr.c_str());
+
+    std::string searchURL = m_root+"/search?content="+bookName+"&pattern="+pattern; // Make a search on this specific book only.
+    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
+
+    return Response::build_404(*this, request, bookName, reader->getTitle(), details);
+  }
+}
+
+}

src/server/internalServer.h

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H
+#define KIWIXLIB_SERVER_INTERNALSERVER_H
+
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "library.h"
+#include "name_mapper.h"
+
+#include <mustache.hpp>
+
+#include <atomic>
+#include <string>
+
+#include "server/request_context.h"
+#include "server/response.h"
+
+namespace kiwix {
+
+typedef kainjow::mustache::data MustacheData;
+
+class Entry;
+class OPDSDumper;
+
+class InternalServer {
+  public:
+    InternalServer(Library* library,
+                   NameMapper* nameMapper,
+                   std::string addr,
+                   int port,
+                   std::string root,
+                   int nbThreads,
+                   bool verbose,
+                   bool withTaskbar,
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
+    virtual ~InternalServer() = default;
+
+    MHD_Result handlerCallback(struct MHD_Connection* connection,
+                               const char* url,
+                               const char* method,
+                               const char* version,
+                               const char* upload_data,
+                               size_t* upload_data_size,
+                               void** cont_cls);
+    bool start();
+    void stop();
+
+  private: // functions
+    std::unique_ptr<Response> handle_request(const RequestContext& request);
+    std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_skin(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
+    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_suggest(const RequestContext& request);
+    std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
+    std::unique_ptr<Response> handle_content(const RequestContext& request);
+
+    std::vector<std::string> search_catalog(const RequestContext& request,
+                                            kiwix::OPDSDumper& opdsDumper);
+
+    MustacheData get_default_data() const;
+
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
+    std::string m_addr;
+    int m_port;
+    std::string m_root;
+    int m_nbThreads;
+    std::atomic_bool m_verbose;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    struct MHD_Daemon* mp_daemon;
+
+    Library* mp_library;
+    NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
+    std::string m_library_id;
+
+    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
+
+};
+
+}
+
+#endif //KIWIXLIB_SERVER_INTERNALSERVER_H

src/server/internalServer_catalog_v2.cpp

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "internalServer.h"
+
+#include "library.h"
+#include "opds_dumper.h"
+#include "request_context.h"
+#include "response.h"
+#include "tools/otherTools.h"
+#include "kiwixlib-resources.h"
+
+#include <mustache.hpp>
+
+#include <string>
+#include <vector>
+
+namespace kiwix {
+
+std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog_v2");
+  }
+
+  std::string url;
+  try {
+    url  = request.get_url_part(2);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, "", "");
+  }
+
+  if (url == "root.xml") {
+    return handle_catalog_v2_root(request);
+  } else if (url == "searchdescription.xml") {
+    const std::string endpoint_root = m_root + "/catalog/v2";
+    return ContentResponse::build(*this,
+        RESOURCE::catalog_v2_searchdescription_xml,
+        kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
+        "application/opensearchdescription+xml"
+    );
+  } else if (url == "entries") {
+    return handle_catalog_v2_entries(request);
+  } else if (url == "categories") {
+    return handle_catalog_v2_categories(request);
+  } else {
+    return Response::build_404(*this, request, "", "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
+{
+  return ContentResponse::build(
+             *this,
+             RESOURCE::templates::catalog_v2_root_xml,
+             kainjow::mustache::object{
+               {"date", gen_date_str()},
+               {"endpoint_root", m_root + "/catalog/v2"},
+               {"feed_id", gen_uuid(m_library_id)},
+               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
+               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")}
+             },
+             "application/atom+xml;profile=opds-catalog;kind=navigation"
+  );
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request)
+{
+  OPDSDumper opdsDumper(mp_library);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(m_library_id);
+  const auto bookIds = search_catalog(request, opdsDumper);
+  const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query());
+  return ContentResponse::build(
+             *this,
+             opdsFeed,
+             "application/atom+xml;profile=opds-catalog;kind=acquisition"
+  );
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
+{
+  OPDSDumper opdsDumper(mp_library);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(m_library_id);
+  return ContentResponse::build(
+             *this,
+             opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()),
+             "application/atom+xml;profile=opds-catalog;kind=navigation"
+  );
+}
+
+} // namespace kiwix

src/server/request_context.cpp

@@ -26,6 +26,8 @@
 #include <cstdio>
 #include <atomic>
 
+#include "tools/stringTools.h"
+
 namespace kiwix {
 
 static std::atomic_ullong s_requestIndex(0);
@@ -74,8 +76,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingDeflate(false),
-  accept_range(false),
-  range_pair(0, -1)
+  byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
@@ -85,43 +86,24 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
         (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
-  /*Check if range is requested. */
   try {
-    auto range = get_header(MHD_HTTP_HEADER_RANGE);
-    int start = 0;
-    int end = -1;
-    std::istringstream iss(range);
-    char c;
-
-    iss >> start >> c;
-    if (iss.good() && c=='-') {
-      iss >> end;
-      if (iss.fail()) {
-        // Something went wrong will extracting.
-        end = -1;
-      }
-      if (iss.eof()) {
-        accept_range = true;
-        range_pair = std::pair<int, int>(start, end);
-      }
-    }
+    byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
 }
 
 RequestContext::~RequestContext()
 {}
 
-
-int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                 const char *key, const char *value)
+MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
+                                       const char *key, const char *value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[key] = value;
+  _this->headers[lcAll(key)] = value;
   return MHD_YES;
 }
 
-int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                   const char *key, const char* value)
+MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key] = value == nullptr ? "" : value;
@@ -146,7 +128,7 @@ void RequestContext::print_debug_info() const {
   printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
-  printf("has_range : %d\n", accept_range);
+  printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
 }
@@ -188,12 +170,8 @@ bool RequestContext::is_valid_url() const {
   return !url.empty();
 }
 
-bool RequestContext::has_range() const {
-  return accept_range;
-}
-
-std::pair<int, int> RequestContext::get_range() const {
-  return range_pair;
+ByteRange RequestContext::get_range() const {
+  return byteRange_;
 }
 
 template<>
@@ -202,7 +180,17 @@ std::string RequestContext::get_argument(const std::string& name) const {
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
-  return headers.at(name);
+  return headers.at(lcAll(name));
+}
+
+std::string RequestContext::get_query() const {
+  std::string q;
+  const char* sep = "";
+  for ( const auto& a : arguments ) {
+    q += sep + a.first + '=' + a.second;
+    sep = "&";
+  }
+  return q;
 }
 
 }

src/server/request_context.h

@@ -27,8 +27,10 @@
 #include <map>
 #include <stdexcept>
 
+#include "byte_range.h"
+
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
@@ -51,9 +53,6 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
-  public: // types
-  typedef std::pair<int, int> ByteRange;
-
   public: // functions
     RequestContext(struct MHD_Connection* connection,
                    std::string rootLocation,
@@ -75,13 +74,22 @@ class RequestContext {
         return v;
     }
 
+    template<class T>
+    T get_optional_param(const std::string& name, T default_value) const
+    {
+      try {
+        return get_argument<T>(name);
+      } catch (...) {}
+      return default_value;
+    }
+
 
     RequestMethod get_method() const;
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
+    std::string get_query() const;
 
-    bool has_range() const;
     ByteRange get_range() const;
 
     bool can_compress() const { return acceptEncodingDeflate; }
@@ -95,14 +103,13 @@ class RequestContext {
 
     bool acceptEncodingDeflate;
 
-    bool accept_range;
-    ByteRange range_pair;
+    ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::string> arguments;
 
   private: // functions
-    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 
 template<> std::string RequestContext::get_argument(const std::string& name) const;

src/server/response.cpp

@@ -1,12 +1,30 @@
-
-
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
 
 #include "response.h"
 #include "request_context.h"
+#include "internalServer.h"
 #include "kiwixlib-resources.h"
 
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
 
 #include "string.h"
 #include <mustache.hpp>
@@ -21,10 +39,10 @@ namespace
 {
 // some utilities
 
-std::string get_mime_type(const kiwix::Entry& entry)
+std::string get_mime_type(const zim::Item& item)
 {
   try {
-    return entry.getMimetype();
+    return item.getMimetype();
   } catch (exception& e) {
     return "application/octet-stream";
   }
@@ -39,34 +57,80 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/json") != string::npos;
 }
 
-int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
-{
-  return range.second == -1
-       ? entry.getSize() - range.first
-       : range.second - range.first;
-}
 
 } // unnamed namespace
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
+Response::Response(bool verbose)
   : m_verbose(verbose),
-    m_root(root),
-    m_content(""),
-    m_mimeType(""),
-    m_returnCode(MHD_HTTP_OK),
-    m_withTaskbar(withTaskbar),
-    m_withLibraryButton(withLibraryButton),
-    m_blockExternalLinks(blockExternalLinks),
-    m_addTaskbar(false),
-    m_bookName(""),
-    m_startRange(0),
-    m_lenRange(0)
+    m_returnCode(MHD_HTTP_OK)
 {
+  add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+}
+
+std::unique_ptr<Response> Response::build(const InternalServer& server)
+{
+  return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
+}
+
+std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
+{
+  auto response = Response::build(server);
+  response->set_code(MHD_HTTP_NOT_MODIFIED);
+  response->m_etag = etag;
+  if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
+{
+  MustacheData results;
+  results.set("url", request.get_full_url());
+  results.set("details", details);
+
+  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
+  response->set_code(MHD_HTTP_NOT_FOUND);
+  response->set_taskbar(bookName, bookTitle);
+
+  return std::move(response);
+}
+
+std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
+{
+  auto response = Response::build(server);
+// [FIXME] (compile with recent enough version of libmicrohttpd)
+//  response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
+  response->set_code(416);
+  std::ostringstream oss;
+  oss << "bytes */" << resourceLength;
+  response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str());
+
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
+{
+  MustacheData data;
+  data.set("error", msg);
+  auto content = render_template(RESOURCE::templates::_500_html, data);
+  std::unique_ptr<Response> response (
+      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
+  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
+  return response;
 }
 
 
-static int print_key_value (void *cls, enum MHD_ValueKind kind,
-                            const char *key, const char *value)
+std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
+{
+  auto response = Response::build(server);
+  response->m_returnCode = MHD_HTTP_FOUND;
+  response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
+  return response;
+}
+
+static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
+                                   const char *key, const char *value)
 {
   printf (" - %s: '%s'\n", key, value);
   return MHD_YES;
@@ -74,17 +138,17 @@ static int print_key_value (void *cls, enum MHD_ValueKind kind,
 
 
 struct RunningResponse {
-   kiwix::Entry entry;
+   zim::Item item;
    int range_start;
 
-   RunningResponse(kiwix::Entry entry,
+   RunningResponse(zim::Item item,
                    int range_start) :
-     entry(entry),
+     item(item),
      range_start(range_start)
    {}
 };
 
-static ssize_t callback_reader_from_entry(void* cls,
+static ssize_t callback_reader_from_item(void* cls,
                                   uint64_t pos,
                                   char* buf,
                                   size_t max)
@@ -93,13 +157,13 @@ static ssize_t callback_reader_from_entry(void* cls,
 
   size_t max_size_to_set = min<size_t>(
     max,
-    response->entry.getSize() - pos - response->range_start);
+    response->item.getSize() - pos - response->range_start);
 
   if (max_size_to_set <= 0) {
     return MHD_CONTENT_READER_END_WITH_ERROR;
   }
 
-  zim::Blob blob = response->entry.getBlob(response->range_start+pos, max_size_to_set);
+  zim::Blob blob = response->item.getData(response->range_start+pos, max_size_to_set);
   memcpy(buf, blob.data(), max_size_to_set);
   return max_size_to_set;
 }
@@ -121,33 +185,18 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-std::string render_template(const std::string& template_str, kainjow::mustache::data data)
+void ContentResponse::introduce_taskbar()
 {
-  kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
-  std::stringstream ss;
-  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
-
-
-void Response::introduce_taskbar()
-{
-  if (! m_withTaskbar)
-    // Taskbar is globally disabled.
-    return;
   kainjow::mustache::data data;
   data.set("root", m_root);
   data.set("content", m_bookName);
-  data.set("hascontent", !m_bookName.empty());
+  data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
   data.set("title", m_bookTitle);
   data.set("withlibrarybutton", m_withLibraryButton);
-  auto head_content = render_template(RESOURCE::templates::head_part_html, data);
-  m_content = appendToFirstOccurence(
+  auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
+  m_content = prependToFirstOccurence(
     m_content,
-    "<head>",
+    "</head[ \\t]*>",
     head_content);
 
   auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
@@ -158,36 +207,61 @@ void Response::introduce_taskbar()
 }
 
 
-void Response::inject_externallinks_blocker()
+void ContentResponse::inject_externallinks_blocker()
 {
   kainjow::mustache::data data;
   data.set("root", m_root);
   auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = appendToFirstOccurence(
+  m_content = prependToFirstOccurence(
     m_content,
-    "<head>",
+    "</head[ \\t]*>",
     script_tag);
 }
 
+void ContentResponse::inject_root_link(){
+  m_content = prependToFirstOccurence(
+    m_content,
+    "</head[ \\t]*>",
+    "<link type=\"root\" href=\"" + m_root + "\">");
+}
+
 bool
-Response::can_compress(const RequestContext& request) const
+ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
 }
 
-MHD_Response*
-Response::create_raw_content_mhd_response(const RequestContext& request)
+bool
+ContentResponse::contentDecorationAllowed() const
 {
-  if (m_addTaskbar) {
-    introduce_taskbar();
-  }
-  if ( m_blockExternalLinks ) {
-    inject_externallinks_blocker();
+    return (startsWith(m_mimeType, "text/html")
+         && m_mimeType.find(";raw=true") == std::string::npos);
+}
+
+MHD_Response*
+Response::create_mhd_response(const RequestContext& request)
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT);
+  return response;
+}
+
+MHD_Response*
+ContentResponse::create_mhd_response(const RequestContext& request)
+{
+  if (contentDecorationAllowed()) {
+    inject_root_link();
+
+    if (m_withTaskbar) {
+      introduce_taskbar();
+    }
+    if (m_blockExternalLinks) {
+      inject_externallinks_blocker();
+    }
   }
 
-  bool shouldCompress = m_compress && can_compress(request);
+  bool shouldCompress = can_compress(request);
   if (shouldCompress) {
     std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
     uLongf comprLen = compr_buffer.capacity();
@@ -211,83 +285,29 @@ Response::create_raw_content_mhd_response(const RequestContext& request)
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
-  // shouldCompress can have different values. This can happen for a 304 (Not
-  // Modified) response generated while handling a conditional If-None-Match
-  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
-  // option) is obtained from the ETag list of the If-None-Match header and the
-  // response has no body (which shouldn't be compressed).
-  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-  }
-  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
   }
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
   return response;
 }
 
-MHD_Response*
-Response::create_redirection_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_entry_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
-                                               16384,
-                                               callback_reader_from_entry,
-                                               new RunningResponse(m_entry, m_startRange),
-                                               callback_free_response);
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-  std::ostringstream oss;
-  oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-      << "/" << m_entry.getSize();
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_mhd_response(const RequestContext& request)
-{
-  switch (m_mode) {
-    case ResponseMode::RAW_CONTENT :
-      return create_raw_content_mhd_response(request);
-
-    case ResponseMode::REDIRECTION :
-      return create_redirection_mhd_response();
-
-    case ResponseMode::ENTRY :
-      return create_entry_mhd_response();
-  }
-  return nullptr;
-}
-
-int Response::send(const RequestContext& request, MHD_Connection* connection)
+MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
 {
   MHD_Response* response = create_mhd_response(request);
 
-  MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
     m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+    MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  for(auto& p: m_customHeaders) {
+    MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
+  }
 
-  if (m_returnCode == MHD_HTTP_OK && request.has_range())
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
   if (m_verbose)
@@ -298,48 +318,103 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
   return ret;
 }
 
-void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
-  set_content(render_template(template_str, data));
-}
-
-void Response::set_content(const std::string& content) {
-  m_content = content;
-  m_mode = ResponseMode::RAW_CONTENT;
-}
-
-void Response::set_redirection(const std::string& url) {
-  m_content = url;
-  m_mode = ResponseMode::REDIRECTION;
-  m_returnCode = MHD_HTTP_FOUND;
-}
-
-void Response::set_entry(const Entry& entry, const RequestContext& request) {
-  m_entry = entry;
-  m_mode = ResponseMode::ENTRY;
-
-  const std::string mimeType = get_mime_type(entry);
-  set_mimeType(mimeType);
-  set_cacheable();
-
-  if ( is_compressible_mime_type(mimeType) ) {
-    zim::Blob raw_content = entry.getBlob();
-    const std::string content = string(raw_content.data(), raw_content.size());
-
-    set_content(content);
-    set_compress(true);
-  } else {
-    const int range_len = get_range_len(entry, request.get_range());
-    set_range_first(request.get_range().first);
-    set_range_len(range_len);
-  }
-}
-
-void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
 {
-  m_addTaskbar = true;
   m_bookName = bookName;
   m_bookTitle = bookTitle;
 }
 
 
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+  Response(verbose),
+  m_root(root),
+  m_content(content),
+  m_mimeType(mimetype),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  m_bookName(""),
+  m_bookTitle("")
+{
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
+{
+   return std::unique_ptr<ContentResponse>(new ContentResponse(
+        server.m_root,
+        server.m_verbose.load(),
+        server.m_withTaskbar && !isHomePage,
+        server.m_withLibraryButton,
+        server.m_blockExternalLinks,
+        content,
+        mimetype));
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
+  auto content = render_template(template_str, data);
+  return ContentResponse::build(server, content, mimetype, isHomePage);
+}
+
+ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
+  Response(verbose),
+  m_item(item),
+  m_mimeType(mimetype)
+{
+  m_byteRange = byterange;
+  set_cacheable();
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
+{
+  const std::string mimetype = get_mime_type(item);
+  auto byteRange = request.get_range().resolve(item.getSize());
+  const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if (noRange && is_compressible_mime_type(mimetype)) {
+    // Return a contentResponse
+    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    response->set_cacheable();
+    response->m_byteRange = byteRange;
+    return std::move(response);
+  }
+
+  if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
+    auto response = Response::build_416(server, item.getSize());
+    response->set_cacheable();
+    return response;
+  }
+
+  return std::unique_ptr<Response>(new ItemResponse(
+        server.m_verbose.load(),
+        item,
+        mimetype,
+        byteRange));
+}
+
+MHD_Response*
+ItemResponse::create_mhd_response(const RequestContext& request)
+{
+  const auto content_length = m_byteRange.length();
+  MHD_Response* response = MHD_create_response_from_callback(content_length,
+                                               16384,
+                                               callback_reader_from_item,
+                                               new RunningResponse(m_item, m_byteRange.first()),
+                                               callback_free_response);
+  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
+  if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
+    std::ostringstream oss;
+    oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
+        << "/" << m_item.getSize();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
+  return response;
+}
+
+
 }

src/server/response.h

@@ -22,80 +22,99 @@
 #define KIWIXLIB_SERVER_RESPONSE_H
 
 #include <string>
+#include <map>
 
 #include <mustache.hpp>
+#include "byte_range.h"
 #include "entry.h"
 #include "etag.h"
 
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
 
-enum class ResponseMode {
-  RAW_CONTENT,
-  REDIRECTION,
-  ENTRY
-};
-
+class InternalServer;
 class RequestContext;
 
+class EntryResponse;
+
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
-    ~Response() = default;
+    Response(bool verbose);
+    virtual ~Response() = default;
 
-    int send(const RequestContext& request, MHD_Connection* connection);
+    static std::unique_ptr<Response> build(const InternalServer& server);
+    static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
+    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
+    static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
+    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
+    static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
-    void set_template(const std::string& template_str, kainjow::mustache::data data);
-    void set_content(const std::string& content);
-    void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry, const RequestContext& request);
+    MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
-
-    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
     void set_code(int code) { m_returnCode = code; }
     void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
     void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
-    void set_etag(const ETag& etag) { m_etag = etag; }
-    void set_compress(bool compress) { m_compress = compress; }
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
-    void set_range_first(uint64_t start) { m_startRange = start; }
-    void set_range_len(uint64_t len) { m_lenRange = len; }
+    void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
-    std::string get_mimeType() const { return m_mimeType; }
+
+  private: // functions
+    virtual MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
+
+  protected: // data
+    bool m_verbose;
+    int m_returnCode;
+    ByteRange m_byteRange;
+    ETag m_etag;
+    std::map<std::string, std::string> m_customHeaders;
+
+    friend class ItemResponse;
+};
+
+
+class ContentResponse : public Response {
+  public:
+    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
+
+    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
 
     void introduce_taskbar();
     void inject_externallinks_blocker();
-
+    void inject_root_link();
     bool can_compress(const RequestContext& request) const;
+    bool contentDecorationAllowed() const;
 
-  private: // functions
-    MHD_Response* create_mhd_response(const RequestContext& request);
-    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
-    MHD_Response* create_redirection_mhd_response() const;
-    MHD_Response* create_entry_mhd_response() const;
 
-  private: // data
-    bool m_verbose;
-    ResponseMode m_mode;
+  private:
     std::string m_root;
     std::string m_content;
-    Entry m_entry;
     std::string m_mimeType;
-    int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
-    bool m_compress;
-    bool m_addTaskbar;
     std::string m_bookName;
     std::string m_bookTitle;
-    uint64_t m_startRange;
-    uint64_t m_lenRange;
-    ETag m_etag;
+ };
+
+class ItemResponse : public Response {
+  public:
+    ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
+
+    zim::Item m_item;
+    std::string m_mimeType;
 };
 
 }

src/subprocess_unix.cpp

@@ -12,45 +12,31 @@
 UnixImpl::UnixImpl():
   m_pid(0),
   m_running(false),
-  m_mutex(PTHREAD_MUTEX_INITIALIZER),
-  m_waitingThread()
+  m_shouldQuit(false)
 {
 }
 
 UnixImpl::~UnixImpl()
 {
   kill();
-// Android has no pthread_cancel :(
-#ifdef __ANDROID__
-  pthread_kill(m_waitingThread, SIGUSR1);
-#else
-  pthread_cancel(m_waitingThread);
-#endif
+  m_shouldQuit = true;
+  m_waitingThread.join();
 }
 
-#ifdef __ANDROID__
-void thread_exit_handler(int sig) {
-  pthread_exit(0);
-}
-#endif
-
 void* UnixImpl::waitForPID(void* _self)
 {
-#ifdef __ANDROID__
-  struct sigaction actions;
-  memset(&actions, 0, sizeof(actions));
-  sigemptyset(&actions.sa_mask);
-  actions.sa_flags = 0;
-  actions.sa_handler = thread_exit_handler;
-  sigaction(SIGUSR1, &actions, NULL);
-#endif
-
   UnixImpl* self = static_cast<UnixImpl*>(_self);
-  waitpid(self->m_pid, NULL, 0);
+  while (true) {
+    if (!waitpid(self->m_pid, NULL, WNOHANG)) {
+      break;
+    }
+    if (self->m_shouldQuit) {
+      return nullptr;
+    }
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  }
 
-  pthread_mutex_lock(&self->m_mutex);
   self->m_running = false;
-  pthread_mutex_unlock(&self->m_mutex);
 
   return self;
 }
@@ -74,7 +60,7 @@ void UnixImpl::run(commandLine_t& commandLine)
     default:
       m_pid = pid;
       m_running = true;
-      pthread_create(&m_waitingThread, NULL, waitForPID, this);
+      m_waitingThread = std::thread(waitForPID, this);
       break;
   }
 }
@@ -86,8 +72,5 @@ bool UnixImpl::kill()
 
 bool UnixImpl::isRunning()
 {
-  pthread_mutex_lock(&m_mutex);
-  bool ret = m_running;
-  pthread_mutex_unlock(&m_mutex);
-  return ret;
+  return m_running;
 }

src/subprocess_unix.h

@@ -3,16 +3,16 @@
 
 #include "subprocess.h"
 
-#include <pthread.h>
-
+#include <atomic>
+#include <thread>
 
 class UnixImpl : public SubprocessImpl
 {
   private:
     int m_pid;
-    bool m_running;
-    pthread_mutex_t m_mutex;
-    pthread_t m_waitingThread;
+    std::atomic<bool> m_running;
+    std::atomic<bool> m_shouldQuit;
+    std::thread m_waitingThread;
 
   public:
     UnixImpl();

src/subprocess_windows.cpp

@@ -11,7 +11,8 @@
 WinImpl::WinImpl():
   m_pid(0),
   m_running(false),
-  m_handle(INVALID_HANDLE_VALUE)
+  m_subprocessHandle(INVALID_HANDLE_VALUE),
+  m_waitingThreadHandle(INVALID_HANDLE_VALUE)
 {
   InitializeCriticalSection(&m_criticalSection);
 }
@@ -19,14 +20,15 @@ WinImpl::WinImpl():
 WinImpl::~WinImpl()
 {
   kill();
-  CloseHandle(m_handle);
+  WaitForSingleObject(m_waitingThreadHandle, INFINITE);
+  CloseHandle(m_subprocessHandle);
   DeleteCriticalSection(&m_criticalSection);
 }
 
 DWORD WINAPI WinImpl::waitForPID(void* _self)
 {
   WinImpl* self = static_cast<WinImpl*>(_self);
-  WaitForSingleObject(self->m_handle, INFINITE);
+  WaitForSingleObject(self->m_subprocessHandle, INFINITE);
 
   EnterCriticalSection(&self->m_criticalSection);
   self->m_running = false;
@@ -79,16 +81,16 @@ void WinImpl::run(commandLine_t& commandLine)
     &procInfo))
   {
     m_pid = procInfo.dwProcessId;
-    m_handle = procInfo.hProcess;
+    m_subprocessHandle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);
     m_running = true;
-    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
+    m_waitingThreadHandle = CreateThread(NULL, 0, &waitForPID, this, 0, NULL);
   }
 }
 
 bool WinImpl::kill()
 {
-  return TerminateProcess(m_handle, 0);
+  return TerminateProcess(m_subprocessHandle, 0);
 }
 
 bool WinImpl::isRunning()

src/subprocess_windows.h

@@ -11,7 +11,8 @@ class WinImpl : public SubprocessImpl
   private:
     int m_pid;
     bool m_running;
-    HANDLE m_handle;
+    HANDLE m_subprocessHandle;
+    HANDLE  m_waitingThreadHandle;
     CRITICAL_SECTION m_criticalSection;
 
   public:

src/tools/otherTools.cpp

@@ -18,6 +18,8 @@
  */
 
 #include "tools/otherTools.h"
+#include <algorithm>
+#include <iomanip>
 
 #ifdef _WIN32
 #include <windows.h>
@@ -31,6 +33,8 @@
 #include <sstream>
 #include <pugixml.hpp>
 
+#include <zim/uuid.h>
+
 
 static std::map<std::string, std::string> codeisomapping {
 { "aa", "aar" },
@@ -280,3 +284,95 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
+namespace
+{
+// The counter metadata format is a list of item separated by a `;` :
+// item0;item1;item2
+// Each item is a "tuple" mimetype=number.
+// However, the mimetype may contains parameters:
+// text/html;raw=true;foo=bar
+// So the final format may be complex to parse:
+// key0=value0;key1;foo=bar=value1;key2=value2
+
+typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
+
+std::string readFullMimetypeAndCounterString(std::istream& in)
+{
+  std::string mtcStr, params;
+  getline(in, mtcStr, ';');
+  if ( mtcStr.find('=') == std::string::npos )
+  {
+    do
+    {
+      if ( !getline(in, params, ';' ) )
+        return std::string();
+      mtcStr += ";" + params;
+    }
+    while ( std::count(params.begin(), params.end(), '=') != 2 );
+  }
+  return mtcStr;
+}
+
+MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
+{
+  const std::string::size_type k = s.find_last_of("=");
+  if ( k != std::string::npos )
+  {
+    const std::string mimeType = s.substr(0, k);
+    std::istringstream counterSS(s.substr(k+1));
+    unsigned int counter;
+    if (counterSS >> counter && counterSS.eof())
+      return MimetypeAndCounter{mimeType, counter};
+  }
+  return MimetypeAndCounter{"", 0};
+}
+
+} // unnamed namespace
+
+kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
+{
+  kiwix::MimeCounterType counters;
+  std::istringstream ss(counterData);
+
+  while (ss)
+  {
+    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
+    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
+    if ( !mtc.first.empty() )
+      counters.insert(mtc);
+  }
+
+  return counters;
+}
+
+std::string kiwix::gen_date_str()
+{
+  auto now = std::time(0);
+  auto tm = std::localtime(&now);
+
+  std::stringstream is;
+  is << std::setw(2) << std::setfill('0')
+     << 1900+tm->tm_year << "-"
+     << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-"
+     << std::setw(2) << std::setfill('0') << tm->tm_mday << "T"
+     << std::setw(2) << std::setfill('0') << tm->tm_hour << ":"
+     << std::setw(2) << std::setfill('0') << tm->tm_min << ":"
+     << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z";
+  return is.str();
+}
+
+std::string kiwix::gen_uuid(const std::string& s)
+{
+  return kiwix::to_string(zim::Uuid::generate(s));
+}
+
+std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
+{
+  kainjow::mustache::mustache tmpl(template_str);
+  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
+                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
+  data.set("urlencoded", urlencode);
+  std::stringstream ss;
+  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}

src/tools/pathTools.cpp

@@ -81,13 +81,19 @@ std::wstring Utf8ToWide(const std::string& str)
 bool isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+  if (path.size() < 3 ) {
+    return true;
+  }
+  if (path.substr(1, 2) == ":\\" || path.substr(0, 2) == "\\\\") {
+    return false;
+  }
+  return true;
 #else
   return path.empty() || path.substr(0, 1) == "/" ? false : true;
 #endif
 }
 
-std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute)
 {
   std::vector<std::string> ret;
 #ifdef _WIN32
@@ -95,10 +101,35 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
   //Starts from there.
   auto it = find_if(parts.rbegin(), parts.rend(),
                     [](const std::string& p) ->bool
-                      { return p.length() == 2 && p[1] == ':'; });
+                      { return ((p.length() == 2 && p[1] == ':')
+                             || (p.length() > 2 && p[0] == '\\' && p[1] == '\\')); });
   if (it != parts.rend()) {
     parts.erase(parts.begin(), it.base()-1);
   }
+  // Special case for samba mount point starting with two "\\" ("\\\\samba\\foo")
+  if (parts.size() > 2  && parts[0].empty() && parts[1].empty()) {
+    parts.erase(parts.begin(), parts.begin()+2);
+    parts[0] = "\\\\" + parts[0];
+  }
+  // Special case if we have a samba drive not at first.
+  // Path is "..\\\\sambdadrive\\..\\.." So we will have an empty part.
+  auto previous_empty = false;
+  for (it = parts.rbegin(); it!=parts.rend(); it++) {
+    if(it->empty()) {
+      if (previous_empty) {
+        it++;
+        break;
+      } else {
+        previous_empty = true;
+      }
+    } else {
+      previous_empty = false;
+    }
+  }
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+    parts[0] = "\\\\" + parts[0];
+  }
 #endif
 
   size_t index = 0;
@@ -144,8 +175,10 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
 
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
-  auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
-  auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto parts = kiwix::split(path, SEPARATOR, false);
+  auto pathParts = normalizeParts(parts, false);
+  parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absolutePathParts = normalizeParts(parts, true);
 
   unsigned int commonCount = 0;
   while (commonCount < pathParts.size()
@@ -172,8 +205,10 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
     absolutePath = getCurrentDirectory();
   }
 
-  auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
-  auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
+  auto parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absoluteParts = normalizeParts(parts, true);
+  parts = kiwix::split(relativePath, SEPARATOR, false);
+  auto relativeParts = normalizeParts(parts, false);
 
   absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
   auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
@@ -182,7 +217,8 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
 
 std::string removeLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  auto parts_ = kiwix::split(path, SEPARATOR, false);
+  auto parts = normalizeParts(parts_, false);
   if (!parts.empty()) {
     parts.pop_back();
   }
@@ -202,7 +238,8 @@ std::string appendToDirectory(const std::string& directoryPath, const std::strin
 
 std::string getLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
+  auto parts_ = kiwix::split(path, SEPARATOR);
+  auto parts = normalizeParts(parts_, false);
   if (parts.empty()) {
     return "";
   }
@@ -302,7 +339,7 @@ std::string makeTmpDirectory()
   _wmkdir(ctmp);
   return WideToUtf8(ctmp);
 #else
-  char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
+  char _template_array[] = {"/tmp/libkiwix_XXXXXX"};
   std::string dir = mkdtemp(_template_array);
   return dir;
 #endif

src/tools/regexTools.cpp

@@ -18,7 +18,6 @@
  */
 
 #include <tools/regexTools.h>
-#include <tools/lock.h>
 
 #include <unicode/regex.h>
 #include <unicode/ucnv.h>
@@ -26,10 +25,10 @@
 #include <memory>
 #include <map>
 #include <stdexcept>
-#include <pthread.h>
+#include <mutex>
 
 std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;
-static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
+static std::mutex regexLock;
 
 std::unique_ptr<icu::RegexMatcher> buildMatcher(const std::string& regex, icu::UnicodeString& content)
 {
@@ -39,7 +38,7 @@ std::unique_ptr<icu::RegexMatcher> buildMatcher(const std::string& regex, icu::U
     pattern = regexCache.at(regex);
   } catch (std::out_of_range&) {
     // Redo the search with a lock to avoid race condition.
-    kiwix::Lock l(&regexLock);
+    std::lock_guard<std::mutex> l(regexLock);
     try {
       pattern = regexCache.at(regex);
     } catch (std::out_of_range&) {
@@ -95,3 +94,22 @@ std::string appendToFirstOccurence(const std::string& content,
 
   return content;
 }
+
+std::string prependToFirstOccurence(const std::string& content,
+                                   const std::string& regex,
+                                   const std::string& replacement)
+{
+  ucnv_setDefaultName("UTF-8");
+  icu::UnicodeString ucontent(content.c_str());
+  icu::UnicodeString ureplacement(replacement.c_str());
+  auto matcher = buildMatcher(regex, ucontent);
+  if (matcher->find()) {
+    UErrorCode status = U_ZERO_ERROR;
+    ucontent.insert(matcher->start(status), ureplacement);
+    std::string tmp;
+    ucontent.toUTF8String(tmp);
+    return tmp;
+  }
+
+  return content;
+}

src/tools/stringTools.cpp

@@ -268,7 +268,8 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
                                       const std::string& delims,
-                                      bool trimEmpty)
+                                      bool trimEmpty,
+                                      bool keepDelim)
 {
   std::string::size_type lastPos = 0;
   std::string::size_type pos = 0;
@@ -279,6 +280,9 @@ std::vector<std::string> kiwix::split(const std::string& str,
     if (!trimEmpty || !token.empty()) {
       tokens.push_back(token);
     }
+    if (keepDelim) {
+      tokens.push_back(str.substr(pos, 1));
+    }
     lastPos = pos + 1;
   }
 

src/wrapper/java/book.cpp

@@ -69,6 +69,7 @@ GETTER(jstring, getDate)
 GETTER(jstring, getUrl)
 GETTER(jstring, getName)
 GETTER(jstring, getFlavour)
+GETTER(jstring, getCategory)
 GETTER(jstring, getTags)
 GETTER(jlong, getArticleCount)
 GETTER(jlong, getMediaCount)

src/wrapper/java/kiwixicu.cpp

@@ -28,11 +28,7 @@
 
 #include "utils.h"
 
-#if __ANDROID__
-pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER;
-#else
-pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
+std::mutex globalLock;
 
 JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
     JNIEnv* env, jclass kclass, jstring dirStr)

src/wrapper/java/kiwixreader.cpp

@@ -20,7 +20,6 @@
 
 
 #include <jni.h>
-#include <zim/file.h>
 #include <exception>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
@@ -46,6 +45,72 @@ JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
   }
 }
 
+namespace
+{
+
+int jni2fd(const jobject& fdObj, JNIEnv* env)
+{
+  jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
+  jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I");
+  if ( field_fd == NULL )
+  {
+    env->ExceptionClear();
+    // Under Android the (private) 'fd' field of java.io.FileDescriptor has been
+    // renamed to 'descriptor'. See, for example,
+    // https://android.googlesource.com/platform/libcore/+/refs/tags/android-8.1.0_r1/ojluni/src/main/java/java/io/FileDescriptor.java#55
+    field_fd = env->GetFieldID(class_fdesc, "descriptor", "I");
+  }
+  return env->GetIntField(fdObj, field_fd);
+}
+
+} // unnamed namespace
+
+JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderByFD(
+    JNIEnv* env, jobject obj, jobject fdObj)
+{
+#ifndef _WIN32
+  int fd = jni2fd(fdObj, env);
+
+  LOG("Attempting to create reader with fd: %d", fd);
+  Lock l;
+  try {
+    kiwix::Reader* reader = new kiwix::Reader(fd);
+    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
+  } catch (std::exception& e) {
+    LOG("Error opening ZIM file");
+    LOG(e.what());
+    return 0;
+  }
+#else
+  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
+  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderByFD() is not supported under Windows");
+  return 0;
+#endif
+}
+
+JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderEmbedded(
+    JNIEnv* env, jobject obj, jobject fdObj, jlong offset, jlong size)
+{
+#ifndef _WIN32
+  int fd = jni2fd(fdObj, env);
+
+  LOG("Attempting to create reader with fd: %d", fd);
+  Lock l;
+  try {
+    kiwix::Reader* reader = new kiwix::Reader(fd, offset, size);
+    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
+  } catch (std::exception& e) {
+    LOG("Error opening ZIM file");
+    LOG(e.what());
+    return 0;
+  }
+#else
+  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
+  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderEmbedded() is not supported under Windows");
+  return 0;
+#endif
+}
+
 JNIEXPORT void JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj)
 {
@@ -326,22 +391,22 @@ JNIEXPORT jobject JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     JNIEnv* env, jobject obj, jstring url)
 {
-   jclass classPair = env->FindClass("org/kiwix/kiwixlib/Pair");
-   jmethodID midPairinit = env->GetMethodID(classPair, "<init>", "()V");
-   jobject pair = env->NewObject(classPair, midPairinit);
-   setPairObjValue("", 0, pair, env);
+   jclass daiClass = env->FindClass("org/kiwix/kiwixlib/DirectAccessInfo");
+   jmethodID daiInitMethod = env->GetMethodID(daiClass, "<init>", "()V");
+   jobject dai = env->NewObject(daiClass, daiInitMethod);
+   setDaiObjValue("", 0, dai, env);
 
    std::string cUrl = jni2c(url, env);
    try {
     auto entry = READER->getEntryFromEncodedPath(cUrl);
     entry = entry.getFinalEntry();
     auto part_info = entry.getDirectAccessInfo();
-    setPairObjValue(part_info.first, part_info.second, pair, env);
+    setDaiObjValue(part_info.first, part_info.second, dai, env);
   } catch (std::exception& e) {
     LOG("Unable to get direct access info for url: %s", cUrl.c_str());
     LOG(e.what());
   }
-  return pair;
+  return dai;
 }
 
 JNIEXPORT jboolean JNICALL
@@ -355,9 +420,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
   unsigned int cCount = jni2c(count, env);
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
     LOG(e.what());
@@ -377,11 +445,14 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
   std::string cUrl;
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
       setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get next suggestion");
     LOG(e.what());

src/wrapper/java/kiwixsearcher.cpp

@@ -19,7 +19,6 @@
  */
 
 
-#include <zim/file.h>
 #include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
 #include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"
 

src/wrapper/java/kiwixserver.cpp

@@ -20,7 +20,6 @@
 
 
 #include <jni.h>
-#include <zim/file.h>
 #include "org_kiwix_kiwixlib_JNIKiwixServer.h"
 
 #include "tools/base64.h"

src/wrapper/java/manager.cpp

@@ -19,7 +19,6 @@
 
 
 #include <jni.h>
-#include <zim/file.h>
 #include "org_kiwix_kiwixlib_Manager.h"
 
 #include "manager.h"

src/wrapper/java/meson.build

@@ -12,7 +12,7 @@ java_sources = files([
   'org/kiwix/kiwixlib/JNIKiwixString.java',
   'org/kiwix/kiwixlib/JNIKiwixBool.java',
   'org/kiwix/kiwixlib/JNIKiwixException.java',
-  'org/kiwix/kiwixlib/Pair.java'
+  'org/kiwix/kiwixlib/DirectAccessInfo.java'
 ])
 
 kiwix_jni = custom_target('jni',

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -24,6 +24,7 @@ public class Book
   public native String getUrl();
   public native String getName();
   public native String getFlavour();
+  public native String getCategory();
   public native String getTags();
   /**
    * Return the value associated to the tag tagName

src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java

@@ -19,7 +19,7 @@
 
 package org.kiwix.kiwixlib;
 
-public class Pair
+public class DirectAccessInfo
 {
   public String filename;
   public long offset;

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -24,7 +24,8 @@ import org.kiwix.kiwixlib.JNIKiwixException;
 import org.kiwix.kiwixlib.JNIKiwixString;
 import org.kiwix.kiwixlib.JNIKiwixInt;
 import org.kiwix.kiwixlib.JNIKiwixSearcher;
-import org.kiwix.kiwixlib.Pair;
+import org.kiwix.kiwixlib.DirectAccessInfo;
+import java.io.FileDescriptor;
 
 public class JNIKiwixReader
 {
@@ -102,13 +103,13 @@ public class JNIKiwixReader
    * the zim file (or zim part) and directly read the content from it (and so
    * bypassing the libzim).
    *
-   * Return a `Pair` (filename, offset) where the content is located.
+   * Return a `DirectAccessInfo` (filename, offset) where the content is located.
    *
    * If the content cannot be directly accessed (content is compressed or zim
    * file is cut in the middle of the content), the filename is an empty string
    * and offset is zero.
    */
-  public native Pair getDirectAccessInformation(String url);
+  public native DirectAccessInfo getDirectAccessInformation(String url);
 
   public native boolean searchSuggestions(String prefix, int count);
 
@@ -151,11 +152,31 @@ public class JNIKiwixReader
         throw new JNIKiwixException("Cannot open zimfile "+filename);
     }
   }
+
+  public JNIKiwixReader(FileDescriptor fd) throws JNIKiwixException
+  {
+    nativeHandle = getNativeReaderByFD(fd);
+    if (nativeHandle == 0) {
+        throw new JNIKiwixException("Cannot open zimfile by fd "+fd.toString());
+    }
+  }
+
+  public JNIKiwixReader(FileDescriptor fd, long offset, long size)
+      throws JNIKiwixException
+  {
+    nativeHandle = getNativeReaderEmbedded(fd, offset, size);
+    if (nativeHandle == 0) {
+        throw new JNIKiwixException(String.format("Cannot open embedded zimfile (fd=%s, offset=%d, size=%d)", fd, offset, size));
+    }
+  }
+
   public JNIKiwixReader() {
 
   }
   public native void dispose();
 
   private native long getNativeReader(String filename);
+  private native long getNativeReaderByFD(FileDescriptor fd);
+  private native long getNativeReaderEmbedded(FileDescriptor fd, long offset, long size);
   private long nativeHandle;
 }

src/wrapper/java/org/kiwix/testing/catalog.xml

@@ -0,0 +1,19 @@
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
+  <id>00000000-0000-0000-0000-000000000000</id>
+  <entry>
+    <title>Test ZIM file</title>
+    <id>urn:uuid:86c91e51-55bf-8882-464e-072aca37a3e8</id>
+    <icon>/meta?name=favicon&amp;content=small</icon>
+    <updated>2020-11-27:00::00:Z</updated>
+    <language>en</language>
+    <summary>This is a ZIM file used in libzim unit-tests</summary>
+    <tags>unit;test</tags>
+    <link type="text/html" href="/small" />
+    <author>
+      <name>Kiwix</name>
+    </author>
+    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="http://localhost/small.zim" length="78982" />
+    <link rel="http://opds-spec.org/image/thumbnail" type="image/png" href="/meta?name=favicon&amp;content=small" />
+  </entry>
+</feed>
+

src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/bash
+
+# This script compiles and runs the unit test to test the java wrapper.
+# This is not integrated in meson because ... this is not so easy.
+
+die()
+{
+  echo >&2 "!!! ERROR: $*"
+  exit 1
+}
+
+
+KIWIX_LIB_JAR=$1
+if [ -z $KIWIX_LIB_JAR ]
+then
+  die "You must give the path to the kiwixlib.jar as first argument"
+fi
+
+KIWIX_LIB_DIR=$2
+if [ -z $KIWIX_LIB_DIR ]
+then
+  die "You must give the path to directory containing libkiwix.so as second argument"
+fi
+
+KIWIX_LIB_JAR=$(readlink -f "$KIWIX_LIB_JAR")
+KIWIX_LIB_DIR=$(readlink -f "$KIWIX_LIB_DIR")
+TEST_SOURCE_DIR=$(dirname "$(readlink -f $0)")
+
+cd "$TEST_SOURCE_DIR"
+
+javac -g -d . -s . -cp "junit-4.13.jar:$KIWIX_LIB_JAR" test.java \
+  || die "Compilation failed"
+
+java -Djava.library.path="$KIWIX_LIB_DIR" \
+     -cp "junit-4.13.jar:hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:." \
+     org.junit.runner.JUnitCore test \
+  || die "Unit test failed"

src/wrapper/java/org/kiwix/testing/compile_test.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/bash
-
-# This script compile the unit test to test the java wrapper.
-# This is not integrated in meson because ... this is not so easy.
-
-
-KIWIX_LIB_JAR=$1
-if [ -z $KIWIX_LIB_JAR ]
-then
-  echo "You must give the path to the kiwixlib.jar as first argument"
-  exit 1
-fi
-
-KIWIX_LIB_DIR=$2
-if [ -z $KIWIX_LIB_DIR ]
-then
-  echo "You must give the path to directory containing libkiwix.so as second argument"
-  exit 1
-fi
-TEST_SOURCE_DIR=$(dirname $(readlink -f $0))
-
-
-javac -g -d . -s . -cp $TEST_SOURCE_DIR/junit-4.13.jar:$KIWIX_LIB_JAR $TEST_SOURCE_DIR/test.java
-
-java -Djava.library.path=$KIWIX_LIB_DIR -cp $TEST_SOURCE_DIR/junit-4.13.jar:$TEST_SOURCE_DIR/hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:. org.junit.runner.JUnitCore test
-

src/wrapper/java/org/kiwix/testing/create_test_zimfiles

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+die()
+{
+  echo >&2 "!!! ERROR: $*"
+  exit 1
+}
+
+cd "$(dirname "$0")"
+rm -f small.zim
+zimwriterfs --withoutFTIndex \
+            -w main.html \
+            -f favicon.png \
+            -l en \
+            -t "Test ZIM file" \
+            -d "N/A" \
+            -c "N/A" \
+            -p "N/A" \
+            small_zimfile_data \
+            small.zim \
+&& echo 'small.zim was successfully created' \
+|| die 'Failed to create small.zim'
+
+printf "BEGINZIM" > small.zim.embedded \
+&& cat small.zim >> small.zim.embedded \
+&& printf "ENDZIM" >> small.zim.embedded \
+&& echo 'small.zim.embedded was successfully created' \
+|| die 'Failed to create small.zim.embedded'

src/wrapper/java/org/kiwix/testing/small_zimfile_data/main.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <title>Test ZIM file</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  </head>
+
+  <body>
+    Test ZIM file
+  </body>
+</html>

src/wrapper/java/org/kiwix/testing/test.java

@@ -10,36 +10,143 @@ static {
   System.loadLibrary("kiwix");
 }
 
-private static String getCatalogContent()
+private static byte[] getFileContent(String path)
 throws IOException
 {
-  BufferedReader reader = new BufferedReader(new FileReader("catalog.xml"));
-  String line;
-  StringBuilder sb = new StringBuilder();
-  while ((line = reader.readLine()) != null)
-  {
-    sb.append(line + "\n");
-  }
-  reader.close();
-  return sb.toString();
+  File file = new File(path);
+  DataInputStream in = new DataInputStream(
+                         new BufferedInputStream(
+                           new FileInputStream(file)));
+  byte[] data = new byte[(int)file.length()];
+  in.read(data);
+  return data;
+}
+
+private static byte[] getFileContentPartial(String path, int offset, int size)
+throws IOException
+{
+  File file = new File(path);
+  DataInputStream in = new DataInputStream(
+                         new BufferedInputStream(
+                           new FileInputStream(file)));
+  byte[] data = new byte[size];
+  in.skipBytes(offset);
+  in.read(data, 0, size);
+  return data;
+}
+
+private static String getTextFileContent(String path)
+throws IOException
+{
+  return new String(getFileContent(path));
 }
 
 @Test
-public void testSome()
+public void testReader()
+throws JNIKiwixException, IOException
+{
+  JNIKiwixReader reader = new JNIKiwixReader("small.zim");
+  assertEquals("Test ZIM file", reader.getTitle());
+  assertEquals(45, reader.getFileSize()); // The file size is in KiB
+  assertEquals("A/main.html", reader.getMainPage());
+  String s = getTextFileContent("small_zimfile_data/main.html");
+  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertEquals(s, new String(c));
+
+  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
+  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
+  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertTrue(Arrays.equals(faviconData, c));
+
+  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
+  assertNotEquals("", dai.filename);
+  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
+  assertTrue(Arrays.equals(faviconData, c));
+}
+
+@Test
+public void testReaderByFd()
+throws JNIKiwixException, IOException
+{
+  FileInputStream fis = new FileInputStream("small.zim");
+  JNIKiwixReader reader = new JNIKiwixReader(fis.getFD());
+  assertEquals("Test ZIM file", reader.getTitle());
+  assertEquals(45, reader.getFileSize()); // The file size is in KiB
+  assertEquals("A/main.html", reader.getMainPage());
+  String s = getTextFileContent("small_zimfile_data/main.html");
+  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertEquals(s, new String(c));
+
+  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
+  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
+  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertTrue(Arrays.equals(faviconData, c));
+
+  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
+  assertNotEquals("", dai.filename);
+  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
+  assertTrue(Arrays.equals(faviconData, c));
+}
+
+@Test
+public void testReaderWithAnEmbeddedArchive()
+throws JNIKiwixException, IOException
+{
+  File plainArchive = new File("small.zim");
+  FileInputStream fis = new FileInputStream("small.zim.embedded");
+  JNIKiwixReader reader = new JNIKiwixReader(fis.getFD(), 8, plainArchive.length());
+  assertEquals("Test ZIM file", reader.getTitle());
+  assertEquals(45, reader.getFileSize()); // The file size is in KiB
+  assertEquals("A/main.html", reader.getMainPage());
+  String s = getTextFileContent("small_zimfile_data/main.html");
+  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertEquals(s, new String(c));
+
+  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
+  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
+  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
+                                new JNIKiwixString(),
+                                new JNIKiwixString(),
+                                new JNIKiwixInt());
+  assertTrue(Arrays.equals(faviconData, c));
+
+  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
+  assertNotEquals("", dai.filename);
+  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
+  assertTrue(Arrays.equals(faviconData, c));
+}
+
+@Test
+public void testLibrary()
 throws IOException
 {
   Library lib = new Library();
   Manager manager = new Manager(lib);
-  String content = getCatalogContent();
-  manager.readOpds(content, "https://library.kiwix.org");
-  assertEquals(lib.getBookCount(true, true), 10);
+  String content = getTextFileContent("catalog.xml");
+  manager.readOpds(content, "http://localhost");
+  assertEquals(lib.getBookCount(true, true), 1);
   String[] bookIds = lib.getBooksIds();
-  assertEquals(bookIds.length, 10);
+  assertEquals(bookIds.length, 1);
   Book book = lib.getBookById(bookIds[0]);
-  assertEquals(book.getTitle(), "Wikisource");
-  assertEquals(book.getTags(), "wikisource;_category:wikisource;_pictures:no;_videos:no;_details:yes;_ftindex:yes");
-  assertEquals(book.getFaviconUrl(), "https://library.kiwix.org/meta?name=favicon&content=wikisource_fr_all_nopic_2020-01");
-  assertEquals(book.getUrl(), "http://download.kiwix.org/zim/wikisource/wikisource_fr_all_nopic_2020-01.zim.meta4");
+  assertEquals(book.getTitle(), "Test ZIM file");
+  assertEquals(book.getTags(), "unit;test");
+  assertEquals(book.getFaviconUrl(), "http://localhost/meta?name=favicon&content=small");
+  assertEquals(book.getUrl(), "http://localhost/small.zim");
 }
 
 static

src/wrapper/java/utils.h

@@ -24,7 +24,7 @@
 
 #include <jni.h>
 
-#include <pthread.h>
+#include <mutex>
 #include <string>
 #include <vector>
 #include <iostream>
@@ -36,7 +36,7 @@
  #define LOG(...)
 #endif
 
-extern pthread_mutex_t globalLock;
+extern std::mutex globalLock;
 
 template<typename T>
 void setPtr(JNIEnv* env, jobject thisObj, T* ptr)
@@ -88,22 +88,10 @@ inline jobjectArray createArray(JNIEnv* env, size_t length, const std::string& t
   return env->NewObjectArray(length, c, NULL);
 }
 
-class Lock
+class Lock : public std::unique_lock<std::mutex>
 {
- protected:
-  pthread_mutex_t* lock;
-
  public:
-  Lock() : lock(&globalLock) { pthread_mutex_lock(lock); }
-  Lock(const Lock&) = delete;
-  Lock& operator=(const Lock&) = delete;
-  Lock(Lock&& other) : lock(&globalLock) { other.lock = nullptr; }
-  virtual ~Lock()
-  {
-    if (lock) {
-      pthread_mutex_unlock(lock);
-    }
-  }
+  Lock() : std::unique_lock<std::mutex>(globalLock) { }
 };
 
 template <class T>
@@ -258,7 +246,7 @@ inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
   env->SetIntField(obj, objFid, c2jni(value, env));
 }
 
-inline void setPairObjValue(const std::string& filename, const long offset,
+inline void setDaiObjValue(const std::string& filename, const long offset,
                             const jobject obj, JNIEnv* env)
 {
   jclass objClass = env->GetObjectClass(obj);

src/xmlrpc.h

@@ -3,6 +3,7 @@
 #ifndef KIWIX_XMLRPC_H_
 #define KIWIX_XMLRPC_H_
 
+#include <stdexcept>
 #include <tools/otherTools.h>
 #include <pugixml.hpp>
 

static/catalog_v2_searchdescription.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+  <ShortName>Zim catalog search</ShortName>
+  <Description>Search zim files in the catalog.</Description>
+  <Url type="application/atom+xml;profile=opds-catalog;kind=acquisition"
+       xmlns:atom="http://www.w3.org/2005/Atom"
+       xmlns:k="http://kiwix.org/opensearchextension/1.0"
+       indexOffset="0"
+       template="{{endpoint_root}}/entries?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/>
+</OpenSearchDescription>

static/opensearchdescription.xml

@@ -6,5 +6,5 @@
        xmlns:atom="http://www.w3.org/2005/Atom"
        xmlns:k="http://kiwix.org/opensearchextension/1.0"
        indexOffset="0"
-       template="/{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&notag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/>
+       template="{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&notag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/>
 </OpenSearchDescription>

static/resources_list.txt

@@ -19,6 +19,11 @@ skin/jquery-ui/jquery-ui.theme.min.css
 skin/jquery-ui/jquery-ui.min.css
 skin/caret.png
 skin/taskbar.js
+skin/langList.js
+skin/categoryList.js
+skin/iso6391To3.js
+skin/isotope.pkgd.min.js
+skin/index.js
 skin/taskbar.css
 skin/block_external.js
 templates/search_result.html
@@ -27,8 +32,13 @@ templates/404.html
 templates/500.html
 templates/index.html
 templates/suggestion.json
-templates/head_part.html
+templates/head_taskbar.html
 templates/taskbar_part.html
 templates/external_blocker_part.html
 templates/captured_external.html
+templates/catalog_entries.xml
+templates/catalog_v2_root.xml
+templates/catalog_v2_entries.xml
+templates/catalog_v2_categories.xml
 opensearchdescription.xml
+catalog_v2_searchdescription.xml

static/skin/block_external.js

@@ -1,4 +1,6 @@
-var block_path = "/catch/external";
+const root = document.querySelector( `link[type='root']` ).getAttribute("href");
+// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
+var block_path = `${root}/catch/external`;
 // called only on external links
 function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
 

static/skin/categoryList.js

@@ -0,0 +1,19 @@
+// eslint-disable-next-line no-unused-vars
+const categoryList = {
+  "other": "Other",
+  "gutenberg": "Gutenberg",
+  "mooc": "Mooc",
+  "phet": "Phet",
+  "psiram": "Psiram",
+  "stack_exchange": "Stack Exchange",
+  "ted": "Ted",
+  "vikidia": "Vikidia",
+  "wikibooks": "Wikibooks",
+  "wikinews": "Wikinews",
+  "wikipedia": "Wikipedia",
+  "wikiquote": "Wikiquote",
+  "wikisource": "Wikisource",
+  "wikiversity": "Wikiversity",
+  "wikivoyage": "Wikivoyage",
+  "wiktionary": "Wiktionary"
+}

static/skin/index.js

@@ -0,0 +1,250 @@
+(function() {
+    const root = $(`link[type='root']`).attr('href');
+    const incrementalLoadingParams = {
+        start: 0,
+        count: viewPortToCount()
+    };
+    const filterTypes = ['lang', 'category', 'q'];
+    const bookOrderMap = new Map();
+    const filterCookieName = 'filters';
+    const oneDayDelta = 86400000;
+    let footer;
+    let fadeOutDiv;
+    let iso;
+    let isFetching = false;
+    let noResultInjected = false;
+    let filters = getCookie(filterCookieName);
+    let params = new URLSearchParams(window.location.search || filters || '');
+    let timer;
+
+    function queryUrlBuilder() {
+        let url = `${root}/catalog/search?`;
+        url += Object.keys(incrementalLoadingParams).map(key => `${key}=${incrementalLoadingParams[key]}`).join("&");
+        params.forEach((value, key) => {url+= value ? `&${key}=${value}` : ''});
+        return (url);
+    }
+
+    function setCookie(cookieName, cookieValue) {
+        const date = new Date();
+        date.setTime(date.getTime() + oneDayDelta);
+        document.cookie = `${cookieName}=${cookieValue};expires=${date.toUTCString()};sameSite=Strict`;
+    }
+
+    function getCookie(cookieName) {
+        const name = cookieName + "=";
+        let result;
+        decodeURIComponent(document.cookie).split('; ').forEach(val => {
+            if (val.indexOf(name) === 0) {
+                result = val.substring(name.length);
+            }
+        });
+        return result;
+    }
+
+    function htmlEncode(str) {
+        return str.replace(/[\u00A0-\u9999<>\&]/gim, (i) => `&#${i.charCodeAt(0)};`);
+    }
+
+    function viewPortToCount(){
+        return Math.floor(window.innerHeight/100 + 1)*(window.innerWidth>1000 ? 3 : 2);
+    }
+
+    function getInnerHtml(node, query) {
+        return node.querySelector(query).innerHTML;
+    }
+
+    function generateBookHtml(book, sort = false) {
+        const link = book.querySelector('link').getAttribute('href');
+        const title =  getInnerHtml(book, 'title');
+        const description = getInnerHtml(book, 'summary');
+        const id = getInnerHtml(book, 'id');
+        const iconUrl = getInnerHtml(book, 'icon');
+        const articleCount = getInnerHtml(book, 'articleCount');
+        const mediaCount = getInnerHtml(book, 'mediaCount');
+
+        const linkTag = document.createElement('a');
+        linkTag.setAttribute('class', 'book');
+        linkTag.setAttribute('data-id', id);
+        linkTag.setAttribute('href', link);
+        if (sort) {
+            linkTag.setAttribute('data-idx', bookOrderMap.get(id));
+        }
+        linkTag.innerHTML = `<div class='book__background' style="background-image: url('${iconUrl}');">
+            <div class='book__title' title='${title}'>${title}</div>
+            <div class='book__description' title='${description}'>${description}</div>
+            <div class='book__info'>${articleCount} articles, ${mediaCount} medias</div>
+        </div>`;
+        return linkTag;
+    }
+
+    function toggleFooter(show=false) {
+        if (show) {
+            footer.style.display = 'block';
+        } else {
+            footer.style.display = 'none';
+            fadeOutDiv.style.display = 'block';
+        }
+    }
+
+    async function loadBooks() {
+        const loader = document.querySelector('.loader');
+        loader.style.display = 'block';
+        return await fetch(queryUrlBuilder()).then(async (resp) => {
+            const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
+            const books = data.querySelectorAll('entry');
+            books.forEach((book, idx) => {
+                bookOrderMap.set(getInnerHtml(book, 'id'), idx);
+            });
+            incrementalLoadingParams.start += books.length;
+            if (parseInt(data.querySelector('totalResults').innerHTML) === bookOrderMap.size) {
+                incrementalLoadingParams.count = 0;
+                toggleFooter(true);
+            } else {
+                toggleFooter();
+            }
+            loader.style.display = 'none';
+            return books;
+        });
+    }
+
+    async function loadAndDisplayOptions(nodeQuery, query) {
+        // currently taking an object in place of query, will replace it with query while fetching data from backend later on.
+        document.querySelector(nodeQuery).innerHTML += Object.keys(query)
+            .map((option) => {return `<option value='${option}'>${htmlEncode(query[option])}</option>`})
+        .join('');
+    }
+
+    function checkAndInjectEmptyMessage() {
+        if (!bookOrderMap.size) {
+            if (!noResultInjected) {
+                noResultInjected = true;
+                iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('a'));
+                iso.layout();
+                const spanTag = document.createElement('span');
+                spanTag.setAttribute('class', 'noResults');
+                spanTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter?</a>`;
+                document.querySelector('body').append(spanTag);
+                spanTag.getElementsByTagName('a')[0].onclick = (event) => {
+                    event.preventDefault();
+                    window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
+                    setCookie(filterCookieName, 'lang=');
+                    resetAndFilter();
+                    filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
+                };
+            }
+            return true;
+        } else if (noResultInjected) {
+            noResultInjected = false;
+            document.getElementsByClassName('noResults')[0].remove();
+        }
+        return false;
+    }
+
+    async function loadAndDisplayBooks(sort = false) {
+        if (isFetching) return;
+        isFetching = true;
+        await loadAndDisplayBooksUnguarded(sort);
+        isFetching = false;
+    }
+
+    async function loadAndDisplayBooksUnguarded(sort) {
+        let books = await loadBooks();
+        if (checkAndInjectEmptyMessage()) {return}
+        const booksToFilter = new Set();
+        const booksToDelete = new Set();
+        iso.arrange({
+            filter: function (idx, elem) {
+                const id = elem.getAttribute('data-id');
+                const retVal = bookOrderMap.has(id);
+                if (retVal) {
+                    booksToFilter.add(id);
+                    if (sort) {
+                        elem.setAttribute('data-idx', bookOrderMap.get(id));
+                        iso.updateSortData(elem);
+                    }
+                } else {
+                    booksToDelete.add(elem);
+                }
+                return retVal;
+            }
+        });
+        books = [...books].filter((book) => {return !booksToFilter.has(getInnerHtml(book, 'id'))});
+        booksToDelete.forEach(book => {iso.remove(book);});
+        books.forEach((book) => {iso.insert(generateBookHtml(book, sort))});
+    }
+
+    async function resetAndFilter(filterType = '', filterValue = '') {
+        isFetching = false;
+        incrementalLoadingParams.start = 0;
+        incrementalLoadingParams.count = viewPortToCount();
+        fadeOutDiv.style.display = 'none';
+        bookOrderMap.clear();
+        params = new URLSearchParams(window.location.search);
+        if (filterType) {
+            params.set(filterType, filterValue);
+            window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
+            setCookie(filterCookieName, params.toString());
+        }
+        await loadAndDisplayBooks(true);
+    }
+
+    window.addEventListener('popstate', async () => {
+        await resetAndFilter();
+        filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
+    });
+
+    async function loadSubset() {
+        if (window.innerHeight + window.scrollY >= document.body.offsetHeight) {
+            if (incrementalLoadingParams.count) {
+                loadAndDisplayBooks();
+            }
+            else {
+                fadeOutDiv.style.display = 'none';
+            }
+        }
+    }
+
+    window.addEventListener('resize', (event) => {
+        if (timer) {clearTimeout(timer)}
+        timer = setTimeout(() => {
+            incrementalLoadingParams.count = incrementalLoadingParams.count && viewPortToCount();
+            loadSubset();
+        }, 100, event);
+    });
+
+    window.addEventListener('scroll', loadSubset);
+
+    window.onload = async () => {
+        iso = new Isotope( '.book__list', {
+            itemSelector: '.book',
+            getSortData:{
+                weight: function( itemElem ) {
+                    const index = itemElem.getAttribute('data-idx');
+                    return index ? parseInt(index) : Infinity;
+                }
+            },
+            sortBy: 'weight'
+        });
+        footer = document.getElementById('kiwixfooter');
+        fadeOutDiv = document.getElementById('fadeOut');
+        await loadAndDisplayBooks();
+        await loadAndDisplayOptions('#languageFilter', langList);
+        await loadAndDisplayOptions('#categoryFilter', categoryList);
+        filterTypes.forEach((filter) => {
+            const filterTag = document.getElementsByName(filter)[0];
+            filterTag.addEventListener('change', () => {resetAndFilter(filterTag.name, filterTag.value)});
+        });
+        if (filters) {
+            window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
+        }
+        params.forEach((value, key) => {document.getElementsByName(key)[0].value = value});
+        document.getElementById('kiwixSearchForm').onsubmit = (event) => {event.preventDefault()};
+        if (!window.location.search) {
+            const browserLang = navigator.language.split('-')[0];
+            const langFilter = document.getElementById('languageFilter');
+            langFilter.value = browserLang.length === 3 ? browserLang : iso6391To3[browserLang];
+            langFilter.dispatchEvent(new Event('change'));
+        }
+        setCookie(filterCookieName, params.toString());
+    }
+})();

static/skin/iso6391To3.js

@@ -0,0 +1,124 @@
+// eslint-disable-next-line no-unused-vars
+const iso6391To3 = {
+    "aa": "aar",
+    "af": "afr",
+    "ak": "aka",
+    "am": "amh",
+    "ar": "ara",
+    "as": "asm",
+    "az": "aze",
+    "ba": "bak",
+    "be": "bel",
+    "bg": "bul",
+    "bm": "bam",
+    "bn": "ben",
+    "bo": "bod",
+    "br": "bre",
+    "bs": "bos",
+    "ca": "cat",
+    "ce": "che",
+    "co": "cos",
+    "cs": "ces",
+    "cv": "chv",
+    "cy": "cym",
+    "da": "dan",
+    "de": "deu",
+    "dz": "dzo",
+    "ee": "ewe",
+    "en": "eng",
+    "es": "spa",
+    "et": "est",
+    "eu": "eus",
+    "fa": "fas",
+    "ff": "ful",
+    "fi": "fin",
+    "fo": "fao",
+    "fr": "fra",
+    "ga": "gle",
+    "gl": "glg",
+    "gn": "grn",
+    "gu": "guj",
+    "gv": "glv",
+    "ha": "hau",
+    "he": "heb",
+    "hi": "hin",
+    "hr": "hrv",
+    "hu": "hun",
+    "hy": "hye",
+    "id": "ind",
+    "ig": "ibo",
+    "is": "isl",
+    "it": "ita",
+    "iu": "iku",
+    "ja": "jpn",
+    "jv": "jav",
+    "ka": "kat",
+    "ki": "kik",
+    "kk": "kaz",
+    "km": "khm",
+    "kn": "kan",
+    "ko": "kor",
+    "ks": "kas",
+    "ku": "kur",
+    "kw": "cor",
+    "ky": "kir",
+    "lb": "ltz",
+    "lg": "lug",
+    "ln": "lin",
+    "lo": "lao",
+    "lt": "lit",
+    "lv": "lav",
+    "mg": "mlg",
+    "mi": "mri",
+    "mk": "mkd",
+    "ml": "mal",
+    "mn": "mon",
+    "mr": "mar",
+    "mt": "mlt",
+    "my": "mya",
+    "nl": "nld",
+    "ny": "nya",
+    "om": "orm",
+    "pl": "pol",
+    "pt": "por",
+    "qu": "que",
+    "rm": "roh",
+    "rn": "run",
+    "ro": "ron",
+    "ru": "rus",
+    "rw": "kin",
+    "sa": "san",
+    "sd": "snd",
+    "sg": "sag",
+    "si": "sin",
+    "sk": "slk",
+    "sl": "slv",
+    "sn": "sna",
+    "so": "som",
+    "sq": "sqi",
+    "sr": "srp",
+    "ss": "ssw",
+    "sv": "swe",
+    "ta": "tam",
+    "te": "tel",
+    "tg": "tgk",
+    "th": "tha",
+    "ti": "tir",
+    "tk": "tuk",
+    "tn": "tsn",
+    "tr": "tur",
+    "ts": "tso",
+    "tt": "tat",
+    "ug": "uig",
+    "uk": "ukr",
+    "ur": "urd",
+    "uz": "uzb",
+    "ve": "ven",
+    "vi": "vie",
+    "wa": "wln",
+    "wo": "wol",
+    "xh": "xho",
+    "yo": "yor",
+    "zh": "zho",
+    "zu": "zul"
+}

static/skin/langList.js

@@ -0,0 +1,124 @@
+// eslint-disable-next-line no-unused-vars
+const langList = {
+  "aar": "Afaraf",
+  "afr": "Afrikaans",
+  "aka": "Akan",
+  "amh": "አማርኛ",
+  "ara": "اللغة العربية",
+  "asm": "অসমীয়া",
+  "aze": "azərbaycan dili",
+  "bak": "башҡорт теле",
+  "bel": "беларуская мова",
+  "bul": "български език",
+  "bam": "bamanankan",
+  "ben": "বাংলা",
+  "bod": "བོད་ཡིག",
+  "bre": "brezhoneg",
+  "bos": "bosanski jezik",
+  "cat": "Català",
+  "che": "нохчийн мотт",
+  "cos": "corsu",
+  "ces": "čeština",
+  "chv": "чӑваш чӗлхи",
+  "cym": "Cymraeg",
+  "dan": "dansk",
+  "deu": "Deutsch",
+  "dzo": "རྫོང་ཁ",
+  "ewe": "Eʋegbe",
+  "eng": "English",
+  "spa": "Español",
+  "est": "eesti",
+  "eus": "euskara",
+  "fas": "فارسی",
+  "ful": "Fulfulde",
+  "fin": "suomi",
+  "fao": "føroyskt",
+  "fra": "Français",
+  "gle": "Gaeilge",
+  "glg": "galego",
+  "grn": "Avañe'ẽ",
+  "guj": "ગુજરાતી",
+  "glv": "Gaelg",
+  "hau": "هَوُسَ",
+  "heb": "עברית",
+  "hin": "हिन्दी",
+  "hrv": "hrvatski jezik",
+  "hun": "magyar",
+  "hye": "Հայերեն",
+  "ind": "Bahasa Indonesia",
+  "ibo": "Asụsụ Igbo",
+  "isl": "Íslenska",
+  "ita": "Italiano",
+  "iku": "ᐃᓄᒃᑎᑐᑦ",
+  "jpn": "日本語",
+  "jav": "basa Jawa",
+  "kat": "ქართული",
+  "kik": "Gĩkũyũ",
+  "kaz": "қазақ тілі",
+  "khm": "ខេមរភាសា",
+  "kan": "ಕನ್ನಡ",
+  "kor": "한국어",
+  "kas": "कश्मीरी",
+  "kur": "Kurdî",
+  "cor": "Kernewek",
+  "kir": "Кыргызча",
+  "ltz": "Lëtzebuergesch",
+  "lug": "Luganda",
+  "lin": "Lingála",
+  "lao": "ພາສາ",
+  "lit": "lietuvių kalba",
+  "lav": "latviešu valoda",
+  "mlg": "fiteny malagasy",
+  "mri": "te reo Māori",
+  "mkd": "македонски јазик",
+  "mal": "മലയാളം",
+  "mon": "Монгол хэл",
+  "mar": "मराठी",
+  "mlt": "Malti",
+  "mya": "ဗမာစာ",
+  "nld": "Nederlands",
+  "nya": "chiCheŵa",
+  "orm": "Afaan Oromoo",
+  "pol": "język polski",
+  "por": "Português",
+  "que": "Runa Simi",
+  "roh": "rumantsch grischun",
+  "run": "Ikirundi",
+  "ron": "Română",
+  "rus": "Русский",
+  "kin": "Ikinyarwanda",
+  "san": "संस्कृतम्",
+  "snd": "सिन्धी",
+  "sag": "yângâ tî sängö",
+  "sin": "සිංහල",
+  "slk": "slovenčina",
+  "slv": "slovenski jezik",
+  "sna": "chiShona",
+  "som": "Soomaaliga",
+  "sqi": "Shqip",
+  "srp": "српски језик",
+  "ssw": "SiSwati",
+  "swe": "svenska",
+  "tam": "தமிழ்",
+  "tel": "తెలుగు",
+  "tgk": "тоҷикӣ",
+  "tha": "ไทย",
+  "tir": "ትግርኛ",
+  "tuk": "Türkmen",
+  "tsn": "Setswana",
+  "tur": "Türkçe",
+  "tso": "Xitsonga",
+  "tat": "татар теле",
+  "uig": "ئۇيغۇرچە‎",
+  "ukr": "Українська",
+  "urd": "اردو",
+  "uzb": "Ўзбек",
+  "ven": "Tshivenḓa",
+  "vie": "Tiếng Việt",
+  "wln": "walon",
+  "wol": "Wollof",
+  "xho": "isiXhosa",
+  "yor": "Yorùbá",
+  "zho": "中文",
+  "zul": "isiZulu"
+}

static/skin/taskbar.css

@@ -124,7 +124,7 @@ label[for=kiwixsearchbox] {
 }
 
 body {
-    padding-top: 3em !important;
+    padding-top: calc(3em - 5px) !important;
 }
 
 /* Try to fix buggy stuff in jquery-ui autocomplete */

static/skin/taskbar.js

@@ -1,49 +1,96 @@
+function htmlDecode(input) {
+    var doc = new DOMParser().parseFromString(input, "text/html");
+    return doc.documentElement.textContent;
+}
 
-(function ($) {
-    if ($(window).width() < 520) {
-        var didScroll;
-        var lastScrollTop = 0;
-        var delta = 5;
-        // on scroll, let the interval function know the user has scrolled
-        $(window).scroll(function (event) {
-            didScroll = true;
+const jq = jQuery.noConflict(true);
+jq(document).ready(() => {
+    (function ($) {
+        const root = $( `link[type='root']` ).attr("href");
+
+        const bookName = (window.location.pathname == `${root}/search`)
+               ? (new URLSearchParams(window.location.search)).get('content') 
+               : window.location.pathname.split(`${root}/`)[1].split('/')[0];
+    
+        $( "#kiwixsearchbox" ).autocomplete({
+    
+            source: `${root}/suggest?content=${bookName}`,
+            dataType: "json",
+            cache: false,
+    
+            response: function( event, ui ) {
+                  for(const item of ui.content) {
+                      item.label = htmlDecode(item.label);
+                      item.value = htmlDecode(item.value);
+                      if (item.path) item.path = htmlDecode(item.path);
+                  }
+            },
+    
+            select: function(event, ui) {
+                if (ui.item.kind === 'path') {
+                    window.location.href = `${root}/${bookName}/${encodeURI(ui.item.path)}`;
+                } else {
+                    $( "#kiwixsearchbox" ).val(ui.item.value);
+                    $( "#kiwixsearchform" ).submit();
+                }
+            },
+        }).data( "ui-autocomplete" )._renderItem = function( ul, item ) {
+            return $( "<li>" )
+              .data( "ui-autocomplete-item", item )
+              .append( item.label )
+              .appendTo( ul );
+        };
+    
+        /* cybook hack */
+        if (navigator.userAgent.indexOf("bookeen/cybook") != -1) {
+            $("html").addClass("cybook");
+        }
+    
+        if ($(window).width() < 520) {
+            var didScroll;
+            var lastScrollTop = 0;
+            var delta = 5;
+            // on scroll, let the interval function know the user has scrolled
+            $(window).scroll(function (event) {
+                didScroll = true;
+            });
+            // run hasScrolled() and reset didScroll status
+            setInterval(function () {
+                if (didScroll) {
+                    hasScrolled();
+                    didScroll = false;
+                }
+            }, 250);
+            function hasScrolled() {
+                var st = $(this).scrollTop();
+    
+                // Make sure they scroll more than delta
+                if (Math.abs(lastScrollTop - st) <= delta)
+                    return;
+    
+                // If they scrolled down and are past the navbar, add class .nav-up.
+                // This is necessary so you never see what is "behind" the navbar.
+                if (st > lastScrollTop) {
+                    // Scroll Down
+                    $('#kiwixtoolbar').css({ top: '-100%' });
+                } else {
+                    // Scroll Up
+                    $('#kiwixtoolbar').css({ top: '0' });
+                }
+    
+                lastScrollTop = st;
+            }
+        }
+    
+        $('#kiwixsearchbox').on({
+            focus: function () {
+                $('.kiwix_searchform').addClass('full_width');
+                $('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').addClass('searching');
+            },
+            blur: function () {
+                $('.kiwix_searchform').removeClass('full_width');
+                $('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').removeClass('searching');
+            }
         });
-        // run hasScrolled() and reset didScroll status
-        setInterval(function () {
-            if (didScroll) {
-                hasScrolled();
-                didScroll = false;
-            }
-        }, 250);
-        function hasScrolled() {
-            var st = $(this).scrollTop();
-
-            // Make sure they scroll more than delta
-            if (Math.abs(lastScrollTop - st) <= delta)
-                return;
-
-            // If they scrolled down and are past the navbar, add class .nav-up.
-            // This is necessary so you never see what is "behind" the navbar.
-            if (st > lastScrollTop) {
-                // Scroll Down
-                $('#kiwixtoolbar').css({ top: '-100%' });
-            } else {
-                // Scroll Up
-                $('#kiwixtoolbar').css({ top: '0' });
-            }
-
-            lastScrollTop = st;
-        }
-    }
-
-    $('#kiwixsearchbox').on({
-        focus: function () {
-            $('.kiwix_searchform').addClass('full_width');
-            $('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').addClass('searching');
-        },
-        blur: function () {
-            $('.kiwix_searchform').removeClass('full_width');
-            $('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').removeClass('searching');
-        }
-    });
-})(jQuery);
+    })(jq);
+})

static/templates/404.html

@@ -9,5 +9,10 @@
     <p>
       The requested URL "{{url}}" was not found on this server.
     </p>
+    {{#details}}
+    <p>
+      {{{details}}}
+    </p>
+    {{/details}}
   </body>
 </html>

static/templates/captured_external.html

@@ -6,7 +6,7 @@
 </head>
 <body class="kiwix">
 <h1>External link blocked</h1>
-<p>This instance of Kiwix protects you from accidentaly going to external (out-of ZIM) links.</p>
+<p>This instance of Kiwix protects you from accidentally going to external (out-of ZIM) links.</p>
 <p>If you intend to go to such locations, please click the link below.</p>
 <p><a href="{{ source }}">Go to {{ source }}</a></p>
 <div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>

static/templates/catalog_entries.xml

@@ -0,0 +1,38 @@
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
+  <id>{{feed_id}}</id>
+  <title>{{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}}</title>
+  <updated>{{date}}</updated>
+{{#filter}}
+  <totalResults>{{totalResults}}</totalResults>
+  <startIndex>{{startIndex}}</startIndex>
+  <itemsPerPage>{{itemsPerPage}}</itemsPerPage>
+{{/filter}}
+  <link rel="self" href="" type="application/atom+xml" />
+  <link rel="search" type="application/opensearchdescription+xml" href="{{root}}/catalog/searchdescription.xml" />
+  {{#books}}
+  <entry>
+    <id>{{id}}</id>
+    <title>{{title}}</title>
+    <summary>{{description}}</summary>
+    <language>{{language}}</language>
+    <updated>{{updated}}</updated>
+    <name>{{name}}</name>
+    <flavour>{{flavour}}</flavour>
+    <category>{{category}}</category>
+    <tags>{{tags}}</tags>
+    <articleCount>{{article_count}}</articleCount>
+    <mediaCount>{{media_count}}</mediaCount>
+    <icon>/meta?name=favicon&amp;content={{{content_id}}}</icon>
+    <link type="text/html" href="/{{{content_id}}}" />
+    <author>
+      <name>{{author_name}}</name>
+    </author>
+    <publisher>
+      <name>{{publisher_name}}</name>
+    </publisher>
+    {{#url}}
+    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
+    {{/url}}
+  </entry>
+  {{/books}}
+</feed>

static/templates/catalog_v2_categories.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:opds="https://specs.opds.io/opds-1.2">
+  <id>{{feed_id}}</id>
+  <link rel="self"
+        href="{{endpoint_root}}/categories"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <link rel="start"
+        href="{{endpoint_root}}/root.xml"
+        type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
+  <title>List of categories</title>
+  <updated>{{date}}</updated>
+
+  {{#categories}}
+  <entry>
+    <title>{{name}}</title>
+    <link rel="subsection"
+          href="{{endpoint_root}}/entries?category={{{urlencoded_name}}}"
+          type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
+    <updated>{{updated}}</updated>
+    <id>{{id}}</id>
+    <content type="text">All entries with category of '{{name}}'.</content>
+  </entry>
+  {{/categories}}
+</feed>