Add documentation for widget

Added documentation for current widget usage, currently supported arguments, using custom CSS/JS

.github/workflows/ci.yml

@@ -1,17 +1,21 @@
 name: CI
 
-on: [push]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
   Macos:
     runs-on: macos-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
-      - name: Setup python 3.5
-        uses: actions/setup-python@v1
+        uses: actions/checkout@v2
+      - name: Setup python 3.10
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.5'
+          python-version: '3.10'
       - name: Install packages
         run: |
           brew update
@@ -37,17 +41,8 @@ jobs:
           export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
           cd build
           meson test --verbose
-          ninja coverage
         env:
           SKIP_BIG_MEMORY_TEST: 1
-      - name: Publish coverage
-        shell: bash
-        run: |
-          curl https://codecov.io/bash -o codecov.sh
-          bash codecov.sh -n osx_native_dyn -Z
-          rm codecov.sh
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   Linux:
     strategy:
@@ -56,7 +51,6 @@ jobs:
         name:
           - native_static
           - native_dyn
-          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
@@ -64,31 +58,27 @@ jobs:
         include:
           - name: native_static
             target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: native_dyn
-            target: native_dyn
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - name: native_dyn_bionic
             target: native_dyn
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: android_arm
             target: android_arm
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/arm-linux-androideabi'
           - name: android_arm64
             target: android_arm64
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/aarch64-linux-android'
           - name: win32_static
             target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - name: win32_dyn
             target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
     env:
       HOME: /home/runner
@@ -96,10 +86,6 @@ jobs:
     container:
       image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
     steps:
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
     - name: Checkout code
       shell: python
       run: |
@@ -109,7 +95,7 @@ jobs:
           'git', 'clone',
           'https://github.com/${{github.repository}}',
           '--depth=1',
-          '--branch', '${{steps.extract_branch.outputs.branch}}'
+          '--branch', '${{ github.head_ref || github.ref_name }}'
         ]
         check_call(command, cwd=environ['HOME'])
     - name: Install deps
@@ -132,7 +118,7 @@ jobs:
           MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
         fi
         if [[ "${{matrix.target}}" =~ android_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+          MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
         fi
         cd $HOME/libkiwix
         meson . build ${MESON_OPTION}
@@ -158,6 +144,6 @@ jobs:
         curl https://codecov.io/bash -o codecov.sh
         bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
         rm codecov.sh
-      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
+      if: startsWith(matrix.target, 'native_')
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -8,8 +8,8 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-jammy
           - ubuntu-impish
-          - ubuntu-hirsute
           - ubuntu-focal
           - ubuntu-bionic
     steps:
@@ -34,18 +34,18 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
+      - uses: legoktm/gh-action-build-deb@ubuntu-impish
+        if: matrix.distro == 'ubuntu-impish'
+        name: Build package for ubuntu-impish
+        id: build-ubuntu-impish
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}

.gitignore

@@ -5,3 +5,5 @@ subprojects/googletest-release*
 build/
 .vscode/
 builddir/
+.cache/
+.clangd/

ChangeLog

@@ -1,7 +1,127 @@
+libkiwix 11.0.0
+===============
+
+ * [server] Add support for internationalization (@veloman-yunkan #679)
+ * [server] Use gzip compression instead of deflat (mgautierfr #757)
+ * [server] Version the static resources. This allow better invalidating
+   browser cache when resources are changed (@veloman-yunkan #712)
+ * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server] Improve multizim search API:
+   - Improvement of the cache system
+   - Better API to select on which books to search in.
+   - SysAdmin is now able to limit the number of book we search in for a multizim search
+ * [server] Introduce a opensearch API for multizim fulltext search
+ * [wrapper] Remove java wrapper
+ * Testing:
+   - Testing of search result pages content (@veloman-yunkan #765)
+   - Better testing structure of xml search result (@veloman-yunkan #780)
+
+libkiwix 10.1.1
+===============
+
+ * Correctly detect the number of article for older zims (<=6) (@mgautier #743)
+ * [server] Fix fulltext search (@mgautierfr #724)
+ * [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
+ * Fix CI (@mgautierfr #736)
+
+libkiwix 10.1.0
+===============
+
+ This release is an important one as it fixes a Xss vulnerability introduced
+ in libkiwix 10.0.0
+
+ * [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
+ * [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
+ * [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
+ * [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
+ * Add `dc:issued` to opds output stream (@veloman-yunkan #715)
+ * Add handling of several languages not provided by ICU (@juuz0 #701)
+ * [server] Add a caching system for search and suggestion (@maneeshpm #620)
+ * Fix cross-compilation (@kelson42 #703)
+ * Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
+ * Better testing system of html response (@veloman-yunkan #725)
+
+libkiwix 10.0.1
+===============
+
+ * [server] The catalog search interpret `count=0` as no limit.
+   This was the case for a long time. This was changed unintentionally
+   (@veloman-yunkan #686)
+ * [server] Correctly generere a human friendly title in the server frontend.
+   (@juuz0 #687, @kelson42 #689)
+ * [server] Fix download button if there is no url do download from.
+   (@juuz0 #691)
+ * Add non-minified isotope.pkdg.js
+   Needed for debian packaging as we need the source and minified version is
+   not the source (@legoktm #693)
+ * [server] Add a tooltip with the full language for the lang tag.
+ * CI fixes (@kelson42 @legoktm)
+
 libkiwix 10.0.0
 ===============
 
- * ...
+ This release is huge release.
+ The project has been renamed to libkiwix, it is more coherent with the library name.
+
+ * Server front page :
+  - Use js in the front page to display the available book,
+     using the OPDS stream as source. The front page is now populated only with
+     the visible books and user can search for books. (@MananJethwany #530, #541, #534)
+     (@kelson42 #628)
+  - Revamp css (@MananJethwany #559)
+  - Correctly Convert 3iso language code to 2iso (@juuz0 #672)
+
+ * Server suggestions search :
+  - Add pagination for suggestion search (@maneeshpm #591)
+  - Fix suggestion system (@MananJethwany #498)
+  - Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
+  - The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
+  - Properly handle html encoding of suggestions (@veloman-yunkan #458)
+
+ * Server improvements :
+  - Remove meta endpoints (@mgautier #669)
+  - Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
+  - Add details on 404 error pages (@soumyankar #490)
+  - Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
+  - Better headbar insertion (after charset definition) (@kelson42 #442)
+
+ * New OPDS Stream v2 :
+  - Add a list of categories (@veloman-yunkan)
+  - Support for partial entries (@veloman-yunkan #602)
+  - Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
+  - Add language endpoint to catalog (@veloman-yunkan #553)
+  - Add illustration API to get the illustration of a book (@mgautierfr #645)
+  - OPDS search can now filter books by category (@veloman-yunkan #459)
+
+ * Library improvements :
+  - Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
+  - Properly handle removing of book from the library (@veloman-yunkan #485)
+  - Use xapian to search for books in the library (@veloman-yunkan #460, #488)
+
+ * Added methods/functions :
+  - Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
+  - Add `getVersions` and `printVersions` functions (@kelson42 #665)
+  - Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
+  - Add `get_zimid()` method to the search result (@maneeshpm #510)
+
+ * Various improvements :
+  - Better secret value for aria2c rpc (@juuz0 #666)
+  - Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
+  - Add basic documentation (@mgautierfr #640)
+  - Do not use Reader internally (@maneeshpm #536 #576)
+  - Remove dependency headers from our public headers (@mgautierfr #574)
+  - Downloader now don't write metalink on the filesystem (@kelson42 #502)
+  - Support opening a zim file using a fd (@veloman-yukan #429)
+  - Use C++11 std::thread instead of pthread (@mgautierfr #445)
+  - [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
+  - Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
+  - Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
+  - Better parsing of the counterMap (@kelson42 #437)
+  - Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
+  - Remove deprecated methods (@mgautierfr)
+  - CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
+  - Fix compilation for FreeBSD (@swills g#432)
+  - Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
 
 kiwix-lib 9.4.1
 ===============

README.md

@@ -5,8 +5,10 @@ The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
 core. It contains the code shared by all Kiwix ports (Windows,
 GNU/Linux, macOS, Android, iOS, ...).
 
+[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
 [![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
 [![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
@@ -91,6 +93,14 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
+with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
+
+```bash
+meson . build -Dwrapper=android -Dwerror=false
+ninja -C build
+```
+
 Testing
 -------
 
@@ -154,7 +164,7 @@ to use custom welcome page mention `customIndexPage` argument in `kiwix::interna
 
 to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
 
-To use JS provided by kiwix-serve you can use the following template to start with -> 
+To use JS provided by kiwix-serve you can use the following template to start with ->
 
 ```
 <!DOCTYPE html>
@@ -163,14 +173,6 @@ To use JS provided by kiwix-serve you can use the following template to start wi
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width,initial-scale=1" />
     <title><-- Custom Tittle --></title>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
-    ></script>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
-    ></script>
     <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
     <script src="{{root}}/skin/iso6391To3.js"></script>
     <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
@@ -184,11 +186,11 @@ To use JS provided by kiwix-serve you can use the following template to start wi
 - To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
 - To add language select box add - `<select id="languageFilter"></select>` under body tag.
 - To add language select box add - `<select id="categoryFilter"></select>` under body tag.
-- To add search box for books use following form - 
+- To add search box for books use following form -
     ```
         <form id='kiwixSearchForm'>
         <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
-        <input type="submit" class="searchButton" value="Search"/>
+        <input type="submit" class="kiwixButton" value="Search"/>
         </form>
     ```
 

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/libkiwix'

debian/control

@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
  meson,
  pkg-config,
- libzim-dev (>= 6.1.8),
+ libzim-dev (>= 7.2.0~),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
@@ -23,7 +23,7 @@ Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
- libzim-dev (>= 6.0.0),
+ libzim-dev (>= 7.2.0~),
  libicu-dev,
  libpugixml-dev,
  libcurl4-gnutls-dev,

debian/libkiwix-dev.manpages

@@ -1 +1,2 @@
 usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

docs/.gitignore

@@ -0,0 +1,2 @@
+api
+xml

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'libkiwix'
+copyright = '2022, libkiwix-team'
+author = 'libkiwix-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+    'exhale'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+breathe_projects = {
+    "libkiwix": "./xml"
+}
+breathe_default_project = 'libkiwix'
+
+exhale_args = {
+    "containmentFolder":   "./api",
+    "rootFileName":        "ref_api.rst",
+    "rootFileTitle":       "Reference API",
+    "doxygenStripFromPath":"..",
+    "treeViewIsBootstrap": True,
+    "createTreeView" : True,
+    "exhaleExecutesDoxygen": True,
+    "exhaleDoxygenStdin":    "INPUT = ../include"
+}
+
+primary_domain = 'cpp'
+
+highlight_language = 'cpp'

docs/index.rst

@@ -0,0 +1,15 @@
+.. libkiwix documentation master file, created by
+   sphinx-quickstart on Fri Jul 24 15:40:50 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libkiwix's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage
+   api/ref_api
+   widget

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+breathe
+exhale

docs/usage.rst

@@ -0,0 +1,15 @@
+Libkiwix programming
+====================
+
+Introduction
+------------
+
+libkiwix is written in C++. To use the library, you need the include files of libkiwix have
+to link against libzim.
+
+Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
+which is always derived from std::exception.
+
+All classes are defined in the namespace kiwix.
+
+libkiwix is a set of tools to manage zim files and provide some common functionnality.

docs/widget.rst

@@ -0,0 +1,82 @@
+Kiwix serve widget
+====================
+
+Introduction
+------------
+
+The kiwix-serve widget provides an easy to embed way to show the `kiwix-serve` homepage.
+
+Usage
+-----
+
+To use the widget, simply add an iframe with its `src` attribute set to the `widget` endpoint.
+Example HTML Page ::
+
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+    <title>Widget Test</title>
+    </head>
+    <body>
+    <iframe src="http://192.168.18.8:8080/widget?disabledesc&disablefilter&disabledownload" width=1000 height=1000></iframe>
+    </body>
+    </html>
+
+This creates an iframe with the kiwix-serve homepage contents.
+
+Arguments are explained below.
+
+Possible Arguments
+-------------------
+
+Currently, the following arguments are supported.
+
+disabledesc (value = N/A)
+    Disables the description part of a tile.
+
+disablefilter (value = N/A)
+    Disables the search filters: language, category, tag and search function.
+
+disableclick (value = N/A)
+    Disables clicking the book to open it for reading.
+
+disabledownload (value = N/A)
+    Disables the download button (if avaialable at all) on the tile.
+
+
+Custom CSS and JS
+-----------------
+
+You can add your custom CSS rules and Javascript code to the widget.
+
+To do that, use the following code as template::
+
+    <iframe id="receiver" src="http://192.168.18.8:8080/widget?disabledesc=&disablefilter=&disabledownload=" width="1000" height="1000">
+    <p>Your browser does not support iframes.</p>
+    </iframe>
+
+    <script>
+    window.onload = function() {
+    var receiver = document.getElementById('receiver').contentWindow;
+    function sendMessage() {
+        let msg = {
+        css: `
+        .book__header {
+            color:red;
+        }`,
+        js: `
+        function widgetTest() {
+        console.log("Testing widget");
+        }
+        widgetTest();
+        `
+        }
+        receiver.postMessage(msg, 'http://192.168.18.8:8080/widget');
+    }
+    sendMessage();
+    }
+    </script>
+
+
+The CSS/JS fields are optional, you may send both or only one.
+

include/book.h

@@ -21,6 +21,10 @@
 #define KIWIX_BOOK_H
 
 #include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include "common.h"
 
 namespace pugi {
 class xml_node;
@@ -34,19 +38,36 @@ namespace kiwix
 {
 
 class OPDSDumper;
-class Reader;
 
 /**
  * A class to store information about a book (a zim file)
  */
 class Book
 {
- public:
+ public: // types
+  class Illustration
+  {
+    friend class Book;
+   public:
+    uint16_t width = 48;
+    uint16_t height = 48;
+    std::string mimeType;
+    std::string url;
+
+    const std::string& getData() const;
+
+   private:
+    mutable std::string data;
+    mutable std::mutex mutex;
+  };
+
+  typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
+
+ public: // functions
   Book();
   ~Book();
 
   bool update(const Book& other);
-  void update(const Reader& reader);
   void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
@@ -73,9 +94,13 @@ class Book
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const;
-  const std::string& getFaviconUrl() const { return m_faviconUrl; }
-  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
+  DEPRECATED const std::string& getFavicon() const;
+  DEPRECATED const std::string& getFaviconUrl() const;
+  DEPRECATED const std::string& getFaviconMimeType() const;
+
+  Illustrations getIllustrations() const;
+  std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
+
   const std::string& getDownloadId() const { return m_downloadId; }
 
   void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
@@ -96,14 +121,13 @@ class Book
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
   void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
   void setSize(uint64_t size) { m_size = size; }
-  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
-  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
   void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
 
- private:
+ private: // functions
   std::string getCategoryFromTags() const;
+  const Illustration& getDefaultIllustration() const;
 
- protected:
+ protected: // data
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
@@ -124,9 +148,11 @@ class Book
   uint64_t m_mediaCount = 0;
   bool m_readOnly = false;
   uint64_t m_size = 0;
-  mutable std::string m_favicon;
-  std::string m_faviconUrl;
-  std::string m_faviconMimeType;
+  Illustrations m_illustrations;
+
+  // Used as the return value of getDefaultIllustration() when no default
+  // illustration is found in the book
+  static const Illustration missingDefaultIllustration;
 };
 
 }

include/entry.h

@@ -1,183 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_ENTRY_H
-#define KIWIX_ENTRY_H
-
-#include <stdio.h>
-#include <zim/entry.h>
-#include <zim/item.h>
-#include <exception>
-#include <string>
-
-#include "common.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-
-class NoEntry : public std::exception {};
-
-/**
- * A entry represent an.. entry in a zim file.
- */
-class Entry
-{
-  public:
-    /**
-     * Construct an entry making reference to an zim article.
-     *
-     * @param article a zim::Article object
-     */
-    Entry(zim::Entry entry);
-    virtual ~Entry() = default;
-
-    /**
-     * Get the path of the entry.
-     *
-     * The path is the "key" of an entry.
-     *
-     * @return the path of the entry.
-     */
-    std::string getPath() const { return entry.getPath(); }
-
-    /**
-     * Get the title of the entry.
-     *
-     * @return the title of the entry.
-     */
-    std::string getTitle() const { return entry.getTitle(); }
-
-    /**
-     * Get the content of the entry.
-     *
-     * The string is a copy of the content.
-     * If you don't want to do a copy, use get_blob.
-     *
-     * @return the content of the entry.
-     */
-    std::string getContent() const { return entry.getItem().getData(); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @param size The size of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
-
-    /**
-     * Get the info for direct access to the content of the entry.
-     *
-     * Some entry (ie binary ones) have their content plain stored
-     * in the zim file. Knowing the offset where the content is stored
-     * an user can directly read the content in the zim file bypassing the
-     * libkiwix/libzim.
-     *
-     * @return A pair specifying where to read the content.
-     *         The string is the real file to read (may be different that .zim
-     *         file if zim is cut).
-     *         The offset is the offset to read in the file.
-     *         Return <"",0> if is not possible to read directly.
-     */
-    zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
-
-    /**
-     * Get the size of the entry.
-     *
-     * @return the size of the entry.
-     */
-    size_type   getSize() const;
-
-    /**
-     * Get the mime_type of the entry.
-     *
-     * @return the mime_type of the entry.
-     */
-    std::string getMimetype() const;
-
-
-    /**
-     * Get if the entry is a redirect entry.
-     *
-     * @return True if the entry is a redirect.
-     */
-    bool isRedirect() const;
-
-    /**
-     * Get if the entry is a link target entry.
-     *
-     * @return True if the entry is a link target.
-     */
-    bool isLinkTarget() const;
-
-    /**
-     * Get if the entry is a deleted entry.
-     *
-     * @return True if the entry is a deleted entry.
-     */
-    bool isDeleted() const;
-
-    /**
-     * Get the entry pointed by this entry.
-     *
-     * @return the entry pointed.
-     * @throw NoEntry if the entry is not a redirected entry.
-     */
-    Entry getRedirectEntry() const;
-
-    /**
-     * Get the final entry pointed by this entry.
-     *
-     * Follow the redirection until a "not redirecting" entry is found.
-     * If the entry is not a redirected entry, return the entry itself.
-     *
-     * @return the final entry.
-     */
-    Entry getFinalEntry() const;
-
-    /**
-     * Get the zim entry wrapped by this (kiwix) entry
-     *
-     * @return the zim entry
-     */
-    const zim::Entry& getZimEntry() const { return entry; }
-
-  private:
-    zim::Entry entry;
-};
-
-}
-
-#endif // KIWIX_ENTRY_H

include/kiwix.h

@@ -22,4 +22,4 @@
 
 #include "library.h"
 
-#endif
+#endif

include/library.h

@@ -24,7 +24,9 @@
 #include <vector>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <zim/archive.h>
+#include <zim/search.h>
 
 #include "book.h"
 #include "bookmark.h"
@@ -140,21 +142,34 @@ private: // functions
 };
 
 
+class ZimSearcher : public zim::Searcher
+{
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
+
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
+
 /**
  * A Library store several books.
  */
 class Library
 {
-  std::map<std::string, kiwix::Book> m_books;
-  std::map<std::string, std::shared_ptr<Reader>> m_readers;
-  std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
-  std::vector<kiwix::Bookmark> m_bookmarks;
-  class BookDB;
-  std::unique_ptr<BookDB> m_bookDB;
+  // all data fields must be added in LibraryBase
+  mutable std::mutex m_mutex;
 
  public:
+  typedef uint64_t Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
 
  public:
   Library();
@@ -180,6 +195,11 @@ class Library
    */
   bool addBook(const Book& book);
 
+  /**
+   * A self-explanatory alias for addBook()
+   */
+  bool addOrUpdateBook(const Book& book) { return addBook(book); }
+
   /**
    * Add a bookmark to the library.
    *
@@ -196,12 +216,18 @@ class Library
    */
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
+  // XXX: This is a non-thread-safe operation
   const Book& getBookById(const std::string& id) const;
-  Book& getBookById(const std::string& id);
+  // XXX: This is a non-thread-safe operation
   const Book& getBookByPath(const std::string& path) const;
-  Book& getBookByPath(const std::string& path);
-  std::shared_ptr<Reader> getReaderById(const std::string& id);
+
+  Book getBookByIdThreadSafe(const std::string& id) const;
+
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
 
   /**
    * Remove a book from the library.
@@ -285,17 +311,6 @@ class Library
    */
   BookIdCollection getBooksIds() const;
 
-  /**
-   * Filter the library and generate a new one with the keep elements.
-   *
-   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
-   *
-   * @param search List only books with search in the title or description.
-   * @return The list of bookIds corresponding to the query.
-   */
-  DEPRECATED BookIdCollection filter(const std::string& search) const;
-
-
   /**
    * Filter the library and return the id of the keep elements.
    *
@@ -315,48 +330,39 @@ class Library
   void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
 
   /**
-   * List books in the library.
+   * Return the current revision of the library.
    *
-   * @param mode The mode of listing :
-   *             - LOCAL    : list only local books (with a path).
-   *             - REMOTE   : list only remote books (with an url).
-   *             - VALID    : list only valid books (without a path or with a
-   *                          path pointing to a valid zim file).
-   *             - NOLOCAL  : list only books without valid path.
-   *             - NOREMOTE : list only books without url.
-   *             - NOVALID  : list only books not valid.
-   *             - ALL : Do not do any filter (LOCAL or REMOTE)
-   *             - Flags can be combined.
-   * @param sortBy Attribute to sort by the book list.
-   * @param search List only books with search in the title, description.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param maxSize Do not list book bigger than maxSize.
-   *                Set to 0 to cancel this filter.
-   * @return The list of bookIds corresponding to the query.
+   * The revision of the library is updated (incremented by one) only by
+   * the addBook() operation.
+   *
+   * @return Current revision of the library.
    */
-  DEPRECATED BookIdCollection listBooksIds(
-    int supportedListMode = ALL,
-    supportedListSortBy sortBy = UNSORTED,
-    const std::string& search = "",
-    const std::string& language = "",
-    const std::string& creator = "",
-    const std::string& publisher = "",
-    const std::vector<std::string>& tags = {},
-    size_t maxSize = 0) const;
+  Revision getRevision() const;
+
+  /**
+   * Remove books that have not been updated since the specified revision.
+   *
+   * @param  rev the library revision to use
+   * @return Count of books that were removed by this operation.
+   */
+  uint32_t removeBooksNotUpdatedSince(Revision rev);
 
   friend class OPDSDumper;
   friend class libXMLDumper;
 
 private: // types
   typedef const std::string& (Book::*BookStrPropMemFn)() const;
+  struct Impl;
 
 private: // functions
   AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
+  void dropCache(const std::string& bookId);
+
+private: //data
+  std::unique_ptr<Impl> mp_impl;
 };
 
 }

include/manager.h

@@ -22,10 +22,10 @@
 
 #include "book.h"
 #include "library.h"
-#include "reader.h"
 
 #include <string>
 #include <vector>
+#include <memory>
 
 namespace pugi {
 class xml_document;
@@ -34,26 +34,25 @@ class xml_document;
 namespace kiwix
 {
 
-class LibraryManipulator {
- public:
-  virtual ~LibraryManipulator() {}
-  virtual bool addBookToLibrary(Book book) = 0;
-  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
-};
+class LibraryManipulator
+{
+ public: // functions
+  explicit LibraryManipulator(Library* library);
+  virtual ~LibraryManipulator();
 
-class DefaultLibraryManipulator : public LibraryManipulator {
- public:
-  DefaultLibraryManipulator(Library* library) :
-    library(library) {}
-  virtual ~DefaultLibraryManipulator() {}
-  bool addBookToLibrary(Book book) {
-    return library->addBook(book);
-  }
-  void addBookmarkToLibrary(Bookmark bookmark) {
-    library->addBookmark(bookmark);
-  }
- private:
-   kiwix::Library* library;
+  Library& getLibrary() const { return library; }
+
+  bool addBookToLibrary(const Book& book);
+  void addBookmarkToLibrary(const Bookmark& bookmark);
+  uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
+
+ protected: // overrides
+  virtual void bookWasAddedToLibrary(const Book& book);
+  virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
+  virtual void booksWereRemovedFromLibrary();
+
+ private: // data
+  kiwix::Library& library;
 };
 
 /**
@@ -61,10 +60,12 @@ class DefaultLibraryManipulator : public LibraryManipulator {
  */
 class Manager
 {
- public:
-  Manager(LibraryManipulator* manipulator);
-  Manager(Library* library);
-  ~Manager();
+ public: // types
+  typedef std::vector<std::string> Paths;
+
+ public: // functions
+  explicit Manager(LibraryManipulator* manipulator);
+  explicit Manager(Library* library);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
@@ -72,10 +73,22 @@ class Manager
    * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
+   * @param trustLibrary use book metadata coming from XML.
    * @return True if file has been properly parsed.
    */
   bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
+  /**
+   * Sync the contents of the library with one or more `library.xml` files.
+   *
+   * The metadata of the library files is trusted unconditionally.
+   * Any books not present in the input library.xml files are removed
+   * from the library.
+   *
+   * @param paths The (utf8) paths to the `library.xml` files.
+   */
+  void reload(const Paths& paths);
+
   /**
    * Load a library content store in the string.
    *
@@ -150,8 +163,7 @@ class Manager
   uint64_t m_itemsPerPage = 0;
 
  protected:
-  kiwix::LibraryManipulator* manipulator;
-  bool mustDeleteManipulator;
+  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,

include/meson.build

@@ -7,14 +7,12 @@ headers = [
   'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
-  'reader.h',
-  'entry.h',
-  'searcher.h',
   'search_renderer.h',
   'server.h',
   'kiwixserve.h',
   'name_mapper.h',
-  'tools.h'
+  'tools.h',
+  'version.h'
 ]
 
 install_headers(headers, subdir:'kiwix')

include/name_mapper.h

@@ -22,6 +22,8 @@
 
 #include <string>
 #include <map>
+#include <memory>
+#include <mutex>
 
 namespace kiwix
 {
@@ -31,15 +33,15 @@ class Library;
 class NameMapper {
   public:
     virtual ~NameMapper() = default;
-    virtual std::string getNameForId(const std::string& id) = 0;
-    virtual std::string getIdForName(const std::string& name) = 0;
+    virtual std::string getNameForId(const std::string& id) const = 0;
+    virtual std::string getIdForName(const std::string& name) const = 0;
 };
 
 
 class IdNameMapper : public NameMapper {
   public:
-    virtual std::string getNameForId(const std::string& id) { return id; };
-    virtual std::string getIdForName(const std::string& name) { return name; };
+    virtual std::string getNameForId(const std::string& id) const { return id; };
+    virtual std::string getIdForName(const std::string& name) const { return name; };
 };
 
 class HumanReadableNameMapper : public NameMapper {
@@ -50,11 +52,29 @@ class HumanReadableNameMapper : public NameMapper {
   public:
     HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
     virtual ~HumanReadableNameMapper() = default;
-    virtual std::string getNameForId(const std::string& id);
-    virtual std::string getIdForName(const std::string& name);
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
 };
 
+class UpdatableNameMapper : public NameMapper {
+    typedef std::shared_ptr<NameMapper> NameMapperHandle;
+  public:
+    UpdatableNameMapper(Library& library, bool withAlias);
 
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+
+    void update();
+
+  private:
+    NameMapperHandle currentNameMapper() const;
+
+  private:
+    mutable std::mutex mutex;
+    Library& library;
+    NameMapperHandle nameMapper;
+    const bool withAlias;
+};
 
 }
 

include/opds_dumper.h

@@ -27,7 +27,6 @@
 #include <pugixml.hpp>
 
 #include "library.h"
-#include "reader.h"
 
 using namespace std;
 

include/reader.h

@@ -1,497 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_READER_H
-#define KIWIX_READER_H
-
-#include <stdio.h>
-#include <zim/zim.h>
-#include <zim/archive.h>
-#include <exception>
-#include <map>
-#include <sstream>
-#include <string>
-#include "common.h"
-#include "entry.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-/**
- * The SuggestionItem is a helper class that contains the info about a single
- * suggestion item.
- */
-class SuggestionItem
-{
-  // Functions
-  // Temporarily making the constructor public until the code move is complete
-  public:
-    // Create a sugggestion item.
-    explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
-                            const std::string& path, const std::string& snippet = "") :
-      title(title),
-      normalizedTitle(normalizedTitle),
-      path(path),
-      snippet(snippet) {}
-
-  public:
-    const std::string& getTitle() const           { return title;}
-    const std::string& getNormalizedTitle() const { return normalizedTitle;}
-    const std::string& getPath() const            { return path;}
-    const std::string& getSnippet() const         { return snippet;}
-
-    bool hasSnippet() const                { return !snippet.empty();}
-
-  // Data
-  private:
-    std::string title;
-    std::string normalizedTitle;
-    std::string path;
-    std::string snippet;
-
-  friend class Reader;
-};
-
-/**
- * The Reader class is the class who allow to get an entry content from a zim
- * file.
- */
-
-using SuggestionsList_t = std::vector<SuggestionItem>;
-class Reader
-{
- public:
-  /**
-   * Create a Reader to read a zim file specified by zimFilePath.
-   *
-   * @param zimFilePath The path to the zim file to read.
-   *                    The zim file can be splitted (.zimaa, .zimab, ...).
-   *                    In this case, the file path must still point to the
-   *                    unsplitted path as if the file were not splitted
-   *                    (.zim extesion).
-   */
-  explicit Reader(const string zimFilePath);
-
-   /**
-   * Create a Reader to read a zim file given by the Archive.
-   *
-   * @param archive The shared pointer to the Archive object.
-   */
-  explicit Reader(const std::shared_ptr<zim::Archive> archive);
-#ifndef _WIN32
-  explicit Reader(int fd);
-  Reader(int fd, zim::offset_type offset, zim::size_type size);
-#endif
-  ~Reader() = default;
-
-  /**
-   * Get the number of "displayable" entries in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'text/html' MIMEtype specified in the metadata.
-   *         Else return the number of entries in the 'A' namespace.
-   */
-  unsigned int getArticleCount() const;
-
-  /**
-   * Get the number of media in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'image/jpeg', 'image/gif' and 'image/png' in
-   *         the metadata.
-   *         Else return the number of entries in the 'I' namespace.
-   */
-  unsigned int getMediaCount() const;
-
-  /**
-   * Get the number of all entries in the zim file.
-   *
-   * @return Return the number of all the entries, whatever their MIMEtype or
-   *         their namespace.
-   */
-  unsigned int getGlobalCount() const;
-
-  /**
-   * Get the path of the zim file.
-   *
-   * @return the path of the zim file as given in the constructor.
-   */
-  string getZimFilePath() const;
-
-  /**
-   * Get the Id of the zim file.
-   *
-   * @return The uuid stored in the zim file.
-   */
-  string getId() const;
-
-  /**
-   * Get a random page.
-   *
-   * @return A random Entry. The entry is picked from all entries in
-   *         the 'A' namespace.
-   *         The main entry is excluded from the potential results.
-   */
-  Entry getRandomPage() const;
-
-  /**
-   * Get the entry of the main page.
-   *
-   * @return Entry of the main page as specified in the zim file.
-   */
-  Entry getMainPage() const;
-
-  /**
-   * Get the content of a metadata.
-   *
-   * @param[in]  name The name of the metadata.
-   * @param[out] value The value will be set to the content of the metadata.
-   * @return True if it was possible to get the content of the metadata.
-   */
-  bool getMetadata(const string& name, string& value) const;
-
-  /**
-   * Get the name of the zim file.
-   *
-   * @return The name of the zim file as specified in the zim metadata.
-   */
-  string getName() const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the creator of the zim file.
-   *
-   * @return The creator of the zim file as specified in the zim metadata.
-   */
-  string getCreator() const;
-
-  /**
-   * Get the publisher of the zim file.
-   *
-   * @return The publisher of the zim file as specified in the zim metadata.
-   */
-  string getPublisher() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the long description of the zim file.
-   *
-   * @return The long description of the zim file as specifed in the zim metadata.
-   */
-  string getLongDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
-
-  /**
-   * Get the license of the zim file.
-   *
-   * @return The license of the zim file as specified in the zim metadata.
-   */
-  string getLicense() const;
-
-  /**
-   * Get the tags of the zim file.
-   *
-   * @param original If true, return the original tags as specified in the zim metadata.
-   *                 Else, try to convert it to the new 'normalized' format.
-   * @return The tags of the zim file.
-   */
-  string getTags(bool original=false) const;
-
-  /**
-   * Get the value (as a string) of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   */
-  string getTagStr(const std::string& tagName) const;
-
-  /**
-   * Get the boolean value of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The boolean value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   *         std::domain_error if the value of the tag cannot be convert to bool.
-   */
-  bool getTagBool(const std::string& tagName) const;
-
-  /**
-   * Get the relations of the zim file.
-   *
-   * @return The relation of the zim file as specified in the zim metadata.
-   */
-  string getRelation() const;
-
-  /**
-   * Get the flavour of the zim file.
-   *
-   * @return The flavour of the zim file as specified in the zim metadata.
-   */
-  string getFlavour() const;
-
-  /**
-   * Get the source of the zim file.
-   *
-   * @return The source of the zim file as specified in the zim metadata.
-   */
-  string getSource() const;
-
-  /**
-   * Get the scraper of the zim file.
-   *
-   * @return The scraper of the zim file as specified in the zim metadata.
-   */
-  string getScraper() const;
-
-  /**
-   * Get the favicon of the zim file.
-   *
-   * @param[out] content The content of the favicon.
-   * @param[out] mimeType The mimeType of the favicon.
-   * @return True if a favicon has been found.
-   */
-  bool getFavicon(string& content, string& mimeType) const;
-
-  /**
-   * Get an entry associated to an path.
-   *
-   * @param path The path of the entry.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromPath(const std::string& path) const;
-
-  /**
-   * Get an entry associated to an url encoded path.
-   *
-   * Equivalent to `getEntryFromPath(urlDecode(path));`
-   *
-   * @param path The url encoded path.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromEncodedPath(const std::string& path) const;
-
-  /**
-   * Get un entry associated to a title.
-   *
-   * @param title The title.
-   * @return The entry
-   * throw NoEntry If no entry correspond to the url.
-   */
-  Entry getEntryFromTitle(const std::string& title) const;
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are stored in an internal vector and can be retrieved using
-   * `getNextSuggestion` method.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestions(const string& prefix,
-   *                        unsigned int suggestionsCount,
-   *                        SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param reset If true, remove previous suggestions in the internal vector.
-   *              If false, add suggestions to the internal vector
-   *              (until internal vector size is suggestionCount (or no more
-   *               suggestion))
-   * @return True if some suggestions have been added to the internal vector.
-   */
-  DEPRECATED bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         const bool reset = true);
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are added to the `result` vector.
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param result The vector where to store the suggestions.
-   * @return True if some suggestions have been added to the vector.
-   */
-
-  bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         SuggestionsList_t& resuls);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestionsSmart(const string& prefix,
-   *                             unsigned int suggestionsCount,
-   *                             SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   */
-  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param results The vector where to store the suggestions
-   * @return True if some suggestions have been added to the results.
-   */
-   bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount,
-                              SuggestionsList_t& results);
-
-
-  /**
-   * Check if the path exists in the zim file.
-   *
-   * @param path the path to check.
-   * @return True if the path exists in the zim file.
-   */
-  bool pathExists(const string& path) const;
-
-  /**
-   * Check if the zim file has a embedded fulltext index.
-   *
-   * @return True if the zim file has a embedded fulltext index
-   *         and is not split (else the fulltext is not accessible).
-   */
-  bool hasFulltextIndex() const;
-
-  /**
-   * Get potential case title variations for a title.
-   *
-   * @param title a title.
-   * @return the list of variantions.
-   */
-  std::vector<std::string> getTitleVariants(const std::string& title) const;
-
-  /**
-   * Get the next suggestion title.
-   *
-   * @param[out] title the title of the suggestion.
-   * @return True if title has been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title);
-
-  /**
-   * Get the next suggestion title and url.
-   *
-   * @param[out] title the title of the suggestion.
-   * @param[out] url the url of the suggestion.
-   * @return True if title and url have been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title, string& url);
-
-  /**
-   * Get if we can check zim file integrity (has a checksum).
-   *
-   * @return True if zim file have a checksum.
-   */
-  bool canCheckIntegrity() const;
-
-  /**
-   * Check is zim file is corrupted.
-   *
-   * @return True if zim file is corrupted.
-   */
-  bool isCorrupted() const;
-
-  /**
-   * Return the total size of the zim file.
-   *
-   * If zim file is split, return the sum of all parts' size.
-   *
-   * @return Size of the size file is KiB.
-   */
-  unsigned int getFileSize() const;
-
-  /**
-   * Get the zim file handler.
-   *
-   * @return The libzim file handler.
-   */
-  zim::Archive* getZimArchive() const;
-
- protected:
-  std::shared_ptr<zim::Archive> zimArchive;
-  std::string zimFilePath;
-
-  SuggestionsList_t suggestions;
-  SuggestionsList_t::iterator suggestionsOffset;
-
- private:
-  std::map<const std::string, unsigned int> parseCounterMetadata() const;
-};
-}
-
-#endif

include/search_renderer.h

@@ -22,11 +22,11 @@
 
 #include <string>
 #include <zim/search.h>
+#include "library.h"
 
 namespace kiwix
 {
 
-class Searcher;
 class NameMapper;
 /**
  * The SearcherRenderer class is used to render a search result to a html page.
@@ -35,23 +35,43 @@ class SearchRenderer
 {
  public:
   /**
-   * The default constructor.
+   * Construct a SearchRenderer from a SearchResultSet.
    *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
+   * The constructed version of the SearchRenderer will not introduce
+   * the book name for each result. It is better to use the other constructor
+   * with a Library pointer to have a better html page.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
    */
-  SearchRenderer(Searcher* searcher, NameMapper* mapper);
   SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                  unsigned int start, unsigned int estimatedResultCount);
 
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                 unsigned int start, unsigned int estimatedResultCount);
+
   ~SearchRenderer();
 
+  /**
+   * Set the search pattern used to do the search
+   */
   void setSearchPattern(const std::string& pattern);
 
   /**
-   * Set the search content id.
+   * Set the querystring used to select books
    */
-  void setSearchContent(const std::string& name);
+  void setSearchBookQuery(const std::string& bookQuery);
 
   /**
    * Set protocol prefix.
@@ -70,16 +90,25 @@ class SearchRenderer
     this->pageLength  = pageLength;
   }
 
+  std::string renderTemplate(const std::string& tmpl_str);
+
   /**
    * Generate the html page with the resutls of the search.
    */
   std::string getHtml();
 
+    /**
+   * Generate the xml page with the resutls of the search.
+   */
+  std::string getXml();
+
+
  protected:
   std::string beautifyInteger(const unsigned int number);
   zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
-  std::string searchContent;
+  Library* mp_library;
+  std::string searchBookQuery;
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;

include/searcher.h

@@ -1,178 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCHER_H
-#define KIWIX_SEARCHER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unicode/putil.h>
-#include <algorithm>
-#include <cctype>
-#include <locale>
-#include <string>
-#include <memory>
-#include <vector>
-
-#include <zim/search.h>
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader;
-class Result
-{
- public:
-  virtual ~Result(){};
-  virtual std::string get_url() = 0;
-  virtual std::string get_title() = 0;
-  virtual int get_score() = 0;
-  virtual std::string get_snippet() = 0;
-  virtual std::string get_content() = 0;
-  virtual int get_wordCount() = 0;
-  virtual int get_size() = 0;
-  virtual std::string get_zimId() = 0;
-};
-
-struct SearcherInternal;
-struct SuggestionInternal;
-/**
- * The Searcher class is reponsible to do different kind of search using the
- * fulltext index.
- */
-class Searcher
-{
- public:
-  /**
-   * The default constructor.
-   */
-  Searcher();
-
-  ~Searcher();
-
-  /**
-   * Add a reader (containing embedded fulltext index) to the search.
-   *
-   * @param reader The Reader for the zim containing the fulltext index.
-   * @return true if the reader has been added.
-   *         false if the reader cannot be added (no embedded fulltext index present)
-   */
-  bool add_reader(Reader* reader);
-
-
-  Reader* get_reader(int index);
-
-  /**
-   * Start a search on the zim associated to the Searcher.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param search The search query.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void search(const std::string& search,
-              unsigned int resultStart,
-              unsigned int maxResultCount,
-              const bool verbose = false);
-
-  /**
-   * Start a geographique search.
-   * The search return result for entry in a disc of center latitude/longitude
-   * and radius distance.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param latitude The latitude of the center point.
-   * @param longitude The longitude of the center point.
-   * @param distance The radius of the disc.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum number of results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void geo_search(float latitude, float longitude, float distance,
-                  unsigned int resultStart,
-                  unsigned int maxResultCount,
-                  const bool verbose = false);
-
-  /**
-   * Start a suggestion search.
-   * The search made depend of the "version" of the embedded index.
-   *  - If the index is newer enough and have a title namespace, the search is
-   *    made in the titles only.
-   *  - Else the search is made on the whole article content.
-   * In any case, the search is made "partial" (as adding '*' at the end of the query)
-   *
-   * @param search The search query.
-   * @param verbose print some info on stdout if true.
-   */
-  void suggestions(std::string& search, const bool verbose = false);
-
-  /**
-   * Get the next result of a started search.
-   * This is the method to use to loop hover the search results.
-   */
-  Result* getNextResult();
-
-  /**
-   * Restart the previous search.
-   * Next call to getNextResult will return the first result.
-   */
-  void restart_search();
-
-  /**
-   * Get a estimation of the result count.
-   */
-  unsigned int getEstimatedResultCount();
-
-  /**
-   * Get a SearchResultSet object for current search
-   */
-  zim::SearchResultSet getSearchResultSet();
-
-  unsigned int getResultStart() { return resultStart; }
-  unsigned int getMaxResultCount() { return maxResultCount; }
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  void closeIndex();
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int maxResultCount,
-                     const bool verbose = false);
-
-  std::vector<Reader*> readers;
-  std::unique_ptr<SearcherInternal> internal;
-  std::unique_ptr<SuggestionInternal> suggestionInternal;
-  std::string searchPattern;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int maxResultCount;
-
- private:
-  void reset();
-
-};
-
-
-}
-
-#endif

include/server.h

@@ -54,12 +54,16 @@ namespace kiwix
        void setAddress(const std::string& addr) { m_addr = addr; }
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
+       void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
        void setBlockExternalLinks(bool blockExternalLinks)
         { m_blockExternalLinks = blockExternalLinks; }
+       int getPort();
+       std::string getAddress();
 
      protected:
        Library* mp_library;
@@ -69,10 +73,12 @@ namespace kiwix
        std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
        bool m_blockExternalLinks = false;
+       int m_ipConnectionLimit = 0;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

include/tools.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <vector>
+#include <map>
 
 namespace kiwix {
 
@@ -177,16 +178,25 @@ std::string converta2toa3(const std::string& a2code);
  */
 std::string getFileContent(const std::string& path);
 
-/** checks if file exists.
+/** Checks if file exists.
  *
- * This function returns boolean stating if file exists or not.
+ * This function returns boolean stating if file exists.
  *
  * @param path The absolute path provided in string format.
  * @return Boolean representing if file exists or not.
  */
 bool fileExists(const std::string& path);
 
-/** provides mimetype from filename.
+/** Checks if file is readable.
+ *
+ * This function returns boolean stating if file is readable.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file is readale or not.
+ */
+bool fileReadable(const std::string& path);
+
+/** Provides mimetype from filename.
  *
  * This function provides mimetype from file-name.
  *
@@ -194,5 +204,17 @@ bool fileExists(const std::string& path);
  * @return mimetype from filename in string format.
  */
 std::string getMimeTypeForFile(const std::string& filename);
+
+/** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 network interfaces
+ */
+std::map<std::string, std::string> getNetworkInterfaces();
+
+/** Provides the best IP address
+ * This function provides the best IP address from the list given by getNetworkInterfaces
+ */
+std::string getBestPublicIp();
+
 }
 #endif // KIWIX_TOOLS_H

include/version.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,21 +17,18 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_VERSION_H
+#define KIWIX_VERSION_H
 
-public class JNIKiwixString
+#include <string>
+#include <vector>
+#include <iostream>
+
+namespace kiwix
 {
-  public String value;
-
-  public JNIKiwixString(String value) {
-    this.value = value;
-  }
-
-  public JNIKiwixString() {
-    this("");
-  }
-
-  public String getValue() {
-    return value;
-  }
+  typedef std::vector<std::pair<std::string, std::string>> LibVersions;
+  LibVersions getVersions();
+  void printVersions(std::ostream& out = std::cout);
 }
+
+#endif // KIWIX_VERSION_H

meson.build

@@ -1,29 +1,20 @@
 project('libkiwix', 'cpp',
-  version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '11.0.0',
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
-wrapper = get_option('wrapper')
+static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
 
-static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
-if wrapper.contains('android')
-  extra_libs = ['-llog']
+# See https://github.com/kiwix/libkiwix/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
+  extra_libs = ['-latomic']
 else
   extra_libs = []
 endif
 
-if wrapper.contains('java')
-  add_languages('java')
-endif
-
-# See https://github.com/kiwix/libkiwix/issues/371
-if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
-  extra_libs += '-latomic'
-endif
-
-if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
   # C++ std::thread is implemented using pthread on linux by gcc
   thread_dep = dependency('threads')
 else
@@ -44,19 +35,19 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif
 
 
 extra_cflags = ''
-if target_machine.system() == 'windows' and static_deps
+if host_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-if target_machine.system() == 'windows'
+if host_machine.system() == 'windows'
   add_project_arguments('-DNOMINMAX', language: 'cpp')
 endif
 
@@ -65,7 +56,7 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microh
 inc = include_directories('include', extra_include)
 
 conf = configuration_data()
-conf.set('VERSION', '"@0@"'.format(meson.project_version()))
+conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -78,6 +69,9 @@ subdir('scripts')
 subdir('static')
 subdir('src')
 subdir('test')
+if get_option('doc')
+  subdir('docs')
+endif
 
 pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
 

meson_options.txt

@@ -1,2 +1,4 @@
-option('wrapper', type:'array', choices:['java', 'android'], value:[],
-  description: 'The wrapper to generate.')
+option('static-linkage', type : 'boolean', value : false,
+  description : 'Link statically with the dependencies.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')

scripts/kiwix-compile-i18n

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.data = f.read()
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_files', nargs='+',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    resources = [Resource(filename) for filename in args.i18n_resource_files]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_files ...
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -102,7 +102,7 @@ class Resource:
 
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>

scripts/kiwix-resources

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_file(src_path, dst_path):
+    with open(src_path, 'rb') as src:
+        with open(dst_path, 'wb') as dst:
+            dst.write(src.read())
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        preprocess_resource(resource)
+    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,6 +1,13 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
 
 install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/aria2.cpp

@@ -3,12 +3,14 @@
 #include "aria2.h"
 #include "xmlrpc.h"
 #include <iostream>
+#include <algorithm>
 #include <sstream>
 #include <thread>
 #include <chrono>
 #include "tools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
 #include "downloader.h" // For AriaError
 
 #ifdef _WIN32
@@ -30,7 +32,7 @@ namespace kiwix {
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret("kiwixariarpc"),
+  m_secret(getNewRpcSecret()),
   m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
   mp_curl(nullptr)
 {
@@ -68,7 +70,7 @@ Aria2::Aria2():
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
   callCmd.push_back(download_dir.c_str());
-  if (fileExists(session_file)) {
+  if (fileReadable(session_file)) {
     callCmd.push_back(inputFile.c_str());
   }
   callCmd.push_back(session.c_str());
@@ -195,6 +197,13 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
   return doRequest(methodCall);
 }
 
+std::string Aria2::getNewRpcSecret()
+{
+  std::string uuid = gen_uuid("");
+  uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
+  return uuid.substr(0, 9);
+}
+
 std::vector<std::string> Aria2::tellActive()
 {
   MethodCall methodCall("aria2.tellActive", m_secret);

src/aria2.h

@@ -37,6 +37,7 @@ class Aria2
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    static std::string getNewRpcSecret();
     std::vector<std::string> tellActive();
     std::vector<std::string> tellWaiting();
     void saveSession();

src/book.cpp

@@ -18,7 +18,6 @@
  */
 
 #include "book.h"
-#include "reader.h"
 
 #include "tools.h"
 #include "tools/base64.h"
@@ -30,7 +29,7 @@
 #include "tools/archiveTools.h"
 
 #include <zim/archive.h>
-
+#include <zim/item.h>
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -41,11 +40,17 @@ Book::Book() :
   m_readOnly(false)
 {
 }
+
 /* Destructor */
 Book::~Book()
 {
 }
 
+Book::Illustrations Book::getIllustrations() const
+{
+  return m_illustrations;
+}
+
 bool Book::update(const kiwix::Book& other)
 {
   if (m_readOnly)
@@ -54,38 +59,10 @@ bool Book::update(const kiwix::Book& other)
   if (m_id != other.m_id)
     return false;
 
-  m_readOnly = other.m_readOnly;
-  m_path = other.m_path;
-  m_pathValid = other.m_pathValid;
-  m_title = other.m_title;
-  m_description = other.m_description;
-  m_language = other.m_language;
-  m_creator = other.m_creator;
-  m_publisher = other.m_publisher;
-  m_date = other.m_date;
-  m_url = other.m_url;
-  m_name = other.m_name;
-  m_flavour = other.m_flavour;
-  m_tags = other.m_tags;
-  m_category = other.m_category;
-  m_origId = other.m_origId;
-  m_articleCount = other.m_articleCount;
-  m_mediaCount = other.m_mediaCount;
-  m_size = other.m_size;
-  m_favicon = other.m_favicon;
-  m_faviconMimeType = other.m_faviconMimeType;
-  m_faviconUrl = other.m_faviconUrl;
-
-  m_downloadId = other.m_downloadId;
-
+  *this = other;
   return true;
 }
 
-void Book::update(const kiwix::Reader& reader)
-{
-  update(*reader.getZimArchive());
-}
-
 void Book::update(const zim::Archive& archive) {
   m_path = archive.getFilename();
   m_pathValid = true;
@@ -100,11 +77,20 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = archive.getArticleCount();
+  m_articleCount = getArchiveArticleCount(archive);
   m_mediaCount = getArchiveMediaCount(archive);
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
-  getArchiveFavicon(archive, 48, m_favicon, m_faviconMimeType);
+  m_illustrations.clear();
+  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+    const auto illustration = std::make_shared<Illustration>();
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
+    illustration->width = illustration->height = illustrationSize;
+    illustration->mimeType = illustrationItem.getMimetype();
+    illustration->data = illustrationItem.getData();
+    // NOTE: illustration->url is left uninitialized
+    m_illustrations.push_back(illustration);
+  }
 }
 
 #define ATTR(name) node.attribute(name).value()
@@ -116,7 +102,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  m_pathValid = fileExists(path);
+  m_pathValid = fileReadable(path);
   m_title = ATTR("title");
   m_description = ATTR("description");
   m_language = ATTR("language");
@@ -131,9 +117,14 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  m_favicon = base64_decode(ATTR("favicon"));
-  m_faviconMimeType = ATTR("faviconMimeType");
-  m_faviconUrl = ATTR("faviconUrl");
+  std::string favicon_mimetype = ATTR("faviconMimeType");
+  if (! favicon_mimetype.empty()) {
+    const auto favicon = std::make_shared<Illustration>();
+    favicon->data = base64_decode(ATTR("favicon"));
+    favicon->mimeType = favicon_mimetype;
+    favicon->url = ATTR("faviconUrl");
+    m_illustrations.assign(1, favicon);
+  }
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -164,7 +155,9 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_language = VALUE("language");
   m_creator = node.child("author").child("name").child_value();
   m_publisher = node.child("publisher").child("name").child_value();
-  m_date = fromOpdsDate(VALUE("updated"));
+  const std::string dcIssuedDate = VALUE("dc:issued");
+  m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
+  m_date = fromOpdsDate(m_date);
   m_name = VALUE("name");
   m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
@@ -181,8 +174,11 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
       m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
     }
     if (rel == "http://opds-spec.org/image/thumbnail") {
-      m_faviconUrl = urlHost + linkNode.attribute("href").value();
-      m_faviconMimeType = linkNode.attribute("type").value();
+      const auto favicon = std::make_shared<Illustration>();
+      favicon->data.clear();
+      favicon->url = urlHost + linkNode.attribute("href").value();
+      favicon->mimeType = linkNode.attribute("type").value();
+      m_illustrations.assign(1, favicon);
     }
  }
 
@@ -193,7 +189,7 @@ std::string Book::getHumanReadableIdFromPath() const
 {
   std::string id = m_path;
   if (!id.empty()) {
-    kiwix::removeAccents(id);
+    id = kiwix::removeAccents(id);
 
 #ifdef _WIN32
     id = replaceRegex(id, "", "^.*\\\\");
@@ -215,15 +211,54 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-const std::string& Book::getFavicon() const {
-  if (m_favicon.empty() && !m_faviconUrl.empty()) {
-    try {
-      m_favicon = download(m_faviconUrl);
-    } catch(...) {
-      std::cerr << "Cannot download favicon from " << m_faviconUrl;
+const Book::Illustration Book::missingDefaultIllustration;
+
+std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
+{
+  for ( const auto& ilPtr : m_illustrations ) {
+    if (ilPtr->width == size && ilPtr->height == size) {
+      return ilPtr;
     }
   }
-  return m_favicon;
+  throw std::runtime_error("Cannot find illustration");
+}
+
+const Book::Illustration& Book::getDefaultIllustration() const
+{
+  try {
+    return *getIllustration(48);
+  } catch (...) {
+    return missingDefaultIllustration;
+  }
+}
+
+const std::string& Book::Illustration::getData() const
+{
+  if (data.empty() && !url.empty()) {
+    const std::lock_guard<std::mutex> l(mutex);
+    if ( data.empty() ) {
+      try {
+        data = download(url);
+      } catch(...) {
+        std::cerr << "Cannot download favicon from " << url;
+      }
+    }
+  }
+  return data;
+}
+
+const std::string& Book::getFavicon() const {
+  return getDefaultIllustration().getData();
+}
+
+const std::string& Book::getFaviconUrl() const
+{
+  return getDefaultIllustration().url;
+}
+
+const std::string& Book::getFaviconMimeType() const
+{
+  return getDefaultIllustration().mimeType;
 }
 
 std::string Book::getTagStr(const std::string& tagName) const {

src/config.h.in

@@ -1,3 +1,3 @@
 
-#mesondefine VERSION
+#mesondefine LIBKIWIX_VERSION
 

src/entry.cpp

@@ -1,73 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-namespace kiwix
-{
-
-Entry::Entry(zim::Entry entry)
-  : entry(entry)
-{
-}
-
-size_type Entry::getSize() const
-{
-  if (entry.isRedirect()) {
-    return 0;
-  } else {
-    return entry.getItem().getSize();
-  }
-}
-
-std::string Entry::getMimetype() const
-{
-  return entry.getItem(true).getMimetype();
-}
-
-bool Entry::isRedirect() const
-{
-  return entry.isRedirect();
-}
-
-Entry Entry::getRedirectEntry() const
-{
-  if ( !entry.isRedirect() ) {
-    throw NoEntry();
-  }
-
-  return entry.getRedirectEntry();
-}
-
-Entry Entry::getFinalEntry() const
-{
-  int loopCounter = 42;
-  auto final_entry = entry;
-  while (final_entry.isRedirect() && loopCounter--) {
-    final_entry = final_entry.getRedirectEntry();
-  }
-  // Prevent infinite loops.
-  if (final_entry.isRedirect()) {
-    throw NoEntry();
-  }
-  return final_entry;
-}
-
-}

src/library.cpp

@@ -19,7 +19,6 @@
 
 #include "library.h"
 #include "book.h"
-#include "reader.h"
 #include "libxml_dumper.h"
 
 #include "tools.h"
@@ -27,10 +26,13 @@
 #include "tools/regexTools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
 #include <set>
+#include <cmath>
 #include <unicode/locid.h>
 #include <xapian.h>
 
@@ -49,54 +51,152 @@ std::string normalizeText(const std::string& text)
   return removeAccents(text);
 }
 
+bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
+{
+  return book1.isPathValid()
+      && book2.isPathValid()
+      && book1.getPath() == book2.getPath();
+}
+
+template<typename Key, typename Value>
+class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
+{
+  public:
+    explicit MultiKeyCache(size_t maxEntries)
+      : ConcurrentCache<std::set<Key>, Value>(maxEntries)
+    {}
+
+    bool drop(const Key& key)
+    {
+      std::unique_lock<std::mutex> l(this->lock_);
+      bool removed = false;
+      for(auto& cache_key: this->impl_.keys()) {
+        if(cache_key.find(key)!=cache_key.end()) {
+          removed |= this->impl_.drop(cache_key);
+        }
+      }
+      return removed;
+    }
+};
+
 } // unnamed namespace
 
-class Library::BookDB : public Xapian::WritableDatabase
+struct Library::Impl
 {
-public:
-  BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
+  struct Entry : Book
+  {
+    Library::Revision lastUpdatedRevision = 0;
+  };
+
+  Library::Revision m_revision;
+  std::map<std::string, Entry> m_books;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+  Xapian::WritableDatabase m_bookDB;
+
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
+
+  Impl();
+  ~Impl();
+
+  Impl(Impl&& );
+  Impl& operator=(Impl&& );
 };
 
+Library::Impl::Impl()
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
+{
+}
+
+Library::Impl::~Impl()
+{
+}
+
+Library::Impl::Impl(Library::Impl&& ) = default;
+Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
+
+unsigned int
+Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
+{
+  unsigned int result = 0;
+  for (auto& pair: m_books) {
+    auto& book = pair.second;
+    if ((!book.getPath().empty() && localBooks)
+        || (!book.getUrl().empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
+
 /* Constructor */
 Library::Library()
-  : m_bookDB(new BookDB)
+  : mp_impl(new Library::Impl)
 {
 }
 
-Library::Library(Library&& ) = default;
+Library::Library(Library&& other)
+  : mp_impl(std::move(other.mp_impl))
+{
+}
 
-Library& Library::operator=(Library&& ) = default;
+Library& Library::operator=(Library&& other)
+{
+  mp_impl = std::move(other.mp_impl);
+  return *this;
+}
 
 /* Destructor */
-Library::~Library()
-{
-}
-
+Library::~Library() = default;
 
 bool Library::addBook(const Book& book)
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
+  ++mp_impl->m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
-    auto& oldbook = m_books.at(book.getId());
-    oldbook.update(book);
+    auto& oldbook = mp_impl->m_books.at(book.getId());
+    if ( ! booksReferToTheSameArchive(oldbook, book) ) {
+      dropCache(book.getId());
+    }
+    oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
+                          // XXX: Then m_bookDB will become out-of-sync with
+                          // XXX: the real contents of the library.
+    oldbook.lastUpdatedRevision = mp_impl->m_revision;
     return false;
   } catch (std::out_of_range&) {
-    m_books[book.getId()] = book;
+    auto& newEntry = mp_impl->m_books[book.getId()];
+    static_cast<Book&>(newEntry) = book;
+    newEntry.lastUpdatedRevision = mp_impl->m_revision;
+    size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
+    if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+    }
+    if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+    }
     return true;
   }
 }
 
 void Library::addBookmark(const Bookmark& bookmark)
 {
-  m_bookmarks.push_back(bookmark);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  mp_impl->m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
-  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
-      m_bookmarks.erase(it);
+      mp_impl->m_bookmarks.erase(it);
       return true;
     }
   }
@@ -104,29 +204,70 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
+void Library::dropCache(const std::string& id)
+{
+  mp_impl->mp_archiveCache->drop(id);
+  mp_impl->mp_searcherCache->drop(id);
+}
 
 bool Library::removeBookById(const std::string& id)
 {
-  m_bookDB->delete_document("Q" + id);
-  m_readers.erase(id);
-  m_archives.erase(id);
-  return m_books.erase(id) == 1;
+  std::lock_guard<std::mutex> lock(m_mutex);
+  mp_impl->m_bookDB.delete_document("Q" + id);
+  dropCache(id);
+  // We do not change the cache size here
+  // Most of the time, the book is remove in case of library refresh, it is
+  // often associated with addBook calls (which will properly set the cache size)
+  // Having a too big cache is not a problem here (or it would have been before)
+  // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+  //  will be removed in put or getOrPut).
+  return mp_impl->m_books.erase(id) == 1;
+}
+
+Library::Revision Library::getRevision() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return mp_impl->m_revision;
+}
+
+uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
+{
+  BookIdCollection booksToRemove;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    for ( const auto& entry : mp_impl->m_books) {
+      if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
+        booksToRemove.push_back(entry.first);
+      }
+    }
+  }
+
+  uint32_t countOfRemovedBooks = 0;
+  for ( const auto& id : booksToRemove ) {
+    if ( removeBookById(id) )
+      ++countOfRemovedBooks;
+  }
+  return countOfRemovedBooks;
 }
 
 const Book& Library::getBookById(const std::string& id) const
 {
-  return m_books.at(id);
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
+  return mp_impl->m_books.at(id);
 }
 
-Book& Library::getBookById(const std::string& id)
+Book Library::getBookByIdThreadSafe(const std::string& id) const
 {
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookById(id));
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return getBookById(id);
 }
 
 const Book& Library::getBookByPath(const std::string& path) const
 {
-  for(auto& it: m_books) {
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
+  for(auto& it: mp_impl->m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
       return book;
@@ -136,83 +277,79 @@ const Book& Library::getBookByPath(const std::string& path) const
   throw std::out_of_range(ss.str());
 }
 
-Book& Library::getBookByPath(const std::string& path)
-{
-  const Library& const_self = *this;
-  return const_cast<Book&>(const_self.getBookByPath(path));
-}
-
-std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
-{
-  try {
-    return m_readers.at(id);
-  } catch (std::out_of_range& e) {}
-
-  try {
-    auto reader = make_shared<Reader>(m_archives.at(id));
-    m_readers[id] = reader;
-    return reader;
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
-    return nullptr;
-
-  auto archive = make_shared<zim::Archive>(book.getPath());
-  m_archives[id] = archive;
-  auto reader = make_shared<Reader>(archive);
-  m_readers[id] = reader;
-  return reader;
-}
-
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
   try {
-    return m_archives.at(id);
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
+    return mp_impl->mp_archiveCache->getOrPut(id,
+    [&](){
+      auto book = getBookById(id);
+      if (!book.isPathValid()) {
+        throw std::invalid_argument("");
+      }
+      return std::make_shared<zim::Archive>(book.getPath());
+    });
+  } catch (std::invalid_argument&) {
     return nullptr;
+  }
+}
 
-  auto sptr = make_shared<zim::Archive>(book.getPath());
-  m_archives[id] = sptr;
-  return sptr;
+std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
+{
+  assert(!ids.empty());
+  try {
+    return mp_impl->mp_searcherCache->getOrPut(ids,
+    [&](){
+      std::vector<zim::Archive> archives;
+      for(auto& id:ids) {
+        auto archive = getArchiveById(id);
+        if(!archive) {
+          throw std::invalid_argument("");
+        }
+        archives.push_back(*archive);
+      }
+      return std::make_shared<ZimSearcher>(zim::Searcher(archives));
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
-  unsigned int result = 0;
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    if ((!book.getPath().empty() && localBooks)
-        || (book.getPath().empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return mp_impl->getBookCount(localBooks, remoteBooks);
 }
 
 bool Library::writeToFile(const std::string& path) const
 {
+  const auto allBookIds = getBooksIds();
+
   auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
   dumper.setBaseDir(baseDir);
-  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
+  std::string xml;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    xml = dumper.dumpLibXMLContent(allBookIds);
+  };
+  return writeTextFile(path, xml);
 }
 
 bool Library::writeBookmarksToFile(const std::string& path) const
 {
   LibXMLDumper dumper(this);
-  return writeTextFile(path, dumper.dumpLibXMLBookmark());
+  // NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
+  const std::string xml = dumper.dumpLibXMLBookmark();
+  return writeTextFile(path, xml);
 }
 
 Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   AttributeCounts propValueCounts;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     if (book.getOrigId().empty()) {
       propValueCounts[(book.*p)()] += 1;
@@ -242,9 +379,10 @@ Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
 
 std::vector<std::string> Library::getBooksCategories() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   std::set<std::string> categories;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     const auto& c = book.getCategory();
     if ( !c.empty() ) {
@@ -268,11 +406,12 @@ std::vector<std::string> Library::getBooksPublishers() const
 const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
 {
   if (!onlyValidBookmarks) {
-    return m_bookmarks;
+    return mp_impl->m_bookmarks;
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
-  for(auto& bookmark:m_bookmarks) {
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto& bookmark:mp_impl->m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
     }
@@ -282,24 +421,16 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
 
 Library::BookIdCollection Library::getBooksIds() const
 {
+  std::lock_guard<std::mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
-  for (auto& pair: m_books) {
+  for (auto& pair: mp_impl->m_books) {
     bookIds.push_back(pair.first);
   }
 
   return bookIds;
 }
 
-Library::BookIdCollection Library::filter(const std::string& search) const
-{
-  if (search.empty()) {
-    return getBooksIds();
-  }
-
-  return filter(Filter().query(search));
-}
-
 
 void Library::updateBookDB(const Book& book)
 {
@@ -331,15 +462,20 @@ void Library::updateBookDB(const Book& book)
   indexer.index_text(normalizeText(book.getName()),      1, "XN");
   indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
 
-  for ( const auto& tag : split(normalizeText(book.getTags()), ";") )
+  for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
     doc.add_boolean_term("XT" + tag);
+    if ( tag[0] != '_' ) {
+      indexer.increase_termpos();
+      indexer.index_text(tag);
+    }
+  }
 
   const std::string idterm = "Q" + book.getId();
   doc.add_boolean_term(idterm);
 
   doc.set_data(book.getId());
 
-  m_bookDB->replace_document(idterm, doc);
+  mp_impl->m_bookDB.replace_document(idterm, doc);
 }
 
 namespace
@@ -471,9 +607,10 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 
   BookIdCollection bookIds;
 
-  Xapian::Enquire enquire(*m_bookDB);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  Xapian::Enquire enquire(mp_impl->m_bookDB);
   enquire.set_query(query);
-  const auto results = enquire.get_mset(0, m_books.size());
+  const auto results = enquire.get_mset(0, mp_impl->m_books.size());
   for ( auto it = results.begin(); it != results.end(); ++it  ) {
     bookIds.push_back(it.get_document().get_data());
   }
@@ -484,8 +621,10 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 Library::BookIdCollection Library::filter(const Filter& filter) const
 {
   BookIdCollection result;
-  for(auto id : filterViaBookDB(filter)) {
-    if(filter.accept(m_books.at(id))) {
+  const auto preliminaryResult = filterViaBookDB(filter);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto id : preliminaryResult) {
+    if(filter.accept(mp_impl->m_books.at(id))) {
       result.push_back(id);
     }
   }
@@ -553,6 +692,11 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
 
 void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
 {
+  // NOTE: Can reimplement this method in a way that doesn't require locking
+  // NOTE: for the entire duration of the sort. Will need to obtain (under a
+  // NOTE: lock) the required atributes from the books once, and then the
+  // NOTE: sorting will run on a copy of data without locking.
+  std::lock_guard<std::mutex> lock(m_mutex);
   switch(sort) {
     case TITLE:
       std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
@@ -575,48 +719,6 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
 }
 
 
-Library::BookIdCollection Library::listBooksIds(
-    int mode,
-    supportedListSortBy sortBy,
-    const std::string& search,
-    const std::string& language,
-    const std::string& creator,
-    const std::string& publisher,
-    const std::vector<std::string>& tags,
-    size_t maxSize) const {
-
-  Filter _filter;
-  if (mode & LOCAL)
-    _filter.local(true);
-  if (mode & NOLOCAL)
-    _filter.local(false);
-  if (mode & VALID)
-    _filter.valid(true);
-  if (mode & NOVALID)
-    _filter.valid(false);
-  if (mode & REMOTE)
-    _filter.remote(true);
-  if (mode & NOREMOTE)
-    _filter.remote(false);
-  if (!tags.empty())
-    _filter.acceptTags(tags);
-  if (maxSize != 0)
-    _filter.maxSize(maxSize);
-  if (!language.empty())
-    _filter.lang(language);
-  if (!publisher.empty())
-    _filter.publisher(publisher);
-  if (!creator.empty())
-    _filter.creator(creator);
-  if (!search.empty())
-    _filter.query(search);
-
-  auto bookIds = filter(_filter);
-
-  sort(bookIds, sortBy, true);
-  return bookIds;
-}
-
 Filter::Filter()
   : activeFilters(0),
     _maxSize(0)

src/libxml_dumper.cpp

@@ -60,10 +60,13 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
     ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
     ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
-    if (!book.getFavicon().empty())
-      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
+    try {
+      auto defaultIllustration = book.getIllustration(48);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
+      if (!defaultIllustration->getData().empty())
+        ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
+    } catch(...) {}
   } else {
     ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
   }
@@ -91,7 +94,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
   auto book_node = entry_node.append_child("book");
 
   try {
-    auto book = library->getBookById(bookmark.getBookId());
+    auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
     ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());

src/manager.cpp

@@ -26,28 +26,81 @@
 
 namespace kiwix
 {
+
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+////////////////////////////////////////////////////////////////////////////////
+// LibraryManipulator
+////////////////////////////////////////////////////////////////////////////////
+
+LibraryManipulator::LibraryManipulator(Library* library)
+  : library(*library)
+{}
+
+LibraryManipulator::~LibraryManipulator()
+{}
+
+bool LibraryManipulator::addBookToLibrary(const Book& book)
+{
+  const auto ret = library.addBook(book);
+  if ( ret ) {
+    bookWasAddedToLibrary(book);
+  }
+  return ret;
+}
+
+void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
+{
+  library.addBookmark(bookmark);
+  bookmarkWasAddedToLibrary(bookmark);
+}
+
+uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
+{
+  const auto n = library.removeBooksNotUpdatedSince(rev);
+  if ( n != 0 ) {
+    booksWereRemovedFromLibrary();
+  }
+  return n;
+}
+
+void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
+{
+}
+
+void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
+{
+}
+
+void LibraryManipulator::booksWereRemovedFromLibrary()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Manager
+////////////////////////////////////////////////////////////////////////////////
+
 /* Constructor */
 Manager::Manager(LibraryManipulator* manipulator):
   writableLibraryPath(""),
-  manipulator(manipulator),
-  mustDeleteManipulator(false)
+  manipulator(manipulator, NoDelete())
 {
 }
 
 Manager::Manager(Library* library) :
   writableLibraryPath(""),
-  manipulator(new DefaultLibraryManipulator(library)),
-  mustDeleteManipulator(true)
+  manipulator(new LibraryManipulator(library))
 {
 }
 
-/* Destructor */
-Manager::~Manager()
-{
-  if (mustDeleteManipulator) {
-    delete manipulator;
-  }
-}
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
                           bool readOnly,
                           const std::string& libraryPath,
@@ -249,4 +302,21 @@ bool Manager::readBookmarkFile(const std::string& path)
   return true;
 }
 
+void Manager::reload(const Paths& paths)
+{
+  const auto libRevision = manipulator->getLibrary().getRevision();
+  for (std::string path : paths) {
+    if (!path.empty()) {
+      if ( kiwix::isRelativePath(path) )
+        path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
+
+      if (!readFile(path, false, true)) {
+        throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
+      }
+    }
+  }
+
+  manipulator->removeBooksNotUpdatedSince(libRevision);
+}
+
 }

src/meson.build

@@ -6,10 +6,7 @@ kiwix_sources = [
   'libxml_dumper.cpp',
   'opds_dumper.cpp',
   'downloader.cpp',
-  'reader.cpp',
-  'entry.cpp',
   'server.cpp',
-  'searcher.cpp',
   'search_renderer.cpp',
   'subprocess.cpp',
   'aria2.cpp',
@@ -28,9 +25,12 @@ kiwix_sources = [
   'server/response.cpp',
   'server/internalServer.cpp',
   'server/internalServer_catalog_v2.cpp',
-  'opds_catalog.cpp'
+  'server/i18n.cpp',
+  'opds_catalog.cpp',
+  'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
 if host_machine.system() == 'windows'
   kiwix_sources += 'subprocess_windows.cpp'
@@ -38,15 +38,7 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if wrapper.contains('android')
-  install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
-else
-  install_dir = get_option('libdir')
-endif
-
-if wrapper.contains('android') or wrapper.contains('java')
-  subdir('wrapper/java')
-endif
+install_dir = get_option('libdir')
 
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,

src/name_mapper.cpp

@@ -51,12 +51,54 @@ HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool w
   }
 }
 
-std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
   return m_idToName.at(id);
 }
 
-std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
   return m_nameToId.at(name);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// UpdatableNameMapper
+////////////////////////////////////////////////////////////////////////////////
+
+UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+  : library(lib)
+  , withAlias(withAlias)
+{
+  update();
+}
+
+void UpdatableNameMapper::update()
+{
+  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  std::lock_guard<std::mutex> lock(mutex);
+  nameMapper.reset(newNameMapper);
+}
+
+UpdatableNameMapper::NameMapperHandle
+UpdatableNameMapper::currentNameMapper() const
+{
+  // Return a copy of the handle to the current NameMapper object. It will
+  // ensure that the object survives any call to UpdatableNameMapper::update()
+  // made before the completion of any pending operation on that object.
+  std::lock_guard<std::mutex> lock(mutex);
+  return nameMapper;
+}
+
+std::string UpdatableNameMapper::getNameForId(const std::string& id) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getNameForId(id);
+}
+
+std::string UpdatableNameMapper::getIdForName(const std::string& name) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getIdForName(name);
+}
+
 }

src/opds_dumper.cpp

@@ -22,7 +22,6 @@
 
 #include "kiwixlib-resources.h"
 #include <mustache.hpp>
-#include <unicode/locid.h>
 
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
@@ -58,11 +57,12 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
 {
     kainjow::mustache::list illustrations;
     if ( book.isPathValid() ) {
-      for ( auto illustration_size : zim::Archive(book.getPath()).getIllustrationSizes() ) {
+      for ( const auto& illustration : book.getIllustrations() ) {
+        // For now, we are handling only sizexsize@1 illustration.
+        // So we can simply pass one size to mustache.
         illustrations.push_back(kainjow::mustache::object{
-          {"icon_width", to_string(illustration_size)},
-          {"icon_height", to_string(illustration_size)},
-          {"icon_scale", "1"},
+          {"icon_size", to_string(illustration->width)},
+          {"icon_mimetype", illustration->mimeType}
         });
       }
     }
@@ -71,17 +71,16 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
 
 kainjow::mustache::object getSingleBookData(const Book& book)
 {
-    const MustacheData bookUrl = book.getUrl().empty()
-                               ? MustacheData(false)
-                               : MustacheData(book.getUrl());
+    const auto bookDate = book.getDate() + "T00:00:00Z";
     return kainjow::mustache::object{
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
       {"language", book.getLanguage()},
-      {"content_id",  book.getHumanReadableIdFromPath()},
-      {"updated", book.getDate() + "T00:00:00Z"},
+      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+      {"book_date", bookDate},
       {"category", book.getCategory()},
       {"flavour", book.getFlavour()},
       {"tags", book.getTags()},
@@ -89,53 +88,103 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"media_count", to_string(book.getMediaCount())},
       {"author_name", book.getCreator()},
       {"publisher_name", book.getPublisher()},
-      {"url", bookUrl},
+      {"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& endpointRoot, bool partial)
+std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
 {
   auto data = getSingleBookData(book);
   data["with_xml_header"] = MustacheData(withXMLHeader);
   data["dump_partial_entries"] = MustacheData(partial);
   data["endpoint_root"] = endpointRoot;
+  data["root"] = rootLocation;
   return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
-    const Book& book = library->getBookById(bookId);
-    booksData.push_back(kainjow::mustache::object{
-        {"entry", getSingleBookEntryXML(book, false, endpointRoot, partial)}
-    });
+    try {
+      const Book book = library->getBookByIdThreadSafe(bookId);
+      booksData.push_back(kainjow::mustache::object{
+          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
+      });
+    } catch ( const std::out_of_range& ) {
+      // the book was removed from the library since its id was obtained
+      // ignore it
+    }
   }
 
   return booksData;
 }
 
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"}
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
+  }
+}
+
 std::string getLanguageSelfName(const std::string& lang) {
-  const icu::Locale locale(lang.c_str());
-  icu::UnicodeString ustring;
-  locale.getDisplayLanguage(locale, ustring);
-  std::string result;
-  ustring.toUTF8String(result);
-  return result;
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
 };
 
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, "", false);
+  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
      {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
@@ -148,14 +197,14 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, endpointRoot, partial);
+  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"query", query.empty() ? "" : "?" + urlEncode(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
@@ -169,7 +218,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, "", false);
+  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const
@@ -201,6 +250,7 @@ std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
   kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
   for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
     const std::string languageCode = langAndBookCount.first;
     const int bookCount = langAndBookCount.second;

src/reader.cpp

@@ -1,472 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-#include <zim/item.h>
-#include <zim/error.h>
-
-#include "tools.h"
-#include "tools/stringTools.h"
-#include "tools/otherTools.h"
-#include "tools/archiveTools.h"
-
-namespace kiwix
-{
-/* Constructor */
-Reader::Reader(const string zimFilePath)
-  :  zimArchive(nullptr),
-     zimFilePath(zimFilePath)
-{
-  string tmpZimFilePath = zimFilePath;
-
-  /* Remove potential trailing zimaa */
-  size_t found = tmpZimFilePath.rfind("zimaa");
-  if (found != string::npos && tmpZimFilePath.size() > 5
-      && found == tmpZimFilePath.size() - 5) {
-    tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
-  }
-
-  zimArchive.reset(new zim::Archive(tmpZimFilePath));
-
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(const std::shared_ptr<zim::Archive> archive)
-  : zimArchive(archive),
-    zimFilePath(archive->getFilename())
-  {}
-
-#ifndef _WIN32
-Reader::Reader(int fd)
-  :  zimArchive(new zim::Archive(fd)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(int fd, zim::offset_type offset, zim::size_type size)
-  :  zimArchive(new zim::Archive(fd, offset, size)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-#endif // #ifndef _WIN32
-
-zim::Archive* Reader::getZimArchive() const
-{
-  return zimArchive.get();
-}
-
-MimeCounterType Reader::parseCounterMetadata() const
-{
-  return kiwix::parseArchiveCounter(*zimArchive);
-}
-
-/* Get the count of articles which can be indexed/displayed */
-unsigned int Reader::getArticleCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  for(auto &pair:counterMap) {
-    if (startsWith(pair.first, "text/html")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-/* Get the count of medias content in the ZIM file */
-unsigned int Reader::getMediaCount() const
-{
-  return kiwix::getArchiveMediaCount(*zimArchive);
-}
-
-/* Get the total of all items of a ZIM file, redirects included */
-unsigned int Reader::getGlobalCount() const
-{
-  return zimArchive->getEntryCount();
-}
-
-/* Return the UID of the ZIM file */
-string Reader::getId() const
-{
-  return kiwix::getArchiveId(*zimArchive);
-}
-
-Entry Reader::getRandomPage() const
-{
-  try {
-    return zimArchive->getRandomEntry();
-  } catch(...) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getMainPage() const
-{
-  return zimArchive->getMainEntry();
-}
-
-bool Reader::getFavicon(string& content, string& mimeType) const
-{
-  return kiwix::getArchiveFavicon(*zimArchive, 48, content, mimeType);
-}
-
-string Reader::getZimFilePath() const
-{
-  return zimFilePath;
-}
-/* Return a metatag value */
-bool Reader::getMetadata(const string& name, string& value) const
-{
-  try {
-    value = zimArchive->getMetadata(name);
-    return true;
-  } catch(zim::EntryNotFound& e) {
-    return false;
-  }
-}
-
-#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
-
-string Reader::getName() const
-{
-  return kiwix::getMetaName(*zimArchive);
-}
-
-string Reader::getTitle() const
-{
-  return kiwix::getArchiveTitle(*zimArchive);
-}
-
-string Reader::getCreator() const
-{
-  return kiwix::getMetaCreator(*zimArchive);
-}
-
-string Reader::getPublisher() const
-{
-  return kiwix::getMetaPublisher(*zimArchive);
-}
-
-string Reader::getDate() const
-{
-  return kiwix::getMetaDate(*zimArchive);
-}
-
-string Reader::getDescription() const
-{
-  return kiwix::getMetaDescription(*zimArchive);
-}
-
-string Reader::getLongDescription() const
-{
-  METADATA("LongDescription")
-}
-
-string Reader::getLanguage() const
-{
-  return kiwix::getMetaLanguage(*zimArchive);
-}
-
-string Reader::getLicense() const
-{
-  METADATA("License")
-}
-
-string Reader::getTags(bool original) const
-{
-  return kiwix::getMetaTags(*zimArchive, original);
-}
-
-
-string Reader::getTagStr(const std::string& tagName) const
-{
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  return getTagValueFromTagList(convertTags(tags_str), tagName);
-}
-
-bool Reader::getTagBool(const std::string& tagName) const
-{
-  return convertStrToBool(getTagStr(tagName));
-}
-
-string Reader::getRelation() const
-{
-  METADATA("Relation")
-}
-
-string Reader::getFlavour() const
-{
-  return kiwix::getMetaFlavour(*zimArchive);
-}
-
-string Reader::getSource() const
-{
-  METADATA("Source")
-}
-
-string Reader::getScraper() const
-{
-  METADATA("Scraper")
-}
-#undef METADATA
-
-Entry Reader::getEntryFromPath(const std::string& path) const
-{
-  try {
-    return kiwix::getEntryFromPath(*zimArchive, path);
-  } catch (zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getEntryFromEncodedPath(const std::string& path) const
-{
-  return getEntryFromPath(urlDecode(path, true));
-}
-
-Entry Reader::getEntryFromTitle(const std::string& title) const
-{
-  try {
-    return zimArchive->getEntryByTitle(title);
-  } catch(zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-bool Reader::pathExists(const string& path) const
-{
-  return zimArchive->hasEntryByPath(path);
-}
-
-/* Does the ZIM file has a fulltext index */
-bool Reader::hasFulltextIndex() const
-{
-  return zimArchive->hasFulltextIndex();
-}
-
-/* Search titles by prefix */
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               const bool reset)
-{
-  /* Reset the suggestions otherwise check if the suggestions number is less
-   * than the suggestionsCount */
-  if (reset) {
-    this->suggestions.clear();
-    this->suggestionsOffset = this->suggestions.begin();
-  } else {
-    if (this->suggestions.size() > suggestionsCount) {
-      return false;
-    }
-  }
-
-  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
-
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               SuggestionsList_t& results)
-{
-  bool retVal = false;
-
-  /* Return if no prefix */
-  if (prefix.size() == 0) {
-    return false;
-  }
-
-  for (auto& entry: zimArchive->findByTitle(prefix)) {
-    if (results.size() >= suggestionsCount) {
-      break;
-    }
-    /* Extract the interesting part of article title & url */
-    std::string normalizedArticleTitle
-        = kiwix::normalize(entry.getTitle());
-
-    // Get the final path.
-    auto item = entry.getItem(true);
-    std::string articleFinalUrl = item.getPath();
-
-    /* Go through all already found suggestions and skip if this
-       article is already in the suggestions list (with an other
-       title) */
-    bool insert = true;
-    std::vector<SuggestionItem>::iterator suggestionItr;
-    for (suggestionItr = results.begin();
-         suggestionItr != results.end();
-         suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr).getNormalizedTitle());
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr).getPath()) == 0) {
-        insert = false;
-        break;
-      } else if (result < 0) {
-        break;
-      }
-    }
-
-    /* Insert if possible */
-    if (insert) {
-      SuggestionItem suggestion(entry.getTitle(), normalizedArticleTitle, articleFinalUrl);
-      results.insert(suggestionItr, suggestion);
-    }
-
-    /* Suggestions where found */
-    retVal = true;
-  }
-
-  return retVal;
-}
-
-std::vector<std::string> Reader::getTitleVariants(
-    const std::string& title) const
-{
-  return kiwix::getTitleVariants(title);
-}
-
-
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount)
-{
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
-
-  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
-
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-/* Try also a few variations of the prefix to have better results */
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount,
-                                    SuggestionsList_t& results)
-{
-  std::vector<std::string> variants = this->getTitleVariants(prefix);
-
-  auto suggestionSearcher = zim::SuggestionSearcher(*zimArchive);
-
-  if (zimArchive->hasTitleIndex()) {
-    auto suggestionSearch = suggestionSearcher.suggest(prefix);
-    const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
-    for (auto current : suggestions) {
-      SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-      results.push_back(suggestion);
-    }
-  } else {
-    // Check some of the variants of the prefix
-    for (std::vector<std::string>::iterator variantsItr = variants.begin();
-         variantsItr != variants.end();
-         variantsItr++) {
-      auto suggestionSearch = suggestionSearcher.suggest(*variantsItr);
-      for (auto current : suggestionSearch.getResults(0, suggestionsCount)) {
-        if (results.size() >= suggestionsCount) {
-          break;
-        }
-
-        SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-        results.push_back(suggestion);
-      }
-    }
-  }
-
-  return results.size() > 0;
-}
-
-/* Get next suggestion */
-bool Reader::getNextSuggestion(string& title)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-bool Reader::getNextSuggestion(string& title, string& url)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-    url = (*(this->suggestionsOffset)).getPath();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-/* Check if the file has as checksum */
-bool Reader::canCheckIntegrity() const
-{
-  return zimArchive->hasChecksum();
-}
-
-/* Return true if corrupted, false otherwise */
-bool Reader::isCorrupted() const
-{
-  try {
-    if (zimArchive->check() == true) {
-      return false;
-    }
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return true;
-  }
-
-  return true;
-}
-
-/* Return the file size, works also for splitted files */
-unsigned int Reader::getFileSize() const
-{
-  return kiwix::getArchiveFileSize(*zimArchive);
-}
-
-}

src/search_renderer.cpp

@@ -21,11 +21,11 @@
 #include <cmath>
 
 #include "search_renderer.h"
-#include "searcher.h"
-#include "reader.h"
 #include "library.h"
 #include "name_mapper.h"
 
+#include "tools/archiveTools.h"
+
 #include <zim/search.h>
 
 #include <mustache.hpp>
@@ -36,21 +36,18 @@ namespace kiwix
 {
 
 /* Constructor */
-SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : m_srs(searcher->getSearchResultSet()),
-      mp_nameMapper(mapper),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      estimatedResultCount(searcher->getEstimatedResultCount()),
-      resultStart(searcher->getResultStart())
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
 {}
 
-SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
                       unsigned int start, unsigned int estimatedResultCount)
     : m_srs(srs),
       mp_nameMapper(mapper),
+      mp_library(library),
       protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
+      searchProtocolPrefix("search://"),
       estimatedResultCount(estimatedResultCount),
       resultStart(start)
 {}
@@ -60,12 +57,12 @@ SearchRenderer::~SearchRenderer() = default;
 
 void SearchRenderer::setSearchPattern(const std::string& pattern)
 {
-  this->searchPattern = pattern;
+  searchPattern = pattern;
 }
 
-void SearchRenderer::setSearchContent(const std::string& name)
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
 {
-  this->searchContent = name;
+  searchBookQuery = bookQuery;
 }
 
 void SearchRenderer::setProtocolPrefix(const std::string& prefix)
@@ -78,81 +75,161 @@ void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
   this->searchProtocolPrefix = prefix;
 }
 
-std::string SearchRenderer::getHtml()
-{
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
+std::string extractValueFromQuery(const std::string& query, const std::string& key) {
+  const std::string p = key + "=";
+  const size_t i = query.find(p);
+  if (i == std::string::npos) {
+    return "";
+  }
+  std::string r = query.substr(i + p.size());
+  return r.substr(0, r.find("&"));
+}
 
+kainjow::mustache::data buildQueryData
+(
+  const std::string& searchProtocolPrefix,
+  const std::string& pattern,
+  const std::string& bookQuery
+) {
+  kainjow::mustache::data query;
+  query.set("pattern", kiwix::encodeDiples(pattern));
+  std::ostringstream ss;
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << "&" << bookQuery;
+  query.set("unpaginatedQuery", ss.str());
+  auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
+  if(!lang.empty()) {
+    query.set("lang", lang);
+  }
+  return query;
+}
+
+kainjow::mustache::data buildPagination(
+  unsigned int pageLength,
+  unsigned int resultsCount,
+  unsigned int resultsStart
+)
+{
+  assert(pageLength!=0);
+  kainjow::mustache::data pagination;
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  if (resultsCount == 0) {
+    // Easy case
+    pagination.set("itemsPerPage", to_string(pageLength));
+    pagination.set("hasPages", false);
+    pagination.set("pages", pages);
+    return pagination;
+  }
+
+  // First we want to display pages starting at a multiple of `pageLength`
+  // so, let's calculate the start index of the current page.
+  auto currentPage = resultsStart/pageLength;
+  auto lastPage = ((resultsCount-1)/pageLength);
+  auto lastPageStart = lastPage*pageLength;
+  auto nbPages = lastPage + 1;
+
+  auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
+  auto lastPageGenerated = std::min(currentPage+4, lastPage);
+
+  if (nbPages != 1) {
+    if (firstPageGenerated!=0) {
+      kainjow::mustache::data page;
+      page.set("label", "◀");
+      page.set("start", to_string(0));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+
+    for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
+      kainjow::mustache::data page;
+      page.set("label", to_string(i+1));
+      page.set("start", to_string(i*pageLength));
+      page.set("current", bool(i == currentPage));
+      pages.push_back(page);
+    }
+
+    if (lastPageGenerated!=lastPage) {
+      kainjow::mustache::data page;
+      page.set("label", "▶");
+      page.set("start", to_string(lastPageStart));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+  }
+
+  pagination.set("itemsPerPage", to_string(pageLength));
+  pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
+  pagination.set("pages", pages);
+  return pagination;
+}
+
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+{
+  const std::string absPathPrefix = protocolPrefix + "content/";
+  // Build the results list
+  kainjow::mustache::data items{kainjow::mustache::data::type::list};
   for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
+    std::string zim_id(it.getZimId());
     result.set("title", it.getTitle());
-    result.set("url", it.getPath());
+    result.set("absolutePath", absPathPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
     result.set("snippet", it.getSnippet());
-    std::ostringstream s;
-    s << it.getZimId();
-    result.set("resultContentId", mp_nameMapper->getNameForId(s.str()));
-
+    if (mp_library) {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
     if (it.getWordCount() >= 0) {
       result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
     }
 
-    results.push_back(result);
+    items.push_back(result);
   }
+  kainjow::mustache::data results;
+  results.set("items", items);
+  results.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  results.set("hasResults", estimatedResultCount != 0);
+  results.set("start", kiwix::beautifyInteger(resultStart));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
 
-  // pages
-  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+  // pagination
+  auto pagination = buildPagination(
+    pageLength,
+    estimatedResultCount,
+    resultStart
+  );
 
-  auto resultEnd = 0U;
-  auto currentPage = 0U;
-  auto pageStart = 0U;
-  auto pageEnd = 0U;
-  auto lastPageStart = 0U;
-  if (pageLength) {
-    currentPage = resultStart/pageLength;
-    pageStart = currentPage > 4 ? currentPage-4 : 0;
-    pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / pageLength) {
-      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
-    }
-    if (estimatedResultCount > pageLength) {
-      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
-    }
-  }
+  kainjow::mustache::data query = buildQueryData(
+    searchProtocolPrefix,
+    searchPattern,
+    searchBookQuery
+  );
 
-  resultEnd = resultStart+pageLength; //setting result end
-
-  for (unsigned int i = pageStart; i < pageEnd; i++) {
-    kainjow::mustache::data page;
-    page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * pageLength));
-
-    if (i == currentPage) {
-      page.set("selected", true);
-    }
-    pages.push_back(page);
-  }
-
-  std::string template_str = RESOURCE::templates::search_result_html;
-  kainjow::mustache::mustache tmpl(template_str);
 
   kainjow::mustache::data allData;
+  allData.set("protocolPrefix", protocolPrefix);
   allData.set("results", results);
-  allData.set("pages", pages);
-  allData.set("hasResults", estimatedResultCount != 0);
-  allData.set("hasPages", pageStart != pageEnd);
-  allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
-  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
-  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
-  allData.set("resultStart", to_string(resultStart + 1));
-  allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("pageLength", to_string(pageLength));
-  allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("protocolPrefix", this->protocolPrefix);
-  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
-  allData.set("contentId", this->searchContent);
+  allData.set("pagination", pagination);
+  allData.set("query", query);
+
+  kainjow::mustache::mustache tmpl(tmpl_str);
 
   std::stringstream ss;
   tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  if (!tmpl.is_valid()) {
+    throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
+  }
   return ss.str();
 }
 
+std::string SearchRenderer::getHtml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_html);
+}
+
+std::string SearchRenderer::getXml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_xml);
+}
+
+
 }

src/searcher.cpp

@@ -1,324 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-
-#include "searcher.h"
-#include "reader.h"
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-
-#include <mustache.hpp>
-#include <cmath>
-#include "tools/stringTools.h"
-#include "kiwixlib-resources.h"
-
-#define MAX_SEARCH_LEN 140
-
-namespace kiwix
-{
-class _Result : public Result
-{
- public:
-  _Result(zim::SearchResultSet::iterator iterator);
-  _Result(SuggestionItem suggestionItem);
-  virtual ~_Result(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual std::string get_zimId();
-
- private:
-  zim::SearchResultSet::iterator iterator;
-  SuggestionItem suggestionItem;
-  bool isSuggestion;
-};
-
-struct SearcherInternal : zim::SearchResultSet {
-  explicit SearcherInternal(const zim::SearchResultSet& srs)
-    : zim::SearchResultSet(srs)
-    , current_iterator(srs.begin())
-  {
-  }
-
-  zim::SearchResultSet::iterator current_iterator;
-};
-
-struct SuggestionInternal : zim::SuggestionResultSet {
-  explicit SuggestionInternal(const zim::SuggestionResultSet& srs)
-    : zim::SuggestionResultSet(srs),
-      currentIterator(srs.begin()) {}
-
-  zim::SuggestionResultSet::iterator currentIterator;
-};
-
-/* Constructor */
-Searcher::Searcher()
-    : searchPattern(""),
-      estimatedResultCount(0),
-      resultStart(0),
-      maxResultCount(0)
-{
-  loadICUExternalTables();
-}
-
-/* Destructor */
-Searcher::~Searcher()
-{
-}
-
-bool Searcher::add_reader(Reader* reader)
-{
-  if (!reader->hasFulltextIndex()) {
-      return false;
-  }
-  this->readers.push_back(reader);
-  return true;
-}
-
-
-Reader* Searcher::get_reader(int readerIndex)
-{
-  return readers.at(readerIndex);
-}
-
-/* Search strings in the database */
-void Searcher::search(const std::string& search,
-                      unsigned int resultStart,
-                      unsigned int maxResultCount,
-                      const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing query `" << search << "'" << endl;
-  }
-
-  this->searchPattern = search;
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-  /* Try to find results */
-  if (maxResultCount != 0) {
-    /* Perform the search */
-    string unaccentedSearch = removeAccents(search);
-    std::vector<zim::Archive> archives;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      if ( (*current)->hasFulltextIndex() ) {
-          archives.push_back(*(*current)->getZimArchive());
-      }
-    }
-    zim::Searcher searcher(archives);
-    searcher.setVerbose(verbose);
-    zim::Query query;
-    query.setQuery(unaccentedSearch);
-    zim::Search search = searcher.search(query);
-    internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-    this->estimatedResultCount = search.getEstimatedMatches();
-  }
-
-  return;
-}
-
-
-void Searcher::geo_search(float latitude, float longitude, float distance,
-                          unsigned int resultStart,
-                          unsigned int maxResultCount,
-                          const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-  }
-
-  /* Perform the search */
-  std::ostringstream oss;
-  oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
-  this->searchPattern = oss.str();
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-
-  /* Try to find results */
-  if (maxResultCount == 0) {
-    return;
-  }
-
-  std::vector<zim::Archive> archives;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    archives.push_back(*(*current)->getZimArchive());
-  }
-  zim::Searcher searcher(archives);
-  searcher.setVerbose(verbose);
-  zim::Query query;
-  query.setQuery("");
-  query.setGeorange(latitude, longitude, distance);
-  zim::Search search = searcher.search(query);
-  internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-
-void Searcher::restart_search()
-{
-  if (internal.get()) {
-    internal->current_iterator = internal->begin();
-  }
-}
-
-Result* Searcher::getNextResult()
-{
-  if (internal.get() && internal->current_iterator != internal->end()) {
-    Result* result = new _Result(internal->current_iterator);
-    internal->current_iterator++;
-    return result;
-  } else if (suggestionInternal.get() &&
-             suggestionInternal->currentIterator != suggestionInternal->end()) {
-    SuggestionItem item(
-                        suggestionInternal->currentIterator->getTitle(),
-                        normalize(suggestionInternal->currentIterator->getTitle()),
-                        suggestionInternal->currentIterator->getPath(),
-                        suggestionInternal->currentIterator->getSnippet()
-                      );
-    Result* result = new _Result(item);
-    suggestionInternal->currentIterator++;
-    return result;
-  }
-  return NULL;
-}
-
-/* Reset the results */
-void Searcher::reset()
-{
-  this->estimatedResultCount = 0;
-  this->searchPattern = "";
-  return;
-}
-
-void Searcher::suggestions(std::string& searchPattern, const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
-  }
-
-  this->searchPattern = searchPattern;
-  this->resultStart = 0;
-  this->maxResultCount = 10;
-  string unaccentedSearch = removeAccents(searchPattern);
-
-  // Multizim suggestion is not supported as of now! taking only one archive
-  zim::Archive archive = *(*this->readers.begin())->getZimArchive();
-  zim::SuggestionSearcher searcher(archive);
-  searcher.setVerbose(verbose);
-  zim::SuggestionSearch search = searcher.suggest(searchPattern);
-  suggestionInternal.reset(new SuggestionInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-/* Return the result count estimation */
-unsigned int Searcher::getEstimatedResultCount()
-{
-  return this->estimatedResultCount;
-}
-
-zim::SearchResultSet Searcher::getSearchResultSet()
-{
-  return *(this->internal);
-}
-
-_Result::_Result(zim::SearchResultSet::iterator iterator)
-    : iterator(iterator),
-      suggestionItem("", "", ""),
-      isSuggestion(false)
-{}
-
-_Result::_Result(SuggestionItem item)
-    : iterator(),
-      suggestionItem(item.getTitle(), item.getNormalizedTitle(), item.getPath(), item.getSnippet()),
-      isSuggestion(true)
-{}
-
-std::string _Result::get_url()
-{
-  if (isSuggestion) {
-    return suggestionItem.getPath();
-  }
-  return iterator.getPath();
-}
-std::string _Result::get_title()
-{
-  if (isSuggestion) {
-    return suggestionItem.getTitle();
-  }
-  return iterator.getTitle();
-}
-int _Result::get_score()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getScore();
-}
-std::string _Result::get_snippet()
-{
-  if (isSuggestion) {
-    return suggestionItem.getSnippet();
-  }
-  return iterator.getSnippet();
-}
-std::string _Result::get_content()
-{
-  if (isSuggestion) return "";
-  return iterator->getItem(true).getData();
-}
-int _Result::get_size()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getSize();
-}
-int _Result::get_wordCount()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getWordCount();
-}
-std::string _Result::get_zimId()
-{
-  if (isSuggestion) {
-    return "";
-  }
-  std::ostringstream s;
-  s << iterator.getZimId();
-  return s.str();
-}
-
-
-}

src/server.cpp

@@ -24,6 +24,7 @@
 
 #include <string>
 
+#include <zim/item.h>
 #include "server/internalServer.h"
 
 namespace kiwix {
@@ -45,11 +46,13 @@ bool Server::start() {
     m_port,
     m_root,
     m_nbThreads,
+    m_multizimSearchLimit,
     m_verbose,
     m_withTaskbar,
     m_withLibraryButton,
     m_blockExternalLinks,
-    m_indexTemplateString));
+    m_indexTemplateString,
+    m_ipConnectionLimit));
   return mp_server->start();
 }
 
@@ -71,4 +74,14 @@ void Server::setRoot(const std::string& root)
   }
 }
 
+int Server::getPort()
+{
+  return mp_server->getPort();
+}
+
+std::string Server::getAddress()
+{
+  return mp_server->getAddress();
+}
+
 }

src/server/i18n.cpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "i18n.h"
+
+#include "tools/otherTools.h"
+
+#include <algorithm>
+#include <map>
+
+namespace kiwix
+{
+
+const char* I18nStringTable::get(const std::string& key) const
+{
+  const I18nString* const begin = entries;
+  const I18nString* const end = begin + entryCount;
+  const I18nString* found = std::lower_bound(begin, end, key,
+      [](const I18nString& a, const std::string& k) {
+        return a.key < k;
+  });
+  return (found == end || found->key != key) ? nullptr : found->value;
+}
+
+namespace i18n
+{
+// this data is generated by the i18n resource compiler
+extern const I18nStringTable stringTables[];
+extern const size_t langCount;
+}
+
+namespace
+{
+
+class I18nStringDB
+{
+public: // functions
+  I18nStringDB() {
+    for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
+      const auto& t = kiwix::i18n::stringTables[i];
+      lang2TableMap[t.lang] = &t;
+    }
+    enStrings = lang2TableMap.at("en");
+  };
+
+  std::string get(const std::string& lang, const std::string& key) const {
+    const char* s = getStringsFor(lang)->get(key);
+    if ( s == nullptr ) {
+      s = enStrings->get(key);
+      if ( s == nullptr ) {
+        throw std::runtime_error("Invalid message id");
+      }
+    }
+    return s;
+  }
+
+private: // functions
+  const I18nStringTable* getStringsFor(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang);
+    } catch(const std::out_of_range&) {
+      return enStrings;
+    }
+  }
+
+private: // data
+  std::map<std::string, const I18nStringTable*> lang2TableMap;
+  const I18nStringTable* enStrings;
+};
+
+} // unnamed namespace
+
+std::string getTranslatedString(const std::string& lang, const std::string& key)
+{
+  static const I18nStringDB stringDb;
+
+  return stringDb.get(lang, key);
+}
+
+namespace i18n
+{
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params)
+{
+  const std::string tmpl = getTranslatedString(lang, key);
+  return render_template(tmpl, params);
+}
+
+} // namespace i18n
+
+std::string ParameterizedMessage::getText(const std::string& lang) const
+{
+  return i18n::expandParameterizedString(lang, msgId, params);
+}
+
+} // namespace kiwix

src/server/i18n.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_I18N
+#define KIWIX_SERVER_I18N
+
+#include <string>
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+struct I18nString {
+  const char* const key;
+  const char* const value;
+};
+
+struct I18nStringTable {
+  const char* const lang;
+  const size_t entryCount;
+  const I18nString* const entries;
+
+  const char* get(const std::string& key) const;
+};
+
+std::string getTranslatedString(const std::string& lang, const std::string& key);
+
+namespace i18n
+{
+
+typedef kainjow::mustache::object Parameters;
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params);
+
+class GetTranslatedString
+{
+public:
+  explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
+
+  std::string operator()(const std::string& key) const
+  {
+    return getTranslatedString(m_lang, key);
+  }
+
+  std::string operator()(const std::string& key, const Parameters& params) const
+  {
+    return expandParameterizedString(m_lang, key, params);
+  }
+
+private:
+  const std::string m_lang;
+};
+
+} // namespace i18n
+
+struct ParameterizedMessage
+{
+public: // types
+  typedef kainjow::mustache::object Parameters;
+
+public: // functions
+  ParameterizedMessage(const std::string& msgId, const Parameters& params)
+    : msgId(msgId)
+    , params(params)
+  {}
+
+  std::string getText(const std::string& lang) const;
+
+private: // data
+  const std::string msgId;
+  const Parameters  params;
+};
+
+} // namespace kiwix
+
+#endif // KIWIX_SERVER_I18N

src/server/internalServer.h

@@ -28,6 +28,9 @@ extern "C" {
 #include "library.h"
 #include "name_mapper.h"
 
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
 #include <mustache.hpp>
 
 #include <atomic>
@@ -36,11 +39,58 @@ extern "C" {
 #include "server/request_context.h"
 #include "server/response.h"
 
+#include "tools/concurrent_cache.h"
+
 namespace kiwix {
 
-typedef kainjow::mustache::data MustacheData;
+struct GeoQuery {
+  GeoQuery()
+    : GeoQuery(0, 0, -1)
+  {}
+
+  GeoQuery(float latitude, float longitude, float distance)
+    : latitude(latitude), longitude(longitude), distance(distance)
+  {}
+  float latitude;
+  float longitude;
+  float distance;
+
+  explicit operator bool() const {
+    return distance >= 0;
+  }
+
+  friend bool operator<(const GeoQuery& l, const GeoQuery& r)
+  {
+      return std::tie(l.latitude, l.longitude, l.distance)
+           < std::tie(r.latitude, r.longitude, r.distance); // keep the same order
+  }
+};
+
+class SearchInfo {
+  public:
+    SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
+
+    zim::Query getZimQuery(bool verbose) const;
+    const Library::BookIdSet& getBookIds() const { return bookIds; }
+
+    friend bool operator<(const SearchInfo& l, const SearchInfo& r)
+    {
+        return std::tie(l.bookIds, l.pattern, l.geoQuery)
+             < std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
+    }
+
+  public: //data
+    std::string pattern;
+    GeoQuery geoQuery;
+    Library::BookIdSet bookIds;
+    std::string bookFilterQuery;
+};
+
+
+typedef kainjow::mustache::data MustacheData;
+typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+typedef ConcurrentCache<std::string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
 
-class Entry;
 class OPDSDumper;
 
 class InternalServer {
@@ -51,12 +101,14 @@ class InternalServer {
                    int port,
                    std::string root,
                    int nbThreads,
+                   unsigned int multizimSearchLimit,
                    bool verbose,
                    bool withTaskbar,
                    bool withLibraryButton,
                    bool blockExternalLinks,
-                   std::string indexTemplateString);
-    virtual ~InternalServer() = default;
+                   std::string indexTemplateString,
+                   int ipConnectionLimit);
+    virtual ~InternalServer();
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
                                const char* url,
@@ -67,6 +119,8 @@ class InternalServer {
                                void** cont_cls);
     bool start();
     void stop();
+    std::string getAddress() { return m_addr; }
+    int getPort() { return m_port; }
 
   private: // functions
     std::unique_ptr<Response> handle_request(const RequestContext& request);
@@ -80,12 +134,16 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
-    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_catch(const RequestContext& request);
     std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
     std::unique_ptr<Response> handle_content(const RequestContext& request);
+    std::unique_ptr<Response> handle_raw(const RequestContext& request);
+    std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
+    std::unique_ptr<Response> handle_widget(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
@@ -94,30 +152,40 @@ class InternalServer {
 
     bool etag_not_needed(const RequestContext& r) const;
     ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+    std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
+    SearchInfo getSearchInfo(const RequestContext& r) const;
+
+    bool isLocallyCustomizedResource(const std::string& url) const;
 
   private: // data
     std::string m_addr;
     int m_port;
     std::string m_root;
     int m_nbThreads;
+    unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
     std::string m_indexTemplateString;
+    int m_ipConnectionLimit;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
     NameMapper* mp_nameMapper;
 
+    SearchCache searchCache;
+    SuggestionSearcherCache suggestionSearcherCache;
+
     std::string m_server_id;
     std::string m_library_id;
 
-    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
-    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
+    class CustomizedResources;
+    std::unique_ptr<CustomizedResources> m_customizedResources;
 
+    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
 };
 
 }

src/server/internalServer_catalog_v2.cpp

@@ -43,7 +43,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "root.xml") {
@@ -66,8 +67,11 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
     return handle_catalog_v2_categories(request);
   } else if (url == "languages") {
     return handle_catalog_v2_languages(request);
+  } else if (url == "illustration") {
+    return handle_catalog_v2_illustration(request);
   } else {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 }
 
@@ -108,7 +112,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
   try {
     mp_library->getBookById(entryId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request, "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   OPDSDumper opdsDumper(mp_library);
@@ -146,4 +151,18 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const Requ
   );
 }
 
+std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
+{
+  try {
+    const auto bookId  = request.get_url_part(3);
+    auto book = mp_library->getBookByIdThreadSafe(bookId);
+    auto size = request.get_argument<unsigned int>("size");
+    auto illustration = book.getIllustration(size);
+    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+  } catch(...) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+}
+
 } // namespace kiwix

src/server/request_context.cpp

@@ -75,15 +75,15 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
-  acceptEncodingDeflate(false),
+  acceptEncodingGzip(false),
   byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
 
   try {
-    acceptEncodingDeflate =
-        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
+    acceptEncodingGzip =
+        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("gzip") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
   try {
@@ -106,7 +106,7 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
                                          const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->arguments[key] = value == nullptr ? "" : value;
+  _this->arguments[key].push_back(value == nullptr ? "" : value);
   return MHD_YES;
 }
 
@@ -121,13 +121,19 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("arguments :\n");
-  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+  for (auto& pair:arguments) {
+    printf(" - %s :", pair.first.c_str());
+    bool first = true;
+    for (auto& v: pair.second) {
+      printf("%s %s", first?"":",", v.c_str());
+      first = false;
+    }
+    printf("\n");
   }
   printf("Parsed : \n");
   printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
-  printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
+  printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
   printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
@@ -176,21 +182,29 @@ ByteRange RequestContext::get_range() const {
 
 template<>
 std::string RequestContext::get_argument(const std::string& name) const {
-  return arguments.at(name);
+  return arguments.at(name)[0];
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
   return headers.at(lcAll(name));
 }
 
-std::string RequestContext::get_query() const {
-  std::string q;
-  const char* sep = "";
-  for ( const auto& a : arguments ) {
-    q += sep + a.first + '=' + a.second;
-    sep = "&";
-  }
-  return q;
+std::string RequestContext::get_user_language() const
+{
+  try {
+    return get_argument("userlang");
+  } catch(const std::out_of_range&) {}
+
+  try {
+    return get_header("Accept-Language");
+  } catch(const std::out_of_range&) {}
+
+  return "en";
+}
+
+std::string RequestContext::get_requested_format() const
+{
+  return get_optional_param<std::string>("format", "html");
 }
 
 }

src/server/request_context.h

@@ -25,9 +25,11 @@
 #include <string>
 #include <sstream>
 #include <map>
+#include <vector>
 #include <stdexcept>
 
 #include "byte_range.h"
+#include "tools/stringTools.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -68,10 +70,11 @@ class RequestContext {
     std::string get_header(const std::string& name) const;
     template<typename T=std::string>
     T get_argument(const std::string& name) const {
-        std::istringstream stream(arguments.at(name));
-        T v;
-        stream >> v;
-        return v;
+        return extractFromString<T>(get_argument(name));
+    }
+
+    std::vector<std::string> get_arguments(const std::string& name) const {
+      return arguments.at(name);
     }
 
     template<class T>
@@ -88,11 +91,34 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
-    std::string get_query() const;
+
+    std::string get_query(bool mustEncode = false) const {
+      return get_query([](const std::string& key) {return true;}, mustEncode);
+    }
+
+    template<class F>
+    std::string get_query(F filter, bool mustEncode) const {
+      std::string q;
+      const char* sep = "";
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
+      for ( const auto& a : arguments ) {
+        if (!filter(a.first)) {
+          continue;
+        }
+        for (const auto& v: a.second) {
+          q += sep + encode(a.first) + '=' + encode(v);
+          sep = "&";
+        }
+      }
+      return q;
+    }
 
     ByteRange get_range() const;
 
-    bool can_compress() const { return acceptEncodingDeflate; }
+    bool can_compress() const { return acceptEncodingGzip; }
+
+    std::string get_user_language() const;
+    std::string get_requested_format() const;
 
   private: // data
     std::string full_url;
@@ -101,11 +127,11 @@ class RequestContext {
     std::string version;
     unsigned long long requestIndex;
 
-    bool acceptEncodingDeflate;
+    bool acceptEncodingGzip;
 
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
-    std::map<std::string, std::string> arguments;
+    std::map<std::string, std::vector<std::string>> arguments;
 
   private: // functions
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);

src/server/response.cpp

@@ -25,13 +25,23 @@
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
+#include "tools/archiveTools.h"
 
 #include "string.h"
 #include <mustache.hpp>
 #include <zlib.h>
 
+#include <array>
 
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+// This is somehow a magic value.
+// If this value is too small, we will compress (and lost cpu time) too much
+// content.
+// If this value is too big, we will not compress enough content and send too
+// much data.
+// If we assume that MTU is 1500 Bytes it is useless to compress
+// content smaller as the content will be sent in one packet anyway.
+// 1400 Bytes seems to be a common accepted limit.
+#define KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS 1400
 
 namespace kiwix {
 
@@ -43,20 +53,55 @@ std::string get_mime_type(const zim::Item& item)
 {
   try {
     return item.getMimetype();
-  } catch (exception& e) {
+  } catch (std::exception& e) {
     return "application/octet-stream";
   }
 }
 
 bool is_compressible_mime_type(const std::string& mimeType)
 {
-  return mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/atom") != string::npos
-      || mimeType.find("application/opensearchdescription") != string::npos
-      || mimeType.find("application/json") != string::npos;
+  return mimeType.find("text/") != std::string::npos
+      || mimeType.find("application/javascript") != std::string::npos
+      || mimeType.find("application/atom") != std::string::npos
+      || mimeType.find("application/opensearchdescription") != std::string::npos
+      || mimeType.find("application/json") != std::string::npos;
 }
 
+bool compress(std::string &content) {
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+
+  auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
+                          Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK) { return false; }
+
+  strm.avail_in = static_cast<decltype(strm.avail_in)>(content.size());
+  strm.next_in =
+      const_cast<Bytef *>(reinterpret_cast<const Bytef *>(content.data()));
+
+  std::string compressed;
+
+  std::array<char, 16384> buff{};
+  do {
+    strm.avail_out = buff.size();
+    strm.next_out = reinterpret_cast<Bytef *>(buff.data());
+    ret = deflate(&strm, Z_FINISH);
+    assert(ret != Z_STREAM_ERROR);
+    compressed.append(buff.data(), buff.size() - strm.avail_out);
+  } while (strm.avail_out == 0);
+
+  assert(ret == Z_STREAM_END);
+  assert(strm.avail_in == 0);
+
+  content.swap(compressed);
+
+  deflateEnd(&strm);
+  return true;
+}
+
+
 
 } // unnamed namespace
 
@@ -83,19 +128,137 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   return response;
 }
 
-std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
+const UrlNotFoundMsg urlNotFoundMsg;
+const InvalidUrlMsg invalidUrlMsg;
+
+std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
 {
-  MustacheData results;
-  results.set("url", request.get_full_url());
-  results.set("details", details);
-
-  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
-  response->set_code(MHD_HTTP_NOT_FOUND);
-  response->set_taskbar(bookName, bookTitle);
-
-  return std::move(response);
+  return getTranslatedString(m_request.get_user_language(), msgId);
 }
 
+std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
+{
+  auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
+  r->set_code(m_httpStatusCode);
+  if ( m_taskbarInfo ) {
+    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
+  }
+  return r;
+}
+
+HTTPErrorResponse::HTTPErrorResponse(const InternalServer& server,
+                                     const RequestContext& request,
+                                     int httpStatusCode,
+                                     const std::string& pageTitleMsgId,
+                                     const std::string& headingMsgId,
+                                     const std::string& cssUrl)
+  : ContentResponseBlueprint(&server,
+                             &request,
+                             httpStatusCode,
+                             request.get_requested_format() == "html" ? "text/html; charset=utf-8" : "application/xml; charset=utf-8",
+                             request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml)
+{
+  kainjow::mustache::list emptyList;
+  this->m_data = kainjow::mustache::object{
+                    {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
+                    {"PAGE_TITLE",   getMessage(pageTitleMsgId)},
+                    {"PAGE_HEADING", getMessage(headingMsgId)},
+                    {"details", emptyList}
+  };
+}
+
+HTTP404Response::HTTP404Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_NOT_FOUND,
+                      "404-page-title",
+                      "404-page-heading")
+{
+}
+
+HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
+{
+  const std::string requestUrl = m_request.get_full_url();
+  return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+(const std::string& msg)
+{
+  m_data["details"].push_back({"p", msg});
+  return *this;
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+(const ParameterizedMessage& details)
+{
+  return *this + details.getText(m_request.get_user_language());
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& details)
+{
+  // operator+() is already a state-modifying operator (akin to operator+=)
+  return *this + details;
+}
+
+
+HTTP400Response::HTTP400Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_BAD_REQUEST,
+                      "400-page-title",
+                      "400-page-heading")
+{
+}
+
+HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
+{
+  std::string requestUrl = m_request.get_full_url();
+  const auto query = m_request.get_query();
+  if (!query.empty()) {
+    requestUrl += "?" + encodeDiples(query);
+  }
+  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
+  return *this + msgTmpl.render({"url", requestUrl});
+}
+
+HTTP500Response::HTTP500Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_INTERNAL_SERVER_ERROR,
+                      "500-page-title",
+                      "500-page-heading")
+{
+  // operator+() is a state-modifying operator (akin to operator+=)
+  *this + "An internal server error occured. We are sorry about that :/";
+}
+
+std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
+{
+  // We want a 500 response to be a minimalistic one (so that the server doesn't
+  // have to provide additional resources required for its proper rendering)
+  // ";raw=true" in the MIME-type below disables response decoration
+  // (see ContentResponse::contentDecorationAllowed())
+  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
+  r->set_code(m_httpStatusCode);
+  return r;
+}
+
+ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
+{
+  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
+  return *this;
+}
+
+ContentResponseBlueprint& ContentResponseBlueprint::operator+=(const TaskbarInfo& taskbarInfo)
+{
+  // operator+() is already a state-modifying operator (akin to operator+=)
+  return *this + taskbarInfo;
+}
+
+
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
 {
   auto response = Response::build(server);
@@ -109,17 +272,6 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
   return response;
 }
 
-std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  auto content = render_template(RESOURCE::templates::_500_html, data);
-  std::unique_ptr<Response> response (
-      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
-  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
 
 std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
 {
@@ -155,7 +307,7 @@ static ssize_t callback_reader_from_item(void* cls,
 {
   RunningResponse* response = static_cast<RunningResponse*>(cls);
 
-  size_t max_size_to_set = min<size_t>(
+  size_t max_size_to_set = std::min<size_t>(
     max,
     response->item.getSize() - pos - response->range_start);
 
@@ -185,14 +337,20 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar()
+void ContentResponse::introduce_taskbar(const std::string& lang)
 {
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  data.set("content", m_bookName);
-  data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
-  data.set("title", m_bookTitle);
-  data.set("withlibrarybutton", m_withLibraryButton);
+  i18n::GetTranslatedString t(lang);
+  kainjow::mustache::object data{
+    {"root", m_root},
+    {"content", m_bookName},
+    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
+    {"title", m_bookTitle},
+    {"withlibrarybutton", m_withLibraryButton},
+    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
+    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
+    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
+    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
+  };
   auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
   m_content = prependToFirstOccurence(
     m_content,
@@ -230,14 +388,17 @@ ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
-      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
 bool
 ContentResponse::contentDecorationAllowed() const
 {
-    return (startsWith(m_mimeType, "text/html")
-         && m_mimeType.find(";raw=true") == std::string::npos);
+  if (m_raw) {
+    return false;
+  }
+  return (startsWith(m_mimeType, "text/html")
+    && m_mimeType.find(";raw=true") == std::string::npos);
 }
 
 MHD_Response*
@@ -254,42 +415,24 @@ ContentResponse::create_mhd_response(const RequestContext& request)
     inject_root_link();
 
     if (m_withTaskbar) {
-      introduce_taskbar();
+      introduce_taskbar(request.get_user_language());
     }
     if (m_blockExternalLinks) {
       inject_externallinks_blocker();
     }
   }
 
-  bool shouldCompress = can_compress(request);
-  if (shouldCompress) {
-    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-    uLongf comprLen = compr_buffer.capacity();
-    int err = compress(&compr_buffer[0],
-                       &comprLen,
-                       (const Bytef*)(m_content.data()),
-                       m_content.size());
-    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-      /* /!\ Internet Explorer has a bug with deflate compression.
-         It can not handle the first two bytes (compression headers)
-         We need to chunk them off (move the content 2bytes)
-         It has no incidence on other browsers
-         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-      m_content = string((char*)&compr_buffer[2], comprLen - 2);
-      m_etag.set_option(ETag::COMPRESSED_CONTENT);
-    } else {
-      shouldCompress = false;
-    }
-  }
+  const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  if (shouldCompress) {
+  if (isCompressed) {
+    m_etag.set_option(ETag::COMPRESSED_CONTENT);
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
     MHD_add_response_header(
-        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "gzip");
   }
   return response;
 }
@@ -318,18 +461,19 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
 {
   m_bookName = bookName;
-  m_bookTitle = bookTitle;
+  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
 }
 
 
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
   m_mimeType(mimetype),
+  m_raw(raw),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
@@ -339,11 +483,17 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool wit
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& content,
+  const std::string& mimetype,
+  bool isHomePage,
+  bool raw)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
+        raw,
         server.m_withTaskbar && !isHomePage,
         server.m_withLibraryButton,
         server.m_blockExternalLinks,
@@ -351,7 +501,13 @@ std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& se
         mimetype));
 }
 
-std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
+std::unique_ptr<ContentResponse> ContentResponse::build(
+  const InternalServer& server,
+  const std::string& template_str,
+  kainjow::mustache::data data,
+  const std::string& mimetype,
+  bool isHomePage)
+{
   auto content = render_template(template_str, data);
   return ContentResponse::build(server, content, mimetype, isHomePage);
 }
@@ -366,14 +522,14 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
     response->set_cacheable();
     response->m_byteRange = byteRange;
     return std::move(response);

src/server/response.h

@@ -26,20 +26,24 @@
 
 #include <mustache.hpp>
 #include "byte_range.h"
-#include "entry.h"
 #include "etag.h"
+#include "i18n.h"
+
+#include <zim/item.h>
 
 extern "C" {
 #include "microhttpd_wrapper.h"
 }
 
+namespace zim {
+class Archive;
+} // namespace zim
+
 namespace kiwix {
 
 class InternalServer;
 class RequestContext;
 
-class EntryResponse;
-
 class Response {
   public:
     Response(bool verbose);
@@ -47,9 +51,7 @@ class Response {
 
     static std::unique_ptr<Response> build(const InternalServer& server);
     static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
-    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
     static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
-    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
     static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
@@ -78,16 +80,34 @@ class Response {
 
 class ContentResponse : public Response {
   public:
-    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
-    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
+    ContentResponse(
+      const std::string& root,
+      bool verbose,
+      bool raw,
+      bool withTaskbar,
+      bool withLibraryButton,
+      bool blockExternalLinks,
+      const std::string& content,
+      const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& content,
+      const std::string& mimetype,
+      bool isHomePage = false,
+      bool raw = false);
+    static std::unique_ptr<ContentResponse> build(
+      const InternalServer& server,
+      const std::string& template_str,
+      kainjow::mustache::data data,
+      const std::string& mimetype,
+      bool isHomePage = false);
 
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar();
+    void introduce_taskbar(const std::string& lang);
     void inject_externallinks_blocker();
     void inject_root_link();
     bool can_compress(const RequestContext& request) const;
@@ -98,6 +118,7 @@ class ContentResponse : public Response {
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
+    bool m_raw;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
@@ -105,10 +126,119 @@ class ContentResponse : public Response {
     std::string m_bookTitle;
  };
 
+struct TaskbarInfo
+{
+  const std::string bookName;
+  const zim::Archive* const archive;
+
+  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
+    : bookName(bookName)
+    , archive(a)
+  {}
+};
+
+class ContentResponseBlueprint
+{
+public: // functions
+  ContentResponseBlueprint(const InternalServer* server,
+                           const RequestContext* request,
+                           int httpStatusCode,
+                           const std::string& mimeType,
+                           const std::string& templateStr)
+    : m_server(*server)
+    , m_request(*request)
+    , m_httpStatusCode(httpStatusCode)
+    , m_mimeType(mimeType)
+    , m_template(templateStr)
+  {}
+
+  virtual ~ContentResponseBlueprint() = default;
+
+  operator std::unique_ptr<ContentResponse>() const
+  {
+    return generateResponseObject();
+  }
+
+  operator std::unique_ptr<Response>() const
+  {
+    return operator std::unique_ptr<ContentResponse>();
+  }
+
+
+  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
+  ContentResponseBlueprint& operator+=(const TaskbarInfo& taskbarInfo);
+
+protected: // functions
+  std::string getMessage(const std::string& msgId) const;
+  virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
+
+public: //data
+  const InternalServer& m_server;
+  const RequestContext& m_request;
+  const int m_httpStatusCode;
+  const std::string m_mimeType;
+  const std::string m_template;
+  kainjow::mustache::data m_data;
+  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
+};
+
+struct HTTPErrorResponse : ContentResponseBlueprint
+{
+  HTTPErrorResponse(const InternalServer& server,
+                    const RequestContext& request,
+                    int httpStatusCode,
+                    const std::string& pageTitleMsgId,
+                    const std::string& headingMsgId,
+                    const std::string& cssUrl = "");
+
+  using ContentResponseBlueprint::operator+;
+  using ContentResponseBlueprint::operator+=;
+  HTTPErrorResponse& operator+(const std::string& msg);
+  HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
+  HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
+};
+
+class UrlNotFoundMsg {};
+
+extern const UrlNotFoundMsg urlNotFoundMsg;
+
+struct HTTP404Response : HTTPErrorResponse
+{
+  HTTP404Response(const InternalServer& server,
+                  const RequestContext& request);
+
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(UrlNotFoundMsg /*unused*/);
+};
+
+class InvalidUrlMsg {};
+
+extern const InvalidUrlMsg invalidUrlMsg;
+
+struct HTTP400Response : HTTPErrorResponse
+{
+  HTTP400Response(const InternalServer& server,
+                  const RequestContext& request);
+
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(InvalidUrlMsg /*unused*/);
+};
+
+struct HTTP500Response : HTTPErrorResponse
+{
+  HTTP500Response(const InternalServer& server,
+                  const RequestContext& request);
+
+private: // overrides
+  // generateResponseObject() is overriden in order to produce a minimal
+  // response without any need for additional resources from the server
+  std::unique_ptr<ContentResponse> generateResponseObject() const override;
+};
+
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -125,6 +125,30 @@ unsigned int getArchiveMediaCount(const zim::Archive& archive) {
   return counter;
 }
 
+unsigned int getArchiveArticleCount(const zim::Archive& archive) {
+  // [HACK]
+  // getArticleCount() returns different things depending of the "version" of the zim.
+  // On old zim (<=6), it returns the number of entry in `A` namespace
+  // On recent zim (>=7), it returns:
+  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
+  //  - the number of front article if a frontArticleIndex is present
+  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
+  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
+  if (archive.hasNewNamespaceScheme()) {
+    //The articleCount is "good"
+    return archive.getArticleCount();
+  } else {
+    // We have to parse the `M/Counter` metadata
+    unsigned int counter = 0;
+    for(const auto& pair:parseArchiveCounter(archive)) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
+    }
+    return counter;
+  }
+}
+
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }

src/tools/archiveTools.h

@@ -46,6 +46,7 @@ namespace kiwix
                            std::string& content, std::string& mimeType);
 
     unsigned int getArchiveMediaCount(const zim::Archive& archive);
+    unsigned int getArchiveArticleCount(const zim::Archive& archive);
     unsigned int getArchiveFileSize(const zim::Archive& archive);
 
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);

src/tools/concurrent_cache.h

@@ -0,0 +1,208 @@
+
+/*
+ * Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (C) 2020 Veloman Yunkan
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
+ * NON-INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifndef ZIM_CONCURRENT_CACHE_H
+#define ZIM_CONCURRENT_CACHE_H
+
+#include "lrucache.h"
+
+#include <future>
+#include <mutex>
+
+namespace kiwix
+{
+
+/**
+   ConcurrentCache implements a concurrent thread-safe cache
+
+   Compared to kiwix::lru_cache, each access operation is slightly more expensive.
+   However, different slots of the cache can be safely accessed concurrently
+   with minimal blocking. Concurrent access to the same element is also
+   safe, and, in case of a cache miss, will block until that element becomes
+   available.
+ */
+template <typename Key, typename Value>
+class ConcurrentCache
+{
+private: // types
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      try {
+        valuePromise.set_value(f());
+      } catch (std::exception& e) {
+        drop(key);
+        throw;
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  Impl impl_;
+  std::mutex lock_;
+};
+
+
+/**
+  WeakStore represent a thread safe store (map) of weak ptr.
+  It allows to store weak_ptr from shared_ptr and retrieve shared_ptr from
+  potential non expired weak_ptr.
+  It is not limited in size.
+*/
+template<typename Key, typename Value>
+class WeakStore {
+  private: // types
+    typedef std::weak_ptr<Value> WeakValue;
+
+  public:
+    explicit WeakStore() = default;
+
+    std::shared_ptr<Value> get(const Key& key)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      auto it = m_weakMap.find(key);
+      if (it != m_weakMap.end()) {
+        auto shared = it->second.lock();
+        if (shared) {
+          return shared;
+        } else {
+          m_weakMap.erase(it);
+        }
+      }
+      throw std::runtime_error("No weak ptr");
+    }
+
+    void add(const Key& key, std::shared_ptr<Value> shared)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      m_weakMap[key] = WeakValue(shared);
+    }
+
+
+  private: //data
+    std::map<Key, WeakValue> m_weakMap;
+    std::mutex m_lock;
+};
+
+template <typename Key, typename RawValue>
+class ConcurrentCache<Key, std::shared_ptr<RawValue>>
+{
+private: // types
+  typedef std::shared_ptr<RawValue> Value;
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      // Try to get back the shared_ptr from the weak_ptr first.
+      try {
+        valuePromise.set_value(m_weakStore.get(key));
+      } catch(const std::runtime_error& e) {
+        try {
+          const auto value = f();
+          valuePromise.set_value(value);
+          m_weakStore.add(key, value);
+        } catch (std::exception& e) {
+          drop(key);
+          throw;
+        }
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  std::mutex lock_;
+  Impl impl_;
+  WeakStore<Key, RawValue> m_weakStore;
+};
+
+} // namespace kiwix
+
+#endif // ZIM_CONCURRENT_CACHE_H
+

src/tools/lrucache.h

@@ -0,0 +1,175 @@
+/*
+ * Copyrigth (c) 2021, Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (c) 2020, Veloman Yunkan
+ * Copyright (c) 2014, lamerman
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of lamerman nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * File:   lrucache.hpp
+ * Author: Alexander Ponomarev
+ *
+ * Created on June 20, 2013, 5:09 PM
+ */
+
+#ifndef _LRUCACHE_HPP_INCLUDED_
+#define _LRUCACHE_HPP_INCLUDED_
+
+#include <map>
+#include <list>
+#include <set>
+#include <cstddef>
+#include <stdexcept>
+#include <cassert>
+
+namespace kiwix {
+
+template<typename key_t, typename value_t>
+class lru_cache {
+public: // types
+  typedef typename std::pair<key_t, value_t> key_value_pair_t;
+  typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+
+  enum AccessStatus {
+    HIT, // key was found in the cache
+    PUT, // key was not in the cache but was created by the getOrPut() access
+    MISS // key was not in the cache; get() access failed
+  };
+
+  class AccessResult
+  {
+    const AccessStatus status_;
+    const value_t val_;
+  public:
+    AccessResult(const value_t& val, AccessStatus status)
+      : status_(status), val_(val)
+    {}
+    AccessResult() : status_(MISS), val_() {}
+
+    bool hit() const { return status_ == HIT; }
+    bool miss() const { return !hit(); }
+    const value_t& value() const
+    {
+      if ( status_ == MISS )
+        throw std::range_error("There is no such key in cache");
+      return val_;
+    }
+
+    operator const value_t& () const { return value(); }
+  };
+
+public: // functions
+  explicit lru_cache(size_t max_size) :
+    _max_size(max_size) {
+  }
+
+  // If 'key' is present in the cache, returns the associated value,
+  // otherwise puts the given value into the cache (and returns it with
+  // a status of a cache miss).
+  AccessResult getOrPut(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    } else {
+      putMissing(key, value);
+      return AccessResult(value, PUT);
+    }
+  }
+
+  void put(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      it->second->second = value;
+    } else {
+      putMissing(key, value);
+    }
+  }
+
+  AccessResult get(const key_t& key) {
+    auto it = _cache_items_map.find(key);
+    if (it == _cache_items_map.end()) {
+      return AccessResult();
+    } else {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    }
+  }
+
+  bool drop(const key_t& key) {
+    try {
+      auto list_it = _cache_items_map.at(key);
+      _cache_items_list.erase(list_it);
+      _cache_items_map.erase(key);
+      return true;
+    } catch (std::out_of_range& e) {
+      return false;
+    }
+  }
+
+  bool exists(const key_t& key) const {
+    return _cache_items_map.find(key) != _cache_items_map.end();
+  }
+
+  size_t size() const {
+    return _cache_items_map.size();
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    size_t previous = _max_size;
+    _max_size = new_size;
+    return previous;
+  }
+
+  std::set<key_t> keys() const  {
+    std::set<key_t> keys;
+    for(auto& item:_cache_items_map) {
+      keys.insert(item.first);
+    }
+    return keys;
+  }
+
+private: // functions
+  void putMissing(const key_t& key, const value_t& value) {
+    assert(_cache_items_map.find(key) == _cache_items_map.end());
+    _cache_items_list.push_front(key_value_pair_t(key, value));
+    _cache_items_map[key] = _cache_items_list.begin();
+    while (_cache_items_map.size() > _max_size) {
+      _cache_items_map.erase(_cache_items_list.back().first);
+      _cache_items_list.pop_back();
+    }
+  }
+
+private: // data
+  std::list<key_value_pair_t> _cache_items_list;
+  std::map<key_t, list_iterator_t> _cache_items_map;
+  size_t _max_size;
+};
+
+} // namespace kiwix
+
+#endif  /* _LRUCACHE_HPP_INCLUDED_ */

src/tools/networkTools.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,6 +18,7 @@
  * MA 02110-1301, USA.
  */
 
+#include "tools.h"
 #include <tools/networkTools.h>
 
 #include <stdio.h>
@@ -29,6 +31,17 @@
 #include <iostream>
 #include <stdexcept>
 
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <iostream>
+#else
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <net/if.h>
+#include <netdb.h>
+#endif
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
@@ -57,3 +70,104 @@ std::string kiwix::download(const std::string& url) {
   }
   return ss.str();
 }
+
+std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
+  std::map<std::string, std::string> interfaces;
+
+#ifdef _WIN32
+  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
+  if (sd == INVALID_SOCKET) {
+    std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  INTERFACE_INFO InterfaceList[20];
+  unsigned long nBytesReturned;
+  if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
+      sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
+    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
+    return interfaces;
+  }
+
+  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
+  for (int i = 0; i < nNumInterfaces; ++i) {
+    sockaddr_in *pAddress;
+    pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
+    if(pAddress->sin_family == AF_INET) {
+    /* Add to the map */
+    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
+    interfaces[interfaceName] = interfaceName;
+    }
+  }
+#else
+  /* Get Network interfaces information */
+  char buf[16384];
+  struct ifconf ifconf;
+  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
+  ifconf.ifc_len = sizeof(buf);
+  ifconf.ifc_buf=buf;
+  if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
+    perror("ioctl(SIOCGIFCONF)");
+  }
+
+  /* Go through each interface */
+  struct ifreq *ifreq;
+  ifreq = ifconf.ifc_req;
+  for (int i = 0; i < ifconf.ifc_len; ) {
+    if (ifreq->ifr_addr.sa_family == AF_INET) {
+      /* Get the network interface ip */
+      char host[128] = { 0 };
+      const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
+                                    host, sizeof(host),
+                                    0, 0, NI_NUMERICHOST);
+      if (!error) {
+        std::string interfaceName = std::string(ifreq->ifr_name);
+        std::string interfaceIp = std::string(host);
+        /* Add to the map */
+        interfaces[interfaceName] = interfaceIp;
+      } else {
+        perror("getnameinfo()");
+      }
+    }
+
+    /* some systems have ifr_addr.sa_len and adjust the length that
+     * way, but not mine. weird */
+    size_t len;
+#ifndef __linux__
+    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
+#else
+    len = sizeof(*ifreq);
+#endif
+    ifreq = (struct ifreq*)((char*)ifreq+len);
+    i += len;
+  }
+#endif
+  return interfaces;
+}
+
+std::string kiwix::getBestPublicIp() {
+  auto interfaces = getNetworkInterfaces();
+
+#ifndef _WIN32
+  const char* const prioritizedNames[] =
+      { "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
+  for(auto name: prioritizedNames) {
+    auto it = interfaces.find(name);
+    if(it != interfaces.end()) {
+      return it->second;
+    }
+  }
+#endif
+
+  const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
+  for(auto prefix : prefixes){
+    for(auto& itr : interfaces) {
+      auto interfaceIp = itr.second;
+      if (interfaceIp.find(prefix) == 0) {
+        return interfaceIp;
+      }
+    }
+  }
+
+  return "127.0.0.1";
+}

src/tools/otherTools.cpp

@@ -370,6 +370,13 @@ std::string kiwix::gen_uuid(const std::string& s)
   return kiwix::to_string(zim::Uuid::generate(s));
 }
 
+kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
+{
+  return s.empty()
+       ? kainjow::mustache::data(false)
+       : kainjow::mustache::data(s);
+}
+
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);

src/tools/otherTools.h

@@ -23,9 +23,12 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <cstdlib>
 #include <zim/zim.h>
 #include <mustache.hpp>
 
+#include "stringTools.h"
+
 namespace pugi {
   class xml_node;
 }
@@ -48,7 +51,25 @@ namespace kiwix
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
+  // if s is empty then returns kainjow::mustache::data(false)
+  // otherwise kainjow::mustache::data(value)
+  kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
+
   std::string render_template(const std::string& template_str, kainjow::mustache::data data);
+
+  template<typename T>
+  T getEnvVar(const char* name, const T& defaultValue)
+  {
+    try {
+      const char* envString = std::getenv(name);
+      if (envString == nullptr) {
+        throw std::runtime_error("Environment variable not set");
+      }
+      return extractFromString<T>(envString);
+    } catch (...) {}
+
+    return defaultValue;
+  }
 }
 
 #endif

src/tools/pathTools.cpp

@@ -305,16 +305,18 @@ std::string kiwix::getFileContent(const std::string& path)
 bool kiwix::fileExists(const std::string& path)
 {
 #ifdef _WIN32
-  return PathFileExistsW(Utf8ToWide(path).c_str());
+  return (_waccess_s(Utf8ToWide(path).c_str(), 0) == 0);
 #else
-  bool flag = false;
-  std::fstream fin;
-  fin.open(path.c_str(), std::ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
+  return (access(path.c_str(), F_OK) == 0);
+#endif
+}
+
+bool kiwix::fileReadable(const std::string& path)
+{
+#ifdef _WIN32
+  return (_waccess_s(Utf8ToWide(path).c_str(), 4) == 0);
+#else
+  return (access(path.c_str(), R_OK) == 0);
 #endif
 }
 

src/tools/stringTools.cpp

@@ -49,6 +49,24 @@ void kiwix::loadICUExternalTables()
 #endif
 }
 
+kiwix::ICULanguageInfo::ICULanguageInfo(const std::string& langCode)
+  : locale(langCode.c_str())
+{}
+
+std::string kiwix::ICULanguageInfo::iso3Code() const
+{
+  return locale.getISO3Language();
+}
+
+std::string kiwix::ICULanguageInfo::selfName() const
+{
+  icu::UnicodeString langSelfNameICUString;
+  locale.getDisplayLanguage(locale, langSelfNameICUString);
+  std::string langSelfName;
+  langSelfNameICUString.toUTF8String(langSelfName);
+  return langSelfName;
+}
+
 std::string kiwix::removeAccents(const std::string& text)
 {
   loadICUExternalTables();
@@ -405,3 +423,8 @@ std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   variants.push_back(kiwix::toTitle(title));
   return variants;
 }
+
+template<>
+std::string kiwix::extractFromString(const std::string& str) {
+  return str;
+}

src/tools/stringTools.h

@@ -21,10 +21,12 @@
 #define KIWIX_STRINGTOOLS_H
 
 #include <unicode/unistr.h>
+#include <unicode/locid.h>
 
 #include <string>
 #include <vector>
 #include <sstream>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -40,6 +42,19 @@ std::string encodeDiples(const std::string& str);
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
 
+class ICULanguageInfo
+{
+public:
+  explicit ICULanguageInfo(const std::string& langCode);
+
+  std::string iso3Code() const;
+  std::string selfName() const;
+
+private:
+  const icu::Locale locale;
+};
+
+
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
@@ -65,9 +80,15 @@ T extractFromString(const std::string& str) {
     std::istringstream iss(str);
     T ret;
     iss >> ret;
+    if(iss.fail() || !iss.eof()) {
+        throw std::invalid_argument("no conversion");
+    }
     return ret;
 }
 
+template<>
+std::string extractFromString(const std::string& str);
+
 bool startsWith(const std::string& base, const std::string& start);
 
 std::vector<std::string> getTitleVariants(const std::string& title);

src/version.cpp

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <iostream>
+#include <sstream>
+
+#include <version.h>
+#include <zim/zim.h>
+#include <kiwix_config.h>
+#include <unicode/uversion.h>
+#include <pugixml.hpp>
+#include <curl/curl.h>
+#include <microhttpd.h>
+#include <xapian.h>
+#include <mustache.hpp>
+#include <zlib.h>
+
+namespace kiwix
+{
+  LibVersions getVersions() {
+    LibVersions versions = {
+      { "libkiwix",      LIBKIWIX_VERSION    },
+      { "libzim",        LIBZIM_VERSION      },
+      { "libxapian",     XAPIAN_VERSION      },
+      { "libcurl",       LIBCURL_VERSION     },
+      { "libmicrohttpd", MHD_get_version()   },
+      { "libz",          ZLIB_VERSION        }
+    };
+
+    // U_ICU_VERSION does not include the patch level if 0
+    std::ostringstream libicu_version;
+    libicu_version << U_ICU_VERSION_MAJOR_NUM << "." << U_ICU_VERSION_MINOR_NUM << "." << U_ICU_VERSION_PATCHLEVEL_NUM;
+    versions.push_back({ "libicu", libicu_version.str() });
+
+    // No human readable version string for pugixml
+    const unsigned pugixml_major = (PUGIXML_VERSION - PUGIXML_VERSION % 1000) / 1000;
+    const unsigned pugixml_minor = (PUGIXML_VERSION - pugixml_major * 1000 - PUGIXML_VERSION % 10) / 10;
+    const unsigned pugixml_patch = PUGIXML_VERSION - pugixml_major * 1000 - pugixml_minor * 10;
+    std::ostringstream libpugixml_version;
+    libpugixml_version << pugixml_major << "." << pugixml_minor << "." << pugixml_patch;
+    versions.push_back({ "libpugixml", libpugixml_version.str() });
+
+    // Needs version 5.0 of Mustache
+#if defined(KAINJOW_MUSTACHE_VERSION_MAJOR)
+    std::ostringstream libmustache_version;
+    libmustache_version << KAINJOW_MUSTACHE_VERSION_MAJOR << "." <<
+      KAINJOW_MUSTACHE_VERSION_MINOR << "." << KAINJOW_MUSTACHE_VERSION_PATCH;
+    versions.push_back({ "libmustache", libmustache_version.str() });
+#endif
+
+    return versions;
+  }
+
+  void printVersions(std::ostream& out) {
+    LibVersions versions = getVersions();
+    for (const auto& iter : versions) {
+      out << (iter != versions.front() ? "+ " : "")
+                << iter.first << " " << iter.second << std::endl;
+    }
+  }
+} //namespace kiwix

src/wrapper/java/AndroidManifest.xml

@@ -1,13 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-
-  package="kiwix.org.kiwixlib"
->
-
-  <application android:allowBackup="true"
-    android:label="@string/app_name"
-    android:supportsRtl="true"
-  >
-
-  </application>
-
-</manifest>

src/wrapper/java/book.cpp

@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Book.h"
-
-#include "utils.h"
-#include "book.h"
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_allocate(
-  JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Book>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Book>(env, thisObj);
-}
-
-#define BOOK (getPtr<kiwix::Book>(env, thisObj))
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
-{
-  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
-}
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
-{
-  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
-}
-
-#define GETTER(retType, name) JNIEXPORT retType JNICALL \
-Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
-{ \
-  auto cRet = BOOK->name(); \
-  retType ret = c2jni(cRet, env); \
-  return ret; \
-}
-
-GETTER(jstring, getId)
-GETTER(jstring, getPath)
-GETTER(jboolean, isPathValid)
-GETTER(jstring, getTitle)
-GETTER(jstring, getDescription)
-GETTER(jstring, getLanguage)
-GETTER(jstring, getCreator)
-GETTER(jstring, getPublisher)
-GETTER(jstring, getDate)
-GETTER(jstring, getUrl)
-GETTER(jstring, getName)
-GETTER(jstring, getFlavour)
-GETTER(jstring, getCategory)
-GETTER(jstring, getTags)
-GETTER(jlong, getArticleCount)
-GETTER(jlong, getMediaCount)
-GETTER(jlong, getSize)
-GETTER(jstring, getFavicon)
-GETTER(jstring, getFaviconUrl)
-GETTER(jstring, getFaviconMimeType)
-
-METHOD(jstring, Book, getTagStr, jstring tagName) try {
-  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
-  return c2jni(cRet, env);
-} catch(...) {
-  return c2jni<std::string>("", env);
-}
-
-#undef GETTER

src/wrapper/java/filter.cpp

@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Filter.h"
-
-#include "library.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-METHOD0(void, Filter, allocate) {
-  allocate<kiwix::Filter>(env, thisObj);
-}
-
-METHOD0(void, Filter, dispose) {
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
-
-#define FORWARD(name, args_type) \
-METHOD(jobject, Filter, name, args_type value) { \
-  FILTER->name(jni2c(value, env)); \
-  return thisObj; \
-}
-
-#define FORWARDA(name, args_type) \
-METHOD(jobject, Filter, name, jobjectArray value) { \
-  FILTER->name(jni2c<args_type>(value, env)); \
-  return thisObj; \
-}
-
-
-
-FORWARD(local, jboolean)
-FORWARD(remote, jboolean)
-FORWARD(valid, jboolean)
-FORWARDA(acceptTags, jstring)
-FORWARDA(rejectTags, jstring)
-FORWARD(lang, jstring)
-FORWARD(publisher, jstring)
-FORWARD(creator, jstring)
-FORWARD(maxSize, jlong)
-FORWARD(query, jstring)
-
-

src/wrapper/java/kiwixicu.cpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIICU.h"
-
-#include <iostream>
-#include <string>
-
-#include "unicode/putil.h"
-
-#include "utils.h"
-
-std::mutex globalLock;
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
-    JNIEnv* env, jclass kclass, jstring dirStr)
-{
-  std::string cPath = jni2c(dirStr, env);
-
-  Lock l;
-  try {
-    u_setDataDirectory(cPath.c_str());
-  } catch (...) {
-    std::cerr << "Unable to set data directory " << cPath << std::endl;
-  }
-}

src/wrapper/java/kiwixreader.cpp

@@ -1,561 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <exception>
-#include "org_kiwix_kiwixlib_JNIKiwixReader.h"
-
-#include "tools/base64.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
-    JNIEnv* env, jobject obj, jstring filename)
-{
-  std::string cPath = jni2c(filename, env);
-
-  LOG("Attempting to create reader with: %s", cPath.c_str());
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(cPath);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-namespace
-{
-
-int jni2fd(const jobject& fdObj, JNIEnv* env)
-{
-  jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
-  jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I");
-  if ( field_fd == NULL )
-  {
-    env->ExceptionClear();
-    // Under Android the (private) 'fd' field of java.io.FileDescriptor has been
-    // renamed to 'descriptor'. See, for example,
-    // https://android.googlesource.com/platform/libcore/+/refs/tags/android-8.1.0_r1/ojluni/src/main/java/java/io/FileDescriptor.java#55
-    field_fd = env->GetFieldID(class_fdesc, "descriptor", "I");
-  }
-  return env->GetIntField(fdObj, field_fd);
-}
-
-} // unnamed namespace
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderByFD(
-    JNIEnv* env, jobject obj, jobject fdObj)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderByFD() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderEmbedded(
-    JNIEnv* env, jobject obj, jobject fdObj, jlong offset, jlong size)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd, offset, size);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderEmbedded() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Reader>::dispose(env, obj);
-}
-
-#define READER (Handle<kiwix::Reader>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
-{
-  jstring url;
-
-  try {
-    std::string cUrl = READER->getMainPage().getPath();
-    url = c2jni(cUrl, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM main page");
-    LOG(e.what());
-    url = NULL;
-  }
-  return url;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
-{
-  jstring id;
-
-  try {
-    std::string cId = READER->getId();
-    id = c2jni(cId, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-    id = NULL;
-  }
-
-  return id;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
-{
-  jint size = 0;
-
-  try {
-    int cSize = READER->getFileSize();
-    size = c2jni(cSize, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return size;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
-{
-  jstring creator;
-
-  try {
-    std::string cCreator = READER->getCreator();
-    creator = c2jni(cCreator, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM creator");
-    LOG(e.what());
-    creator = NULL;
-  }
-
-  return creator;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
-{
-  jstring publisher;
-
-  try {
-    std::string cPublisher = READER->getPublisher();
-    publisher = c2jni(cPublisher, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM publish");
-    LOG(e.what());
-    publisher = NULL;
-  }
-  return publisher;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
-{
-  jstring name;
-
-  try {
-    std::string cName = READER->getName();
-    name = c2jni(cName, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM name");
-    LOG(e.what());
-    name = NULL;
-  }
-  return name;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
-{
-  jstring favicon;
-
-  try {
-    std::string cContent;
-    std::string cMime;
-    READER->getFavicon(cContent, cMime);
-    favicon = c2jni(
-        base64_encode(cContent),
-        env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM favicon");
-    LOG(e.what());
-    favicon = NULL;
-  }
-  return favicon;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
-{
-  jstring date;
-
-  try {
-    std::string cDate = READER->getDate();
-    date = c2jni(cDate, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM date");
-    LOG(e.what());
-    date = NULL;
-  }
-  return date;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
-{
-  jstring language;
-
-  try {
-    std::string cLanguage = READER->getLanguage();
-    language = c2jni(cLanguage, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM language");
-    LOG(e.what());
-    language = NULL;
-  }
-
-  return language;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  jstring mimeType;
-
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    auto cMimeType = entry.getMimetype();
-    mimeType = c2jni(cMimeType, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
-    LOG(e.what());
-    mimeType = NULL;
-  }
-  return mimeType;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
-     JNIEnv* env, jobject obj, jstring url)
-{
-  jstring finalUrl;
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    finalUrl = c2jni(entry.getPath(), env);
-  } catch (std::exception& e) {
-    finalUrl = c2jni(std::string(), env);
-  }
-  return finalUrl;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
-    JNIEnv* env, jobject obj, jobject url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
-{
-  /* Default values */
-  setStringObjValue("", titleObj, env);
-  setStringObjValue("", mimeTypeObj, env);
-  setIntObjValue(0, sizeObj, env);
-  jbyteArray data = env->NewByteArray(0);
-
-  /* Retrieve the content */
-  std::string cUrl = getStringObjValue(url, env);
-  unsigned int cSize = 0;
-
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    bool isRedirect = entry.isRedirect();
-    entry = entry.getFinalEntry();
-    cSize = entry.getSize();
-    setIntObjValue(cSize, sizeObj, env);
-    setStringObjValue(entry.getMimetype(), mimeTypeObj, env);
-    setStringObjValue(entry.getTitle(), titleObj, env);
-    if (isRedirect) {
-      setStringObjValue(entry.getPath(), url, env);
-    } else {
-      data = env->NewByteArray(cSize);
-      env->SetByteArrayRegion(
-          data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get content for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-
-  return data;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPart(
-    JNIEnv* env, jobject obj, jstring url, jint offset, jint len, jobject sizeObj)
-{
-  jbyteArray data = env->NewByteArray(0);
-  setIntObjValue(0, sizeObj, env);
-
-  /* Default values */
-  /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset, env);
-  unsigned int cLen = jni2c(len, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-
-    if (cLen == 0) {
-      setIntObjValue(entry.getSize(), sizeObj, env);
-    } else if (cOffset+cLen < entry.getSize()) {
-      auto blob = entry.getBlob(cOffset, cLen);
-      data = env->NewByteArray(cLen);
-      env->SetByteArrayRegion(
-          data, 0, cLen, reinterpret_cast<const jbyte*>(blob.data()));
-      setIntObjValue(cLen, sizeObj, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    LOG(e.what());
-  }
-  return data;
-}
-
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    return c2jni(entry.getSize(), env);
-  } catch(std::exception& e) {
-    LOG("Unable to get size for url : %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return c2jni(0, env);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
-    JNIEnv* env, jobject obj, jstring url)
-{
-   jclass daiClass = env->FindClass("org/kiwix/kiwixlib/DirectAccessInfo");
-   jmethodID daiInitMethod = env->GetMethodID(daiClass, "<init>", "()V");
-   jobject dai = env->NewObject(daiClass, daiInitMethod);
-   setDaiObjValue("", 0, dai, env);
-
-   std::string cUrl = jni2c(url, env);
-   try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    auto part_info = entry.getDirectAccessInfo();
-    setDaiObjValue(part_info.first, part_info.second, dai, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return dai;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
-                                                         jobject obj,
-                                                         jstring prefix,
-                                                         jint count)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count, env);
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
-                                                         jobject obj,
-                                                         jobject titleObj,
-                                                         jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-  std::string cUrl;
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->getNextSuggestion(cTitle, cUrl)) {
-      setStringObjValue(cTitle, titleObj, env);
-      setStringObjValue(cUrl, urlObj, env);
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get next suggestion");
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
-                                                           jobject obj,
-                                                           jstring title,
-                                                           jobject urlObj)
-{
-  std::string cTitle = jni2c(title, env);
-
-  try {
-    auto entry = READER->getEntryFromTitle(cTitle);
-    entry = entry.getFinalEntry();
-    setStringObjValue(entry.getPath(), urlObj, env);
-    return JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get url for title %s: ", cTitle.c_str());
-    LOG(e.what());
-  }
-
-  return JNI_FALSE;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
-    JNIEnv* env, jobject obj)
-{
-  jstring title;
-
-  try {
-    std::string cTitle = READER->getTitle();
-    title = c2jni(cTitle, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim title");
-    LOG(e.what());
-    title = NULL;
-  }
-  return title;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
-{
-  jstring description;
-
-  try {
-    std::string cDescription = READER->getDescription();
-    description = c2jni(cDescription, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim description");
-    LOG(e.what());
-    description = NULL;
-  }
-  return description;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
-{
-  jint articleCount = 0;
-  try {
-    auto cArticleCount = READER->getArticleCount();
-    articleCount = c2jni(cArticleCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get article count.");
-    LOG(e.what());
-  }
-  return articleCount;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
-{
-  jint mediaCount = 0;
-  try {
-    auto cMediaCount = READER->getMediaCount();
-    mediaCount = c2jni(cMediaCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get media count.");
-    LOG(e.what());
-  }
-  return mediaCount;
-}
-
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
-    JNIEnv* env, jobject obj, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cUrl;
-
-  try {
-    std::string cUrl = READER->getRandomPage().getPath();
-    setStringObjValue(cUrl, urlObj, env);
-    retVal = JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get random page");
-    LOG(e.what());
-  }
-  return retVal;
-}

src/wrapper/java/kiwixsearcher.cpp

@@ -1,123 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"
-
-#include "reader.h"
-#include "searcher.h"
-#include "utils.h"
-
-#define SEARCHER (Handle<kiwix::Searcher>::getHandle(env, obj))
-#define RESULT (Handle<kiwix::Result>::getHandle(env, obj))
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Searcher>::dispose(env, obj);
-}
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNativeHandle(JNIEnv* env,
-                                                          jobject obj)
-{
-  kiwix::Searcher* searcher = new kiwix::Searcher();
-  return reinterpret_cast<jlong>(new Handle<kiwix::Searcher>(searcher));
-}
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
-    JNIEnv* env, jobject obj, jobject reader)
-{
-  auto searcher = SEARCHER;
-
-  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)));
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
-    JNIEnv* env, jobject obj, jstring query, jint count)
-{
-  std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count, env);
-
-  SEARCHER->search(cquery, 0, ccount);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNextResult(JNIEnv* env,
-                                                         jobject obj)
-{
-  jobject result = nullptr;
-
-  kiwix::Result* cresult = SEARCHER->getNextResult();
-  if (cresult != nullptr) {
-    jclass resultclass
-        = env->FindClass("org/kiwix/kiwixlib/JNIKiwixSearcher$Result");
-    jmethodID ctor = env->GetMethodID(
-        resultclass, "<init>", "(Lorg/kiwix/kiwixlib/JNIKiwixSearcher;JLorg/kiwix/kiwixlib/JNIKiwixSearcher;)V");
-    result = env->NewObject(resultclass, ctor, obj, reinterpret_cast<jlong>(new Handle<kiwix::Result>(cresult)), obj);
-  }
-  return result;
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_dispose(
-    JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Result>::dispose(env, obj);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getUrl(JNIEnv* env,
-                                                        jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_url(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getTitle(JNIEnv* env,
-                                                          jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_title(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getSnippet(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_snippet(), env);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getContent(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_content(), env);
-}

src/wrapper/java/kiwixserver.cpp

@@ -1,103 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwixServer.h"
-
-#include "tools/base64.h"
-#include "server.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
-    JNIEnv* env, jobject obj, jobject jLibrary)
-{
-  LOG("Attempting to create server");
-  Lock l;
-  try {
-    auto library = getPtr<kiwix::Library>(env, jLibrary);
-    kiwix::Server* server = new kiwix::Server(library);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
-  } catch (std::exception& e) {
-    LOG("Error creating the server");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Server>::dispose(env, obj);
-}
-
-#define SERVER (Handle<kiwix::Server>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setRoot(JNIEnv* env, jobject obj, jstring jRoot)
-{
-  std::string root = jni2c(jRoot, env);
-  SERVER->setRoot(root);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setAddress(JNIEnv* env, jobject obj, jstring jAddress)
-{
-  std::string address = jni2c(jAddress, env);
-  SERVER->setAddress(address);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setPort(JNIEnv* env, jobject obj, int port)
-{
-  SERVER->setPort(port);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setNbThreads(JNIEnv* env, jobject obj, int threads)
-{
-  SERVER->setNbThreads(threads);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboolean withTaskbar, jboolean withLibraryButton)
-{
-  SERVER->setTaskbar(withTaskbar, withLibraryButton);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
-{
-  SERVER->setBlockExternalLinks(blockExternalLinks);
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
-{
-  return SERVER->start();
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_stop(JNIEnv* env, jobject obj)
-{
-  SERVER->stop();
-}

src/wrapper/java/library.cpp

@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Library.h"
-
-#include "library.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_allocate(
-    JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Library>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
-
-/* Kiwix library functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Library_addBook(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    kiwix::Reader reader(cPath);
-    kiwix::Book book;
-    book.update(reader);
-    return LIBRARY->addBook(book);
-  } catch (std::exception& e) {
-    LOG("Unable to add the book");
-    LOG(e.what()); }
-  return false;
-}
-
-METHOD(jobject, Library, getBookById, jstring id) {
-  auto cId = jni2c(id, env);
-  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
-  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
-  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
-  jobject book = env->NewObject(cls, constructorId);
-  setPtr(env, book, cBook);
-  return book;
-}
-
-METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
-  return LIBRARY->getBookCount(localBooks, remoteBooks);
-}
-
-METHOD0(jobjectArray, Library, getBooksIds) {
-  return c2jni(LIBRARY->getBooksIds(), env);
-}
-
-METHOD(jobjectArray, Library, filter, jobject filterObj) {
-  auto filter = getPtr<kiwix::Filter>(env, filterObj);
-  return c2jni(LIBRARY->filter(*filter), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksLanguages) {
-  return c2jni(LIBRARY->getBooksLanguages(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksCreators) {
-  return c2jni(LIBRARY->getBooksCreators(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksPublisher) {
-  return c2jni(LIBRARY->getBooksPublishers(), env);
-}
-

src/wrapper/java/manager.cpp

@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Manager.h"
-
-#include "manager.h"
-#include "utils.h"
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_allocate(
-  JNIEnv* env, jobject thisObj, jobject libraryObj)
-{
-  auto lib = getPtr<kiwix::Library>(env, libraryObj);
-  allocate<kiwix::Manager>(env, thisObj, lib);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Manager>(env, thisObj);
-}
-
-#define MANAGER (getPtr<kiwix::Manager>(env, thisObj))
-
-/* Kiwix manager functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get readFile");
-    LOG(e.what());
-  }
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readXml(
-  JNIEnv* env, jobject thisObj, jstring content, jstring libraryPath)
-{
-  auto cContent = jni2c(content, env);
-  auto cPath = jni2c(libraryPath, env);
-
-  try {
-    return MANAGER->readXml(cContent, false, cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readOpds(
-  JNIEnv* env, jobject thisObj, jstring content, jstring urlHost)
-{
-  auto cContent = jni2c(content, env);
-  auto cUrl = jni2c(urlHost, env);
-
-  try {
-    return MANAGER->readOpds(cContent, cUrl);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readBookmarkFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readBookmarkFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_Manager_addBookFromPath(
-  JNIEnv* env, jobject thisObj,
-  jstring pathToOpen, jstring pathToSave, jstring url, jboolean checkMetaData)
-{
-  auto cPathToOpen = jni2c(pathToOpen, env);
-  auto cPathToSave = jni2c(pathToSave, env);
-  auto cUrl = jni2c(url, env);
-  jstring id = NULL;
-
-  try {
-    auto cId = MANAGER->addBookFromPathAndGetId(cPathToOpen, cPathToSave, cUrl, checkMetaData);
-    if ( !cId.empty() ) {
-      id = c2jni(cId, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return id;
-}

src/wrapper/java/meson.build

@@ -1,55 +0,0 @@
-
-java_sources = files([
-  'org/kiwix/kiwixlib/JNIICU.java',
-  'org/kiwix/kiwixlib/Book.java',
-  'org/kiwix/kiwixlib/JNIKiwixReader.java',
-  'org/kiwix/kiwixlib/Library.java',
-  'org/kiwix/kiwixlib/Manager.java',
-  'org/kiwix/kiwixlib/Filter.java',
-  'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-  'org/kiwix/kiwixlib/JNIKiwixServer.java',
-  'org/kiwix/kiwixlib/JNIKiwixInt.java',
-  'org/kiwix/kiwixlib/JNIKiwixString.java',
-  'org/kiwix/kiwixlib/JNIKiwixBool.java',
-  'org/kiwix/kiwixlib/JNIKiwixException.java',
-  'org/kiwix/kiwixlib/DirectAccessInfo.java'
-])
-
-kiwix_jni = custom_target('jni',
-  input: java_sources,
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_Book.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_Library.h',
-           'org_kiwix_kiwixlib_Manager.h',
-           'org_kiwix_kiwixlib_Filter.h',
-           'org_kiwix_kiwixlib_JNIKiwixServer.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-jni_sources = files([
-     'kiwixicu.cpp',
-     'book.cpp',
-     'kiwixreader.cpp',
-     'library.cpp',
-     'manager.cpp',
-     'filter.cpp',
-     'kiwixsearcher.cpp',
-     'kiwixserver.cpp',
-])
-
-kiwix_sources += jni_sources + [kiwix_jni]
-
-if 'java' in wrapper
-  kiwix_jar = jar('kiwixlib', java_sources)
-  #junit_jar = files('org/kiwix/testing/junit-4.13.jar')
-  #test_jar = jar('testing', 'org/kiwix/testing/test.java',
-  #               link_with: [kiwix_jar, junit_jar])
-  #test('javatest', test_jar)
-endif
-
-install_subdir('org', install_dir: 'kiwix-lib/java', exclude_directories: ['kiwix/testing'])
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -1,48 +0,0 @@
-
-package org.kiwix.kiwixlib;
-
-public class Book
-{
-  public Book() { allocate(); }
-
-
-  public native void update(Book book);
-  public native void update(JNIKiwixReader reader);
-
-  @Override
-  protected void finalize() { dispose();  }
-
-  public native String getId();
-  public native String getPath();
-  public native boolean isPathValid();
-  public native String getTitle();
-  public native String getDescription();
-  public native String getLanguage();
-  public native String getCreator();
-  public native String getPublisher();
-  public native String getDate();
-  public native String getUrl();
-  public native String getName();
-  public native String getFlavour();
-  public native String getCategory();
-  public native String getTags();
-  /**
-   * Return the value associated to the tag tagName
-   *
-   * @param tagName the tag name to search for.
-   * @return The value of the tag. If the tag is not found, return empty string.
-   */
-  public native String getTagStr(String tagName);
-
-  public native long getArticleCount();
-  public native long getMediaCount();
-  public native long getSize();
-
-  public native String getFavicon();
-  public native String getFaviconUrl();
-  public native String getFaviconMimeType();
-
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class DirectAccessInfo
-{
-  public String filename;
-  public long offset;
-}

src/wrapper/java/org/kiwix/kiwixlib/Filter.java

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class Filter
-{
-
-  public native Filter local(boolean accept);
-  public native Filter remote(boolean accept);
-  public native Filter valid(boolean accept);
-  public native Filter acceptTags(String[] tags);
-  public native Filter rejectTags(String[] tags);
-  public native Filter lang(String lang);
-  public native Filter publisher(String publisher);
-  public native Filter creator(String creator);
-  public native Filter maxSize(long size);
-  public native Filter query(String query);
-
-
-  public Filter() { allocate(); }
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIICU.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIICU
-{
-  static public native void setDataDirectory(String icuDataDir);
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwix.java

@@ -1,36 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import android.content.Context;
-import com.getkeepsafe.relinker.ReLinker;
-import org.kiwix.kiwixlib.JNIICU;
-
-public class JNIKiwix
-{
-  public JNIKiwix(final Context context){
-    ReLinker.loadLibrary(context, "kiwix");
-  }
-
-  public void setDataDirectory(String icuDataDir) {
-    JNIICU.setDataDirectory(icuDataDir);
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixBool.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixBool
-{
-  public boolean value;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixException.java

@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixException extends Exception
-{
-  public JNIKiwixException(String message) {
-    super(message);
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixInt.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixInt
-{
-  public int value;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -1,182 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.JNIKiwixString;
-import org.kiwix.kiwixlib.JNIKiwixInt;
-import org.kiwix.kiwixlib.JNIKiwixSearcher;
-import org.kiwix.kiwixlib.DirectAccessInfo;
-import java.io.FileDescriptor;
-
-public class JNIKiwixReader
-{
-  public native String getMainPage();
-
-  public native String getTitle();
-
-  public native String getId();
-
-  public native String getLanguage();
-
-  public native String getMimeType(String url);
-
-  /**
-   * Check if a url exists and is a redirect or not.
-   *
-   * Return an empty string if the url doesn't exist in the reader.
-   * Return the url of the "final" entry.
-   * - equal to the input url if the entry is not a redirection.
-   * - different if the url is a redirection (and the webview should redirect to it).
-   */
-  public native String checkUrl(String url);
-
-  /**
-   * Get the content of a article.
-   *
-   * Return a byte array of the content of the article.
-   * Set the title, mimeType to the title and mimeType of the article.
-   * Set the size to the size of the returned array.
-   *
-   * If the entry doesn't exist :
-   *  - return a empty byte array
-   *  - set all arguments (except url) to empty/0.
-   * If the entry exist but is a redirection :
-   *  - return an empty byte array
-   *  - set all arguments (including url) to information of the targeted article.
-   */
-  public native byte[] getContent(JNIKiwixString url,
-                                  JNIKiwixString title,
-                                  JNIKiwixString mimeType,
-                                  JNIKiwixInt size);
-
-  /**
-   * getContentPart.
-   *
-   * Get only a part of the content of the article.
-   * Return a byte array of `len` size starting from offset `offset`.
-   * Set `size` to the number of bytes read
-   * (`len` if everything is ok, 0 in case of error).
-   * If `len` == 0, no bytes are read but `size` is set to the total size of the
-   * article.
-   */
-  public native byte[] getContentPart(String url,
-                                      int offest,
-                                      int len,
-                                      JNIKiwixInt size);
-
-  /**
-   *
-   * Get the size of an article.
-   *
-   * @param   url The url of the article.
-   * @return  The size of the final (redirections are resolved) article (in byte).
-   *          Return 0 if the article is not found.
-   */
-  public native long getArticleSize(String url);
-
-  /**
-   * getDirectAccessInformation.
-   *
-   * Return information giving where the content is located in the zim file.
-   *
-   * Some contents (binary content) are stored uncompressed in the zim file.
-   * Knowing this information, it could be interesting to directly open
-   * the zim file (or zim part) and directly read the content from it (and so
-   * bypassing the libzim).
-   *
-   * Return a `DirectAccessInfo` (filename, offset) where the content is located.
-   *
-   * If the content cannot be directly accessed (content is compressed or zim
-   * file is cut in the middle of the content), the filename is an empty string
-   * and offset is zero.
-   */
-  public native DirectAccessInfo getDirectAccessInformation(String url);
-
-  public native boolean searchSuggestions(String prefix, int count);
-
-  public native boolean getNextSuggestion(JNIKiwixString title, JNIKiwixString url);
-
-  public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
-
-  public native String getDescription();
-
-  public native String getDate();
-
-  public native String getFavicon();
-
-  public native String getCreator();
-
-  public native String getPublisher();
-
-  public native String getName();
-
-  public native int getFileSize();
-
-  public native int getArticleCount();
-
-  public native int getMediaCount();
-
-  public native boolean getRandomPage(JNIKiwixString url);
-
-  public JNIKiwixSearcher search(String query, int count)
-  {
-    JNIKiwixSearcher searcher = new JNIKiwixSearcher();
-    searcher.addKiwixReader(this);
-    searcher.search(query, count);
-    return searcher;
-  }
-
-  public JNIKiwixReader(String filename) throws JNIKiwixException
-  {
-    nativeHandle = getNativeReader(filename);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException("Cannot open zimfile "+filename);
-    }
-  }
-
-  public JNIKiwixReader(FileDescriptor fd) throws JNIKiwixException
-  {
-    nativeHandle = getNativeReaderByFD(fd);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException("Cannot open zimfile by fd "+fd.toString());
-    }
-  }
-
-  public JNIKiwixReader(FileDescriptor fd, long offset, long size)
-      throws JNIKiwixException
-  {
-    nativeHandle = getNativeReaderEmbedded(fd, offset, size);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException(String.format("Cannot open embedded zimfile (fd=%s, offset=%d, size=%d)", fd, offset, size));
-    }
-  }
-
-  public JNIKiwixReader() {
-
-  }
-  public native void dispose();
-
-  private native long getNativeReader(String filename);
-  private native long getNativeReaderByFD(FileDescriptor fd);
-  private native long getNativeReaderEmbedded(FileDescriptor fd, long offset, long size);
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixSearcher.java

@@ -1,67 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixReader;
-import java.util.Vector;
-
-public class JNIKiwixSearcher
-{
-  public class Result
-  {
-    private long nativeHandle;
-    private JNIKiwixSearcher searcher;
-    public Result(long handle, JNIKiwixSearcher _searcher)
-    {
-      nativeHandle = handle;
-      searcher = _searcher;
-    }
-    public native String getUrl();
-    public native String getTitle();
-    public native String getContent();
-    public native String getSnippet();
-    public native void dispose();
-  }
-
-  public JNIKiwixSearcher()
-  {
-    nativeHandle = getNativeHandle();
-    usedReaders = new Vector();
-  }
-  public native void dispose();
-
-  private native long getNativeHandle();
-  private long nativeHandle;
-  private Vector usedReaders;
-
-  public native void addReader(JNIKiwixReader reader);
-  public void addKiwixReader(JNIKiwixReader reader)
-  {
-    addReader(reader);
-    usedReaders.addElement(reader);
-  };
-
-  public native void search(String query, int count);
-
-  public native Result getNextResult();
-  public native boolean hasMoreResult();
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixServer.java

@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.Library;
-
-public class JNIKiwixServer
-{
-  public native void setRoot(String root);
-
-  public native void setAddress(String address);
-
-  public native void setPort(int port);
-
-  public native void setNbThreads(int nbTreads);
-
-  public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
-
-  public native void setBlockExternalLinks(boolean blockExternalLinks);
-
-  public native boolean start();
-
-  public native void stop();
-
-  public JNIKiwixServer(Library library)
-  {
-    nativeHandle = getNativeServer(library);
-  }
-
-  private native long getNativeServer(Library library);
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/Library.java

@@ -1,49 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.Book;
-import org.kiwix.kiwixlib.JNIKiwixException;
-
-public class Library
-{
-  public native boolean addBook(String path) throws JNIKiwixException;
-
-  public native Book getBookById(String id);
-  public native int  getBookCount(boolean localBooks, boolean remoteBooks);
-
-  public native String[] getBooksIds();
-  public native String[] filter(Filter filter);
-
-  public native String[] getBooksLanguages();
-  public native String[] getBooksCreators();
-  public native String[] getBooksPublishers();
-
-  public Library()
-  {
-    allocate();
-  }
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/Manager.java

@@ -1,90 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.Library;
-
-public class Manager
-{
-  /**
-   * Read a `library.xml` file and add books in the library.
-   *
-   * @param path The (utf8) path to the `library.xml` file.
-   * @return True if the file has been properly parsed.
-   */
-  public native boolean readFile(String path);
-
-  /**
-   * Load a library content stored in a string (at `library.xml` format).
-   *
-   * @param content The content corresponding of the library xml.
-   * @param libraryPath The library path (used to resolve relative paths)
-   * @return True if the content has been properly parsed.
-   */
-  public native boolean readXml(String content, String libraryPath);
-
-  /**
-   * Load a library content stored in a string (at OPDS stream format)
-   *
-   * @param content the content of the OPDS stream.
-   * @param urlHost the url of the stream (used to resolve relative url)
-   * @return True if the content has been properly parsed.
-   */
-  public native boolean readOpds(String content, String urlHost);
-
-  /**
-   * Load a bookmark file
-   *
-   * @param path The path of the file to read.
-   * @return True if the content has been properly parsed
-   */
-  public native boolean readBookmarkFile(String path);
-
-  /**
-   * Add a book to the library.
-   *
-   * @param pathToOpen    The path of the zim file to add.
-   * @param pathToSave    The path to store in the library in place of
-   *                      pathToOpen.
-   * @param url           The url of the book to store in the library
-   *                      (useful for kiiwix-serve catalog)
-   * @param checkMetaData Tell if we check metadata before adding a book to the
-   *                      library.
-   * @return The id of te book if the book has been added to the library.
-   *         Empty string if not.
-   */
-  public native String addBookFromPath(String pathToOpen,
-                                       String pathToSave,
-                                       String url,
-                                       boolean checkMetaData);
-
-  public Manager(Library library) {
-    allocate(library);
-    _library = library;
-  }
-
-  private Library _library;
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate(Library library);
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/testing/catalog.xml

@@ -1,19 +0,0 @@
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
-  <id>00000000-0000-0000-0000-000000000000</id>
-  <entry>
-    <title>Test ZIM file</title>
-    <id>urn:uuid:86c91e51-55bf-8882-464e-072aca37a3e8</id>
-    <icon>/meta?name=favicon&amp;content=small</icon>
-    <updated>2020-11-27:00::00:Z</updated>
-    <language>en</language>
-    <summary>This is a ZIM file used in libzim unit-tests</summary>
-    <tags>unit;test</tags>
-    <link type="text/html" href="/small" />
-    <author>
-      <name>Kiwix</name>
-    </author>
-    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="http://localhost/small.zim" length="78982" />
-    <link rel="http://opds-spec.org/image/thumbnail" type="image/png" href="/meta?name=favicon&amp;content=small" />
-  </entry>
-</feed>
-

src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/bash
-
-# This script compiles and runs the unit test to test the java wrapper.
-# This is not integrated in meson because ... this is not so easy.
-
-die()
-{
-  echo >&2 "!!! ERROR: $*"
-  exit 1
-}
-
-
-KIWIX_LIB_JAR=$1
-if [ -z $KIWIX_LIB_JAR ]
-then
-  die "You must give the path to the kiwixlib.jar as first argument"
-fi
-
-KIWIX_LIB_DIR=$2
-if [ -z $KIWIX_LIB_DIR ]
-then
-  die "You must give the path to directory containing libkiwix.so as second argument"
-fi
-
-KIWIX_LIB_JAR=$(readlink -f "$KIWIX_LIB_JAR")
-KIWIX_LIB_DIR=$(readlink -f "$KIWIX_LIB_DIR")
-TEST_SOURCE_DIR=$(dirname "$(readlink -f $0)")
-
-cd "$TEST_SOURCE_DIR"
-
-javac -g -d . -s . -cp "junit-4.13.jar:$KIWIX_LIB_JAR" test.java \
-  || die "Compilation failed"
-
-java -Djava.library.path="$KIWIX_LIB_DIR" \
-     -cp "junit-4.13.jar:hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:." \
-     org.junit.runner.JUnitCore test \
-  || die "Unit test failed"

src/wrapper/java/org/kiwix/testing/create_test_zimfiles

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-die()
-{
-  echo >&2 "!!! ERROR: $*"
-  exit 1
-}
-
-cd "$(dirname "$0")"
-rm -f small.zim
-zimwriterfs --withoutFTIndex \
-            -w main.html \
-            -f favicon.png \
-            -l en \
-            -t "Test ZIM file" \
-            -d "N/A" \
-            -c "N/A" \
-            -p "N/A" \
-            small_zimfile_data \
-            small.zim \
-&& echo 'small.zim was successfully created' \
-|| die 'Failed to create small.zim'
-
-printf "BEGINZIM" > small.zim.embedded \
-&& cat small.zim >> small.zim.embedded \
-&& printf "ENDZIM" >> small.zim.embedded \
-&& echo 'small.zim.embedded was successfully created' \
-|| die 'Failed to create small.zim.embedded'