fixup! Fixed external links in the viewer iframe

Before this fix clicking an external link in the viewer iframe had no
effect (other than an error being reported in the browser dev tools
console) because the attempt to navigate the top browser context was
suppressed due to sandboxing.

.github/move.yml

@@ -1,27 +0,0 @@
-# Configuration for Move Issues - https://github.com/dessant/move-issues
-
-# Delete the command comment when it contains no other content
-deleteCommand: true
-
-# Close the source issue after moving
-closeSourceIssue: true
-
-# Lock the source issue after moving
-lockSourceIssue: false
-
-# Mention issue and comment authors
-mentionAuthors: true
-
-# Preserve mentions in the issue content
-keepContentMentions: true
-
-# Move labels that also exist on the target repository
-moveLabels: true
-
-# Set custom aliases for targets
-# aliases:
-#   r: repo
-#   or: owner/repo
-
-# Repository to extend settings from
-# _extends: repo

.github/workflows/ci.yml

@@ -1,44 +1,44 @@
 name: CI
 
-on: [push]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
-  Macos:
-    runs-on: macos-latest
+  macOS:
+    runs-on: macos-12
+    env:
+      HOME: /Users/runner
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v1
-      - name: Setup python 3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
+      - name: Retrieve source code
+        uses: actions/checkout@v3
+
       - name: Install packages
         run: |
           brew update
-          brew install gcovr pkg-config ninja
-      - name: Install python modules
-        run: pip3 install meson==0.49.2 pytest
-      - name: Install deps
-        shell: bash
+          brew install pkg-config ninja meson
+
+      - name: Install dependencies
+        env:
+          ARCHIVE_NAME: deps2_osx_native_dyn_libkiwix.tar.xz
         run: |
-          ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
-          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
-      - name: Compile
-        shell: bash
+          wget -O- https://tmp.kiwix.org/ci/${{env.ARCHIVE_NAME}} | tar -xJ -C ${{env.HOME}}
+
+      - name: Compile source code
+        env:
+          PKG_CONFIG_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          CPPFLAGS: -I${{env.HOME}}/BUILD_native_dyn/INSTALL/include
         run: |
-          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
-          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
           meson . build --default-library=shared -Db_coverage=true
-          cd build
-          ninja
-      - name: Test
-        shell: bash
-        run: |
-          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
-          cd build
-          meson test --verbose
+          ninja -C build
+
+      - name: Test libkiwix
         env:
           SKIP_BIG_MEMORY_TEST: 1
+          LD_LIBRARY_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib:${{env.HOME}}/BUILD_native_dyn/INSTALL/lib64
+        run: meson test -C build --verbose
 
   Linux:
     strategy:
@@ -47,7 +47,6 @@ jobs:
         name:
           - native_static
           - native_dyn
-          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
@@ -55,42 +54,34 @@ jobs:
         include:
           - name: native_static
             target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: native_dyn
-            target: native_dyn
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - name: native_dyn_bionic
             target: native_dyn
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: android_arm
             target: android_arm
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/arm-linux-androideabi'
           - name: android_arm64
             target: android_arm64
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/aarch64-linux-android'
           - name: win32_static
             target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - name: win32_dyn
             target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
     env:
       HOME: /home/runner
     runs-on: ubuntu-latest
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:36"
     steps:
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
     - name: Checkout code
       shell: python
       run: |
@@ -100,7 +91,7 @@ jobs:
           'git', 'clone',
           'https://github.com/${{github.repository}}',
           '--depth=1',
-          '--branch', '${{steps.extract_branch.outputs.branch}}'
+          '--branch', '${{ github.head_ref || github.ref_name }}'
         ]
         check_call(command, cwd=environ['HOME'])
     - name: Install deps
@@ -123,7 +114,7 @@ jobs:
           MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
         fi
         if [[ "${{matrix.target}}" =~ android_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+          MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
         fi
         cd $HOME/libkiwix
         meson . build ${MESON_OPTION}
@@ -142,13 +133,9 @@ jobs:
       env:
         LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
         SKIP_BIG_MEMORY_TEST: 1
+
     - name: Publish coverage
-      shell: bash
-      run: |
-        cd $HOME/libkiwix
-        curl https://codecov.io/bash -o codecov.sh
-        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
-        rm codecov.sh
-      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      if: startsWith(matrix.target, 'native_')
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -1,5 +1,10 @@
 name: Packages
-on: [push, pull_request]
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   build-deb:
@@ -8,12 +13,12 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-kinetic
           - ubuntu-jammy
-          - ubuntu-impish
           - ubuntu-focal
           - ubuntu-bionic
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Determine which PPA we should upload to
       - name: PPA
@@ -34,18 +39,18 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
-        if: matrix.distro == 'ubuntu-jammy'
-        name: Build package for ubuntu-jammy
-        id: build-ubuntu-jammy
+      - uses: legoktm/gh-action-build-deb@ubuntu-kinetic
+        if: matrix.distro == 'ubuntu-kinetic'
+        name: Build package for ubuntu-kinetic
+        id: build-ubuntu-kinetic
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
@@ -66,15 +71,15 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: Packages for ${{ matrix.distro }}
           path: output
 
       - uses: legoktm/gh-action-dput@master
         name: Upload dev package
-        # Only upload on pushes to master
-        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/dev

ChangeLog

@@ -1,3 +1,61 @@
+libkiwix 12.0.0
+===============
+
+ * [API Break] Remove wrapper around libzim (@mgautierfr #789)
+ * Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
+ * Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
+ * Prevent search on multi language content (@veloman-yunkan #838)
+ * Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
+ * Catalog:
+  - Include tags in free text catalog search (@veloman-yunkan #802)
+  - Illustration's url is based on book's uuid (@veloman-yunkan #804)
+  - Cleanup of the opds-dumper (@veloman-yunkan #829)
+  - Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
+  - Make opds-dumper respect the namemapper (@mgautierfr #837)
+ * Server:
+  - Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
+  - Better http caching (@veloman-yunkan #833)
+  - Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
+  - Better redirection of main page (@veloman-yunkan #827)
+  - Remove jquery (@mgautierfr @juuz0 #796)
+  - Better Viewer of zim content :
+    . Introduce `/content` endpoints (@veloman-yunkan #806)
+    . Switch to iframe based content viewer (@veloman-yunkan #716)
+  - Optimised design of the welcome page:
+    . Alignement (@juuz0 @kelson42 #786)
+    . Exit download modal on pressing escape key (@juzz0 #800)
+    . Add favicon for different devices (@juzz0 #805)
+    . Fix auto hidding of the toolbar (@veloman-yunkan #821)
+    . Allow user to filter books by tags in the front page (@juuz0 #711)
+ * CI :
+   - Trigger CI on pull_request (@kelson42 #791)
+   - Drop Ubuntu Impish packaging (@legoktm #825)
+   - Add Ubuntu Kinetic packaging (@legoktm #801)
+ * Testing:
+   - Test ICULanguageInfo (@veloman-yunkan #795)
+   - Introduce fake `test` language to test i18n (@veloman-yunkan #848)
+ * Fix documentation (@kelson42 #816)
+ * Udpate translation (#787 #839 #847)
+
+
+libkiwix 11.0.0
+===============
+
+ * [server] Add support for internationalization (@veloman-yunkan #679)
+ * [server] Use gzip compression instead of deflat (mgautierfr #757)
+ * [server] Version the static resources. This allow better invalidating
+   browser cache when resources are changed (@veloman-yunkan #712)
+ * [server|front] Use integer to query the host for page length (@juuz0 #772)
+ * [server] Improve multizim search API:
+   - Improvement of the cache system
+   - Better API to select on which books to search in.
+   - SysAdmin is now able to limit the number of book we search in for a multizim search
+ * [server] Introduce a opensearch API for multizim fulltext search
+ * [wrapper] Remove java wrapper
+ * Testing:
+   - Testing of search result pages content (@veloman-yunkan #765)
+   - Better testing structure of xml search result (@veloman-yunkan #780)
+
 libkiwix 10.1.1
 ===============
 

README.md

@@ -7,9 +7,10 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
-[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
+[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
-[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -100,6 +101,33 @@ meson . build -Dwrapper=android -Dwerror=false
 ninja -C build
 ```
 
+Static files compilation
+------------------------
+
+Libkiwix has a few static files 'compiled' within the binary
+code. This is mostly Javascript/HTML/pictures necessary for the HTTP
+daemon.
+
+These static files are available in the `static` directory and are
+compiled by custom Python code available in this repository `scripts`
+directory. This happens automatically at compilation time without any
+additional command to run.
+
+To avoid HTTP caching issues, the URLs (to the static content) are
+appended with a `cacheid` parameter (this is called "cache
+busting"). This `cacheid` value derived from the
+[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
+static file. As a consequence, each time you change a static file, the
+corresponding `cacheid` value will change.
+
+To properly test this feature, this `cacheid` needs to be added
+manually to the automated tests and has to be commited. After
+modifying the needed static file, [run the automated
+tests](#Testing). They will fail, but the inspection of the testing
+log will give you the new `cacheid` value(s). Finally update
+`test/server.cpp` with the appropriate `cacheid` value(s) which have
+changed.
+
 Testing
 -------
 
@@ -123,7 +151,7 @@ where you want to install the libraries. After the installation
 succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
-------------
+--------------
 
 If you want to uninstall the Kiwix library:
 ```bash
@@ -133,6 +161,55 @@ ninja -C build uninstall
 Like for the installation, you might need to run the command as `root`
 (or using `sudo`).
 
+Custom Index Page
+-----------------
+
+to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
+(note - while using custom html file please mention all external links as absolute path.)
+
+to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
+
+To use JS provided by kiwix-serve you can use the following template to start with ->
+
+```
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title><-- Custom Tittle --></title>
+    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+  </head>
+  <body>
+  </body>
+</html>
+```
+
+- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
+- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
+- To add language select box add - `<select id="languageFilter"></select>` under body tag.
+- To add category select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add search box for books use following form -
+    ```
+        <form id='kiwixSearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
+        <input type="submit" class="kiwixButton" value="Search"/>
+        </form>
+    ```
+
+
+If you compile manually Libmicrohttpd, you might need to compile it
+without GNU TLS, a bug here will empeach further compilation
+otherwise.
+
+If the compilation still fails, you might need to get a more recent
+version of a dependency than the one packaged by your Linux
+distribution. Try then with a source tarball distributed by the
+problematic upstream project or even directly from the source code
+repository.
+
 Troubleshooting
 ---------------
 
@@ -155,63 +232,6 @@ cp ninja ../bin
 cd ..
 ```
 
-Custom Index Page
------------------
-
-to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
-(note - while using custom html file please mention all external links as absolute path.)
-
-to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
-
-To use JS provided by kiwix-serve you can use the following template to start with ->
-
-```
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width,initial-scale=1" />
-    <title><-- Custom Tittle --></title>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
-    ></script>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
-    ></script>
-    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
-    <script src="{{root}}/skin/iso6391To3.js"></script>
-    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
-  </head>
-  <body>
-  </body>
-</html>
-```
-
-- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
-- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
-- To add language select box add - `<select id="languageFilter"></select>` under body tag.
-- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
-- To add search box for books use following form -
-    ```
-        <form id='kiwixSearchForm'>
-        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
-        <input type="submit" class="searchButton" value="Search"/>
-        </form>
-    ```
-
-
-If you compile manually Libmicrohttpd, you might need to compile it
-without GNU TLS, a bug here will empeach further compilation
-otherwise.
-
-If the compilation still fails, you might need to get a more recent
-version of a dependency than the one packaged by your Linux
-distribution. Try then with a source tarball distributed by the
-problematic upstream project or even directly from the source code
-repository.
-
 License
 -------
 

android-kiwix-lib-publisher/.gitignore

@@ -1,13 +0,0 @@
-*.iml
-.gradle
-/local.properties
-/.idea/caches
-/.idea/libraries
-/.idea/modules.xml
-/.idea/workspace.xml
-/.idea/navEditor.xml
-/.idea/assetWizardSettings.xml
-.DS_Store
-/build
-/captures
-.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -1,25 +0,0 @@
-// Top-level build file where you can add configuration options common to all sub-projects/modules.
-
-buildscript {
-    repositories {
-        google()
-        jcenter()
-
-    }
-    dependencies {
-        classpath 'com.android.tools.build:gradle:3.4.1'
-        // NOTE: Do not place your application dependencies here; they belong
-        // in the individual module build.gradle files
-    }
-}
-
-allprojects {
-    repositories {
-        google()
-        jcenter()
-    }
-}
-
-task clean(type: Delete) {
-    delete rootProject.buildDir
-}

android-kiwix-lib-publisher/gradle.properties

@@ -1,15 +0,0 @@
-# Project-wide Gradle settings.
-# IDE (e.g. Android Studio) users:
-# Gradle settings configured through the IDE *will override*
-# any settings specified in this file.
-# For more details on how to configure your build environment visit
-# http://www.gradle.org/docs/current/userguide/build_environment.html
-# Specifies the JVM arguments used for the daemon process.
-# The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx1536m
-# When configured, Gradle will run in incubating parallel mode.
-# This option should only be used with decoupled projects. More details, visit
-# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
-# org.gradle.parallel=true
-# Kotlin code style for this project: "official" or "obsolete":
-kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +0,0 @@
-#Wed Jun 19 15:28:39 BST 2019
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -1,172 +0,0 @@
-#!/usr/bin/env sh
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
-exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -1,84 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -1 +0,0 @@
-/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -1,64 +0,0 @@
-apply plugin: 'com.android.library'
-apply plugin: 'maven'
-
-android {
-    compileSdkVersion 28
-    defaultConfig {
-        minSdkVersion 15
-        targetSdkVersion 28
-        versionCode 1
-        versionName "1.0"
-    }
-    buildTypes {
-        release {
-            minifyEnabled false
-            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
-        }
-    }
-}
-
-dependencies {
-    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
-}
-
-task writePom {
-  pom {
-    project {
-        groupId 'org.kiwix.kiwixlib'
-        artifactId 'kiwixlib'
-        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
-        packaging 'aar'
-        name 'kiwixlib'
-        url 'https://github.com/kiwix/libkiwix'
-        licenses {
-            license {
-                name 'GPLv3'
-                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
-            }
-        }
-      developers {
-          developer {
-              id 'kiwix'
-              name 'kiwix'
-              email 'contact@kiwix.org'
-          }
-      }
-      scm {
-          connection 'https://github.com/kiwix/libkiwix.git'
-          developerConnection 'https://github.com/kiwix/libkiwix.git'
-          url 'https://github.com/kiwix/libkiwix'
-      }
-    }
-  }.withXml {
-    def dependenciesNode = asNode().appendNode('dependencies')
-
-    //Iterate over the implementation dependencies, adding a <dependency> node for each
-    configurations.implementation.allDependencies.each {
-      def dependencyNode = dependenciesNode.appendNode('dependency')
-      dependencyNode.appendNode('groupId', it.group)
-      dependencyNode.appendNode('artifactId', it.name)
-      dependencyNode.appendNode('version', it.version)
-    }
-  }.writeTo("$buildDir/pom.xml")
-}
-

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -1,21 +0,0 @@
-# Add project specific ProGuard rules here.
-# You can control the set of applied configuration files using the
-# proguardFiles setting in build.gradle.
-#
-# For more details, see
-#   http://developer.android.com/guide/developing/tools/proguard.html
-
-# If your project uses WebView with JS, uncomment the following
-# and specify the fully qualified class name to the JavaScript interface
-# class:
-#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
-#   public *;
-#}
-
-# Uncomment this to preserve the line number information for
-# debugging stack traces.
-#-keepattributes SourceFile,LineNumberTable
-
-# If you keep the line number information, uncomment this to
-# hide the original source file name.
-#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,10 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="org.kiwix.kiwixlib">
-
-    <application
-        android:allowBackup="true"
-        android:supportsRtl="true">
-
-    </application>
-
-</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -1 +0,0 @@
-include ':kiwixLibAndroid'

debian/libkiwix-dev.manpages

@@ -1 +1,2 @@
 usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

docs/conf.py

@@ -19,7 +19,7 @@ import os
 
 project = 'libkiwix'
 copyright = '2022, libkiwix-team'
-author = 'libzim-team'
+author = 'libkiwix-team'
 
 
 # -- General configuration ---------------------------------------------------

docs/index.rst

@@ -3,7 +3,7 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to libzim's documentation!
+Welcome to libkiwix's documentation!
 ==================================
 
 .. toctree::

docs/usage.rst

@@ -7,11 +7,9 @@ Introduction
 libkiwix is written in C++. To use the library, you need the include files of libkiwix have
 to link against libzim.
 
-Errors are handled with exceptions. When something goes wrong, libzim throws an error,
+Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
 which is always derived from std::exception.
 
 All classes are defined in the namespace kiwix.
 
 libkiwix is a set of tools to manage zim files and provide some common functionnality.
-While libkiwix has some wrappers around libzim classes, they are deprecated and will be removed
-in the future.

format_code.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/bash
-
-files=(
-"include/library.h"
-"include/common/stringTools.h"
-"include/common/pathTools.h"
-"include/common/otherTools.h"
-"include/common/regexTools.h"
-"include/common/networkTools.h"
-"include/common/archiveTools.h"
-"include/manager.h"
-"include/reader.h"
-"include/kiwix.h"
-"include/xapianSearcher.h"
-"include/searcher.h"
-"src/library.cpp"
-"src/android/kiwix.cpp"
-"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
-"src/searcher.cpp"
-"src/common/pathTools.cpp"
-"src/common/regexTools.cpp"
-"src/common/otherTools.cpp"
-"src/common/archiveTools.cpp"
-"src/common/networkTools.cpp"
-"src/common/stringTools.cpp"
-"src/xapianSearcher.cpp"
-"src/manager.cpp"
-"src/reader.cpp"
-)
-
-for i in "${files[@]}"
-do
-  echo $i
-  clang-format -i -style=file $i
-done

include/book.h

@@ -38,7 +38,6 @@ namespace kiwix
 {
 
 class OPDSDumper;
-class Reader;
 
 /**
  * A class to store information about a book (a zim file)
@@ -69,7 +68,6 @@ class Book
   ~Book();
 
   bool update(const Book& other);
-  DEPRECATED void update(const Reader& reader);
   void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
@@ -81,7 +79,9 @@ class Book
   bool isPathValid() const { return m_pathValid; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
-  const std::string& getLanguage() const { return m_language; }
+  DEPRECATED const std::string& getLanguage() const { return m_language; }
+  const std::string& getCommaSeparatedLanguages() const { return m_language; }
+  const std::vector<std::string> getLanguages() const;
   const std::string& getCreator() const { return m_creator; }
   const std::string& getPublisher() const { return m_publisher; }
   const std::string& getDate() const { return m_date; }

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <memory>
 #include <stdexcept>
+#include <mutex>
 
 namespace kiwix
 {
@@ -43,6 +44,14 @@ class AriaError : public std::runtime_error {
 };
 
 
+/**
+ * A representation of a current download.
+ *
+ * `Download` is not thread safe. User must care to not call method on a
+ * same download from different threads.
+ * However, it is safe to use different `Download`s from different threads.
+ */
+
 class Download {
  public:
   typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
@@ -53,19 +62,89 @@ class Download {
     : mp_aria(p_aria),
       m_status(K_UNKNOWN),
       m_did(did) {};
-  void updateStatus(bool follow=false);
+
+  /**
+   * Update the status of the download.
+   *
+   * This call make an aria rpc call and is blocking.
+   * Some download (started with a metalink) are in fact several downloads.
+   * - A first one to download the metadlink.
+   * - A second one to download the real file.
+   *
+   * If `follow` is true, updateStatus tries to detect that and tracks
+   * the second download when the first one is finished.
+   * By passing false to `follow`, `Download` will only track the first download.
+   *
+   * `getFoo` methods are based on the last statusUpdate.
+   *
+   * @param follow: Do we have to follow following downloads.
+   */
+  void updateStatus(bool follow);
+
+  /**
+   * Pause the download (and call updateStatus)
+   */
   void pauseDownload();
+
+  /**
+   * Resume the download (and call updateStatus)
+   */
   void resumeDownload();
+
+  /**
+   * Cancel the download.
+   *
+   * A canceled downlod cannot be resume and updateStatus does nothing.
+   * However, you can still get information based on the last known information.
+   */
   void cancelDownload();
-  StatusResult getStatus()          { return m_status; }
-  std::string  getDid()             { return m_did; }
-  std::string  getFollowedBy()      { return m_followedBy; }
-  uint64_t     getTotalLength()     { return m_totalLength; }
-  uint64_t     getCompletedLength() { return m_completedLength; }
-  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
-  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
-  std::string  getPath()            { return m_path; }
-  std::vector<std::string>&  getUris()             { return m_uris; }
+
+  /*
+   * Get the status of the download.
+   */
+  StatusResult getStatus() const             { return m_status; }
+
+  /*
+   * Get the id of the download.
+   */
+  const std::string&  getDid() const         { return m_did; }
+
+  /*
+   * Get the id of the "second" download.
+   *
+   * Set only if the "first" download is a metalink and is complete.
+   */
+  const std::string&  getFollowedBy() const  { return m_followedBy; }
+
+  /*
+   * Get the total length of the download.
+   */
+  uint64_t     getTotalLength() const        { return m_totalLength; }
+
+  /*
+   * Get the completed length of the download.
+   */
+  uint64_t     getCompletedLength() const    { return m_completedLength; }
+
+  /*
+   * Get the download speed of the download.
+   */
+  uint64_t     getDownloadSpeed() const      { return m_downloadSpeed; }
+
+  /*
+   * Get the verified length of the download.
+   */
+  uint64_t     getVerifiedLength() const     { return m_verifiedLength; }
+
+  /*
+   * Get the path (local file) of the download.
+   */
+  const std::string&  getPath() const        { return m_path; }
+
+  /*
+   * Get the download uris of the download.
+   */
+  const std::vector<std::string>&  getUris() const { return m_uris; }
 
  protected:
   std::shared_ptr<Aria2> mp_aria;
@@ -83,6 +162,9 @@ class Download {
 /**
  * A tool to download things.
  *
+ * A Downloader manages `Download` using aria2 in the background.
+ * `Downloader` is threadsafe.
+ * However, the returned `Download`s are NOT threadsafe.
  */
 class Downloader
 {
@@ -92,14 +174,41 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
-  Download* getDownload(const std::string& did);
+  /**
+   * Start a new download.
+   *
+   * This method is thread safe and return a pointer to a newly created `Download`.
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param uri: The uri of the thing to download.
+   * @param options: A series of pair <option_name, option_value> to pass to aria.
+   * @return: The newly created Download.
+   */
+  std::shared_ptr<Download> startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
 
-  size_t getNbDownload() { return m_knownDownloads.size(); }
-  std::vector<std::string> getDownloadIds();
+  /**
+   * Get a download corrsponding to a download id (did)
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param did: The download id to search for.
+   * @return: The Download corresponding to did.
+   * @throw: Throw std::out_of_range if did is not found.
+   */
+  std::shared_ptr<Download> getDownload(const std::string& did);
+
+  /**
+   * Get the number of downloads currently managed.
+   */
+  size_t getNbDownload() const;
+
+  /**
+   * Get the ids of the managed downloads.
+   */
+  std::vector<std::string> getDownloadIds() const;
 
  private:
-  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
+  mutable std::mutex m_lock;
+  std::map<std::string, std::shared_ptr<Download>> m_knownDownloads;
   std::shared_ptr<Aria2> mp_aria;
 };
 }

include/entry.h

@@ -1,193 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_ENTRY_H
-#define KIWIX_ENTRY_H
-
-#include <stdio.h>
-#include <zim/entry.h>
-#include <zim/item.h>
-#include <exception>
-#include <string>
-
-#include "common.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-
-class NoEntry : public std::exception {};
-
-/**
- * A entry represent an.. entry in a zim file.
- */
-class Entry
-{
-  public:
-    /**
-     * Construct an entry making reference to an zim article.
-     *
-     * @param article a zim::Article object
-     */
-    DEPRECATED Entry(zim::Entry entry) : Entry(entry, true) {};
-    virtual ~Entry() = default;
-
-    /**
-     * Get the path of the entry.
-     *
-     * The path is the "key" of an entry.
-     *
-     * @return the path of the entry.
-     */
-    std::string getPath() const { return entry.getPath(); }
-
-    /**
-     * Get the title of the entry.
-     *
-     * @return the title of the entry.
-     */
-    std::string getTitle() const { return entry.getTitle(); }
-
-    /**
-     * Get the content of the entry.
-     *
-     * The string is a copy of the content.
-     * If you don't want to do a copy, use get_blob.
-     *
-     * @return the content of the entry.
-     */
-    std::string getContent() const { return entry.getItem().getData(); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @param size The size of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
-
-    /**
-     * Get the info for direct access to the content of the entry.
-     *
-     * Some entry (ie binary ones) have their content plain stored
-     * in the zim file. Knowing the offset where the content is stored
-     * an user can directly read the content in the zim file bypassing the
-     * libkiwix/libzim.
-     *
-     * @return A pair specifying where to read the content.
-     *         The string is the real file to read (may be different that .zim
-     *         file if zim is cut).
-     *         The offset is the offset to read in the file.
-     *         Return <"",0> if is not possible to read directly.
-     */
-    zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
-
-    /**
-     * Get the size of the entry.
-     *
-     * @return the size of the entry.
-     */
-    size_type   getSize() const;
-
-    /**
-     * Get the mime_type of the entry.
-     *
-     * @return the mime_type of the entry.
-     */
-    std::string getMimetype() const;
-
-
-    /**
-     * Get if the entry is a redirect entry.
-     *
-     * @return True if the entry is a redirect.
-     */
-    bool isRedirect() const;
-
-    /**
-     * Get if the entry is a link target entry.
-     *
-     * @return True if the entry is a link target.
-     */
-    bool isLinkTarget() const;
-
-    /**
-     * Get if the entry is a deleted entry.
-     *
-     * @return True if the entry is a deleted entry.
-     */
-    bool isDeleted() const;
-
-    /**
-     * Get the entry pointed by this entry.
-     *
-     * @return the entry pointed.
-     * @throw NoEntry if the entry is not a redirected entry.
-     */
-    Entry getRedirectEntry() const;
-
-    /**
-     * Get the final entry pointed by this entry.
-     *
-     * Follow the redirection until a "not redirecting" entry is found.
-     * If the entry is not a redirected entry, return the entry itself.
-     *
-     * @return the final entry.
-     */
-    Entry getFinalEntry() const;
-
-    /**
-     * Get the zim entry wrapped by this (kiwix) entry
-     *
-     * @return the zim entry
-     */
-    const zim::Entry& getZimEntry() const { return entry; }
-
-  private:
-    zim::Entry entry;
-
-  private:
-    // Entry is deprecated, so we've marked the constructor as deprecated.
-    // But we still need to construct the entry (in our deprecated code)
-    // To avoid warning because we use deprecated function, we create a second
-    // constructor not deprecated. The `bool marker` is unused, it sole purpose
-    // is to change the signature to have two different constructor.
-    // This one is not deprecated and we must use it in our private code.
-    Entry(zim::Entry entry, bool marker);
-    friend class Reader;
-};
-
-}
-
-#endif // KIWIX_ENTRY_H

include/html_dumper.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,21 +17,34 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_HTML_DUMPER_H
+#define KIWIX_HTML_DUMPER_H
 
-public class JNIKiwixString
+#include <string>
+
+#include "library_dumper.h"
+
+namespace kiwix
 {
-  public String value;
 
-  public JNIKiwixString(String value) {
-    this.value = value;
-  }
+/**
+ * A class to dump Library in HTML format.
+ */
+class HTMLDumper : public LibraryDumper
+{
+ public:
+  HTMLDumper(const Library* library, const NameMapper* NameMapper);
+  ~HTMLDumper();
 
-  public JNIKiwixString() {
-    this("");
-  }
 
-  public String getValue() {
-    return value;
-  }
+  /**
+   * Dump library in HTML
+   *
+   * @return HTML content
+   */
+  std::string dumpPlainHTML(kiwix::Filter filter) const;
+};
+
 }
+
+#endif // KIWIX_HTML_DUMPER_H

include/library.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <mutex>
 #include <zim/archive.h>
+#include <zim/search.h>
 
 #include "book.h"
 #include "bookmark.h"
@@ -105,12 +106,22 @@ class Filter {
     Filter& rejectTags(const Tags& tags);
 
     Filter& category(std::string category);
+
+    /**
+     *  Set the filter to only accept books in the specified language.
+     *
+     *  Multiple languages can be specified as a comma-separated list (in
+     *  which case a book in any of those languages will match).
+     */
     Filter& lang(std::string lang);
+
     Filter& publisher(std::string publisher);
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
     Filter& query(std::string query, bool partial=true);
     Filter& name(std::string name);
+    Filter& clearLang();
+    Filter& clearCategory();
 
     bool hasQuery() const;
     const std::string& getQuery() const { return _query; }
@@ -140,53 +151,35 @@ private: // functions
     bool accept(const Book& book) const;
 };
 
-/**
- * This class is not part of the libkiwix API. Its only purpose is
- * to simplify the implementation of the Library's move operations
- * and avoid bugs should new data members be added to Library.
- */
-class LibraryBase
+
+class ZimSearcher : public zim::Searcher
 {
-protected: // types
-  typedef uint64_t LibraryRevision;
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
 
-  struct Entry : Book
-  {
-    LibraryRevision lastUpdatedRevision = 0;
-
-    // May also keep the Archive and Reader pointers here and get
-    // rid of the m_readers and m_archives data members in Library
-  };
-
-protected: // data
-  LibraryRevision m_revision;
-  std::map<std::string, Entry> m_books;
-  std::map<std::string, std::shared_ptr<Reader>> m_readers;
-  std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
-  std::vector<kiwix::Bookmark> m_bookmarks;
-  class BookDB;
-  std::unique_ptr<BookDB> m_bookDB;
-
-protected: // functions
-  LibraryBase();
-  ~LibraryBase();
-
-  LibraryBase(LibraryBase&& );
-  LibraryBase& operator=(LibraryBase&& );
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
 };
 
 /**
  * A Library store several books.
  */
-class Library : private LibraryBase
+class Library
 {
   // all data fields must be added in LibraryBase
   mutable std::mutex m_mutex;
 
  public:
-  typedef LibraryRevision Revision;
+  typedef uint64_t Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
 
  public:
   Library();
@@ -240,8 +233,11 @@ class Library : private LibraryBase
 
   Book getBookByIdThreadSafe(const std::string& id) const;
 
-  DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
 
   /**
    * Remove a book from the library.
@@ -346,12 +342,12 @@ class Library : private LibraryBase
   /**
    * Return the current revision of the library.
    *
-   * The revision of the library is updated (incremented by one) only by
-   * the addBook() operation.
+   * The revision of the library is updated (incremented by one) by
+   * the addBook() and removeBookById() operations.
    *
    * @return Current revision of the library.
    */
-  LibraryRevision getRevision() const;
+  Revision getRevision() const;
 
   /**
    * Remove books that have not been updated since the specified revision.
@@ -359,20 +355,24 @@ class Library : private LibraryBase
    * @param  rev the library revision to use
    * @return Count of books that were removed by this operation.
    */
-  uint32_t removeBooksNotUpdatedSince(LibraryRevision rev);
+  uint32_t removeBooksNotUpdatedSince(Revision rev);
 
   friend class OPDSDumper;
   friend class libXMLDumper;
 
 private: // types
   typedef const std::string& (Book::*BookStrPropMemFn)() const;
+  struct Impl;
 
 private: // functions
   AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
-  void dropReader(const std::string& bookId);
+  void dropCache(const std::string& bookId);
+
+private: //data
+  std::unique_ptr<Impl> mp_impl;
 };
 
 }

include/library_dumper.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBRARY_DUMPER_H
+#define KIWIX_LIBRARY_DUMPER_H
+
+#include <string>
+
+
+#include "library.h"
+#include "name_mapper.h"
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+/**
+ * A base class to dump Library in various formats.
+ *
+ */
+class LibraryDumper
+{
+ public:
+  LibraryDumper(const Library* library, const NameMapper* NameMapper);
+  ~LibraryDumper();
+
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
+
+  /**
+   * Set the root location used when generating url.
+   *
+   * @param rootLocation the root location to use.
+   */
+  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+  
+  /**
+   * Sets user default language
+   * 
+   * @param userLang the user language to be set
+   */
+  void setUserLanguage(std::string userLang) { this->m_userLang = userLang; }
+
+  /**
+   * Get the data of categories
+   */
+  kainjow::mustache::list getCategoryData() const;
+
+  /**
+   * Get the data of languages
+   */
+  kainjow::mustache::list getLanguageData() const;
+
+ protected:
+  const kiwix::Library* const library;
+  const kiwix::NameMapper* const nameMapper;
+  std::string libraryId;
+  std::string rootLocation;
+  std::string m_userLang;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+};
+}
+
+#endif // KIWIX_LIBRARY_DUMPER_H

include/manager.h

@@ -22,7 +22,6 @@
 
 #include "book.h"
 #include "library.h"
-#include "reader.h"
 
 #include <string>
 #include <vector>

include/meson.build

@@ -7,9 +7,6 @@ headers = [
   'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
-  'reader.h',
-  'entry.h',
-  'searcher.h',
   'search_renderer.h',
   'server.h',
   'kiwixserve.h',

include/opds_dumper.h

@@ -27,7 +27,8 @@
 #include <pugixml.hpp>
 
 #include "library.h"
-#include "reader.h"
+#include "name_mapper.h"
+#include "library_dumper.h"
 
 using namespace std;
 
@@ -38,11 +39,10 @@ namespace kiwix
  * A tool to dump a `Library` into a opds stream.
  *
  */
-class OPDSDumper
+class OPDSDumper : public LibraryDumper
 {
  public:
-  OPDSDumper() = default;
-  OPDSDumper(Library* library);
+  OPDSDumper(const Library* library, const NameMapper* NameMapper);
   ~OPDSDumper();
 
   /**
@@ -85,37 +85,6 @@ class OPDSDumper
    * @return The OPDS feed.
    */
   std::string languagesOPDSFeed() const;
-
-  /**
-   * Set the id of the library.
-   *
-   * @param id the id to use.
-   */
-  void setLibraryId(const std::string& id) { this->libraryId = id;}
-
-  /**
-   * Set the root location used when generating url.
-   *
-   * @param rootLocation the root location to use.
-   */
-  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
-
-  /**
-   * Set some informations about the search results.
-   *
-   * @param totalResult the total number of results of the search.
-   * @param startIndex the start index of the result.
-   * @param count the number of result of the current set (or page).
-   */
-  void setOpenSearchInfo(int totalResult, int startIndex, int count);
-
- protected:
-  kiwix::Library* library;
-  std::string libraryId;
-  std::string rootLocation;
-  int m_totalResults;
-  int m_startIndex;
-  int m_count;
 };
 }
 

include/reader.h

@@ -1,506 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_READER_H
-#define KIWIX_READER_H
-
-#include <stdio.h>
-#include <zim/zim.h>
-#include <zim/archive.h>
-#include <exception>
-#include <map>
-#include <sstream>
-#include <string>
-#include "common.h"
-#include "entry.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-/**
- * The SuggestionItem is a helper class that contains the info about a single
- * suggestion item.
- */
-class SuggestionItem
-{
-  // Functions
-  public:
-    // Create a sugggestion item.
-    explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
-                            const std::string& path, const std::string& snippet = "") :
-      title(title),
-      normalizedTitle(normalizedTitle),
-      path(path),
-      snippet(snippet) {}
-
-  public:
-    const std::string& getTitle() const           { return title;}
-    const std::string& getNormalizedTitle() const { return normalizedTitle;}
-    const std::string& getPath() const            { return path;}
-    const std::string& getSnippet() const         { return snippet;}
-
-    bool hasSnippet() const                { return !snippet.empty();}
-
-  // Data
-  private:
-    std::string title;
-    std::string normalizedTitle;
-    std::string path;
-    std::string snippet;
-};
-
-/**
- * The Reader class is the class who allow to get an entry content from a zim
- * file.
- *
- * Reader is now deprecated. Directly use `zim::Archive`.
- */
-
-using SuggestionsList_t = std::vector<SuggestionItem>;
-class Reader
-{
- public:
-  /**
-   * Create a Reader to read a zim file specified by zimFilePath.
-   *
-   * @param zimFilePath The path to the zim file to read.
-   *                    The zim file can be splitted (.zimaa, .zimab, ...).
-   *                    In this case, the file path must still point to the
-   *                    unsplitted path as if the file were not splitted
-   *                    (.zim extesion).
-   */
-  explicit DEPRECATED Reader(const string zimFilePath);
-
-   /**
-   * Create a Reader to read a zim file given by the Archive.
-   *
-   * @param archive The shared pointer to the Archive object.
-   */
-  explicit DEPRECATED Reader(const std::shared_ptr<zim::Archive> archive)
-   : Reader(archive, true) {};
-#ifndef _WIN32
-  explicit DEPRECATED Reader(int fd);
-  DEPRECATED Reader(int fd, zim::offset_type offset, zim::size_type size);
-#endif
-  ~Reader() = default;
-
-  /**
-   * Get the number of "displayable" entries in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'text/html' MIMEtype specified in the metadata.
-   *         Else return the number of entries in the 'A' namespace.
-   */
-  unsigned int getArticleCount() const;
-
-  /**
-   * Get the number of media in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'image/jpeg', 'image/gif' and 'image/png' in
-   *         the metadata.
-   *         Else return the number of entries in the 'I' namespace.
-   */
-  unsigned int getMediaCount() const;
-
-  /**
-   * Get the number of all entries in the zim file.
-   *
-   * @return Return the number of all the entries, whatever their MIMEtype or
-   *         their namespace.
-   */
-  unsigned int getGlobalCount() const;
-
-  /**
-   * Get the path of the zim file.
-   *
-   * @return the path of the zim file as given in the constructor.
-   */
-  string getZimFilePath() const;
-
-  /**
-   * Get the Id of the zim file.
-   *
-   * @return The uuid stored in the zim file.
-   */
-  string getId() const;
-
-  /**
-   * Get a random page.
-   *
-   * @return A random Entry. The entry is picked from all entries in
-   *         the 'A' namespace.
-   *         The main entry is excluded from the potential results.
-   */
-  Entry getRandomPage() const;
-
-  /**
-   * Get the entry of the main page.
-   *
-   * @return Entry of the main page as specified in the zim file.
-   */
-  Entry getMainPage() const;
-
-  /**
-   * Get the content of a metadata.
-   *
-   * @param[in]  name The name of the metadata.
-   * @param[out] value The value will be set to the content of the metadata.
-   * @return True if it was possible to get the content of the metadata.
-   */
-  bool getMetadata(const string& name, string& value) const;
-
-  /**
-   * Get the name of the zim file.
-   *
-   * @return The name of the zim file as specified in the zim metadata.
-   */
-  string getName() const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the creator of the zim file.
-   *
-   * @return The creator of the zim file as specified in the zim metadata.
-   */
-  string getCreator() const;
-
-  /**
-   * Get the publisher of the zim file.
-   *
-   * @return The publisher of the zim file as specified in the zim metadata.
-   */
-  string getPublisher() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the long description of the zim file.
-   *
-   * @return The long description of the zim file as specifed in the zim metadata.
-   */
-  string getLongDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
-
-  /**
-   * Get the license of the zim file.
-   *
-   * @return The license of the zim file as specified in the zim metadata.
-   */
-  string getLicense() const;
-
-  /**
-   * Get the tags of the zim file.
-   *
-   * @param original If true, return the original tags as specified in the zim metadata.
-   *                 Else, try to convert it to the new 'normalized' format.
-   * @return The tags of the zim file.
-   */
-  string getTags(bool original=false) const;
-
-  /**
-   * Get the value (as a string) of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   */
-  string getTagStr(const std::string& tagName) const;
-
-  /**
-   * Get the boolean value of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The boolean value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   *         std::domain_error if the value of the tag cannot be convert to bool.
-   */
-  bool getTagBool(const std::string& tagName) const;
-
-  /**
-   * Get the relations of the zim file.
-   *
-   * @return The relation of the zim file as specified in the zim metadata.
-   */
-  string getRelation() const;
-
-  /**
-   * Get the flavour of the zim file.
-   *
-   * @return The flavour of the zim file as specified in the zim metadata.
-   */
-  string getFlavour() const;
-
-  /**
-   * Get the source of the zim file.
-   *
-   * @return The source of the zim file as specified in the zim metadata.
-   */
-  string getSource() const;
-
-  /**
-   * Get the scraper of the zim file.
-   *
-   * @return The scraper of the zim file as specified in the zim metadata.
-   */
-  string getScraper() const;
-
-  /**
-   * Get the favicon of the zim file.
-   *
-   * @param[out] content The content of the favicon.
-   * @param[out] mimeType The mimeType of the favicon.
-   * @return True if a favicon has been found.
-   */
-  bool getFavicon(string& content, string& mimeType) const;
-
-  /**
-   * Get an entry associated to an path.
-   *
-   * @param path The path of the entry.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromPath(const std::string& path) const;
-
-  /**
-   * Get an entry associated to an url encoded path.
-   *
-   * Equivalent to `getEntryFromPath(urlDecode(path));`
-   *
-   * @param path The url encoded path.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromEncodedPath(const std::string& path) const;
-
-  /**
-   * Get un entry associated to a title.
-   *
-   * @param title The title.
-   * @return The entry
-   * throw NoEntry If no entry correspond to the url.
-   */
-  Entry getEntryFromTitle(const std::string& title) const;
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are stored in an internal vector and can be retrieved using
-   * `getNextSuggestion` method.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestions(const string& prefix,
-   *                        unsigned int suggestionsCount,
-   *                        SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param reset If true, remove previous suggestions in the internal vector.
-   *              If false, add suggestions to the internal vector
-   *              (until internal vector size is suggestionCount (or no more
-   *               suggestion))
-   * @return True if some suggestions have been added to the internal vector.
-   */
-  DEPRECATED bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         const bool reset = true);
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are added to the `result` vector.
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param result The vector where to store the suggestions.
-   * @return True if some suggestions have been added to the vector.
-   */
-
-  bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         SuggestionsList_t& resuls);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestionsSmart(const string& prefix,
-   *                             unsigned int suggestionsCount,
-   *                             SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   */
-  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param results The vector where to store the suggestions
-   * @return True if some suggestions have been added to the results.
-   */
-   bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount,
-                              SuggestionsList_t& results);
-
-
-  /**
-   * Check if the path exists in the zim file.
-   *
-   * @param path the path to check.
-   * @return True if the path exists in the zim file.
-   */
-  bool pathExists(const string& path) const;
-
-  /**
-   * Check if the zim file has a embedded fulltext index.
-   *
-   * @return True if the zim file has a embedded fulltext index
-   *         and is not split (else the fulltext is not accessible).
-   */
-  bool hasFulltextIndex() const;
-
-  /**
-   * Get potential case title variations for a title.
-   *
-   * @param title a title.
-   * @return the list of variantions.
-   */
-  std::vector<std::string> getTitleVariants(const std::string& title) const;
-
-  /**
-   * Get the next suggestion title.
-   *
-   * @param[out] title the title of the suggestion.
-   * @return True if title has been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title);
-
-  /**
-   * Get the next suggestion title and url.
-   *
-   * @param[out] title the title of the suggestion.
-   * @param[out] url the url of the suggestion.
-   * @return True if title and url have been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title, string& url);
-
-  /**
-   * Get if we can check zim file integrity (has a checksum).
-   *
-   * @return True if zim file have a checksum.
-   */
-  bool canCheckIntegrity() const;
-
-  /**
-   * Check is zim file is corrupted.
-   *
-   * @return True if zim file is corrupted.
-   */
-  bool isCorrupted() const;
-
-  /**
-   * Return the total size of the zim file.
-   *
-   * If zim file is split, return the sum of all parts' size.
-   *
-   * @return Size of the size file is KiB.
-   */
-  unsigned int getFileSize() const;
-
-  /**
-   * Get the zim file handler.
-   *
-   * @return The libzim file handler.
-   */
-  zim::Archive* getZimArchive() const;
-
- protected:
-  std::shared_ptr<zim::Archive> zimArchive;
-  std::string zimFilePath;
-
-  SuggestionsList_t suggestions;
-  SuggestionsList_t::iterator suggestionsOffset;
-
- private:
-  std::map<const std::string, unsigned int> parseCounterMetadata() const;
-
-  // Reader is deprecated, so we've marked the constructor as deprecated.
-  // But we still need to construct the reader (in our deprecated code)
-  // To avoid warning because we use deprecated function, we create a
-  // constructor not deprecated. The `bool marker` is unused, it sole purpose
-  // is to change the signature to have a different constructor.
-  // This one is not deprecated and we must use it in our private code.
-  Reader(const std::shared_ptr<zim::Archive> archive, bool marker);
-  friend class Library;
-};
-}
-
-#endif

include/search_renderer.h

@@ -27,7 +27,6 @@
 namespace kiwix
 {
 
-class Searcher;
 class NameMapper;
 /**
  * The SearcherRenderer class is used to render a search result to a html page.
@@ -35,17 +34,6 @@ class NameMapper;
 class SearchRenderer
 {
  public:
-  /**
-   * Construct a SearchRenderer from a Searcher.
-   *
-   * This method is now deprecated. Construct the renderer from a
-   * `zim::SearchResultSet`
-   *
-   * @param searcher The `Searcher` to render.
-   * @param mapper The `NameMapper` to use to do the rendering.
-   */
-  DEPRECATED SearchRenderer(Searcher* searcher, NameMapper* mapper);
-
   /**
    * Construct a SearchRenderer from a SearchResultSet.
    *
@@ -75,12 +63,15 @@ class SearchRenderer
 
   ~SearchRenderer();
 
+  /**
+   * Set the search pattern used to do the search
+   */
   void setSearchPattern(const std::string& pattern);
 
   /**
-   * Set the search content id.
+   * Set the querystring used to select books
    */
-  void setSearchContent(const std::string& name);
+  void setSearchBookQuery(const std::string& bookQuery);
 
   /**
    * Set protocol prefix.
@@ -99,17 +90,25 @@ class SearchRenderer
     this->pageLength  = pageLength;
   }
 
+  std::string renderTemplate(const std::string& tmpl_str);
+
   /**
    * Generate the html page with the resutls of the search.
    */
   std::string getHtml();
 
+    /**
+   * Generate the xml page with the resutls of the search.
+   */
+  std::string getXml();
+
+
  protected:
   std::string beautifyInteger(const unsigned int number);
   zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
   Library* mp_library;
-  std::string searchContent;
+  std::string searchBookQuery;
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;

include/searcher.h

@@ -1,180 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCHER_H
-#define KIWIX_SEARCHER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unicode/putil.h>
-#include <algorithm>
-#include <cctype>
-#include <locale>
-#include <string>
-#include <memory>
-#include <vector>
-
-#include <zim/search.h>
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader;
-class Result
-{
- public:
-  virtual ~Result(){};
-  virtual std::string get_url() = 0;
-  virtual std::string get_title() = 0;
-  virtual int get_score() = 0;
-  virtual std::string get_snippet() = 0;
-  virtual std::string get_content() = 0;
-  virtual int get_wordCount() = 0;
-  virtual int get_size() = 0;
-  virtual std::string get_zimId() = 0;
-};
-
-struct SearcherInternal;
-struct SuggestionInternal;
-/**
- * The Searcher class is reponsible to do different kind of search using the
- * fulltext index.
- *
- * The Searcher is now deprecated. Use libzim search feature.
- */
-class Searcher
-{
- public:
-  /**
-   * The default constructor.
-   */
-  DEPRECATED Searcher();
-
-  ~Searcher();
-
-  /**
-   * Add a reader (containing embedded fulltext index) to the search.
-   *
-   * @param reader The Reader for the zim containing the fulltext index.
-   * @return true if the reader has been added.
-   *         false if the reader cannot be added (no embedded fulltext index present)
-   */
-  bool add_reader(Reader* reader);
-
-
-  Reader* get_reader(int index);
-
-  /**
-   * Start a search on the zim associated to the Searcher.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param search The search query.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void search(const std::string& search,
-              unsigned int resultStart,
-              unsigned int maxResultCount,
-              const bool verbose = false);
-
-  /**
-   * Start a geographique search.
-   * The search return result for entry in a disc of center latitude/longitude
-   * and radius distance.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param latitude The latitude of the center point.
-   * @param longitude The longitude of the center point.
-   * @param distance The radius of the disc.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum number of results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void geo_search(float latitude, float longitude, float distance,
-                  unsigned int resultStart,
-                  unsigned int maxResultCount,
-                  const bool verbose = false);
-
-  /**
-   * Start a suggestion search.
-   * The search made depend of the "version" of the embedded index.
-   *  - If the index is newer enough and have a title namespace, the search is
-   *    made in the titles only.
-   *  - Else the search is made on the whole article content.
-   * In any case, the search is made "partial" (as adding '*' at the end of the query)
-   *
-   * @param search The search query.
-   * @param verbose print some info on stdout if true.
-   */
-  void suggestions(std::string& search, const bool verbose = false);
-
-  /**
-   * Get the next result of a started search.
-   * This is the method to use to loop hover the search results.
-   */
-  Result* getNextResult();
-
-  /**
-   * Restart the previous search.
-   * Next call to getNextResult will return the first result.
-   */
-  void restart_search();
-
-  /**
-   * Get a estimation of the result count.
-   */
-  unsigned int getEstimatedResultCount();
-
-  /**
-   * Get a SearchResultSet object for current search
-   */
-  zim::SearchResultSet getSearchResultSet();
-
-  unsigned int getResultStart() { return resultStart; }
-  unsigned int getMaxResultCount() { return maxResultCount; }
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  void closeIndex();
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int maxResultCount,
-                     const bool verbose = false);
-
-  std::vector<Reader*> readers;
-  std::unique_ptr<SearcherInternal> internal;
-  std::unique_ptr<SuggestionInternal> suggestionInternal;
-  std::string searchPattern;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int maxResultCount;
-
- private:
-  void reset();
-
-};
-
-
-}
-
-#endif

include/server.h

@@ -54,6 +54,7 @@ namespace kiwix
        void setAddress(const std::string& addr) { m_addr = addr; }
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
        void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
@@ -63,7 +64,7 @@ namespace kiwix
         { m_blockExternalLinks = blockExternalLinks; }
        int getPort();
        std::string getAddress();
-       
+
      protected:
        Library* mp_library;
        NameMapper* mp_nameMapper;
@@ -72,6 +73,7 @@ namespace kiwix
        std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;

include/version.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <vector>
+#include <iostream>
 
 namespace kiwix
 {

meson.build

@@ -1,26 +1,17 @@
 project('libkiwix', 'cpp',
-  version : '10.1.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '12.0.0',
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
-wrapper = get_option('wrapper')
-
-static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
-if wrapper.contains('android')
-  extra_libs = ['-llog']
-else
-  extra_libs = []
-endif
-
-if wrapper.contains('java')
-  add_languages('java')
-endif
+static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
 
 # See https://github.com/kiwix/libkiwix/issues/371
 if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
-  extra_libs += '-latomic'
+  extra_libs = ['-latomic']
+else
+  extra_libs = []
 endif
 
 if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
@@ -44,7 +35,7 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif

meson_options.txt

@@ -1,4 +1,4 @@
-option('wrapper', type:'array', choices:['java', 'android'], value:[],
-  description: 'The wrapper to generate.')
+option('static-linkage', type : 'boolean', value : false,
+  description : 'Link statically with the dependencies.')
 option('doc', type : 'boolean', value : false,
   description : 'Build the documentations.')

scripts/format_code.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+
+# Compute 'src' path
+SCRIPT_DIR=$(dirname "$0")
+REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
+DIRS="src include"
+
+# Apply formating to all *.cpp and *.h files
+cd "$REPO_DIR"
+for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
+do
+  echo $FILE
+  clang-format -i -style=file "$FILE"
+done

scripts/kiwix-compile-i18n

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.data = f.read()
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_files', nargs='+',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    resources = [Resource(filename) for filename in args.i18n_resource_files]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_files ...
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -52,15 +52,21 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
 class Resource:
-    def __init__(self, base_dirs, filename):
-        filename = filename.strip()
+    def __init__(self, base_dirs, filename, cacheid=None):
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
@@ -71,7 +77,7 @@ class Resource:
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -93,6 +99,12 @@ class Resource:
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
             namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@@ -102,7 +114,7 @@ class Resource:
 
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>
@@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
@@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -182,15 +202,17 @@ if __name__ == "__main__":
     parser.add_argument('--source_dir',
                         help="Additional directory where to look for resources.",
                         action='append')
-    parser.add_argument('resource_file',
+    parser.add_argument('resource_files', nargs='+',
                         help='The list of resources to compile.')
     args = parser.parse_args()
 
-    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
-    with open(args.resource_file, 'r') as f:
-        resources = [Resource([base_dir]+source_dir, filename)
-                        for filename in f.readlines()]
+    resources = []
+    for resfile in args.resource_files:
+        base_dir = os.path.dirname(os.path.realpath(resfile))
+        with open(resfile, 'r') as f:
+            resources += [Resource([base_dir]+source_dir, *line.strip().split())
+                            for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-compile-resources.1

@@ -2,7 +2,7 @@
 .SH NAME
 kiwix-compile-resources \- helper to compile and generate some Kiwix resources
 .SH SYNOPSIS
-\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
 .SH DESCRIPTION
 .TP
 resource_file

scripts/kiwix-resources

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_resource_list_file(src_path, dst_path):
+    with open(src_path, 'r') as src:
+        with open(dst_path, 'w') as dst:
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        if resource.startswith('skin/'):
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,6 +1,13 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
 
 install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/aria2.cpp

@@ -24,7 +24,7 @@
 #define LOG_ARIA_ERROR() \
   { \
     std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
-    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+    std::cerr << (curlErrorBuffer[0] ? curlErrorBuffer : curl_easy_strerror(res)) << std::endl; \
   }
 
 namespace kiwix {
@@ -32,9 +32,7 @@ namespace kiwix {
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret(getNewRpcSecret()),
-  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
-  mp_curl(nullptr)
+  m_secret(getNewRpcSecret())
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -91,36 +89,32 @@ Aria2::Aria2():
       launchCmd.append(cmd).append(" ");
   }
   mp_aria = Subprocess::run(callCmd);
-  mp_curl = curl_easy_init();
 
-  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
-  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
-  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
+  CURL* p_curl = curl_easy_init();
+  char curlErrorBuffer[CURL_ERROR_SIZE];
+
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
-    m_curlErrorBuffer[0] = 0;
-    auto res = curl_easy_perform(mp_curl);
+    curlErrorBuffer[0] = 0;
+    auto res = curl_easy_perform(p_curl);
     if (res == CURLE_OK) {
       break;
     } else if (watchdog == 1) {
       LOG_ARIA_ERROR();
     }
   }
+  curl_easy_cleanup(p_curl);
   if (!watchdog) {
-    curl_easy_cleanup(mp_curl);
     throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
-Aria2::~Aria2()
-{
-  std::unique_lock<std::mutex> lock(m_lock);
-  curl_easy_cleanup(mp_curl);
-}
-
 void Aria2::close()
 {
   saveSession();
@@ -140,20 +134,25 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
   std::stringstream outStream;
   CURLcode res;
   long response_code;
-  {
-    std::unique_lock<std::mutex> lock(m_lock);
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
-    m_curlErrorBuffer[0] = 0;
-    res = curl_easy_perform(mp_curl);
-    if (res != CURLE_OK) {
-      LOG_ARIA_ERROR();
-      throw std::runtime_error("Cannot perform request");
-    }
-    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  char curlErrorBuffer[CURL_ERROR_SIZE];
+  CURL* p_curl = curl_easy_init();
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+  curl_easy_setopt(p_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(p_curl, CURLOPT_WRITEDATA, &outStream);
+  curlErrorBuffer[0] = 0;
+  res = curl_easy_perform(p_curl);
+  if (res != CURLE_OK) {
+    LOG_ARIA_ERROR();
+    curl_easy_cleanup(p_curl);
+    throw std::runtime_error("Cannot perform request");
   }
+  curl_easy_getinfo(p_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(p_curl);
 
   auto responseContent = outStream.str();
   if (response_code != 200) {

src/aria2.h

@@ -12,7 +12,6 @@
 #include "xmlrpc.h"
 
 #include <memory>
-#include <mutex>
 #include <curl/curl.h>
 
 namespace kiwix {
@@ -24,15 +23,11 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
-    std::unique_ptr<char[]> m_curlErrorBuffer;
-    CURL* mp_curl;
-    std::mutex m_lock;
-
     std::string doRequest(const MethodCall& methodCall);
 
   public:
     Aria2();
-    virtual ~Aria2();
+    virtual ~Aria2() = default;
     void close();
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});

src/book.cpp

@@ -18,7 +18,6 @@
  */
 
 #include "book.h"
-#include "reader.h"
 
 #include "tools.h"
 #include "tools/base64.h"
@@ -30,7 +29,7 @@
 #include "tools/archiveTools.h"
 
 #include <zim/archive.h>
-
+#include <zim/item.h>
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -64,15 +63,10 @@ bool Book::update(const kiwix::Book& other)
   return true;
 }
 
-void Book::update(const kiwix::Reader& reader)
-{
-  update(*reader.getZimArchive());
-}
-
 void Book::update(const zim::Archive& archive) {
   m_path = archive.getFilename();
   m_pathValid = true;
-  m_id = getArchiveId(archive);
+  m_id = std::string(archive.getUuid());
   m_title = getArchiveTitle(archive);
   m_description = getMetaDescription(archive);
   m_language = getMetaLanguage(archive);
@@ -83,8 +77,8 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = getArchiveArticleCount(archive);
-  m_mediaCount = getArchiveMediaCount(archive);
+  m_articleCount = archive.getArticleCount();
+  m_mediaCount = archive.getMediaCount();
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
   m_illustrations.clear();
@@ -292,4 +286,9 @@ std::string Book::getCategoryFromTags() const
   }
 }
 
+const std::vector<std::string> Book::getLanguages() const
+{
+  return kiwix::split(m_language, ",");
+}
+
 }

src/downloader.cpp

@@ -127,22 +127,24 @@ void Download::cancelDownload()
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  try {
-    for (auto gid : mp_aria->tellActive()) {
-      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
-    }
-  } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
-  }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
       m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
+      m_knownDownloads[gid]->updateStatus(false);
     }
   } catch (std::exception& e) {
     std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      if( m_knownDownloads.find(gid) == m_knownDownloads.end()) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads[gid]->updateStatus(false);
+      }
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
+  }
 }
 
 /* Destructor */
@@ -155,7 +157,8 @@ void Downloader::close()
   mp_aria->close();
 }
 
-std::vector<std::string> Downloader::getDownloadIds() {
+std::vector<std::string> Downloader::getDownloadIds() const {
+  std::unique_lock<std::mutex> lock(m_lock);
   std::vector<std::string> ret;
   for(auto& p:m_knownDownloads) {
     ret.push_back(p.first);
@@ -163,42 +166,46 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
+std::shared_ptr<Download> Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
     auto& uris = d->getUris();
     if (std::find(uris.begin(), uris.end(), uri) != uris.end())
-      return d.get();
+      return d;
   }
   std::vector<std::string> uris = {uri};
   auto gid = mp_aria->addUri(uris, options);
-  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-  return m_knownDownloads[gid].get();
+  m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+  return m_knownDownloads[gid];
 }
 
-Download* Downloader::getDownload(const std::string& did)
+std::shared_ptr<Download> Downloader::getDownload(const std::string& did)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   try {
-    m_knownDownloads.at(did).get()->updateStatus(true);
-    return m_knownDownloads.at(did).get();
+    return m_knownDownloads.at(did);
   } catch(std::exception& e) {
-    for (auto gid : mp_aria->tellActive()) {
-      if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
-      }
-    }
     for (auto gid : mp_aria->tellWaiting()) {
       if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
       }
-  }
+    }
+    for (auto gid : mp_aria->tellActive()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
+      }
+    }
     throw e;
   }
 }
 
+size_t Downloader::getNbDownload() const {
+  std::unique_lock<std::mutex> lock(m_lock);
+  return m_knownDownloads.size();
+}
+
 }

src/entry.cpp

@@ -1,73 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-namespace kiwix
-{
-
-Entry::Entry(zim::Entry entry, bool _marker)
-  : entry(entry)
-{
-}
-
-size_type Entry::getSize() const
-{
-  if (entry.isRedirect()) {
-    return 0;
-  } else {
-    return entry.getItem().getSize();
-  }
-}
-
-std::string Entry::getMimetype() const
-{
-  return entry.getItem(true).getMimetype();
-}
-
-bool Entry::isRedirect() const
-{
-  return entry.isRedirect();
-}
-
-Entry Entry::getRedirectEntry() const
-{
-  if ( !entry.isRedirect() ) {
-    throw NoEntry();
-  }
-
-  return Entry(entry.getRedirectEntry(), true);
-}
-
-Entry Entry::getFinalEntry() const
-{
-  int loopCounter = 42;
-  auto final_entry = entry;
-  while (final_entry.isRedirect() && loopCounter--) {
-    final_entry = final_entry.getRedirectEntry();
-  }
-  // Prevent infinite loops.
-  if (final_entry.isRedirect()) {
-    throw NoEntry();
-  }
-  return Entry(final_entry, true);
-}
-
-}

src/html_dumper.cpp

@@ -0,0 +1,120 @@
+#include "html_dumper.h"
+#include "libkiwix-resources.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+#include "tools/regexTools.h"
+#include "server/i18n.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+HTMLDumper::HTMLDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
+{
+}
+/* Destructor */
+HTMLDumper::~HTMLDumper()
+{
+}
+
+namespace {
+
+std::string humanFriendlyTitle(std::string title)
+{
+  std::string humanFriendlyString = replaceRegex(title, "_", " ");
+  humanFriendlyString[0] = toupper(humanFriendlyString[0]);
+  return humanFriendlyString;
+}
+
+kainjow::mustache::list getTagList(std::string tags)
+{
+  const auto tagsList = kiwix::split(tags, ";", true, false);
+  kainjow::mustache::list finalTagList;
+  for (auto tag : tagsList) {
+  if (tag[0] != '_')
+    finalTagList.push_back(kainjow::mustache::object{
+      {"tag", tag}
+    });
+  }
+  return finalTagList;
+}
+
+} // unnamed namespace
+
+std::string HTMLDumper::dumpPlainHTML(kiwix::Filter filter) const
+{
+  kainjow::mustache::list booksData;
+  const auto filteredBooks = library->filter(filter);
+  const auto searchQuery = filter.getQuery();
+  auto languages = getLanguageData();
+  auto categories = getCategoryData();
+
+  for (auto &category : categories) {
+    const auto categoryName = category.get("name")->string_value();
+    if (categoryName == filter.getCategory()) {
+      category["selected"] = true;
+    }
+    category["hf_name"] = humanFriendlyTitle(categoryName);
+  }
+
+  for (auto &language : languages) {
+    if (language.get("lang_code")->string_value() == filter.getLang()) {
+      language["selected"] = true;
+    }
+  }
+
+  for ( const auto& bookId : filteredBooks ) {
+    const auto bookObj = library->getBookById(bookId);
+    const auto bookTitle = bookObj.getTitle();
+    std::string contentId = "";
+    try {
+      contentId = urlEncode(nameMapper->getNameForId(bookId));
+    } catch (...) {}
+    const auto bookDescription = bookObj.getDescription();
+    const auto langCode = bookObj.getCommaSeparatedLanguages();
+    const auto bookIconUrl = rootLocation + "/catalog/v2/illustration/" + bookId +  "/?size=48";
+    const auto tags = bookObj.getTags();
+    const auto downloadAvailable = (bookObj.getUrl() != "");
+    std::string faviconAttr = "style=background-image:url(" + bookIconUrl + ")";
+    
+    booksData.push_back(kainjow::mustache::object{
+      {"id", contentId},
+      {"title", bookTitle},
+      {"description", bookDescription},
+      {"langCode", langCode},
+      {"faviconAttr", faviconAttr},
+      {"tagList", getTagList(tags)},
+      {"downloadAvailable", downloadAvailable}
+    });
+  }
+
+  auto getTranslation = i18n::GetTranslatedStringWithMsgId(m_userLang);
+
+  const auto translations = kainjow::mustache::object{
+                    getTranslation("search"),
+                    getTranslation("download"),
+                    getTranslation("count-of-matching-books", {{"COUNT", to_string(filteredBooks.size())}}),
+                    getTranslation("book-filtering-all-categories"),
+                    getTranslation("book-filtering-all-languages"),
+                    getTranslation("powered-by-kiwix-html"),
+                    getTranslation("welcome-to-kiwix-server"),
+                    getTranslation("preview-book"),
+                    getTranslation("welcome-page-overzealous-filter", {{"URL", "?lang="}})
+  };
+
+  return render_template(
+             RESOURCE::templates::no_js_library_page_html,
+             kainjow::mustache::object{
+               {"root", rootLocation},
+               {"books", booksData },
+               {"searchQuery", searchQuery},
+               {"languages", languages},
+               {"categories", categories},
+               {"noResults", filteredBooks.size() == 0},
+               {"translations", translations}
+             }
+  );
+}
+
+} // namespace kiwix

src/library.cpp

@@ -19,7 +19,6 @@
 
 #include "library.h"
 #include "book.h"
-#include "reader.h"
 #include "libxml_dumper.h"
 
 #include "tools.h"
@@ -27,10 +26,13 @@
 #include "tools/regexTools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
 #include <set>
+#include <cmath>
 #include <unicode/locid.h>
 #include <xapian.h>
 
@@ -56,68 +58,129 @@ bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
       && book1.getPath() == book2.getPath();
 }
 
-} // unnamed namespace
-
-class LibraryBase::BookDB : public Xapian::WritableDatabase
+template<typename Key, typename Value>
+class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
 {
-public:
-  BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
+  public:
+    explicit MultiKeyCache(size_t maxEntries)
+      : ConcurrentCache<std::set<Key>, Value>(maxEntries)
+    {}
+
+    bool drop(const Key& key)
+    {
+      std::unique_lock<std::mutex> l(this->lock_);
+      bool removed = false;
+      for(auto& cache_key: this->impl_.keys()) {
+        if(cache_key.find(key)!=cache_key.end()) {
+          removed |= this->impl_.drop(cache_key);
+        }
+      }
+      return removed;
+    }
 };
 
-LibraryBase::LibraryBase()
-  : m_bookDB(new BookDB)
+} // unnamed namespace
+
+struct Library::Impl
+{
+  struct Entry : Book
+  {
+    Library::Revision lastUpdatedRevision = 0;
+  };
+
+  Library::Revision m_revision;
+  std::map<std::string, Entry> m_books;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+  Xapian::WritableDatabase m_bookDB;
+
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
+
+  Impl();
+  ~Impl();
+
+  Impl(Impl&& );
+  Impl& operator=(Impl&& );
+};
+
+Library::Impl::Impl()
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
 {
 }
 
-LibraryBase::~LibraryBase()
+Library::Impl::~Impl()
 {
 }
 
-LibraryBase::LibraryBase(LibraryBase&& ) = default;
-LibraryBase& LibraryBase::operator=(LibraryBase&& ) = default;
+Library::Impl::Impl(Library::Impl&& ) = default;
+Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
+
+unsigned int
+Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
+{
+  unsigned int result = 0;
+  for (auto& pair: m_books) {
+    auto& book = pair.second;
+    if ((!book.getPath().empty() && localBooks)
+        || (!book.getUrl().empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
 
 /* Constructor */
 Library::Library()
+  : mp_impl(new Library::Impl)
 {
 }
 
 Library::Library(Library&& other)
-  : LibraryBase(std::move(other))
+  : mp_impl(std::move(other.mp_impl))
 {
 }
 
 Library& Library::operator=(Library&& other)
 {
-  LibraryBase::operator=(std::move(other));
+  mp_impl = std::move(other.mp_impl);
   return *this;
 }
 
 /* Destructor */
-Library::~Library()
-{
-}
-
+Library::~Library() = default;
 
 bool Library::addBook(const Book& book)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  ++m_revision;
+  ++mp_impl->m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
-    auto& oldbook = m_books.at(book.getId());
+    auto& oldbook = mp_impl->m_books.at(book.getId());
     if ( ! booksReferToTheSameArchive(oldbook, book) ) {
-      dropReader(book.getId());
+      dropCache(book.getId());
     }
     oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
                           // XXX: Then m_bookDB will become out-of-sync with
                           // XXX: the real contents of the library.
-    oldbook.lastUpdatedRevision = m_revision;
+    oldbook.lastUpdatedRevision = mp_impl->m_revision;
     return false;
   } catch (std::out_of_range&) {
-    Entry& newEntry = m_books[book.getId()];
+    auto& newEntry = mp_impl->m_books[book.getId()];
     static_cast<Book&>(newEntry) = book;
-    newEntry.lastUpdatedRevision = m_revision;
+    newEntry.lastUpdatedRevision = mp_impl->m_revision;
+    size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
+    if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+    }
+    if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+    }
     return true;
   }
 }
@@ -125,15 +188,15 @@ bool Library::addBook(const Book& book)
 void Library::addBookmark(const Bookmark& bookmark)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  m_bookmarks.push_back(bookmark);
+  mp_impl->m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
+  for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
-      m_bookmarks.erase(it);
+      mp_impl->m_bookmarks.erase(it);
       return true;
     }
   }
@@ -141,32 +204,42 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
-void Library::dropReader(const std::string& id)
+void Library::dropCache(const std::string& id)
 {
-  m_readers.erase(id);
-  m_archives.erase(id);
+  mp_impl->mp_archiveCache->drop(id);
+  mp_impl->mp_searcherCache->drop(id);
 }
 
 bool Library::removeBookById(const std::string& id)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  m_bookDB->delete_document("Q" + id);
-  dropReader(id);
-  return m_books.erase(id) == 1;
+  mp_impl->m_bookDB.delete_document("Q" + id);
+  dropCache(id);
+  // We do not change the cache size here
+  // Most of the time, the book is remove in case of library refresh, it is
+  // often associated with addBook calls (which will properly set the cache size)
+  // Having a too big cache is not a problem here (or it would have been before)
+  // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+  //  will be removed in put or getOrPut).
+  const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
+  if ( bookWasRemoved ) {
+    ++mp_impl->m_revision;
+  }
+  return bookWasRemoved;
 }
 
 Library::Revision Library::getRevision() const
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  return m_revision;
+  return mp_impl->m_revision;
 }
 
-uint32_t Library::removeBooksNotUpdatedSince(LibraryRevision libraryRevision)
+uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
 {
   BookIdCollection booksToRemove;
   {
     std::lock_guard<std::mutex> lock(m_mutex);
-    for ( const auto& entry : m_books) {
+    for ( const auto& entry : mp_impl->m_books) {
       if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
         booksToRemove.push_back(entry.first);
       }
@@ -185,7 +258,7 @@ const Book& Library::getBookById(const std::string& id) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  return m_books.at(id);
+  return mp_impl->m_books.at(id);
 }
 
 Book Library::getBookByIdThreadSafe(const std::string& id) const
@@ -198,7 +271,7 @@ const Book& Library::getBookByPath(const std::string& path) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  for(auto& it: m_books) {
+  for(auto& it: mp_impl->m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
       return book;
@@ -208,52 +281,48 @@ const Book& Library::getBookByPath(const std::string& path) const
   throw std::out_of_range(ss.str());
 }
 
-std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
-{
-  try {
-    std::lock_guard<std::mutex> lock(m_mutex);
-    return m_readers.at(id);
-  } catch (std::out_of_range& e) {}
-
-  const auto archive = getArchiveById(id);
-  if ( !archive )
-    return nullptr;
-
-  const shared_ptr<Reader> reader(new Reader(archive, true));
-  std::lock_guard<std::mutex> lock(m_mutex);
-  m_readers[id] = reader;
-  return reader;
-}
-
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
   try {
-    return m_archives.at(id);
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
+    return mp_impl->mp_archiveCache->getOrPut(id,
+    [&](){
+      auto book = getBookById(id);
+      if (!book.isPathValid()) {
+        throw std::invalid_argument("");
+      }
+      return std::make_shared<zim::Archive>(book.getPath());
+    });
+  } catch (std::invalid_argument&) {
     return nullptr;
+  }
+}
 
-  auto sptr = make_shared<zim::Archive>(book.getPath());
-  m_archives[id] = sptr;
-  return sptr;
+std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
+{
+  assert(!ids.empty());
+  try {
+    return mp_impl->mp_searcherCache->getOrPut(ids,
+    [&](){
+      std::vector<zim::Archive> archives;
+      for(auto& id:ids) {
+        auto archive = getArchiveById(id);
+        if(!archive) {
+          throw std::invalid_argument("");
+        }
+        archives.push_back(*archive);
+      }
+      return std::make_shared<ZimSearcher>(zim::Searcher(archives));
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  unsigned int result = 0;
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    if ((!book.getPath().empty() && localBooks)
-        || (book.getPath().empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
+  return mp_impl->getBookCount(localBooks, remoteBooks);
 }
 
 bool Library::writeToFile(const std::string& path) const
@@ -284,7 +353,7 @@ Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) con
   std::lock_guard<std::mutex> lock(m_mutex);
   AttributeCounts propValueCounts;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     if (book.getOrigId().empty()) {
       propValueCounts[(book.*p)()] += 1;
@@ -304,12 +373,27 @@ std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
 
 std::vector<std::string> Library::getBooksLanguages() const
 {
-  return getBookPropValueSet(&Book::getLanguage);
+  std::vector<std::string> langs;
+  for ( const auto& langAndCount : getBooksLanguagesWithCounts() ) {
+    langs.push_back(langAndCount.first);
+  }
+  return langs;
 }
 
 Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
 {
-  return getBookAttributeCounts(&Book::getLanguage);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  AttributeCounts langsWithCounts;
+
+  for (const auto& pair: mp_impl->m_books) {
+    const auto& book = pair.second;
+    if (book.getOrigId().empty()) {
+      for ( const auto& lang : book.getLanguages() ) {
+        ++langsWithCounts[lang];
+      }
+    }
+  }
+  return langsWithCounts;
 }
 
 std::vector<std::string> Library::getBooksCategories() const
@@ -317,7 +401,7 @@ std::vector<std::string> Library::getBooksCategories() const
   std::lock_guard<std::mutex> lock(m_mutex);
   std::set<std::string> categories;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     const auto& c = book.getCategory();
     if ( !c.empty() ) {
@@ -341,12 +425,12 @@ std::vector<std::string> Library::getBooksPublishers() const
 const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
 {
   if (!onlyValidBookmarks) {
-    return m_bookmarks;
+    return mp_impl->m_bookmarks;
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
   std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto& bookmark:m_bookmarks) {
+  for(auto& bookmark:mp_impl->m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
     }
@@ -359,7 +443,7 @@ Library::BookIdCollection Library::getBooksIds() const
   std::lock_guard<std::mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
-  for (auto& pair: m_books) {
+  for (auto& pair: mp_impl->m_books) {
     bookIds.push_back(pair.first);
   }
 
@@ -371,12 +455,14 @@ void Library::updateBookDB(const Book& book)
 {
   Xapian::Stem stemmer;
   Xapian::TermGenerator indexer;
-  const std::string lang = book.getLanguage();
-  try {
-    stemmer = Xapian::Stem(iso639_3ToXapian(lang));
-    indexer.set_stemmer(stemmer);
-    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
-  } catch (...) {}
+  const auto langs = book.getLanguages();
+  if ( langs.size() == 1 ) {
+    try {
+      stemmer = Xapian::Stem(iso639_3ToXapian(langs[0]));
+      indexer.set_stemmer(stemmer);
+      indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
+    } catch (...) {}
+  }
   Xapian::Document doc;
   indexer.set_document(doc);
 
@@ -391,21 +477,28 @@ void Library::updateBookDB(const Book& book)
   // Index all fields for field-based search
   indexer.index_text(title, 1, "S");
   indexer.index_text(desc,  1, "XD");
-  indexer.index_text(lang,  1, "L");
+  for ( const auto& lang : langs ) {
+    indexer.index_text(lang,  1, "L");
+  }
   indexer.index_text(normalizeText(book.getCreator()),   1, "A");
   indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
   indexer.index_text(normalizeText(book.getName()),      1, "XN");
   indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
 
-  for ( const auto& tag : split(normalizeText(book.getTags()), ";") )
+  for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
     doc.add_boolean_term("XT" + tag);
+    if ( tag[0] != '_' ) {
+      indexer.increase_termpos();
+      indexer.index_text(tag);
+    }
+  }
 
   const std::string idterm = "Q" + book.getId();
   doc.add_boolean_term(idterm);
 
   doc.set_data(book.getId());
 
-  m_bookDB->replace_document(idterm, doc);
+  mp_impl->m_bookDB.replace_document(idterm, doc);
 }
 
 namespace
@@ -461,9 +554,20 @@ Xapian::Query categoryQuery(const std::string& category)
   return Xapian::Query("XC" + normalizeText(category));
 }
 
-Xapian::Query langQuery(const std::string& lang)
+Xapian::Query langQuery(const std::string& commaSeparatedLanguageList)
 {
-  return Xapian::Query("L" + normalizeText(lang));
+  Xapian::Query q;
+  bool firstIteration = true;
+  for ( const auto& lang : kiwix::split(commaSeparatedLanguageList, ",") ) {
+    const Xapian::Query singleLangQuery("L" + normalizeText(lang));
+    if ( firstIteration ) {
+      q = singleLangQuery;
+      firstIteration = false;
+    } else {
+      q = Xapian::Query(Xapian::Query::OP_OR, q, singleLangQuery);
+    }
+  }
+  return q;
 }
 
 Xapian::Query publisherQuery(const std::string& publisher)
@@ -538,9 +642,9 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
   BookIdCollection bookIds;
 
   std::lock_guard<std::mutex> lock(m_mutex);
-  Xapian::Enquire enquire(*m_bookDB);
+  Xapian::Enquire enquire(mp_impl->m_bookDB);
   enquire.set_query(query);
-  const auto results = enquire.get_mset(0, m_books.size());
+  const auto results = enquire.get_mset(0, mp_impl->m_books.size());
   for ( auto it = results.begin(); it != results.end(); ++it  ) {
     bookIds.push_back(it.get_document().get_data());
   }
@@ -554,7 +658,7 @@ Library::BookIdCollection Library::filter(const Filter& filter) const
   const auto preliminaryResult = filterViaBookDB(filter);
   std::lock_guard<std::mutex> lock(m_mutex);
   for(auto id : preliminaryResult) {
-    if(filter.accept(m_books.at(id))) {
+    if(filter.accept(mp_impl->m_books.at(id))) {
       result.push_back(id);
     }
   }
@@ -774,6 +878,18 @@ Filter& Filter::name(std::string name)
   return *this;
 }
 
+Filter& Filter::clearLang()
+{
+  activeFilters &= ~LANG;
+  return *this;
+}
+
+Filter& Filter::clearCategory()
+{
+  activeFilters &= ~CATEGORY;
+  return *this;
+}
+
 #define ACTIVE(X) (activeFilters & (X))
 #define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
 bool Filter::hasQuery() const

src/library_dumper.cpp

@@ -0,0 +1,120 @@
+#include "library_dumper.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibraryDumper::LibraryDumper(const Library* library, const NameMapper* nameMapper)
+  : library(library),
+    nameMapper(nameMapper)
+{
+}
+/* Destructor */
+LibraryDumper::~LibraryDumper()
+{
+}
+
+void LibraryDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+}
+
+namespace {
+
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"hbs", "srpskohrvatski"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"},
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
+  }
+}
+
+std::string getLanguageSelfName(const std::string& lang) {
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
+};
+
+} // unnamed namespace
+
+kainjow::mustache::list LibraryDumper::getCategoryData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : library->getBooksCategories() ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
+  }
+  return categoryData;
+}
+
+kainjow::mustache::list LibraryDumper::getLanguageData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+  return languageData;
+}
+
+} // namespace kiwix

src/libxml_dumper.cpp

@@ -54,7 +54,7 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
   if (book.getOrigId().empty()) {
     ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
     ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
-    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getCommaSeparatedLanguages());
     ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
     ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
@@ -97,7 +97,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
     auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
-    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "language", book.getCommaSeparatedLanguages());
     ADD_TEXT_ENTRY(book_node, "date", book.getDate());
   } catch (...) {
     ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());

src/manager.cpp

@@ -238,7 +238,7 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
+        || (!book.getTitle().empty() && !book.getLanguages().empty()
             && !book.getDate().empty())) {
       book.setUrl(url);
       manipulator->addBookToLibrary(book);

src/meson.build

@@ -5,11 +5,10 @@ kiwix_sources = [
   'manager.cpp',
   'libxml_dumper.cpp',
   'opds_dumper.cpp',
+  'html_dumper.cpp',
+  'library_dumper.cpp',
   'downloader.cpp',
-  'reader.cpp',
-  'entry.cpp',
   'server.cpp',
-  'searcher.cpp',
   'search_renderer.cpp',
   'subprocess.cpp',
   'aria2.cpp',
@@ -27,11 +26,13 @@ kiwix_sources = [
   'server/request_context.cpp',
   'server/response.cpp',
   'server/internalServer.cpp',
-  'server/internalServer_catalog_v2.cpp',
+  'server/internalServer_catalog.cpp',
+  'server/i18n.cpp',
   'opds_catalog.cpp',
   'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
 if host_machine.system() == 'windows'
   kiwix_sources += 'subprocess_windows.cpp'
@@ -39,22 +40,14 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if wrapper.contains('android')
-  install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
-else
-  install_dir = get_option('libdir')
-endif
-
-if wrapper.contains('android') or wrapper.contains('java')
-  subdir('wrapper/java')
-endif
+install_dir = get_option('libdir')
 
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
                input : 'config.h.in')
 install_headers(config_h, subdir:'kiwix')
 
-kiwixlib = library('kiwix',
+libkiwix = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,

src/opds_dumper.cpp

@@ -20,9 +20,8 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include <mustache.hpp>
-#include <unicode/locid.h>
 
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
@@ -31,8 +30,8 @@ namespace kiwix
 {
 
 /* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
-  : library(library)
+OPDSDumper::OPDSDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
 {
 }
 /* Destructor */
@@ -40,16 +39,11 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
-{
-  m_totalResults = totalResults;
-  m_startIndex = startIndex,
-  m_count = count;
-}
-
 namespace
 {
 
+const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
+
 typedef kainjow::mustache::data MustacheData;
 typedef kainjow::mustache::list BooksData;
 typedef kainjow::mustache::list IllustrationInfo;
@@ -70,16 +64,17 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
     return illustrations;
 }
 
-kainjow::mustache::object getSingleBookData(const Book& book)
+std::string fullEntryXML(const Book& book, const std::string& rootLocation, const std::string& contentId)
 {
     const auto bookDate = book.getDate() + "T00:00:00Z";
-    return kainjow::mustache::object{
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
-      {"language", book.getLanguage()},
-      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"language", book.getCommaSeparatedLanguages()},
+      {"content_id",  urlEncode(contentId)},
       {"updated", bookDate}, // XXX: this should be the entry update datetime
       {"book_date", bookDate},
       {"category", book.getCategory()},
@@ -93,27 +88,34 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
+    return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+std::string partialEntryXML(const Book& book, const std::string& rootLocation)
 {
-  auto data = getSingleBookData(book);
-  data["with_xml_header"] = MustacheData(withXMLHeader);
-  data["dump_partial_entries"] = MustacheData(partial);
-  data["endpoint_root"] = endpointRoot;
-  data["root"] = rootLocation;
-  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+    const auto bookDate = book.getDate() + "T00:00:00Z";
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
+      {"endpoint_root", rootLocation + "/catalog/v2"},
+      {"id", book.getId()},
+      {"title", book.getTitle()},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+    };
+    const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
+    return render_template(xmlTemplate, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library, const NameMapper* nameMapper, const std::vector<std::string>& bookIds, const std::string& rootLocation, bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
     try {
       const Book book = library->getBookByIdThreadSafe(bookId);
-      booksData.push_back(kainjow::mustache::object{
-          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
-      });
+      const std::string contentId = nameMapper->getNameForId(bookId);
+      const auto entryXML = partial
+                          ? partialEntryXML(book, rootLocation)
+                          : fullEntryXML(book, rootLocation, contentId);
+      booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
     } catch ( const std::out_of_range& ) {
       // the book was removed from the library since its id was obtained
       // ignore it
@@ -123,70 +125,11 @@ BooksData getBooksData(const Library* library, const std::vector<std::string>& b
   return booksData;
 }
 
-std::map<std::string, std::string> iso639_3 = {
-  {"atj", "atikamekw"},
-  {"azb", "آذربایجان دیلی"},
-  {"bcl", "central bikol"},
-  {"bgs", "tagabawa"},
-  {"bxr", "буряад хэлэн"},
-  {"cbk", "chavacano"},
-  {"cdo", "閩東語"},
-  {"dag", "Dagbani"},
-  {"diq", "dimli"},
-  {"dty", "डोटेली"},
-  {"eml", "emiliân-rumagnōl"},
-  {"fbs", "српскохрватски"},
-  {"ido", "ido"},
-  {"kbp", "kabɩyɛ"},
-  {"kld", "Gamilaraay"},
-  {"lbe", "лакку маз"},
-  {"lbj", "ལ་དྭགས་སྐད་"},
-  {"map", "Austronesian"},
-  {"mhr", "марий йылме"},
-  {"mnw", "ဘာသာမန်"},
-  {"myn", "mayan"},
-  {"nah", "nahuatl"},
-  {"nai", "north American Indian"},
-  {"nds", "plattdütsch"},
-  {"nrm", "bhasa narom"},
-  {"olo", "livvi"},
-  {"pih", "Pitcairn-Norfolk"},
-  {"pnb", "Western Panjabi"},
-  {"rmr", "Caló"},
-  {"rmy", "romani shib"},
-  {"roa", "romance languages"},
-  {"twi", "twi"}
-};
-
-std::once_flag fillLanguagesFlag;
-
-void fillLanguagesMap()
-{
-  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
-    auto lang = *icuLangPtr;
-    const icu::Locale locale(lang);
-    icu::UnicodeString ustring;
-    locale.getDisplayLanguage(locale, ustring);
-    std::string displayLanguage;
-    ustring.toUTF8String(displayLanguage);
-    std::string iso3LangCode = locale.getISO3Language();
-    iso639_3.insert({iso3LangCode, displayLanguage});
-  }
-}
-
-std::string getLanguageSelfName(const std::string& lang) {
-  const auto itr = iso639_3.find(lang);
-  if (itr != iso639_3.end()) {
-    return itr->second;
-  }
-  return lang;
-};
-
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
@@ -204,20 +147,20 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
+  const std::string url = endpoint + (query.empty() ? "" : "?" + query);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
      {"filter", onlyAsNonEmptyMustacheValue(query)},
-     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"self_url", url},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
-     {"books", booksData },
-     {"dump_partial_entries", MustacheData(partial)}
+     {"books", booksData }
   };
 
   return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
@@ -225,23 +168,17 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+  const auto book = library->getBookById(bookId);
+  const std::string contentId = nameMapper->getNameForId(bookId);
+  return XML_HEADER
+         + "\n"
+         + fullEntryXML(book, rootLocation, contentId);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list categoryData;
-  for ( const auto& category : library->getBooksCategories() ) {
-    const auto urlencodedCategoryName = urlEncode(category);
-    categoryData.push_back(kainjow::mustache::object{
-      {"name", category},
-      {"urlencoded_name",  urlencodedCategoryName},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
-    });
-  }
-
+  kainjow::mustache::list categoryData = getCategoryData();
   return render_template(
              RESOURCE::templates::catalog_v2_categories_xml,
              kainjow::mustache::object{
@@ -256,21 +193,7 @@ std::string OPDSDumper::categoriesOPDSFeed() const
 std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list languageData;
-  std::call_once(fillLanguagesFlag, fillLanguagesMap);
-  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
-    const std::string languageCode = langAndBookCount.first;
-    const int bookCount = langAndBookCount.second;
-    const auto languageSelfName = getLanguageSelfName(languageCode);
-    languageData.push_back(kainjow::mustache::object{
-      {"lang_code",  languageCode},
-      {"lang_self_name", languageSelfName},
-      {"book_count", to_string(bookCount)},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
-    });
-  }
-
+  kainjow::mustache::list languageData = getLanguageData();
   return render_template(
              RESOURCE::templates::catalog_v2_languages_xml,
              kainjow::mustache::object{

src/reader.cpp

@@ -1,472 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-#include <zim/item.h>
-#include <zim/error.h>
-
-#include "tools.h"
-#include "tools/stringTools.h"
-#include "tools/otherTools.h"
-#include "tools/archiveTools.h"
-
-namespace kiwix
-{
-/* Constructor */
-Reader::Reader(const string zimFilePath)
-  :  zimArchive(nullptr),
-     zimFilePath(zimFilePath)
-{
-  string tmpZimFilePath = zimFilePath;
-
-  /* Remove potential trailing zimaa */
-  size_t found = tmpZimFilePath.rfind("zimaa");
-  if (found != string::npos && tmpZimFilePath.size() > 5
-      && found == tmpZimFilePath.size() - 5) {
-    tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
-  }
-
-  zimArchive.reset(new zim::Archive(tmpZimFilePath));
-
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(const std::shared_ptr<zim::Archive> archive, bool _marker)
-  : zimArchive(archive),
-    zimFilePath(archive->getFilename())
-  {}
-
-#ifndef _WIN32
-Reader::Reader(int fd)
-  :  zimArchive(new zim::Archive(fd)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(int fd, zim::offset_type offset, zim::size_type size)
-  :  zimArchive(new zim::Archive(fd, offset, size)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-#endif // #ifndef _WIN32
-
-zim::Archive* Reader::getZimArchive() const
-{
-  return zimArchive.get();
-}
-
-MimeCounterType Reader::parseCounterMetadata() const
-{
-  return kiwix::parseArchiveCounter(*zimArchive);
-}
-
-/* Get the count of articles which can be indexed/displayed */
-unsigned int Reader::getArticleCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  for(auto &pair:counterMap) {
-    if (startsWith(pair.first, "text/html")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-/* Get the count of medias content in the ZIM file */
-unsigned int Reader::getMediaCount() const
-{
-  return kiwix::getArchiveMediaCount(*zimArchive);
-}
-
-/* Get the total of all items of a ZIM file, redirects included */
-unsigned int Reader::getGlobalCount() const
-{
-  return zimArchive->getEntryCount();
-}
-
-/* Return the UID of the ZIM file */
-string Reader::getId() const
-{
-  return kiwix::getArchiveId(*zimArchive);
-}
-
-Entry Reader::getRandomPage() const
-{
-  try {
-    return Entry(zimArchive->getRandomEntry(), true);
-  } catch(...) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getMainPage() const
-{
-  return Entry(zimArchive->getMainEntry(), true);
-}
-
-bool Reader::getFavicon(string& content, string& mimeType) const
-{
-  return kiwix::getArchiveFavicon(*zimArchive, 48, content, mimeType);
-}
-
-string Reader::getZimFilePath() const
-{
-  return zimFilePath;
-}
-/* Return a metatag value */
-bool Reader::getMetadata(const string& name, string& value) const
-{
-  try {
-    value = zimArchive->getMetadata(name);
-    return true;
-  } catch(zim::EntryNotFound& e) {
-    return false;
-  }
-}
-
-#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
-
-string Reader::getName() const
-{
-  return kiwix::getMetaName(*zimArchive);
-}
-
-string Reader::getTitle() const
-{
-  return kiwix::getArchiveTitle(*zimArchive);
-}
-
-string Reader::getCreator() const
-{
-  return kiwix::getMetaCreator(*zimArchive);
-}
-
-string Reader::getPublisher() const
-{
-  return kiwix::getMetaPublisher(*zimArchive);
-}
-
-string Reader::getDate() const
-{
-  return kiwix::getMetaDate(*zimArchive);
-}
-
-string Reader::getDescription() const
-{
-  return kiwix::getMetaDescription(*zimArchive);
-}
-
-string Reader::getLongDescription() const
-{
-  METADATA("LongDescription")
-}
-
-string Reader::getLanguage() const
-{
-  return kiwix::getMetaLanguage(*zimArchive);
-}
-
-string Reader::getLicense() const
-{
-  METADATA("License")
-}
-
-string Reader::getTags(bool original) const
-{
-  return kiwix::getMetaTags(*zimArchive, original);
-}
-
-
-string Reader::getTagStr(const std::string& tagName) const
-{
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  return getTagValueFromTagList(convertTags(tags_str), tagName);
-}
-
-bool Reader::getTagBool(const std::string& tagName) const
-{
-  return convertStrToBool(getTagStr(tagName));
-}
-
-string Reader::getRelation() const
-{
-  METADATA("Relation")
-}
-
-string Reader::getFlavour() const
-{
-  return kiwix::getMetaFlavour(*zimArchive);
-}
-
-string Reader::getSource() const
-{
-  METADATA("Source")
-}
-
-string Reader::getScraper() const
-{
-  METADATA("Scraper")
-}
-#undef METADATA
-
-Entry Reader::getEntryFromPath(const std::string& path) const
-{
-  try {
-    return Entry(kiwix::getEntryFromPath(*zimArchive, path), true);
-  } catch (zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getEntryFromEncodedPath(const std::string& path) const
-{
-  return getEntryFromPath(urlDecode(path, true));
-}
-
-Entry Reader::getEntryFromTitle(const std::string& title) const
-{
-  try {
-    return Entry(zimArchive->getEntryByTitle(title), true);
-  } catch(zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-bool Reader::pathExists(const string& path) const
-{
-  return zimArchive->hasEntryByPath(path);
-}
-
-/* Does the ZIM file has a fulltext index */
-bool Reader::hasFulltextIndex() const
-{
-  return zimArchive->hasFulltextIndex();
-}
-
-/* Search titles by prefix */
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               const bool reset)
-{
-  /* Reset the suggestions otherwise check if the suggestions number is less
-   * than the suggestionsCount */
-  if (reset) {
-    this->suggestions.clear();
-    this->suggestionsOffset = this->suggestions.begin();
-  } else {
-    if (this->suggestions.size() > suggestionsCount) {
-      return false;
-    }
-  }
-
-  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
-
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               SuggestionsList_t& results)
-{
-  bool retVal = false;
-
-  /* Return if no prefix */
-  if (prefix.size() == 0) {
-    return false;
-  }
-
-  for (auto& entry: zimArchive->findByTitle(prefix)) {
-    if (results.size() >= suggestionsCount) {
-      break;
-    }
-    /* Extract the interesting part of article title & url */
-    std::string normalizedArticleTitle
-        = kiwix::normalize(entry.getTitle());
-
-    // Get the final path.
-    auto item = entry.getItem(true);
-    std::string articleFinalUrl = item.getPath();
-
-    /* Go through all already found suggestions and skip if this
-       article is already in the suggestions list (with an other
-       title) */
-    bool insert = true;
-    std::vector<SuggestionItem>::iterator suggestionItr;
-    for (suggestionItr = results.begin();
-         suggestionItr != results.end();
-         suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr).getNormalizedTitle());
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr).getPath()) == 0) {
-        insert = false;
-        break;
-      } else if (result < 0) {
-        break;
-      }
-    }
-
-    /* Insert if possible */
-    if (insert) {
-      SuggestionItem suggestion(entry.getTitle(), normalizedArticleTitle, articleFinalUrl);
-      results.insert(suggestionItr, suggestion);
-    }
-
-    /* Suggestions where found */
-    retVal = true;
-  }
-
-  return retVal;
-}
-
-std::vector<std::string> Reader::getTitleVariants(
-    const std::string& title) const
-{
-  return kiwix::getTitleVariants(title);
-}
-
-
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount)
-{
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
-
-  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
-
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-/* Try also a few variations of the prefix to have better results */
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount,
-                                    SuggestionsList_t& results)
-{
-  std::vector<std::string> variants = this->getTitleVariants(prefix);
-
-  auto suggestionSearcher = zim::SuggestionSearcher(*zimArchive);
-
-  if (zimArchive->hasTitleIndex()) {
-    auto suggestionSearch = suggestionSearcher.suggest(prefix);
-    const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
-    for (auto current : suggestions) {
-      SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-      results.push_back(suggestion);
-    }
-  } else {
-    // Check some of the variants of the prefix
-    for (std::vector<std::string>::iterator variantsItr = variants.begin();
-         variantsItr != variants.end();
-         variantsItr++) {
-      auto suggestionSearch = suggestionSearcher.suggest(*variantsItr);
-      for (auto current : suggestionSearch.getResults(0, suggestionsCount)) {
-        if (results.size() >= suggestionsCount) {
-          break;
-        }
-
-        SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-        results.push_back(suggestion);
-      }
-    }
-  }
-
-  return results.size() > 0;
-}
-
-/* Get next suggestion */
-bool Reader::getNextSuggestion(string& title)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-bool Reader::getNextSuggestion(string& title, string& url)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-    url = (*(this->suggestionsOffset)).getPath();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-/* Check if the file has as checksum */
-bool Reader::canCheckIntegrity() const
-{
-  return zimArchive->hasChecksum();
-}
-
-/* Return true if corrupted, false otherwise */
-bool Reader::isCorrupted() const
-{
-  try {
-    if (zimArchive->check() == true) {
-      return false;
-    }
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return true;
-  }
-
-  return true;
-}
-
-/* Return the file size, works also for splitted files */
-unsigned int Reader::getFileSize() const
-{
-  return kiwix::getArchiveFileSize(*zimArchive);
-}
-
-}

src/search_renderer.cpp

@@ -21,8 +21,6 @@
 #include <cmath>
 
 #include "search_renderer.h"
-#include "searcher.h"
-#include "reader.h"
 #include "library.h"
 #include "name_mapper.h"
 
@@ -31,22 +29,13 @@
 #include <zim/search.h>
 
 #include <mustache.hpp>
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include "tools/stringTools.h"
 
 namespace kiwix
 {
 
 /* Constructor */
-SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : SearchRenderer(
-        searcher->getSearchResultSet(),
-        mapper,
-        nullptr,
-        searcher->getEstimatedResultCount(),
-        searcher->getResultStart())
-{}
-
 SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                       unsigned int start, unsigned int estimatedResultCount)
     : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
@@ -58,7 +47,7 @@ SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Lib
       mp_nameMapper(mapper),
       mp_library(library),
       protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
+      searchProtocolPrefix("search://"),
       estimatedResultCount(estimatedResultCount),
       resultStart(start)
 {}
@@ -68,12 +57,12 @@ SearchRenderer::~SearchRenderer() = default;
 
 void SearchRenderer::setSearchPattern(const std::string& pattern)
 {
-  this->searchPattern = pattern;
+  searchPattern = pattern;
 }
 
-void SearchRenderer::setSearchContent(const std::string& name)
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
 {
-  this->searchContent = name;
+  searchBookQuery = bookQuery;
 }
 
 void SearchRenderer::setProtocolPrefix(const std::string& prefix)
@@ -86,85 +75,162 @@ void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
   this->searchProtocolPrefix = prefix;
 }
 
-std::string SearchRenderer::getHtml()
-{
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
+std::string extractValueFromQuery(const std::string& query, const std::string& key) {
+  const std::string p = key + "=";
+  const size_t i = query.find(p);
+  if (i == std::string::npos) {
+    return "";
+  }
+  std::string r = query.substr(i + p.size());
+  return r.substr(0, r.find("&"));
+}
 
-  for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
-    kainjow::mustache::data result;
-    result.set("title", it.getTitle());
-    result.set("url", it.getPath());
-    result.set("snippet", it.getSnippet());
-    std::string zim_id(it.getZimId());
-    result.set("resultContentId", mp_nameMapper->getNameForId(zim_id));
-    if (!mp_library) {
-      result.set("bookTitle", kainjow::mustache::data(false));
-    } else {
-      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+kainjow::mustache::data buildQueryData
+(
+  const std::string& searchProtocolPrefix,
+  const std::string& pattern,
+  const std::string& bookQuery
+) {
+  kainjow::mustache::data query;
+  query.set("pattern", kiwix::encodeDiples(pattern));
+  std::ostringstream ss;
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
+  ss << "&" << bookQuery;
+  query.set("unpaginatedQuery", ss.str());
+  auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
+  if(!lang.empty()) {
+    query.set("lang", lang);
+  }
+  return query;
+}
+
+kainjow::mustache::data buildPagination(
+  unsigned int pageLength,
+  unsigned int resultsCount,
+  unsigned int resultsStart
+)
+{
+  assert(pageLength!=0);
+  kainjow::mustache::data pagination;
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  if (resultsCount == 0) {
+    // Easy case
+    pagination.set("itemsPerPage", to_string(pageLength));
+    pagination.set("hasPages", false);
+    pagination.set("pages", pages);
+    return pagination;
+  }
+
+  // First we want to display pages starting at a multiple of `pageLength`
+  // so, let's calculate the start index of the current page.
+  auto currentPage = resultsStart/pageLength;
+  auto lastPage = ((resultsCount-1)/pageLength);
+  auto lastPageStart = lastPage*pageLength;
+  auto nbPages = lastPage + 1;
+
+  auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
+  auto lastPageGenerated = std::min(currentPage+4, lastPage);
+
+  if (nbPages != 1) {
+    if (firstPageGenerated!=0) {
+      kainjow::mustache::data page;
+      page.set("label", "◀");
+      page.set("start", to_string(0));
+      page.set("current", false);
+      pages.push_back(page);
     }
 
+    for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
+      kainjow::mustache::data page;
+      page.set("label", to_string(i+1));
+      page.set("start", to_string(i*pageLength));
+      page.set("current", bool(i == currentPage));
+      pages.push_back(page);
+    }
+
+    if (lastPageGenerated!=lastPage) {
+      kainjow::mustache::data page;
+      page.set("label", "▶");
+      page.set("start", to_string(lastPageStart));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+  }
+
+  pagination.set("itemsPerPage", to_string(pageLength));
+  pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
+  pagination.set("pages", pages);
+  return pagination;
+}
+
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+{
+  const std::string absPathPrefix = protocolPrefix;
+  // Build the results list
+  kainjow::mustache::data items{kainjow::mustache::data::type::list};
+  for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
+    kainjow::mustache::data result;
+    const std::string zim_id(it.getZimId());
+    const auto path = mp_nameMapper->getNameForId(zim_id) + "/" + it.getPath();
+    result.set("title", it.getTitle());
+    result.set("absolutePath", absPathPrefix + urlEncode(path));
+    result.set("snippet", it.getSnippet());
+    if (mp_library) {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
     if (it.getWordCount() >= 0) {
       result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
     }
 
-    results.push_back(result);
+    items.push_back(result);
   }
+  kainjow::mustache::data results;
+  results.set("items", items);
+  results.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  results.set("hasResults", estimatedResultCount != 0);
+  results.set("start", kiwix::beautifyInteger(resultStart));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
 
-  // pages
-  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+  // pagination
+  auto pagination = buildPagination(
+    pageLength,
+    estimatedResultCount,
+    resultStart
+  );
 
-  auto resultEnd = 0U;
-  auto currentPage = 0U;
-  auto pageStart = 0U;
-  auto pageEnd = 0U;
-  auto lastPageStart = 0U;
-  if (pageLength) {
-    currentPage = resultStart/pageLength;
-    pageStart = currentPage > 4 ? currentPage-4 : 0;
-    pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / pageLength) {
-      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
-    }
-    if (estimatedResultCount > pageLength) {
-      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
-    }
-  }
+  kainjow::mustache::data query = buildQueryData(
+    searchProtocolPrefix,
+    searchPattern,
+    searchBookQuery
+  );
 
-  resultEnd = resultStart+pageLength; //setting result end
-
-  for (unsigned int i = pageStart; i < pageEnd; i++) {
-    kainjow::mustache::data page;
-    page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * pageLength));
-
-    if (i == currentPage) {
-      page.set("selected", true);
-    }
-    pages.push_back(page);
-  }
-
-  std::string template_str = RESOURCE::templates::search_result_html;
-  kainjow::mustache::mustache tmpl(template_str);
 
   kainjow::mustache::data allData;
+  allData.set("searchProtocolPrefix", searchProtocolPrefix);
   allData.set("results", results);
-  allData.set("pages", pages);
-  allData.set("hasResults", estimatedResultCount != 0);
-  allData.set("hasPages", pageStart != pageEnd);
-  allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
-  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
-  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
-  allData.set("resultStart", to_string(resultStart + 1));
-  allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("pageLength", to_string(pageLength));
-  allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("protocolPrefix", this->protocolPrefix);
-  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
-  allData.set("contentId", this->searchContent);
+  allData.set("pagination", pagination);
+  allData.set("query", query);
+
+  kainjow::mustache::mustache tmpl(tmpl_str);
 
   std::stringstream ss;
   tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  if (!tmpl.is_valid()) {
+    throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
+  }
   return ss.str();
 }
 
+std::string SearchRenderer::getHtml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_html);
+}
+
+std::string SearchRenderer::getXml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_xml);
+}
+
+
 }

src/searcher.cpp

@@ -1,330 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-
-#include "searcher.h"
-#include "reader.h"
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-
-#include <mustache.hpp>
-#include <cmath>
-#include "tools/stringTools.h"
-#include "kiwixlib-resources.h"
-
-#define MAX_SEARCH_LEN 140
-
-namespace kiwix
-{
-class _Result : public Result
-{
- public:
-  _Result(zim::SearchResultSet::iterator iterator);
-  _Result(SuggestionItem suggestionItem);
-  virtual ~_Result(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual std::string get_zimId();
-
- private:
-  zim::SearchResultSet::iterator iterator;
-  SuggestionItem suggestionItem;
-  bool isSuggestion;
-};
-
-struct SearcherInternal : zim::SearchResultSet {
-  explicit SearcherInternal(const zim::SearchResultSet& srs)
-    : zim::SearchResultSet(srs)
-    , current_iterator(srs.begin())
-  {
-  }
-
-  zim::SearchResultSet::iterator current_iterator;
-};
-
-struct SuggestionInternal : zim::SuggestionResultSet {
-  explicit SuggestionInternal(const zim::SuggestionResultSet& srs)
-    : zim::SuggestionResultSet(srs),
-      currentIterator(srs.begin()) {}
-
-  zim::SuggestionResultSet::iterator currentIterator;
-};
-
-/* Constructor */
-Searcher::Searcher()
-    : searchPattern(""),
-      estimatedResultCount(0),
-      resultStart(0),
-      maxResultCount(0)
-{
-  loadICUExternalTables();
-}
-
-/* Destructor */
-Searcher::~Searcher()
-{
-}
-
-bool Searcher::add_reader(Reader* reader)
-{
-  if (!reader->hasFulltextIndex()) {
-      return false;
-  }
-
-  for ( const Reader* const existing_reader : readers ) {
-    if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
-      return false;
-  }
-
-  this->readers.push_back(reader);
-  return true;
-}
-
-
-Reader* Searcher::get_reader(int readerIndex)
-{
-  return readers.at(readerIndex);
-}
-
-/* Search strings in the database */
-void Searcher::search(const std::string& search,
-                      unsigned int resultStart,
-                      unsigned int maxResultCount,
-                      const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing query `" << search << "'" << endl;
-  }
-
-  this->searchPattern = search;
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-  /* Try to find results */
-  if (maxResultCount != 0) {
-    /* Perform the search */
-    string unaccentedSearch = removeAccents(search);
-    std::vector<zim::Archive> archives;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      if ( (*current)->hasFulltextIndex() ) {
-          archives.push_back(*(*current)->getZimArchive());
-      }
-    }
-    zim::Searcher searcher(archives);
-    searcher.setVerbose(verbose);
-    zim::Query query;
-    query.setQuery(unaccentedSearch);
-    zim::Search search = searcher.search(query);
-    internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-    this->estimatedResultCount = search.getEstimatedMatches();
-  }
-
-  return;
-}
-
-
-void Searcher::geo_search(float latitude, float longitude, float distance,
-                          unsigned int resultStart,
-                          unsigned int maxResultCount,
-                          const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-  }
-
-  /* Perform the search */
-  std::ostringstream oss;
-  oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
-  this->searchPattern = oss.str();
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-
-  /* Try to find results */
-  if (maxResultCount == 0) {
-    return;
-  }
-
-  std::vector<zim::Archive> archives;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    archives.push_back(*(*current)->getZimArchive());
-  }
-  zim::Searcher searcher(archives);
-  searcher.setVerbose(verbose);
-  zim::Query query;
-  query.setQuery("");
-  query.setGeorange(latitude, longitude, distance);
-  zim::Search search = searcher.search(query);
-  internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-
-void Searcher::restart_search()
-{
-  if (internal.get()) {
-    internal->current_iterator = internal->begin();
-  }
-}
-
-Result* Searcher::getNextResult()
-{
-  if (internal.get() && internal->current_iterator != internal->end()) {
-    Result* result = new _Result(internal->current_iterator);
-    internal->current_iterator++;
-    return result;
-  } else if (suggestionInternal.get() &&
-             suggestionInternal->currentIterator != suggestionInternal->end()) {
-    SuggestionItem item(
-                        suggestionInternal->currentIterator->getTitle(),
-                        normalize(suggestionInternal->currentIterator->getTitle()),
-                        suggestionInternal->currentIterator->getPath(),
-                        suggestionInternal->currentIterator->getSnippet()
-                      );
-    Result* result = new _Result(item);
-    suggestionInternal->currentIterator++;
-    return result;
-  }
-  return NULL;
-}
-
-/* Reset the results */
-void Searcher::reset()
-{
-  this->estimatedResultCount = 0;
-  this->searchPattern = "";
-  return;
-}
-
-void Searcher::suggestions(std::string& searchPattern, const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
-  }
-
-  this->searchPattern = searchPattern;
-  this->resultStart = 0;
-  this->maxResultCount = 10;
-  string unaccentedSearch = removeAccents(searchPattern);
-
-  // Multizim suggestion is not supported as of now! taking only one archive
-  zim::Archive archive = *(*this->readers.begin())->getZimArchive();
-  zim::SuggestionSearcher searcher(archive);
-  searcher.setVerbose(verbose);
-  zim::SuggestionSearch search = searcher.suggest(searchPattern);
-  suggestionInternal.reset(new SuggestionInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-/* Return the result count estimation */
-unsigned int Searcher::getEstimatedResultCount()
-{
-  return this->estimatedResultCount;
-}
-
-zim::SearchResultSet Searcher::getSearchResultSet()
-{
-  return *(this->internal);
-}
-
-_Result::_Result(zim::SearchResultSet::iterator iterator)
-    : iterator(iterator),
-      suggestionItem("", "", ""),
-      isSuggestion(false)
-{}
-
-_Result::_Result(SuggestionItem item)
-    : iterator(),
-      suggestionItem(item.getTitle(), item.getNormalizedTitle(), item.getPath(), item.getSnippet()),
-      isSuggestion(true)
-{}
-
-std::string _Result::get_url()
-{
-  if (isSuggestion) {
-    return suggestionItem.getPath();
-  }
-  return iterator.getPath();
-}
-std::string _Result::get_title()
-{
-  if (isSuggestion) {
-    return suggestionItem.getTitle();
-  }
-  return iterator.getTitle();
-}
-int _Result::get_score()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getScore();
-}
-std::string _Result::get_snippet()
-{
-  if (isSuggestion) {
-    return suggestionItem.getSnippet();
-  }
-  return iterator.getSnippet();
-}
-std::string _Result::get_content()
-{
-  if (isSuggestion) return "";
-  return iterator->getItem(true).getData();
-}
-int _Result::get_size()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getSize();
-}
-int _Result::get_wordCount()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getWordCount();
-}
-std::string _Result::get_zimId()
-{
-  if (isSuggestion) {
-    return "";
-  }
-  std::ostringstream s;
-  s << iterator.getZimId();
-  return s.str();
-}
-
-
-}

src/server.cpp

@@ -24,6 +24,7 @@
 
 #include <string>
 
+#include <zim/item.h>
 #include "server/internalServer.h"
 
 namespace kiwix {
@@ -45,6 +46,7 @@ bool Server::start() {
     m_port,
     m_root,
     m_nbThreads,
+    m_multizimSearchLimit,
     m_verbose,
     m_withTaskbar,
     m_withLibraryButton,

src/server/etag.cpp

@@ -37,11 +37,11 @@ namespace {
 // into the ETag for ETag::Option opt.
 // IMPORTANT: The characters in all_options must come in sorted order (so that
 // IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
+const char all_options[] = "Zz";
 
 static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
 
-bool isValidServerId(const std::string& s)
+bool isValidETagBody(const std::string& s)
 {
   return !s.empty() && s.find_first_of("\"/") == std::string::npos;
 }
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
 
 std::string ETag::get_etag() const
 {
-  if ( m_serverId.empty() )
+  if ( m_body.empty() )
     return std::string();
 
-  return "\"" + m_serverId + "/" + m_options + "\"";
+  return "\"" + m_body + "/" + m_options + "\"";
 }
 
-ETag::ETag(const std::string& serverId, const std::string& options)
+ETag::ETag(const std::string& body, const std::string& options)
 {
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
   {
-    m_serverId = serverId;
+    m_body = body;
     m_options = options;
   }
 }
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
   return ETag(s.substr(0, i), s.substr(i+1));
 }
 
-ETag ETag::match(const std::string& etags, const std::string& server_id)
+ETag ETag::match(const std::string& etags, const std::string& body)
 {
   std::istringstream ss(etags);
   std::string etag_str;
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
       etag_str.pop_back();
 
     const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
+    if ( etag && etag.m_body == body )
       return etag;
   }
 

src/server/etag.h

@@ -28,10 +28,11 @@ namespace kiwix {
 // The ETag string used by Kiwix server (more precisely, its value inside the
 // double quotes) consists of two parts:
 //
-// 1. ServerId - The string obtained on server start up
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
 //
-// 2. Options -  Zero or more characters encoding the values of some of the
-//               headers of the response
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
+//               the values of some of the headers of the response
 //
 // The two parts are separated with a slash (/) symbol (which is always present,
 // even when the the options part is empty). Neither portion of a Kiwix ETag
@@ -40,7 +41,7 @@ namespace kiwix {
 //
 //   "abcdefghijklmn/"
 //   "1234567890/z"
-//   "1234567890/cz"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
 //
 // The options part of the Kiwix ETag allows to correctly set the required
 // headers when responding to a conditional If-None-Match request with a 304
@@ -51,7 +52,7 @@ class ETag
 {
   public: // types
     enum Option {
-      CACHEABLE_ENTITY,
+      ZIM_CONTENT,
       COMPRESSED_CONTENT,
       OPTION_COUNT
     };
@@ -59,10 +60,10 @@ class ETag
   public: // functions
     ETag() {}
 
-    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_body(const std::string& s) { m_body = s; }
     void set_option(Option opt);
 
-    explicit operator bool() const { return !m_serverId.empty(); }
+    explicit operator bool() const { return !m_body.empty(); }
 
     bool get_option(Option opt) const;
     std::string get_etag() const;
@@ -76,7 +77,7 @@ class ETag
     static ETag parse(std::string s);
 
   private: // data
-    std::string m_serverId;
+    std::string m_body;
     std::string m_options;
 };
 

src/server/i18n.cpp

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "i18n.h"
+
+#include "tools/otherTools.h"
+
+#include <algorithm>
+#include <map>
+
+namespace kiwix
+{
+
+const char* I18nStringTable::get(const std::string& key) const
+{
+  const I18nString* const begin = entries;
+  const I18nString* const end = begin + entryCount;
+  const I18nString* found = std::lower_bound(begin, end, key,
+      [](const I18nString& a, const std::string& k) {
+        return a.key < k;
+  });
+  return (found == end || found->key != key) ? nullptr : found->value;
+}
+
+namespace i18n
+{
+// this data is generated by the i18n resource compiler
+extern const I18nStringTable stringTables[];
+extern const size_t langCount;
+}
+
+namespace
+{
+
+class I18nStringDB
+{
+public: // functions
+  I18nStringDB() {
+    for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
+      const auto& t = kiwix::i18n::stringTables[i];
+      lang2TableMap[t.lang] = &t;
+    }
+    enStrings = lang2TableMap.at("en");
+  };
+
+  std::string get(const std::string& lang, const std::string& key) const {
+    const char* s = getStringsFor(lang)->get(key);
+    if ( s == nullptr ) {
+      s = enStrings->get(key);
+      if ( s == nullptr ) {
+        throw std::runtime_error("Invalid message id");
+      }
+    }
+    return s;
+  }
+
+  size_t getStringCount(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang)->entryCount;
+    } catch(const std::out_of_range&) {
+      return 0;
+    }
+  }
+
+private: // functions
+  const I18nStringTable* getStringsFor(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang);
+    } catch(const std::out_of_range&) {
+      return enStrings;
+    }
+  }
+
+private: // data
+  std::map<std::string, const I18nStringTable*> lang2TableMap;
+  const I18nStringTable* enStrings;
+};
+
+const I18nStringDB& getStringDb()
+{
+  static const I18nStringDB stringDb;
+  return stringDb;
+}
+
+} // unnamed namespace
+
+std::string getTranslatedString(const std::string& lang, const std::string& key)
+{
+  return getStringDb().get(lang, key);
+}
+
+namespace i18n
+{
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params)
+{
+  const std::string tmpl = getTranslatedString(lang, key);
+  return render_template(tmpl, params);
+}
+
+} // namespace i18n
+
+std::string ParameterizedMessage::getText(const std::string& lang) const
+{
+  return i18n::expandParameterizedString(lang, msgId, params);
+}
+
+namespace
+{
+
+LangPreference parseSingleLanguagePreference(const std::string& s)
+{
+  const size_t langStart = s.find_first_not_of(" \t\n");
+  if ( langStart == std::string::npos ) {
+    return {"", 0};
+  }
+
+  const size_t langEnd = s.find(';', langStart);
+  if ( langEnd == std::string::npos ) {
+    return {s.substr(langStart), 1};
+  }
+
+  const std::string lang = s.substr(langStart, langEnd - langStart);
+  // We don't care about langEnd == langStart which will result in an empty
+  // language name - it will be dismissed by parseUserLanguagePreferences()
+
+  float q = 1.0;
+  int nCharsScanned;
+  if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
+       && langEnd + 1 + nCharsScanned == s.size() ) {
+    return {lang, q};
+  }
+
+  return {"", 0};
+}
+
+} // unnamed namespace
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s)
+{
+  UserLangPreferences result;
+  std::istringstream iss(s);
+  std::string singleLangPrefStr;
+  while ( std::getline(iss, singleLangPrefStr, ',') )
+  {
+    const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
+    if ( !langPref.lang.empty() && langPref.preference > 0 ) {
+      result.push_back(langPref);
+    }
+  }
+
+  return result;
+}
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
+{
+  if ( prefs.empty() ) {
+    return "en";
+  }
+
+  std::string bestLangSoFar("en");
+  float bestScoreSoFar = 0;
+  const auto& stringDb = getStringDb();
+  for ( const auto& entry : prefs ) {
+    const float score = entry.preference * stringDb.getStringCount(entry.lang);
+    if ( score > bestScoreSoFar ) {
+      bestScoreSoFar = score;
+      bestLangSoFar = entry.lang;
+    }
+  }
+  return bestLangSoFar;
+}
+
+} // namespace kiwix

src/server/i18n.h

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_I18N
+#define KIWIX_SERVER_I18N
+
+#include <string>
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+struct I18nString {
+  const char* const key;
+  const char* const value;
+};
+
+struct I18nStringTable {
+  const char* const lang;
+  const size_t entryCount;
+  const I18nString* const entries;
+
+  const char* get(const std::string& key) const;
+};
+
+std::string getTranslatedString(const std::string& lang, const std::string& key);
+
+namespace i18n
+{
+
+typedef kainjow::mustache::object Parameters;
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params);
+
+class GetTranslatedString
+{
+public:
+  explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
+
+  std::string operator()(const std::string& key) const
+  {
+    return getTranslatedString(m_lang, key);
+  }
+
+  std::string operator()(const std::string& key, const Parameters& params) const
+  {
+    return expandParameterizedString(m_lang, key, params);
+  }
+
+private:
+  const std::string m_lang;
+};
+
+class GetTranslatedStringWithMsgId
+{
+  typedef kainjow::mustache::basic_data<std::string> MustacheString;
+  typedef std::pair<std::string, MustacheString> MsgIdAndTranslation;
+
+public:
+  explicit GetTranslatedStringWithMsgId(const std::string& lang) : m_lang(lang) {}
+
+  MsgIdAndTranslation operator()(const std::string& key) const
+  {
+    return {key, getTranslatedString(m_lang, key)};
+  }
+
+  MsgIdAndTranslation operator()(const std::string& key, const Parameters& params) const
+  {
+    return {key, expandParameterizedString(m_lang, key, params)};
+  }
+
+private:
+  const std::string m_lang;
+};
+
+} // namespace i18n
+
+struct ParameterizedMessage
+{
+public: // types
+  typedef kainjow::mustache::object Parameters;
+
+public: // functions
+  ParameterizedMessage(const std::string& msgId, const Parameters& params)
+    : msgId(msgId)
+    , params(params)
+  {}
+
+  std::string getText(const std::string& lang) const;
+
+private: // data
+  const std::string msgId;
+  const Parameters  params;
+};
+
+struct LangPreference
+{
+  const std::string lang;
+  const float preference;
+};
+
+typedef std::vector<LangPreference> UserLangPreferences;
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s);
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs);
+
+} // namespace kiwix
+
+#endif // KIWIX_SERVER_I18N

src/server/internalServer.h

@@ -68,31 +68,26 @@ struct GeoQuery {
 
 class SearchInfo {
   public:
-    SearchInfo(const std::string& pattern);
-    SearchInfo(const std::string& pattern, GeoQuery geoQuery);
-    SearchInfo(const RequestContext& request);
+    SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
 
     zim::Query getZimQuery(bool verbose) const;
+    const Library::BookIdSet& getBookIds() const { return bookIds; }
 
     friend bool operator<(const SearchInfo& l, const SearchInfo& r)
     {
-        return std::tie(l.bookName, l.pattern, l.geoQuery)
-             < std::tie(r.bookName, r.pattern, r.geoQuery); // keep the same order
+        return std::tie(l.bookIds, l.pattern, l.geoQuery)
+             < std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
     }
 
   public: //data
     std::string pattern;
     GeoQuery geoQuery;
-    std::string bookName;
+    Library::BookIdSet bookIds;
+    std::string bookFilterQuery;
 };
 
 
 typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
-typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
-typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
-
-class Entry;
 class OPDSDumper;
 
 class InternalServer {
@@ -103,13 +98,14 @@ class InternalServer {
                    int port,
                    std::string root,
                    int nbThreads,
+                   unsigned int multizimSearchLimit,
                    bool verbose,
                    bool withTaskbar,
                    bool withLibraryButton,
                    bool blockExternalLinks,
                    std::string indexTemplateString,
                    int ipConnectionLimit);
-    virtual ~InternalServer() = default;
+    virtual ~InternalServer();
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
                                const char* url,
@@ -127,6 +123,7 @@ class InternalServer {
     std::unique_ptr<Response> handle_request(const RequestContext& request);
     std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
     std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_viewer_settings(const RequestContext& request);
     std::unique_ptr<Response> handle_skin(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
@@ -134,28 +131,45 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
     std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
+    std::unique_ptr<Response> handle_no_js(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_search_request(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_catch(const RequestContext& request);
     std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
     std::unique_ptr<Response> handle_content(const RequestContext& request);
     std::unique_ptr<Response> handle_raw(const RequestContext& request);
+    std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
 
     MustacheData get_default_data() const;
 
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+    std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
+    SearchInfo getSearchInfo(const RequestContext& r) const;
+
+    bool isLocallyCustomizedResource(const std::string& url) const;
+
+    std::string getLibraryId() const;
+
+    std::string getNoJSDownloadPageHTML(const std::string& bookId, const std::string& userLang) const;
+
+  private: // types
+    class LockableSuggestionSearcher;
+    typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+    typedef ConcurrentCache<std::string, std::shared_ptr<LockableSuggestionSearcher>> SuggestionSearcherCache;
 
   private: // data
     std::string m_addr;
     int m_port;
-    std::string m_root;
+    std::string m_root;                   // URI-encoded
+    std::string m_rootPrefixOfDecodedURL; // URI-decoded
     int m_nbThreads;
+    unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
@@ -167,16 +181,17 @@ class InternalServer {
     Library* mp_library;
     NameMapper* mp_nameMapper;
 
-    SearcherCache searcherCache;
     SearchCache searchCache;
     SuggestionSearcherCache suggestionSearcherCache;
 
     std::string m_server_id;
-    std::string m_library_id;
+
+    class CustomizedResources;
+    std::unique_ptr<CustomizedResources> m_customizedResources;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 };
 
 }

src/server/internalServer_catalog.cpp

@@ -24,7 +24,7 @@
 #include "request_context.h"
 #include "response.h"
 #include "tools/otherTools.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include <mustache.hpp>
 
@@ -33,6 +33,74 @@
 
 namespace kiwix {
 
+namespace
+{
+
+enum OPDSResponseKind
+{
+  OPDS_ENTRY,
+  OPDS_NAVIGATION_FEED,
+  OPDS_ACQUISITION_FEED
+};
+
+const std::string opdsMimeType[] = {
+  "application/atom+xml;type=entry;profile=opds-catalog;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=navigation;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=acquisition;charset=utf-8"
+};
+
+} // unnamed namespace
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  if (url == "v2") {
+    return handle_catalog_v2(request);
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper(mp_library, mp_nameMapper);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(getLibraryId());
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    bookIdsToDump = search_catalog(request, opdsDumper);
+    uuid = zim::Uuid::generate();
+  }
+
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()),
+      opdsMimeType[OPDS_ACQUISITION_FEED]);
+  return std::move(response);
+}
+
 std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -43,7 +111,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return HTTP404HtmlResponse(*this, request)
+    return HTTP404Response(*this, request)
            + urlNotFoundMsg;
   }
 
@@ -70,40 +138,41 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   } else if (url == "illustration") {
     return handle_catalog_v2_illustration(request);
   } else {
-    return HTTP404HtmlResponse(*this, request)
+    return HTTP404Response(*this, request)
            + urlNotFoundMsg;
   }
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
 {
+  const std::string libraryId = getLibraryId();
   return ContentResponse::build(
              *this,
              RESOURCE::templates::catalog_v2_root_xml,
              kainjow::mustache::object{
                {"date", gen_date_str()},
                {"endpoint_root", m_root + "/catalog/v2"},
-               {"feed_id", gen_uuid(m_library_id)},
-               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
-               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
-               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
+               {"feed_id", gen_uuid(libraryId)},
+               {"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
+               {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
+               {"category_list_feed_id", gen_uuid(libraryId + "/categories")},
+               {"language_list_feed_id", gen_uuid(libraryId + "/languages")}
              },
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto bookIds = search_catalog(request, opdsDumper);
   const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
              *this,
              opdsFeed,
-             "application/atom+xml;profile=opds-catalog;kind=acquisition"
+             opdsMimeType[OPDS_ACQUISITION_FEED]
   );
 }
 
@@ -112,56 +181,59 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
   try {
     mp_library->getBookById(entryId);
   } catch (const std::out_of_range&) {
-    return HTTP404HtmlResponse(*this, request)
+    return HTTP404Response(*this, request)
            + urlNotFoundMsg;
   }
 
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
   return ContentResponse::build(
              *this,
              opdsFeed,
-             "application/atom+xml;type=entry;profile=opds-catalog"
+             opdsMimeType[OPDS_ENTRY]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.categoriesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.languagesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
 {
   try {
-    const auto bookName  = request.get_url_part(3);
-    const auto bookId = mp_nameMapper->getIdForName(bookName);
+    const auto bookId  = request.get_url_part(3);
     auto book = mp_library->getBookByIdThreadSafe(bookId);
     auto size = request.get_argument<unsigned int>("size");
     auto illustration = book.getIllustration(size);
-    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+    return ContentResponse::build(
+               *this,
+               illustration->getData(),
+               illustration->mimeType
+    );
   } catch(...) {
-    return HTTP404HtmlResponse(*this, request)
+    return HTTP404Response(*this, request)
            + urlNotFoundMsg;
   }
 }

src/server/request_context.cpp

@@ -25,8 +25,10 @@
 #include <sstream>
 #include <cstdio>
 #include <atomic>
+#include <cctype>
 
 #include "tools/stringTools.h"
+#include "i18n.h"
 
 namespace kiwix {
 
@@ -47,48 +49,35 @@ RequestMethod str2RequestMethod(const std::string& method) {
   else                          return RequestMethod::OTHER;
 }
 
-std::string
-fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
-{
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    return full_url;
-  } else if (full_url == rootLocation) {
-    return "/";
-  } else if (full_url.size() > rootLocation.size() &&
-             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-    return full_url.substr(rootLocation.size());
-  } else {
-    return "";
-  }
-}
-
 } // unnamed namespace
 
 RequestContext::RequestContext(struct MHD_Connection* connection,
-                               std::string rootLocation,
-                               const std::string& _url,
+                               const std::string& _rootLocation, // URI-encoded
+                               const std::string& unrootedUrl,   // URI-decoded
                                const std::string& _method,
                                const std::string& version) :
-  full_url(_url),
-  url(fullURL2LocalURL(_url, rootLocation)),
+  rootLocation(_rootLocation),
+  url(unrootedUrl),
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
-  acceptEncodingDeflate(false),
+  acceptEncodingGzip(false),
   byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
+  MHD_get_connection_values(connection, MHD_COOKIE_KIND, &RequestContext::fill_cookie, this);
 
   try {
-    acceptEncodingDeflate =
-        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
+    acceptEncodingGzip =
+        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("gzip") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
   try {
     byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
+
+  userlang = determine_user_language();
 }
 
 RequestContext::~RequestContext()
@@ -106,7 +95,23 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
                                          const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->arguments[key] = value == nullptr ? "" : value;
+  _this->arguments[key].push_back(value == nullptr ? "" : value);
+  if ( ! _this->queryString.empty() ) {
+    _this->queryString += "&";
+  }
+  _this->queryString += urlEncode(key);
+  if ( value ) {
+    _this->queryString += "=";
+    _this->queryString += urlEncode(value);
+  }
+  return MHD_YES;
+}
+
+MHD_Result RequestContext::fill_cookie(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
+{
+  RequestContext *_this = static_cast<RequestContext*>(__this);
+  _this->cookies[key] = value == nullptr ? "" : value;
   return MHD_YES;
 }
 
@@ -121,13 +126,18 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("arguments :\n");
-  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+  for (auto& pair:arguments) {
+    printf(" - %s :", pair.first.c_str());
+    bool first = true;
+    for (auto& v: pair.second) {
+      printf("%s %s", first?"":",", v.c_str());
+      first = false;
+    }
+    printf("\n");
   }
   printf("Parsed : \n");
-  printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
-  printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
+  printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
   printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
@@ -163,11 +173,15 @@ std::string RequestContext::get_url_part(int number) const {
 }
 
 std::string RequestContext::get_full_url() const {
-  return full_url;
+  return rootLocation + urlEncode(url);
+}
+
+std::string RequestContext::get_root_path() const {
+  return rootLocation.empty() ? "/" : rootLocation;
 }
 
 bool RequestContext::is_valid_url() const {
-  return !url.empty();
+  return url.empty() || url[0] == '/';
 }
 
 ByteRange RequestContext::get_range() const {
@@ -176,21 +190,46 @@ ByteRange RequestContext::get_range() const {
 
 template<>
 std::string RequestContext::get_argument(const std::string& name) const {
-  return arguments.at(name);
+  return arguments.at(name)[0];
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
   return headers.at(lcAll(name));
 }
 
-std::string RequestContext::get_query() const {
-  std::string q;
-  const char* sep = "";
-  for ( const auto& a : arguments ) {
-    q += sep + a.first + '=' + a.second;
-    sep = "&";
-  }
-  return q;
+std::string RequestContext::get_user_language() const
+{
+  return userlang.lang;
+}
+
+bool RequestContext::user_language_comes_from_cookie() const
+{
+  return userlang.selectedBy == UserLanguage::SelectorKind::COOKIE;
+}
+
+RequestContext::UserLanguage RequestContext::determine_user_language() const
+{
+  try {
+    return {UserLanguage::SelectorKind::QUERY_PARAM, get_argument("userlang")};
+  } catch(const std::out_of_range&) {}
+
+  try {
+     return {UserLanguage::SelectorKind::COOKIE, cookies.at("userlang")};
+  } catch(const std::out_of_range&) {}
+
+  try {
+    const std::string acceptLanguage = get_header("Accept-Language");
+    const auto userLangPrefs = parseUserLanguagePreferences(acceptLanguage);
+    const auto lang = selectMostSuitableLanguage(userLangPrefs);
+    return {UserLanguage::SelectorKind::ACCEPT_LANGUAGE_HEADER, lang};
+  } catch(const std::out_of_range&) {}
+
+  return {UserLanguage::SelectorKind::DEFAULT, "en"};
+}
+
+std::string RequestContext::get_requested_format() const
+{
+  return get_optional_param<std::string>("format", "html");
 }
 
 }

src/server/request_context.h

@@ -25,9 +25,11 @@
 #include <string>
 #include <sstream>
 #include <map>
+#include <vector>
 #include <stdexcept>
 
 #include "byte_range.h"
+#include "tools/stringTools.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -55,8 +57,8 @@ class IndexError: public std::runtime_error {};
 class RequestContext {
   public: // functions
     RequestContext(struct MHD_Connection* connection,
-                   std::string rootLocation,
-                   const std::string& url,
+                   const std::string& rootLocation, // URI-encoded
+                   const std::string& unrootedUrl,  // URI-decoded
                    const std::string& method,
                    const std::string& version);
     ~RequestContext();
@@ -68,10 +70,11 @@ class RequestContext {
     std::string get_header(const std::string& name) const;
     template<typename T=std::string>
     T get_argument(const std::string& name) const {
-        std::istringstream stream(arguments.at(name));
-        T v;
-        stream >> v;
-        return v;
+        return extractFromString<T>(get_argument(name));
+    }
+
+    std::vector<std::string> get_arguments(const std::string& name) const {
+      return arguments.at(name);
     }
 
     template<class T>
@@ -88,27 +91,72 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
-    std::string get_query() const;
+    std::string get_root_path() const;
+
+    std::string get_query() const { return queryString; }
+
+    template<class F>
+    std::string get_query(F filter, bool mustEncode) const {
+      std::string q;
+      const char* sep = "";
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value):value; };
+      for ( const auto& a : arguments ) {
+        if (!filter(a.first)) {
+          continue;
+        }
+        for (const auto& v: a.second) {
+          q += sep + encode(a.first) + '=' + encode(v);
+          sep = "&";
+        }
+      }
+      return q;
+    }
 
     ByteRange get_range() const;
 
-    bool can_compress() const { return acceptEncodingDeflate; }
+    bool can_compress() const { return acceptEncodingGzip; }
+
+    std::string get_user_language() const;
+    std::string get_requested_format() const;
+
+    bool user_language_comes_from_cookie() const;
+
+  private: // types
+    struct UserLanguage
+    {
+      enum SelectorKind
+      {
+        QUERY_PARAM,
+        COOKIE,
+        ACCEPT_LANGUAGE_HEADER,
+        DEFAULT
+      };
+
+      SelectorKind selectedBy;
+      std::string  lang;
+    };
 
   private: // data
-    std::string full_url;
+    std::string rootLocation;
     std::string url;
     RequestMethod method;
     std::string version;
     unsigned long long requestIndex;
 
-    bool acceptEncodingDeflate;
+    bool acceptEncodingGzip;
 
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
-    std::map<std::string, std::string> arguments;
+    std::map<std::string, std::vector<std::string>> arguments;
+    std::map<std::string, std::string> cookies;
+    std::string queryString;
+    UserLanguage userlang;
 
   private: // functions
+    UserLanguage determine_user_language() const;
+
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_cookie(void *, enum MHD_ValueKind, const char*, const char*);
     static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 

src/server/response.cpp

@@ -20,7 +20,7 @@
 #include "response.h"
 #include "request_context.h"
 #include "internalServer.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
@@ -31,8 +31,17 @@
 #include <mustache.hpp>
 #include <zlib.h>
 
+#include <array>
 
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+// This is somehow a magic value.
+// If this value is too small, we will compress (and lost cpu time) too much
+// content.
+// If this value is too big, we will not compress enough content and send too
+// much data.
+// If we assume that MTU is 1500 Bytes it is useless to compress
+// content smaller as the content will be sent in one packet anyway.
+// 1400 Bytes seems to be a common accepted limit.
+#define KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS 1400
 
 namespace kiwix {
 
@@ -44,20 +53,69 @@ std::string get_mime_type(const zim::Item& item)
 {
   try {
     return item.getMimetype();
-  } catch (exception& e) {
+  } catch (std::exception& e) {
     return "application/octet-stream";
   }
 }
 
 bool is_compressible_mime_type(const std::string& mimeType)
 {
-  return mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/atom") != string::npos
-      || mimeType.find("application/opensearchdescription") != string::npos
-      || mimeType.find("application/json") != string::npos;
+  return mimeType.find("text/") != std::string::npos
+      || mimeType.find("application/javascript") != std::string::npos
+      || mimeType.find("application/atom") != std::string::npos
+      || mimeType.find("application/opensearchdescription") != std::string::npos
+      || mimeType.find("application/json") != std::string::npos
+
+      // Web fonts
+      || mimeType.find("application/font-") != std::string::npos
+      || mimeType.find("application/x-font-") != std::string::npos
+      || mimeType.find("application/vnd.ms-fontobject") != std::string::npos
+      || mimeType.find("font/") != std::string::npos;
 }
 
+bool compress(std::string &content) {
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+
+  auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
+                          Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK) { return false; }
+
+  strm.avail_in = static_cast<decltype(strm.avail_in)>(content.size());
+  strm.next_in =
+      const_cast<Bytef *>(reinterpret_cast<const Bytef *>(content.data()));
+
+  std::string compressed;
+
+  std::array<char, 16384> buff{};
+  do {
+    strm.avail_out = buff.size();
+    strm.next_out = reinterpret_cast<Bytef *>(buff.data());
+    ret = deflate(&strm, Z_FINISH);
+    assert(ret != Z_STREAM_ERROR);
+    compressed.append(buff.data(), buff.size() - strm.avail_out);
+  } while (strm.avail_out == 0);
+
+  assert(ret == Z_STREAM_END);
+  assert(strm.avail_in == 0);
+
+  content.swap(compressed);
+
+  deflateEnd(&strm);
+  return true;
+}
+
+
+const char* getCacheControlHeader(Response::Kind k)
+{
+  switch(k) {
+    case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
+    case Response::ZIM_CONTENT:     return "max-age=3600, must-revalidate";
+    default:                        return "max-age=0, must-revalidate";
+  }
+}
 
 } // unnamed namespace
 
@@ -68,6 +126,13 @@ Response::Response(bool verbose)
   add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
+void Response::set_kind(Kind k)
+{
+  m_kind = k;
+  if ( k == ZIM_CONTENT )
+    m_etag.set_option(ETag::ZIM_CONTENT);
+}
+
 std::unique_ptr<Response> Response::build(const InternalServer& server)
 {
   return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
@@ -78,6 +143,9 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   auto response = Response::build(server);
   response->set_code(MHD_HTTP_NOT_MODIFIED);
   response->m_etag = etag;
+  if ( etag.get_option(ETag::ZIM_CONTENT) ) {
+    response->set_kind(Response::ZIM_CONTENT);
+  }
   if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
     response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
   }
@@ -87,73 +155,86 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
 const UrlNotFoundMsg urlNotFoundMsg;
 const InvalidUrlMsg invalidUrlMsg;
 
+std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
+{
+  return getTranslatedString(m_request.get_user_language(), msgId);
+}
+
 std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
 {
   auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
   r->set_code(m_httpStatusCode);
-  if ( m_taskbarInfo ) {
-    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
-  }
   return r;
 }
 
-HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
-                                             const RequestContext& request,
-                                             int httpStatusCode,
-                                             const std::string& pageTitleMsg,
-                                             const std::string& headingMsg,
-                                             const std::string& cssUrl)
+HTTPErrorResponse::HTTPErrorResponse(const InternalServer& server,
+                                     const RequestContext& request,
+                                     int httpStatusCode,
+                                     const std::string& pageTitleMsgId,
+                                     const std::string& headingMsgId,
+                                     const std::string& cssUrl)
   : ContentResponseBlueprint(&server,
                              &request,
                              httpStatusCode,
-                             "text/html; charset=utf-8",
-                             RESOURCE::templates::error_html)
+                             request.get_requested_format() == "html" ? "text/html; charset=utf-8" : "application/xml; charset=utf-8",
+                             request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml)
 {
   kainjow::mustache::list emptyList;
   this->m_data = kainjow::mustache::object{
                     {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
-                    {"PAGE_TITLE",   pageTitleMsg},
-                    {"PAGE_HEADING", headingMsg},
+                    {"PAGE_TITLE",   getMessage(pageTitleMsgId)},
+                    {"PAGE_HEADING", getMessage(headingMsgId)},
                     {"details", emptyList}
   };
 }
 
-HTTP404HtmlResponse::HTTP404HtmlResponse(const InternalServer& server,
-                                             const RequestContext& request)
-  : HTTPErrorHtmlResponse(server,
-                          request,
-                          MHD_HTTP_NOT_FOUND,
-                          "Content not found",
-                          "Not Found")
+HTTP404Response::HTTP404Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_NOT_FOUND,
+                      "404-page-title",
+                      "404-page-heading")
 {
 }
 
-HTTPErrorHtmlResponse& HTTP404HtmlResponse::operator+(UrlNotFoundMsg /*unused*/)
+HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
 {
-  const std::string requestUrl = m_request.get_full_url();
-  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{url}}" was not found on this server.)");
-  return *this + msgTmpl.render({"url", requestUrl});
+  const std::string requestUrl = urlDecode(m_request.get_full_url(), false);
+  return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
 }
 
-HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
+HTTPErrorResponse& HTTPErrorResponse::operator+(const std::string& msg)
 {
   m_data["details"].push_back({"p", msg});
   return *this;
 }
 
-HTTP400HtmlResponse::HTTP400HtmlResponse(const InternalServer& server,
-                                         const RequestContext& request)
-  : HTTPErrorHtmlResponse(server,
-                          request,
-                          MHD_HTTP_BAD_REQUEST,
-                          "Invalid request",
-                          "Invalid request")
+HTTPErrorResponse& HTTPErrorResponse::operator+(const ParameterizedMessage& details)
+{
+  return *this + details.getText(m_request.get_user_language());
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& details)
+{
+  // operator+() is already a state-modifying operator (akin to operator+=)
+  return *this + details;
+}
+
+
+HTTP400Response::HTTP400Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_BAD_REQUEST,
+                      "400-page-title",
+                      "400-page-heading")
 {
 }
 
-HTTPErrorHtmlResponse& HTTP400HtmlResponse::operator+(InvalidUrlMsg /*unused*/)
+HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
 {
-  std::string requestUrl = m_request.get_full_url();
+  std::string requestUrl = urlDecode(m_request.get_full_url(), false);
   const auto query = m_request.get_query();
   if (!query.empty()) {
     requestUrl += "?" + encodeDiples(query);
@@ -162,36 +243,26 @@ HTTPErrorHtmlResponse& HTTP400HtmlResponse::operator+(InvalidUrlMsg /*unused*/)
   return *this + msgTmpl.render({"url", requestUrl});
 }
 
-HTTP500HtmlResponse::HTTP500HtmlResponse(const InternalServer& server,
-                                         const RequestContext& request)
-  : HTTPErrorHtmlResponse(server,
-                          request,
-                          MHD_HTTP_INTERNAL_SERVER_ERROR,
-                          "Internal Server Error",
-                          "Internal Server Error")
+HTTP500Response::HTTP500Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_INTERNAL_SERVER_ERROR,
+                      "500-page-title",
+                      "500-page-heading")
 {
   // operator+() is a state-modifying operator (akin to operator+=)
   *this + "An internal server error occured. We are sorry about that :/";
 }
 
-std::unique_ptr<ContentResponse> HTTP500HtmlResponse::generateResponseObject() const
+std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
 {
-  // We want a 500 response to be a minimalistic one (so that the server doesn't
-  // have to provide additional resources required for its proper rendering)
-  // ";raw=true" in the MIME-type below disables response decoration
-  // (see ContentResponse::contentDecorationAllowed())
-  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  const std::string mimeType = "text/html;charset=utf-8";
   auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
   r->set_code(m_httpStatusCode);
   return r;
 }
 
-ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
-{
-  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
-  return *this;
-}
-
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
 {
   auto response = Response::build(server);
@@ -240,7 +311,7 @@ static ssize_t callback_reader_from_item(void* cls,
 {
   RunningResponse* response = static_cast<RunningResponse*>(cls);
 
-  size_t max_size_to_set = min<size_t>(
+  size_t max_size_to_set = std::min<size_t>(
     max,
     response->item.getSize() - pos - response->range_start);
 
@@ -270,62 +341,12 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  data.set("content", m_bookName);
-  data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
-  data.set("title", m_bookTitle);
-  data.set("withlibrarybutton", m_withLibraryButton);
-  auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    head_content);
-
-  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<body[^>]*>",
-    taskbar_part);
-}
-
-
-void ContentResponse::inject_externallinks_blocker()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    script_tag);
-}
-
-void ContentResponse::inject_root_link(){
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    "<link type=\"root\" href=\"" + m_root + "\">");
-}
-
 bool
 ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
-      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
-}
-
-bool
-ContentResponse::contentDecorationAllowed() const
-{
-  if (m_raw) {
-    return false;
-  }
-  return (startsWith(m_mimeType, "text/html")
-    && m_mimeType.find(";raw=true") == std::string::npos);
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
 MHD_Response*
@@ -338,46 +359,17 @@ Response::create_mhd_response(const RequestContext& request)
 MHD_Response*
 ContentResponse::create_mhd_response(const RequestContext& request)
 {
-  if (contentDecorationAllowed()) {
-    inject_root_link();
-
-    if (m_withTaskbar) {
-      introduce_taskbar();
-    }
-    if (m_blockExternalLinks) {
-      inject_externallinks_blocker();
-    }
-  }
-
-  bool shouldCompress = can_compress(request);
-  if (shouldCompress) {
-    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-    uLongf comprLen = compr_buffer.capacity();
-    int err = compress(&compr_buffer[0],
-                       &comprLen,
-                       (const Bytef*)(m_content.data()),
-                       m_content.size());
-    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-      /* /!\ Internet Explorer has a bug with deflate compression.
-         It can not handle the first two bytes (compression headers)
-         We need to chunk them off (move the content 2bytes)
-         It has no incidence on other browsers
-         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-      m_content = string((char*)&compr_buffer[2], comprLen - 2);
-      m_etag.set_option(ETag::COMPRESSED_CONTENT);
-    } else {
-      shouldCompress = false;
-    }
-  }
+  const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  if (shouldCompress) {
+  if (isCompressed) {
+    m_etag.set_option(ETag::COMPRESSED_CONTENT);
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
     MHD_add_response_header(
-        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "gzip");
   }
   return response;
 }
@@ -387,7 +379,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   MHD_Response* response = create_mhd_response(request);
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+                          getCacheControlHeader(m_kind));
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
     MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
@@ -395,6 +387,13 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
     MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
   }
 
+  if ( ! request.user_language_comes_from_cookie() ) {
+    const std::string cookie = "userlang=" + request.get_user_language()
+                               + ";Path=" + request.get_root_path()
+                               + ";Max-Age=31536000";
+    MHD_add_response_header(response, MHD_HTTP_HEADER_SET_COOKIE, cookie.c_str());
+  }
+
   if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
@@ -406,24 +405,11 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
-{
-  m_bookName = bookName;
-  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
-}
-
-
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
-  m_mimeType(mimetype),
-  m_raw(raw),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  m_bookName(""),
-  m_bookTitle("")
+  m_mimeType(mimetype)
 {
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
@@ -431,17 +417,11 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw
 std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& content,
-  const std::string& mimetype,
-  bool isHomePage,
-  bool raw)
+  const std::string& mimetype)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
-        raw,
-        server.m_withTaskbar && !isHomePage,
-        server.m_withLibraryButton,
-        server.m_blockExternalLinks,
         content,
         mimetype));
 }
@@ -450,11 +430,10 @@ std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& template_str,
   kainjow::mustache::data data,
-  const std::string& mimetype,
-  bool isHomePage)
+  const std::string& mimetype)
 {
   auto content = render_template(template_str, data);
-  return ContentResponse::build(server, content, mimetype, isHomePage);
+  return ContentResponse::build(server, content, mimetype);
 }
 
 ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
@@ -463,26 +442,26 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   m_mimeType(mimetype)
 {
   m_byteRange = byterange;
-  set_cacheable();
+  set_kind(Response::ZIM_CONTENT);
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
-    response->set_cacheable();
+    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    response->set_kind(Response::ZIM_CONTENT);
     response->m_byteRange = byteRange;
     return std::move(response);
   }
 
   if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
     auto response = Response::build_416(server, item.getSize());
-    response->set_cacheable();
+    response->set_kind(Response::ZIM_CONTENT);
     return response;
   }
 

src/server/response.h

@@ -26,8 +26,10 @@
 
 #include <mustache.hpp>
 #include "byte_range.h"
-#include "entry.h"
 #include "etag.h"
+#include "i18n.h"
+
+#include <zim/item.h>
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -43,6 +45,14 @@ class InternalServer;
 class RequestContext;
 
 class Response {
+  public:
+    enum Kind
+    {
+      STATIC_RESOURCE,
+      ZIM_CONTENT,
+      DYNAMIC_CONTENT
+    };
+
   public:
     Response(bool verbose);
     virtual ~Response() = default;
@@ -55,8 +65,9 @@ class Response {
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
     void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_kind(Kind k);
+    Kind get_kind() const { return m_kind; }
+    void set_etag_body(const std::string& id) { m_etag.set_body(id); }
     void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
@@ -66,6 +77,7 @@ class Response {
     MHD_Response* create_error_response(const RequestContext& request) const;
 
   protected: // data
+    Kind m_kind = DYNAMIC_CONTENT;
     bool m_verbose;
     int m_returnCode;
     ByteRange m_byteRange;
@@ -81,60 +93,32 @@ class ContentResponse : public Response {
     ContentResponse(
       const std::string& root,
       bool verbose,
-      bool raw,
-      bool withTaskbar,
-      bool withLibraryButton,
-      bool blockExternalLinks,
       const std::string& content,
       const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& content,
-      const std::string& mimetype,
-      bool isHomePage = false,
-      bool raw = false);
+      const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& template_str,
       kainjow::mustache::data data,
-      const std::string& mimetype,
-      bool isHomePage = false);
-
-    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
+      const std::string& mimetype);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar();
-    void inject_externallinks_blocker();
-    void inject_root_link();
     bool can_compress(const RequestContext& request) const;
-    bool contentDecorationAllowed() const;
 
 
   private:
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
-    bool m_raw;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    std::string m_bookName;
-    std::string m_bookTitle;
  };
 
-struct TaskbarInfo
-{
-  const std::string bookName;
-  const zim::Archive* const archive;
-
-  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
-    : bookName(bookName)
-    , archive(a)
-  {}
-};
-
 class ContentResponseBlueprint
 {
 public: // functions
@@ -163,9 +147,8 @@ public: // functions
   }
 
 
-  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
-
 protected: // functions
+  std::string getMessage(const std::string& msgId) const;
   virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
 
 public: //data
@@ -175,52 +158,52 @@ public: //data
   const std::string m_mimeType;
   const std::string m_template;
   kainjow::mustache::data m_data;
-  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
 };
 
-struct HTTPErrorHtmlResponse : ContentResponseBlueprint
+struct HTTPErrorResponse : ContentResponseBlueprint
 {
-  HTTPErrorHtmlResponse(const InternalServer& server,
-                      const RequestContext& request,
-                      int httpStatusCode,
-                      const std::string& pageTitleMsg,
-                      const std::string& headingMsg,
-                      const std::string& cssUrl = "");
+  HTTPErrorResponse(const InternalServer& server,
+                    const RequestContext& request,
+                    int httpStatusCode,
+                    const std::string& pageTitleMsgId,
+                    const std::string& headingMsgId,
+                    const std::string& cssUrl = "");
 
-  using ContentResponseBlueprint::operator+;
-  HTTPErrorHtmlResponse& operator+(const std::string& msg);
+  HTTPErrorResponse& operator+(const std::string& msg);
+  HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
+  HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
 };
 
 class UrlNotFoundMsg {};
 
 extern const UrlNotFoundMsg urlNotFoundMsg;
 
-struct HTTP404HtmlResponse : HTTPErrorHtmlResponse
+struct HTTP404Response : HTTPErrorResponse
 {
-  HTTP404HtmlResponse(const InternalServer& server,
-                      const RequestContext& request);
+  HTTP404Response(const InternalServer& server,
+                  const RequestContext& request);
 
-  using HTTPErrorHtmlResponse::operator+;
-  HTTPErrorHtmlResponse& operator+(UrlNotFoundMsg /*unused*/);
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(UrlNotFoundMsg /*unused*/);
 };
 
 class InvalidUrlMsg {};
 
 extern const InvalidUrlMsg invalidUrlMsg;
 
-struct HTTP400HtmlResponse : HTTPErrorHtmlResponse
+struct HTTP400Response : HTTPErrorResponse
 {
-  HTTP400HtmlResponse(const InternalServer& server,
-                      const RequestContext& request);
+  HTTP400Response(const InternalServer& server,
+                  const RequestContext& request);
 
-  using HTTPErrorHtmlResponse::operator+;
-  HTTPErrorHtmlResponse& operator+(InvalidUrlMsg /*unused*/);
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(InvalidUrlMsg /*unused*/);
 };
 
-struct HTTP500HtmlResponse : HTTPErrorHtmlResponse
+struct HTTP500Response : HTTPErrorResponse
 {
-  HTTP500HtmlResponse(const InternalServer& server,
-                      const RequestContext& request);
+  HTTP500Response(const InternalServer& server,
+                  const RequestContext& request);
 
 private: // overrides
   // generateResponseObject() is overriden in order to produce a minimal
@@ -231,7 +214,7 @@ private: // overrides
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -93,10 +93,6 @@ std::string getMetaFlavour(const zim::Archive& archive) {
   return getMetadata(archive, "Flavour");
 }
 
-std::string getArchiveId(const zim::Archive& archive) {
-  return (std::string) archive.getUuid();
-}
-
 bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType){
   try {
@@ -109,46 +105,6 @@ bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
   return false;
 }
 
-// should this be in libzim
-unsigned int getArchiveMediaCount(const zim::Archive& archive) {
-  std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
-  unsigned int counter = 0;
-
-  for (auto &pair:counterMap) {
-    if (startsWith(pair.first, "image/") ||
-        startsWith(pair.first, "video/") ||
-        startsWith(pair.first, "audio/")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-unsigned int getArchiveArticleCount(const zim::Archive& archive) {
-  // [HACK]
-  // getArticleCount() returns different things depending of the "version" of the zim.
-  // On old zim (<=6), it returns the number of entry in `A` namespace
-  // On recent zim (>=7), it returns:
-  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
-  //  - the number of front article if a frontArticleIndex is present
-  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
-  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
-  if (archive.hasNewNamespaceScheme()) {
-    //The articleCount is "good"
-    return archive.getArticleCount();
-  } else {
-    // We have to parse the `M/Counter` metadata
-    unsigned int counter = 0;
-    for(const auto& pair:parseArchiveCounter(archive)) {
-      if (startsWith(pair.first, "text/html")) {
-        counter += pair.second;
-      }
-    }
-    return counter;
-  }
-}
-
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }
@@ -169,14 +125,4 @@ zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path
   }
   throw zim::EntryNotFound("Cannot find entry for non empty path");
 }
-
-MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
-  try {
-    auto counterContent = archive.getMetadata("Counter");
-    return parseMimetypeCounter(counterContent);
-  } catch (zim::EntryNotFound& e) {
-    return {};
-  }
-}
-
 } // kiwix

src/tools/archiveTools.h

@@ -40,7 +40,6 @@ namespace kiwix
     std::string getMetaCreator(const zim::Archive& archive);
     std::string getMetaPublisher(const zim::Archive& archive);
     std::string getMetaFlavour(const zim::Archive& archive);
-    std::string getArchiveId(const zim::Archive& archive);
 
     bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType);
@@ -52,9 +51,6 @@ namespace kiwix
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
 
     zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
-
-    MimeCounterType parseArchiveCounter(const zim::Archive& archive);
-
 }
 
 #endif

src/tools/concurrent_cache.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
  * Copyright (C) 2020 Veloman Yunkan
@@ -84,11 +85,123 @@ public: // types
     return impl_.drop(key);
   }
 
-private: // data
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
   Impl impl_;
   std::mutex lock_;
 };
 
+
+/**
+  WeakStore represent a thread safe store (map) of weak ptr.
+  It allows to store weak_ptr from shared_ptr and retrieve shared_ptr from
+  potential non expired weak_ptr.
+  It is not limited in size.
+*/
+template<typename Key, typename Value>
+class WeakStore {
+  private: // types
+    typedef std::weak_ptr<Value> WeakValue;
+
+  public:
+    explicit WeakStore() = default;
+
+    std::shared_ptr<Value> get(const Key& key)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      auto it = m_weakMap.find(key);
+      if (it != m_weakMap.end()) {
+        auto shared = it->second.lock();
+        if (shared) {
+          return shared;
+        } else {
+          m_weakMap.erase(it);
+        }
+      }
+      throw std::runtime_error("No weak ptr");
+    }
+
+    void add(const Key& key, std::shared_ptr<Value> shared)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      m_weakMap[key] = WeakValue(shared);
+    }
+
+
+  private: //data
+    std::map<Key, WeakValue> m_weakMap;
+    std::mutex m_lock;
+};
+
+template <typename Key, typename RawValue>
+class ConcurrentCache<Key, std::shared_ptr<RawValue>>
+{
+private: // types
+  typedef std::shared_ptr<RawValue> Value;
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      // Try to get back the shared_ptr from the weak_ptr first.
+      try {
+        valuePromise.set_value(m_weakStore.get(key));
+      } catch(const std::runtime_error& e) {
+        try {
+          const auto value = f();
+          valuePromise.set_value(value);
+          m_weakStore.add(key, value);
+        } catch (std::exception& e) {
+          drop(key);
+          throw;
+        }
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  std::mutex lock_;
+  Impl impl_;
+  WeakStore<Key, RawValue> m_weakStore;
+};
+
 } // namespace kiwix
 
 #endif // ZIM_CONCURRENT_CACHE_H

src/tools/lrucache.h

@@ -40,6 +40,7 @@
 
 #include <map>
 #include <list>
+#include <set>
 #include <cstddef>
 #include <stdexcept>
 #include <cassert>
@@ -138,12 +139,26 @@ public: // functions
     return _cache_items_map.size();
   }
 
+  size_t setMaxSize(size_t new_size) {
+    size_t previous = _max_size;
+    _max_size = new_size;
+    return previous;
+  }
+
+  std::set<key_t> keys() const  {
+    std::set<key_t> keys;
+    for(auto& item:_cache_items_map) {
+      keys.insert(item.first);
+    }
+    return keys;
+  }
+
 private: // functions
   void putMissing(const key_t& key, const value_t& value) {
     assert(_cache_items_map.find(key) == _cache_items_map.end());
     _cache_items_list.push_front(key_value_pair_t(key, value));
     _cache_items_map[key] = _cache_items_list.begin();
-    if (_cache_items_map.size() > _max_size) {
+    while (_cache_items_map.size() > _max_size) {
       _cache_items_map.erase(_cache_items_list.back().first);
       _cache_items_list.pop_back();
     }

src/tools/otherTools.cpp

@@ -32,12 +32,15 @@
 #endif
 
 #include "tools/stringTools.h"
+#include "server/i18n.h"
+#include "libkiwix-resources.h"
 
 #include <map>
 #include <sstream>
 #include <pugixml.hpp>
 
 #include <zim/uuid.h>
+#include <zim/suggestion_iterator.h>
 
 
 static std::map<std::string, std::string> codeisomapping {
@@ -288,67 +291,6 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
-namespace
-{
-// The counter metadata format is a list of item separated by a `;` :
-// item0;item1;item2
-// Each item is a "tuple" mimetype=number.
-// However, the mimetype may contains parameters:
-// text/html;raw=true;foo=bar
-// So the final format may be complex to parse:
-// key0=value0;key1;foo=bar=value1;key2=value2
-
-typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
-
-std::string readFullMimetypeAndCounterString(std::istream& in)
-{
-  std::string mtcStr, params;
-  getline(in, mtcStr, ';');
-  if ( mtcStr.find('=') == std::string::npos )
-  {
-    do
-    {
-      if ( !getline(in, params, ';' ) )
-        return std::string();
-      mtcStr += ";" + params;
-    }
-    while ( std::count(params.begin(), params.end(), '=') != 2 );
-  }
-  return mtcStr;
-}
-
-MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
-{
-  const std::string::size_type k = s.find_last_of("=");
-  if ( k != std::string::npos )
-  {
-    const std::string mimeType = s.substr(0, k);
-    std::istringstream counterSS(s.substr(k+1));
-    unsigned int counter;
-    if (counterSS >> counter && counterSS.eof())
-      return MimetypeAndCounter{mimeType, counter};
-  }
-  return MimetypeAndCounter{"", 0};
-}
-
-} // unnamed namespace
-
-kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
-{
-  kiwix::MimeCounterType counters;
-  std::istringstream ss(counterData);
-
-  while (ss)
-  {
-    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
-    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
-    if ( !mtc.first.empty() )
-      counters.insert(mtc);
-  }
-
-  return counters;
-}
-
 std::string kiwix::gen_date_str()
 {
   auto now = std::time(0);
@@ -380,10 +322,76 @@ kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
   std::stringstream ss;
   tmpl.render(data, [&ss](const std::string& str) { ss << str; });
   return ss.str();
 }
+
+namespace
+{
+
+std::string escapeBackslashes(const std::string& s)
+{
+  std::string es;
+  es.reserve(s.size());
+  for (char c : s) {
+    if ( c == '\\' ) {
+      es.push_back('\\');
+    }
+    es.push_back(c);
+  }
+  return es;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang,
+                                         const std::string& queryString)
+{
+  return kiwix::i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+} // unnamed namespace
+
+kiwix::Suggestions::Suggestions()
+  : m_data(kainjow::mustache::data::type::list)
+{
+}
+
+void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
+{
+  kainjow::mustache::data result;
+
+  const std::string label = suggestion.hasSnippet()
+                          ? suggestion.getSnippet()
+                          : suggestion.getTitle();
+
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(suggestion.getTitle()));
+  result.set("kind", "path");
+  result.set("path", escapeBackslashes(suggestion.getPath()));
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
+                                               const std::string& queryString)
+{
+  kainjow::mustache::data result;
+  const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(queryString + " "));
+  result.set("kind", "pattern");
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+std::string kiwix::Suggestions::getJSON() const
+{
+  kainjow::mustache::data data;
+  data.set("suggestions", m_data);
+
+  return render_template(RESOURCE::templates::suggestion_json, data);
+}

src/tools/otherTools.h

@@ -23,13 +23,20 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <cstdlib>
 #include <zim/zim.h>
 #include <mustache.hpp>
 
+#include "stringTools.h"
+
 namespace pugi {
   class xml_node;
 }
 
+namespace zim {
+  class SuggestionItem;
+}
+
 namespace kiwix
 {
   std::string nodeToString(const pugi::xml_node& node);
@@ -42,9 +49,6 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
-  using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
-  MimeCounterType parseMimetypeCounter(const std::string& counterData);
-
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
@@ -53,6 +57,36 @@ namespace kiwix
   kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
 
   std::string render_template(const std::string& template_str, kainjow::mustache::data data);
+
+  template<typename T>
+  T getEnvVar(const char* name, const T& defaultValue)
+  {
+    try {
+      const char* envString = std::getenv(name);
+      if (envString == nullptr) {
+        throw std::runtime_error("Environment variable not set");
+      }
+      return extractFromString<T>(envString);
+    } catch (...) {}
+
+    return defaultValue;
+  }
+
+  class Suggestions
+  {
+  public:
+    Suggestions();
+
+    void add(const zim::SuggestionItem& suggestion);
+
+    void addFTSearchSuggestion(const std::string& uiLang,
+                               const std::string& query);
+
+    std::string getJSON() const;
+
+  private:
+    kainjow::mustache::data m_data;
+  };
 }
 
 #endif

src/tools/pathTools.cpp

@@ -493,12 +493,14 @@ static std::map<std::string, std::string> extMimeTypes = {
  { "jpeg", "image/jpeg"},
  { "jpg", "image/jpeg"},
  { "gif", "image/gif"},
+ { "ico", "image/x-icon"},
  { "svg", "image/svg+xml"},
  { "txt", "text/plain"},
  { "xml", "text/xml"},
  { "pdf", "application/pdf"},
  { "ogg", "application/ogg"},
  { "js", "application/javascript"},
+ { "json", "application/json"},
  { "css", "text/css"},
  { "otf", "application/vnd.ms-opentype"},
  { "ttf", "application/font-ttf"},

src/tools/regexTools.cpp

@@ -75,41 +75,3 @@ std::string replaceRegex(const std::string& content,
   uresult.toUTF8String(tmp);
   return tmp;
 }
-
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->end(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}
-
-std::string prependToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->start(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}

src/tools/regexTools.h

@@ -26,11 +26,5 @@ bool matchRegex(const std::string& content, const std::string& regex);
 std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement);
-std::string prependToFirstOccurence(const std::string& content,
-                                    const std::string& regex,
-                                    const std::string& replacement);
 
 #endif

src/tools/stringTools.cpp

@@ -49,6 +49,24 @@ void kiwix::loadICUExternalTables()
 #endif
 }
 
+kiwix::ICULanguageInfo::ICULanguageInfo(const std::string& langCode)
+  : locale(langCode.c_str())
+{}
+
+std::string kiwix::ICULanguageInfo::iso3Code() const
+{
+  return locale.getISO3Language();
+}
+
+std::string kiwix::ICULanguageInfo::selfName() const
+{
+  icu::UnicodeString langSelfNameICUString;
+  locale.getDisplayLanguage(locale, langSelfNameICUString);
+  std::string langSelfName;
+  langSelfNameICUString.toUTF8String(langSelfName);
+  return langSelfName;
+}
+
 std::string kiwix::removeAccents(const std::string& text)
 {
   loadICUExternalTables();
@@ -143,15 +161,14 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-/* urlEncode() based on javascript encodeURI() &
-   encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
+namespace
+{
 
 bool isReservedUrlChar(char c)
 {
   switch (c) {
   case ';':
   case ',':
-  case '/':
   case '?':
   case ':':
   case '@':
@@ -159,22 +176,22 @@ bool isReservedUrlChar(char c)
   case '=':
   case '+':
   case '$':
+  case '#':
     return true;
   default:
     return false;
   }
 }
 
-bool needsEscape(char c, bool encodeReserved)
+bool isHarmlessUriChar(char c)
 {
   if (c >= 'a' && c <= 'z')
-    return false;
+    return true;
   if (c >= 'A' && c <= 'Z')
-    return false;
+    return true;
   if (c >= '0' && c <= '9')
-    return false;
-  if (isReservedUrlChar(c))
-    return encodeReserved;
+    return true;
+
   switch (c) {
   case '-':
   case '_':
@@ -185,9 +202,10 @@ bool needsEscape(char c, bool encodeReserved)
   case '\'':
   case '(':
   case ')':
-    return false;
+  case '/':
+    return true;
   }
-  return true;
+  return false;
 }
 
 int hexToInt(char c) {
@@ -212,18 +230,18 @@ int hexToInt(char c) {
   }
 }
 
-std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
+} // unnamed namespace
+
+std::string kiwix::urlEncode(const std::string& value)
 {
   std::ostringstream os;
   os << std::hex << std::uppercase;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
-
-    if (!needsEscape(*it, encodeReserved)) {
-      os << *it;
+  for (const char c : value) {
+    if (isHarmlessUriChar(c)) {
+      os << c;
     } else {
-      os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
+      const unsigned int charVal = static_cast<unsigned char>(c);
+      os << '%' << std::setw(2) << std::setfill('0') << charVal;
     }
   }
   return os.str();
@@ -249,15 +267,15 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
       int iHi = hexToInt(hi);
       int iLo = hexToInt(lo);
       if (iHi < 0 || iLo < 0) {
-	// Invalid escape sequence
-	os << '%' << hi << lo;
-	continue;
+        // Invalid escape sequence
+        os << '%' << hi << lo;
+        continue;
       }
       char c = (char)(iHi << 4 | iLo);
       if (!component && isReservedUrlChar(c)) {
-	os << '%' << hi << lo;
+        os << '%' << hi << lo;
       } else {
-	os << c;
+        os << c;
       }
     } else {
       os << *it;
@@ -397,6 +415,17 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
         && std::equal(start.begin(), start.end(), base.begin());
 }
 
+std::string kiwix::stripSuffix(const std::string& str, const std::string& suffix)
+{
+  if (str.size() > suffix.size()) {
+    const auto subStr = str.substr(str.size() - suffix.size(), str.size());
+    if (subStr == suffix) {
+      return str.substr(0, str.size() - suffix.size());
+    }
+  }
+  return str;
+}
+
 std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   std::vector<std::string> variants;
   variants.push_back(title);
@@ -405,3 +434,8 @@ std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   variants.push_back(kiwix::toTitle(title));
   return variants;
 }
+
+template<>
+std::string kiwix::extractFromString(const std::string& str) {
+  return str;
+}

src/tools/stringTools.h

@@ -21,10 +21,12 @@
 #define KIWIX_STRINGTOOLS_H
 
 #include <unicode/unistr.h>
+#include <unicode/locid.h>
 
 #include <string>
 #include <vector>
 #include <sstream>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -40,7 +42,22 @@ std::string encodeDiples(const std::string& str);
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
 
-std::string urlEncode(const std::string& value, bool encodeReserved = false);
+class ICULanguageInfo
+{
+public:
+  explicit ICULanguageInfo(const std::string& langCode);
+
+  std::string iso3Code() const;
+  std::string selfName() const;
+
+private:
+  const icu::Locale locale;
+};
+
+
+/* urlEncode() is the equivalent of JS encodeURIComponent(), with the only
+ * difference that the slash (/) symbol is NOT encoded. */
+std::string urlEncode(const std::string& value);
 std::string urlDecode(const std::string& value, bool component = false);
 
 std::string join(const std::vector<std::string>& list, const std::string& sep);
@@ -65,11 +82,19 @@ T extractFromString(const std::string& str) {
     std::istringstream iss(str);
     T ret;
     iss >> ret;
+    if(iss.fail() || !iss.eof()) {
+        throw std::invalid_argument("no conversion");
+    }
     return ret;
 }
 
+template<>
+std::string extractFromString(const std::string& str);
+
 bool startsWith(const std::string& base, const std::string& start);
 
+std::string stripSuffix(const std::string& str, const std::string& suffix);
+
 std::vector<std::string> getTitleVariants(const std::string& title);
 } //namespace kiwix
 #endif

src/wrapper/java/AndroidManifest.xml

@@ -1,13 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-
-  package="kiwix.org.kiwixlib"
->
-
-  <application android:allowBackup="true"
-    android:label="@string/app_name"
-    android:supportsRtl="true"
-  >
-
-  </application>
-
-</manifest>

src/wrapper/java/book.cpp

@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Book.h"
-
-#include "utils.h"
-#include "book.h"
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_allocate(
-  JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Book>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Book>(env, thisObj);
-}
-
-#define BOOK (getPtr<kiwix::Book>(env, thisObj))
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
-{
-  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
-}
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
-{
-  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
-}
-
-#define GETTER(retType, name) JNIEXPORT retType JNICALL \
-Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
-{ \
-  auto cRet = BOOK->name(); \
-  retType ret = c2jni(cRet, env); \
-  return ret; \
-}
-
-GETTER(jstring, getId)
-GETTER(jstring, getPath)
-GETTER(jboolean, isPathValid)
-GETTER(jstring, getTitle)
-GETTER(jstring, getDescription)
-GETTER(jstring, getLanguage)
-GETTER(jstring, getCreator)
-GETTER(jstring, getPublisher)
-GETTER(jstring, getDate)
-GETTER(jstring, getUrl)
-GETTER(jstring, getName)
-GETTER(jstring, getFlavour)
-GETTER(jstring, getCategory)
-GETTER(jstring, getTags)
-GETTER(jlong, getArticleCount)
-GETTER(jlong, getMediaCount)
-GETTER(jlong, getSize)
-GETTER(jstring, getFavicon)
-GETTER(jstring, getFaviconUrl)
-GETTER(jstring, getFaviconMimeType)
-
-METHOD(jstring, Book, getTagStr, jstring tagName) try {
-  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
-  return c2jni(cRet, env);
-} catch(...) {
-  return c2jni<std::string>("", env);
-}
-
-#undef GETTER

src/wrapper/java/filter.cpp

@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Filter.h"
-
-#include "library.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-METHOD0(void, Filter, allocate) {
-  allocate<kiwix::Filter>(env, thisObj);
-}
-
-METHOD0(void, Filter, dispose) {
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
-
-#define FORWARD(name, args_type) \
-METHOD(jobject, Filter, name, args_type value) { \
-  FILTER->name(jni2c(value, env)); \
-  return thisObj; \
-}
-
-#define FORWARDA(name, args_type) \
-METHOD(jobject, Filter, name, jobjectArray value) { \
-  FILTER->name(jni2c<args_type>(value, env)); \
-  return thisObj; \
-}
-
-
-
-FORWARD(local, jboolean)
-FORWARD(remote, jboolean)
-FORWARD(valid, jboolean)
-FORWARDA(acceptTags, jstring)
-FORWARDA(rejectTags, jstring)
-FORWARD(lang, jstring)
-FORWARD(publisher, jstring)
-FORWARD(creator, jstring)
-FORWARD(maxSize, jlong)
-FORWARD(query, jstring)
-
-

src/wrapper/java/kiwixicu.cpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIICU.h"
-
-#include <iostream>
-#include <string>
-
-#include "unicode/putil.h"
-
-#include "utils.h"
-
-std::mutex globalLock;
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
-    JNIEnv* env, jclass kclass, jstring dirStr)
-{
-  std::string cPath = jni2c(dirStr, env);
-
-  Lock l;
-  try {
-    u_setDataDirectory(cPath.c_str());
-  } catch (...) {
-    std::cerr << "Unable to set data directory " << cPath << std::endl;
-  }
-}

src/wrapper/java/kiwixreader.cpp

@@ -1,561 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <exception>
-#include "org_kiwix_kiwixlib_JNIKiwixReader.h"
-
-#include "tools/base64.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
-    JNIEnv* env, jobject obj, jstring filename)
-{
-  std::string cPath = jni2c(filename, env);
-
-  LOG("Attempting to create reader with: %s", cPath.c_str());
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(cPath);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-namespace
-{
-
-int jni2fd(const jobject& fdObj, JNIEnv* env)
-{
-  jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
-  jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I");
-  if ( field_fd == NULL )
-  {
-    env->ExceptionClear();
-    // Under Android the (private) 'fd' field of java.io.FileDescriptor has been
-    // renamed to 'descriptor'. See, for example,
-    // https://android.googlesource.com/platform/libcore/+/refs/tags/android-8.1.0_r1/ojluni/src/main/java/java/io/FileDescriptor.java#55
-    field_fd = env->GetFieldID(class_fdesc, "descriptor", "I");
-  }
-  return env->GetIntField(fdObj, field_fd);
-}
-
-} // unnamed namespace
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderByFD(
-    JNIEnv* env, jobject obj, jobject fdObj)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderByFD() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderEmbedded(
-    JNIEnv* env, jobject obj, jobject fdObj, jlong offset, jlong size)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd, offset, size);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderEmbedded() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Reader>::dispose(env, obj);
-}
-
-#define READER (Handle<kiwix::Reader>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
-{
-  jstring url;
-
-  try {
-    std::string cUrl = READER->getMainPage().getPath();
-    url = c2jni(cUrl, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM main page");
-    LOG(e.what());
-    url = NULL;
-  }
-  return url;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
-{
-  jstring id;
-
-  try {
-    std::string cId = READER->getId();
-    id = c2jni(cId, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-    id = NULL;
-  }
-
-  return id;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
-{
-  jint size = 0;
-
-  try {
-    int cSize = READER->getFileSize();
-    size = c2jni(cSize, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return size;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
-{
-  jstring creator;
-
-  try {
-    std::string cCreator = READER->getCreator();
-    creator = c2jni(cCreator, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM creator");
-    LOG(e.what());
-    creator = NULL;
-  }
-
-  return creator;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
-{
-  jstring publisher;
-
-  try {
-    std::string cPublisher = READER->getPublisher();
-    publisher = c2jni(cPublisher, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM publish");
-    LOG(e.what());
-    publisher = NULL;
-  }
-  return publisher;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
-{
-  jstring name;
-
-  try {
-    std::string cName = READER->getName();
-    name = c2jni(cName, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM name");
-    LOG(e.what());
-    name = NULL;
-  }
-  return name;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
-{
-  jstring favicon;
-
-  try {
-    std::string cContent;
-    std::string cMime;
-    READER->getFavicon(cContent, cMime);
-    favicon = c2jni(
-        base64_encode(cContent),
-        env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM favicon");
-    LOG(e.what());
-    favicon = NULL;
-  }
-  return favicon;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
-{
-  jstring date;
-
-  try {
-    std::string cDate = READER->getDate();
-    date = c2jni(cDate, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM date");
-    LOG(e.what());
-    date = NULL;
-  }
-  return date;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
-{
-  jstring language;
-
-  try {
-    std::string cLanguage = READER->getLanguage();
-    language = c2jni(cLanguage, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM language");
-    LOG(e.what());
-    language = NULL;
-  }
-
-  return language;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  jstring mimeType;
-
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    auto cMimeType = entry.getMimetype();
-    mimeType = c2jni(cMimeType, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
-    LOG(e.what());
-    mimeType = NULL;
-  }
-  return mimeType;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
-     JNIEnv* env, jobject obj, jstring url)
-{
-  jstring finalUrl;
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    finalUrl = c2jni(entry.getPath(), env);
-  } catch (std::exception& e) {
-    finalUrl = c2jni(std::string(), env);
-  }
-  return finalUrl;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
-    JNIEnv* env, jobject obj, jobject url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
-{
-  /* Default values */
-  setStringObjValue("", titleObj, env);
-  setStringObjValue("", mimeTypeObj, env);
-  setIntObjValue(0, sizeObj, env);
-  jbyteArray data = env->NewByteArray(0);
-
-  /* Retrieve the content */
-  std::string cUrl = getStringObjValue(url, env);
-  unsigned int cSize = 0;
-
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    bool isRedirect = entry.isRedirect();
-    entry = entry.getFinalEntry();
-    cSize = entry.getSize();
-    setIntObjValue(cSize, sizeObj, env);
-    setStringObjValue(entry.getMimetype(), mimeTypeObj, env);
-    setStringObjValue(entry.getTitle(), titleObj, env);
-    if (isRedirect) {
-      setStringObjValue(entry.getPath(), url, env);
-    } else {
-      data = env->NewByteArray(cSize);
-      env->SetByteArrayRegion(
-          data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get content for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-
-  return data;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPart(
-    JNIEnv* env, jobject obj, jstring url, jint offset, jint len, jobject sizeObj)
-{
-  jbyteArray data = env->NewByteArray(0);
-  setIntObjValue(0, sizeObj, env);
-
-  /* Default values */
-  /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset, env);
-  unsigned int cLen = jni2c(len, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-
-    if (cLen == 0) {
-      setIntObjValue(entry.getSize(), sizeObj, env);
-    } else if (cOffset+cLen < entry.getSize()) {
-      auto blob = entry.getBlob(cOffset, cLen);
-      data = env->NewByteArray(cLen);
-      env->SetByteArrayRegion(
-          data, 0, cLen, reinterpret_cast<const jbyte*>(blob.data()));
-      setIntObjValue(cLen, sizeObj, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    LOG(e.what());
-  }
-  return data;
-}
-
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    return c2jni(entry.getSize(), env);
-  } catch(std::exception& e) {
-    LOG("Unable to get size for url : %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return c2jni(0, env);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
-    JNIEnv* env, jobject obj, jstring url)
-{
-   jclass daiClass = env->FindClass("org/kiwix/kiwixlib/DirectAccessInfo");
-   jmethodID daiInitMethod = env->GetMethodID(daiClass, "<init>", "()V");
-   jobject dai = env->NewObject(daiClass, daiInitMethod);
-   setDaiObjValue("", 0, dai, env);
-
-   std::string cUrl = jni2c(url, env);
-   try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    auto part_info = entry.getDirectAccessInfo();
-    setDaiObjValue(part_info.first, part_info.second, dai, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return dai;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
-                                                         jobject obj,
-                                                         jstring prefix,
-                                                         jint count)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count, env);
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
-                                                         jobject obj,
-                                                         jobject titleObj,
-                                                         jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-  std::string cUrl;
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->getNextSuggestion(cTitle, cUrl)) {
-      setStringObjValue(cTitle, titleObj, env);
-      setStringObjValue(cUrl, urlObj, env);
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get next suggestion");
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
-                                                           jobject obj,
-                                                           jstring title,
-                                                           jobject urlObj)
-{
-  std::string cTitle = jni2c(title, env);
-
-  try {
-    auto entry = READER->getEntryFromTitle(cTitle);
-    entry = entry.getFinalEntry();
-    setStringObjValue(entry.getPath(), urlObj, env);
-    return JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get url for title %s: ", cTitle.c_str());
-    LOG(e.what());
-  }
-
-  return JNI_FALSE;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
-    JNIEnv* env, jobject obj)
-{
-  jstring title;
-
-  try {
-    std::string cTitle = READER->getTitle();
-    title = c2jni(cTitle, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim title");
-    LOG(e.what());
-    title = NULL;
-  }
-  return title;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
-{
-  jstring description;
-
-  try {
-    std::string cDescription = READER->getDescription();
-    description = c2jni(cDescription, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim description");
-    LOG(e.what());
-    description = NULL;
-  }
-  return description;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
-{
-  jint articleCount = 0;
-  try {
-    auto cArticleCount = READER->getArticleCount();
-    articleCount = c2jni(cArticleCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get article count.");
-    LOG(e.what());
-  }
-  return articleCount;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
-{
-  jint mediaCount = 0;
-  try {
-    auto cMediaCount = READER->getMediaCount();
-    mediaCount = c2jni(cMediaCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get media count.");
-    LOG(e.what());
-  }
-  return mediaCount;
-}
-
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
-    JNIEnv* env, jobject obj, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cUrl;
-
-  try {
-    std::string cUrl = READER->getRandomPage().getPath();
-    setStringObjValue(cUrl, urlObj, env);
-    retVal = JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get random page");
-    LOG(e.what());
-  }
-  return retVal;
-}

src/wrapper/java/kiwixsearcher.cpp

@@ -1,123 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"
-
-#include "reader.h"
-#include "searcher.h"
-#include "utils.h"
-
-#define SEARCHER (Handle<kiwix::Searcher>::getHandle(env, obj))
-#define RESULT (Handle<kiwix::Result>::getHandle(env, obj))
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Searcher>::dispose(env, obj);
-}
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNativeHandle(JNIEnv* env,
-                                                          jobject obj)
-{
-  kiwix::Searcher* searcher = new kiwix::Searcher();
-  return reinterpret_cast<jlong>(new Handle<kiwix::Searcher>(searcher));
-}
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
-    JNIEnv* env, jobject obj, jobject reader)
-{
-  auto searcher = SEARCHER;
-
-  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)));
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
-    JNIEnv* env, jobject obj, jstring query, jint count)
-{
-  std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count, env);
-
-  SEARCHER->search(cquery, 0, ccount);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNextResult(JNIEnv* env,
-                                                         jobject obj)
-{
-  jobject result = nullptr;
-
-  kiwix::Result* cresult = SEARCHER->getNextResult();
-  if (cresult != nullptr) {
-    jclass resultclass
-        = env->FindClass("org/kiwix/kiwixlib/JNIKiwixSearcher$Result");
-    jmethodID ctor = env->GetMethodID(
-        resultclass, "<init>", "(Lorg/kiwix/kiwixlib/JNIKiwixSearcher;JLorg/kiwix/kiwixlib/JNIKiwixSearcher;)V");
-    result = env->NewObject(resultclass, ctor, obj, reinterpret_cast<jlong>(new Handle<kiwix::Result>(cresult)), obj);
-  }
-  return result;
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_dispose(
-    JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Result>::dispose(env, obj);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getUrl(JNIEnv* env,
-                                                        jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_url(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getTitle(JNIEnv* env,
-                                                          jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_title(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getSnippet(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_snippet(), env);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getContent(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_content(), env);
-}

src/wrapper/java/kiwixserver.cpp

@@ -1,103 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwixServer.h"
-
-#include "tools/base64.h"
-#include "server.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
-    JNIEnv* env, jobject obj, jobject jLibrary)
-{
-  LOG("Attempting to create server");
-  Lock l;
-  try {
-    auto library = getPtr<kiwix::Library>(env, jLibrary);
-    kiwix::Server* server = new kiwix::Server(library);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
-  } catch (std::exception& e) {
-    LOG("Error creating the server");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Server>::dispose(env, obj);
-}
-
-#define SERVER (Handle<kiwix::Server>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setRoot(JNIEnv* env, jobject obj, jstring jRoot)
-{
-  std::string root = jni2c(jRoot, env);
-  SERVER->setRoot(root);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setAddress(JNIEnv* env, jobject obj, jstring jAddress)
-{
-  std::string address = jni2c(jAddress, env);
-  SERVER->setAddress(address);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setPort(JNIEnv* env, jobject obj, int port)
-{
-  SERVER->setPort(port);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setNbThreads(JNIEnv* env, jobject obj, int threads)
-{
-  SERVER->setNbThreads(threads);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboolean withTaskbar, jboolean withLibraryButton)
-{
-  SERVER->setTaskbar(withTaskbar, withLibraryButton);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
-{
-  SERVER->setBlockExternalLinks(blockExternalLinks);
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
-{
-  return SERVER->start();
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_stop(JNIEnv* env, jobject obj)
-{
-  SERVER->stop();
-}

src/wrapper/java/library.cpp

@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Library.h"
-
-#include "library.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_allocate(
-    JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Library>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
-
-/* Kiwix library functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Library_addBook(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    kiwix::Reader reader(cPath);
-    kiwix::Book book;
-    book.update(reader);
-    return LIBRARY->addBook(book);
-  } catch (std::exception& e) {
-    LOG("Unable to add the book");
-    LOG(e.what()); }
-  return false;
-}
-
-METHOD(jobject, Library, getBookById, jstring id) {
-  auto cId = jni2c(id, env);
-  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
-  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
-  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
-  jobject book = env->NewObject(cls, constructorId);
-  setPtr(env, book, cBook);
-  return book;
-}
-
-METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
-  return LIBRARY->getBookCount(localBooks, remoteBooks);
-}
-
-METHOD0(jobjectArray, Library, getBooksIds) {
-  return c2jni(LIBRARY->getBooksIds(), env);
-}
-
-METHOD(jobjectArray, Library, filter, jobject filterObj) {
-  auto filter = getPtr<kiwix::Filter>(env, filterObj);
-  return c2jni(LIBRARY->filter(*filter), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksLanguages) {
-  return c2jni(LIBRARY->getBooksLanguages(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksCreators) {
-  return c2jni(LIBRARY->getBooksCreators(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksPublisher) {
-  return c2jni(LIBRARY->getBooksPublishers(), env);
-}
-

src/wrapper/java/manager.cpp

@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Manager.h"
-
-#include "manager.h"
-#include "utils.h"
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_allocate(
-  JNIEnv* env, jobject thisObj, jobject libraryObj)
-{
-  auto lib = getPtr<kiwix::Library>(env, libraryObj);
-  allocate<kiwix::Manager>(env, thisObj, lib);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Manager>(env, thisObj);
-}
-
-#define MANAGER (getPtr<kiwix::Manager>(env, thisObj))
-
-/* Kiwix manager functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get readFile");
-    LOG(e.what());
-  }
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readXml(
-  JNIEnv* env, jobject thisObj, jstring content, jstring libraryPath)
-{
-  auto cContent = jni2c(content, env);
-  auto cPath = jni2c(libraryPath, env);
-
-  try {
-    return MANAGER->readXml(cContent, false, cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readOpds(
-  JNIEnv* env, jobject thisObj, jstring content, jstring urlHost)
-{
-  auto cContent = jni2c(content, env);
-  auto cUrl = jni2c(urlHost, env);
-
-  try {
-    return MANAGER->readOpds(cContent, cUrl);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readBookmarkFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readBookmarkFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_Manager_addBookFromPath(
-  JNIEnv* env, jobject thisObj,
-  jstring pathToOpen, jstring pathToSave, jstring url, jboolean checkMetaData)
-{
-  auto cPathToOpen = jni2c(pathToOpen, env);
-  auto cPathToSave = jni2c(pathToSave, env);
-  auto cUrl = jni2c(url, env);
-  jstring id = NULL;
-
-  try {
-    auto cId = MANAGER->addBookFromPathAndGetId(cPathToOpen, cPathToSave, cUrl, checkMetaData);
-    if ( !cId.empty() ) {
-      id = c2jni(cId, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return id;
-}

src/wrapper/java/meson.build

@@ -1,55 +0,0 @@
-
-java_sources = files([
-  'org/kiwix/kiwixlib/JNIICU.java',
-  'org/kiwix/kiwixlib/Book.java',
-  'org/kiwix/kiwixlib/JNIKiwixReader.java',
-  'org/kiwix/kiwixlib/Library.java',
-  'org/kiwix/kiwixlib/Manager.java',
-  'org/kiwix/kiwixlib/Filter.java',
-  'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-  'org/kiwix/kiwixlib/JNIKiwixServer.java',
-  'org/kiwix/kiwixlib/JNIKiwixInt.java',
-  'org/kiwix/kiwixlib/JNIKiwixString.java',
-  'org/kiwix/kiwixlib/JNIKiwixBool.java',
-  'org/kiwix/kiwixlib/JNIKiwixException.java',
-  'org/kiwix/kiwixlib/DirectAccessInfo.java'
-])
-
-kiwix_jni = custom_target('jni',
-  input: java_sources,
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_Book.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_Library.h',
-           'org_kiwix_kiwixlib_Manager.h',
-           'org_kiwix_kiwixlib_Filter.h',
-           'org_kiwix_kiwixlib_JNIKiwixServer.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-jni_sources = files([
-     'kiwixicu.cpp',
-     'book.cpp',
-     'kiwixreader.cpp',
-     'library.cpp',
-     'manager.cpp',
-     'filter.cpp',
-     'kiwixsearcher.cpp',
-     'kiwixserver.cpp',
-])
-
-kiwix_sources += jni_sources + [kiwix_jni]
-
-if 'java' in wrapper
-  kiwix_jar = jar('kiwixlib', java_sources)
-  #junit_jar = files('org/kiwix/testing/junit-4.13.jar')
-  #test_jar = jar('testing', 'org/kiwix/testing/test.java',
-  #               link_with: [kiwix_jar, junit_jar])
-  #test('javatest', test_jar)
-endif
-
-install_subdir('org', install_dir: 'kiwix-lib/java', exclude_directories: ['kiwix/testing'])
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -1,48 +0,0 @@
-
-package org.kiwix.kiwixlib;
-
-public class Book
-{
-  public Book() { allocate(); }
-
-
-  public native void update(Book book);
-  public native void update(JNIKiwixReader reader);
-
-  @Override
-  protected void finalize() { dispose();  }
-
-  public native String getId();
-  public native String getPath();
-  public native boolean isPathValid();
-  public native String getTitle();
-  public native String getDescription();
-  public native String getLanguage();
-  public native String getCreator();
-  public native String getPublisher();
-  public native String getDate();
-  public native String getUrl();
-  public native String getName();
-  public native String getFlavour();
-  public native String getCategory();
-  public native String getTags();
-  /**
-   * Return the value associated to the tag tagName
-   *
-   * @param tagName the tag name to search for.
-   * @return The value of the tag. If the tag is not found, return empty string.
-   */
-  public native String getTagStr(String tagName);
-
-  public native long getArticleCount();
-  public native long getMediaCount();
-  public native long getSize();
-
-  public native String getFavicon();
-  public native String getFaviconUrl();
-  public native String getFaviconMimeType();
-
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class DirectAccessInfo
-{
-  public String filename;
-  public long offset;
-}

src/wrapper/java/org/kiwix/kiwixlib/Filter.java

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class Filter
-{
-
-  public native Filter local(boolean accept);
-  public native Filter remote(boolean accept);
-  public native Filter valid(boolean accept);
-  public native Filter acceptTags(String[] tags);
-  public native Filter rejectTags(String[] tags);
-  public native Filter lang(String lang);
-  public native Filter publisher(String publisher);
-  public native Filter creator(String creator);
-  public native Filter maxSize(long size);
-  public native Filter query(String query);
-
-
-  public Filter() { allocate(); }
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIICU.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIICU
-{
-  static public native void setDataDirectory(String icuDataDir);
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwix.java

@@ -1,36 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import android.content.Context;
-import com.getkeepsafe.relinker.ReLinker;
-import org.kiwix.kiwixlib.JNIICU;
-
-public class JNIKiwix
-{
-  public JNIKiwix(final Context context){
-    ReLinker.loadLibrary(context, "kiwix");
-  }
-
-  public void setDataDirectory(String icuDataDir) {
-    JNIICU.setDataDirectory(icuDataDir);
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixBool.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixBool
-{
-  public boolean value;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixException.java

@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixException extends Exception
-{
-  public JNIKiwixException(String message) {
-    super(message);
-  }
-}