New version 9.4.1

libzim were dependent of zlib and we were (wrongly) using its
dependency declaration to link with zlib.

Now that libzim doesn't depends on zlib, we need to fix our build system
and explicitly depend on it.

.codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+
+ignore:
+  - "test"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/

.github/move.yml

@@ -0,0 +1,27 @@
+# Configuration for Move Issues - https://github.com/dessant/move-issues
+
+# Delete the command comment when it contains no other content
+deleteCommand: true
+
+# Close the source issue after moving
+closeSourceIssue: true
+
+# Lock the source issue after moving
+lockSourceIssue: false
+
+# Mention issue and comment authors
+mentionAuthors: true
+
+# Preserve mentions in the issue content
+keepContentMentions: true
+
+# Move labels that also exist on the target repository
+moveLabels: true
+
+# Set custom aliases for targets
+# aliases:
+#   r: repo
+#   or: owner/repo
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

.github/workflows/ci.yml

@@ -0,0 +1,162 @@
+name: CI
+
+on: [push]
+
+jobs:
+  Macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v1
+      - name: Setup python 3.5
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.5'
+      - name: Install packages
+        run:
+          brew install gcovr pkg-config ninja
+      - name: Install python modules
+        run: pip3 install meson==0.49.2 pytest
+      - name: Install deps
+        shell: bash
+        run: |
+          ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
+          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
+      - name: Compile
+        shell: bash
+        run: |
+          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
+          meson . build --default-library=shared -Db_coverage=true
+          cd build
+          ninja
+      - name: Test
+        shell: bash
+        run: |
+          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
+          cd build
+          meson test --verbose
+          ninja coverage
+        env:
+          SKIP_BIG_MEMORY_TEST: 1
+      - name: Publish coverage
+        shell: bash
+        run: |
+          curl https://codecov.io/bash -o codecov.sh
+          bash codecov.sh -n osx_native_dyn -Z
+          rm codecov.sh
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - native_static
+          - native_dyn
+          - native_dyn_bionic
+          - android_arm
+          - android_arm64
+          - win32_static
+          - win32_dyn
+        include:
+          - name: native_static
+            target: native_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: native_dyn
+            target: native_dyn
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: native_dyn_bionic
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm64
+            target: android_arm64
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: win32_static
+            target: win32_static
+            image_variant: f31
+            lib_postfix: '64'
+          - name: win32_dyn
+            target: win32_dyn
+            image_variant: f31
+            lib_postfix: '64'
+    env:
+      HOME: /home/runner
+    runs-on: ubuntu-latest
+    container:
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
+    steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+    - name: Checkout code
+      shell: python
+      run: |
+        from subprocess import check_call
+        from os import environ
+        command = [
+          'git', 'clone',
+          'https://github.com/${{github.repository}}',
+          '--depth=1',
+          '--branch', '${{steps.extract_branch.outputs.branch}}'
+        ]
+        check_call(command, cwd=environ['HOME'])
+    - name: Install deps
+      shell: bash
+      run: |
+        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
+        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
+    - name: Compile
+      shell: bash
+      run: |
+        meson --version
+        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
+          MESON_OPTION="--default-library=shared"
+        else
+          MESON_OPTION="--default-library=static"
+        fi
+        if [[ "${{matrix.target}}" =~ native_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
+        else
+          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
+        fi
+        if [[ "${{matrix.target}}" =~ android_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+        fi
+        cd $HOME/kiwix-lib
+        meson . build ${MESON_OPTION}
+        cd build
+        ninja
+      env:
+        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
+    - name: Test
+      if: startsWith(matrix.target, 'native_')
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib/build
+        meson test --verbose
+        ninja coverage
+      env:
+        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
+        SKIP_BIG_MEMORY_TEST: 1
+    - name: Publish coverage
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib
+        curl https://codecov.io/bash -o codecov.sh
+        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
+        rm codecov.sh
+      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -0,0 +1,79 @@
+name: Packages
+on: [push, pull_request]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distro: [ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
+    steps:
+      - uses: actions/checkout@v2
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-groovy
+        if: matrix.distro == 'ubuntu-groovy'
+        name: Build package for ubuntu-groovy
+        id: build-ubuntu-groovy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to master
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.gitignore

@@ -1,2 +1,6 @@
 .idea/
 *.swp
+subprojects/googletest-release*
+*.class
+build/
+.vscode/

.travis.yml

@@ -1,37 +0,0 @@
-language: cpp
-dist: xenial
-sudo: true
-cache: ccache
-before_install:
-- PATH=$PATH:$HOME/bin
-install: travis/install_deps.sh
-script: travis/compile.sh
-env:
-  matrix:
-    - PLATFORM="native_static"
-    - PLATFORM="native_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-addons:
-  apt:
-    packages:
-      - cmake
-      - python3.5
-      - python3-pip
-      - libbz2-dev
-      - ccache
-      - zlib1g-dev
-      - uuid-dev
-      - libctpp2-dev
-      - ctpp2-utils
-      - libmicrohttpd-dev
-      - g++-mingw-w64-i686
-      - gcc-mingw-w64-i686
-      - gcc-mingw-w64-base
-      - mingw-w64-tools
-matrix:
-  include:
-    - env: PLATFORM="native_dyn"
-      os: osx

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,185 @@
+kiwix-lib 9.4.1
+===============
+
+ * Fix `M/Counter` parsing.
+ * [SERVER] Adjust body padding-top for taskbar
+ * Fix potential crash when stoping a server not started.
+ * Various fix in build system and the CI.
+
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
+kiwix-lib 9.2
+=============
+
+ * Add tests
+ * Refactoring server code.
+ * [SERVER] Add HEAD, Etag and If-None-Match support.
+ * [SERVER] Compress opds catalog answers.
+
+kiwix-lib 9.1.2
+===============
+
+ * Do not use the pathToSave if it is empty.
+
+kiwix-lib 9.1.1
+===============
+
+ * Fix the detection of the dataDirectory on windows.
+
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
+kiwix-lib 9.0.1
+===============
+
+ * [JAVA] Use a long to store the offset of an article in the zim file instead
+    of an int.
+
+kiwix-lib 9.0.0
+===============
+
+ * [OPDS] Correctly set the id of the OPDS stream.
+ * [OPDS] Do not try to filter the catalog if no filter field is given in the
+   request.
+ * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
+ * [LIBRARY] Remove the function the read file using a native path.
+   All path must be utf8, no need to pass a native path along the utf8 path.
+ * [TEST] Fix tests using the main function of gtest instead of custom one.
+ * [CI] Move to github CI instead of Travis.
+ * The `Book::update` method always update the book's fields. Even if they are
+   not empty.
+ * [JAVA] Add wrapping around the library manager (opds parsing)
+ * [ARIA2] Add api option to start download with option (destination folder)
+ * [OPDS] Fixes about opds parsing, generation (missing attributes)
+   and requesting (server)
+ * Add methods on `Book` to get specific tag values (was on `Reader` only)
+ * Add flavour attribute to `Book`
+ * Fix opensearch description.
+ * Trust the given library.xml (by default) instead of reading the value from
+   the zim files.
+ * [OPDS] Be able to filter the content by name or size.
+ * [WINDOWS] Fix launching subcommand when there is spaces in the path.
+
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
+kiwix-lib 8.1.0
+===============
+
+ * Fix pathTools manipulation.
+ * Add missing implementation of getArticleCount and getMediaCount on android.
+ * Correctly convert windows path to utf8.
+ * Add code coverage in the CI
+
+kiwix-lib 8.0.1
+===============
+
+ * Fix join function
+
+kiwix-lib 8.0.0
+===============
+
+ * Add new methods to get all (and new) metadata from the zim file.
+ * Add methods to get the value of a specific tag.
+ * [API Change] Convert tags value to the new convention.
+ * [API Change] Rename `getMatatag` method to `getMetadata`
+ * [ABI Change] Correctly detect executable path in appimage.
+
+kiwix-lib 7.0.0
+===============
+
+ * [API break] Add a argument to kiwix-serve to specify the library to use.
+
+kiwix-lib 6.0.4
+===============
+
+ * Fix HTML rendering of the search result if there is no result.
+ * Do not crash at html rendering if request ask for 0 results (start == end)
+ * Correctly find the executable path if we are using AppImage
+
+kiwix-lib 6.0.3
+===============
+
+ * force one column suggestion in kiwix-serve suggestions
+ * fix fulltext search link in suggestions
+ * UI fixes in kiwix-serve rendering
+
+kiwix-lib 6.0.2
+===============
+
+ * Correctly set the groupId in the pom file.
+
 kiwix-lib 6.0.1
 ===============
 

README.md

@@ -1,14 +1,18 @@
 Kiwix library
 =============
 
-The Kiwix library provides the Kiwix software core. It contains the
-code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
+The Kiwix library provides the [Kiwix](https://kiwix.org) software
+suite core. It contains the code shared by all Kiwix ports (Windows,
+GNU/Linux, macOS, Android, iOS, ...).
 
 [![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
-[![Build Status](https://travis-ci.org/kiwix/kiwix-lib.svg?branch=master)](https://travis-ci.org/kiwix/kiwix-lib)
+[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
+[![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
+[![Packaging status](https://repology.org/badge/vertical-allrepos/kiwix-lib.svg)](https://repology.org/project/kiwix-lib/versions)
+
 Disclaimer
 ----------
 
@@ -35,10 +39,15 @@ libraries need to be available:
 * [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
 * [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
 * [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
-* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
-set CPPFLAGS with correct `-I` option)
+set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
+* [libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
+* [microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
+* [zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
+
+The following dependency needs to be available at runtime:
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -53,13 +62,13 @@ Environment
 -------------
 
 The Kiwix library builds using [Meson](https://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
-* Meson
-* Ninja
-* Pkg-config
+* [Meson](https://mesonbuild.com/)
+* [Ninja](https://ninja-build.org/)
+* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
 
 These tools should be packaged if you use a cutting edge operating
 system. If not, have a look to the [Troubleshooting](#Troubleshooting)
@@ -70,7 +79,6 @@ Compilation
 
 Once all dependencies are installed, you can compile the Kiwix library
 with:
-
 ```bash
 meson . build
 ninja -C build
@@ -83,37 +91,43 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+Testing
+-------
+
+To run the automated tests:
+```bash
+cd build
+meson test
+```
+
 Installation
 ------------
 
 If you want to install the Kiwix library and the headers you just have
 compiled on your system, here we go:
-
 ```bash
 ninja -C build install
 ```
 
 You might need to run the command as root (or using `sudo`), depending
 where you want to install the libraries. After the installation
-succeeded, you may need to run `ldconfig` (as root).
+succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
 ------------
 
 If you want to uninstall the Kiwix library:
-
 ```bash
 ninja -C build uninstall
 ```
 
-Like for the installation, you might need to run the command as root
+Like for the installation, you might need to run the command as `root`
 (or using `sudo`).
 
 Troubleshooting
 ---------------
 
 If you need to install Meson "manually":
-
 ```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
@@ -122,7 +136,6 @@ hash -r                  # Refresh bash paths
 ```
 
 If you need to install Ninja "manually":
-
 ```bash
 git clone git://github.com/ninja-build/ninja.git
 cd ninja
@@ -142,4 +155,5 @@ repository.
 License
 -------
 
-GPLv3 or later, see COPYING for more details.
+[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
+[COPYING](COPYING) for more details.

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -24,9 +24,9 @@ dependencies {
 task writePom {
   pom {
     project {
-        group 'org.kiwix.kiwixlib'
+        groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '6.0.1'
+        version '9.4.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,5 +1,5 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="kiwix.org.kiwixlib">
+    package="org.kiwix.kiwixlib">
 
     <application
         android:allowBackup="true"

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,46 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+ meson,
+ pkg-config,
+ libzim-dev (>= 6.1.8),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/kiwix-lib
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 6.0.0),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev,
+ libmicrohttpd-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix9
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
+Conflicts: libkiwix0, libkiwix3
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix-dev.manpages

@@ -0,0 +1 @@
+usr/share/man/man1/kiwix-compile-resources.1*

debian/libkiwix9.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

include/book.h

@@ -60,6 +60,9 @@ class Book
   const std::string& getUrl() const { return m_url; }
   const std::string& getName() const { return m_name; }
   const std::string& getTags() const { return m_tags; }
+  std::string getTagStr(const std::string& tagName) const;
+  bool getTagBool(const std::string& tagName) const;
+  const std::string& getFlavour() const { return m_flavour; }
   const std::string& getOrigId() const { return m_origId; }
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
@@ -81,6 +84,7 @@ class Book
   void setDate(const std::string& date) { m_date = date; }
   void setUrl(const std::string& url) { m_url = url; }
   void setName(const std::string& name) { m_name = name; }
+  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
   void setTags(const std::string& tags) { m_tags = tags; }
   void setOrigId(const std::string& origId) { m_origId = origId; }
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
@@ -94,7 +98,7 @@ class Book
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
-  bool m_pathValid;
+  bool m_pathValid = false;
   std::string m_title;
   std::string m_description;
   std::string m_language;
@@ -103,12 +107,13 @@ class Book
   std::string m_date;
   std::string m_url;
   std::string m_name;
+  std::string m_flavour;
   std::string m_tags;
   std::string m_origId;
-  uint64_t m_articleCount;
-  uint64_t m_mediaCount;
-  bool m_readOnly;
-  uint64_t m_size;
+  uint64_t m_articleCount = 0;
+  uint64_t m_mediaCount = 0;
+  bool m_readOnly = false;
+  uint64_t m_size = 0;
   mutable std::string m_favicon;
   std::string m_faviconUrl;
   std::string m_faviconMimeType;

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <pthread.h>
 #include <memory>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -92,7 +93,7 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri);
+  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
   Download* getDownload(const std::string& did);
 
   size_t getNbDownload() { return m_knownDownloads.size(); }

include/entry.h

@@ -64,14 +64,14 @@ class Entry
      * @return the path of the entry.
      */
     std::string getPath() const;
-    
+
     /**
      * Get the title of the entry.
      *
      * @return the title of the entry.
      */
     std::string getTitle() const;
-        
+
     /**
      * Get the content of the entry.
      *
@@ -81,7 +81,7 @@ class Entry
      * @return the content of the entry.
      */
     std::string getContent() const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -91,7 +91,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset = 0) const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -102,7 +102,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset, size_type size) const;
-        
+
     /**
      * Get the info for direct access to the content of the entry.
      *
@@ -118,7 +118,7 @@ class Entry
      *         Return <"",0> if is not possible to read directly.
      */
     std::pair<std::string, offset_type> getDirectAccessInfo() const;
-        
+
     /**
      * Get the size of the entry.
      *
@@ -132,8 +132,8 @@ class Entry
      * @return the mime_type of the entry.
      */
     std::string getMimetype() const;
-    
-    
+
+
     /**
      * Get if the entry is a redirect entry.
      *

include/kiwixserve.h

@@ -2,6 +2,7 @@
 #define KIWIXLIB_KIWIX_SERVE_H_
 
 #include <memory>
+#include <string>
 
 class Subprocess;
 namespace kiwix {
@@ -9,17 +10,19 @@ namespace kiwix {
 class KiwixServe
 {
   public:
-    KiwixServe(int port = 8181);
+    KiwixServe(const std::string& libraryPath, int port = 8181);
     ~KiwixServe();
 
     void run();
     void shutDown();
     bool isRunning();
     int getPort() { return m_port; }
+    int setPort(int port);
 
   private:
   std::unique_ptr<Subprocess> mp_kiwixServe;
   int m_port;
+  std::string m_libraryPath;
 };
 
 }; //end namespace kiwix

include/library.h

@@ -57,6 +57,7 @@ class Filter {
     std::string _creator;
     size_t _maxSize;
     std::string _query;
+    std::string _name;
 
   public:
     Filter();
@@ -100,6 +101,7 @@ class Filter {
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
     Filter& query(std::string query);
+    Filter& name(std::string name);
 
     bool accept(const Book& book) const;
 };
@@ -147,6 +149,7 @@ class Library
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
   Book& getBookById(const std::string& id);
+  Book& getBookByPath(const std::string& path);
   std::shared_ptr<Reader> getReaderById(const std::string& id);
 
   /**
@@ -208,7 +211,7 @@ class Library
    *
    * @return A list of bookmarks
    */
-  const std::vector<kiwix::Bookmark>& getBookmarks() { return m_bookmarks; }
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
 
   /**
    * Get all book ids of the books in the library.

include/manager.h

@@ -69,26 +69,12 @@ class Manager
   /**
    * Read a `library.xml` and add book in the file to the library.
    *
-   * @param path The path to the `library.xml`.
+   * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const std::string& path, const bool readOnly = true);
-
-  /**
-   * Read a `library.xml` and add book in the file to the library.
-   *
-   * @param nativePath The path of the `library.xml`
-   * @param UTF8Path The utf8 version (?) of the path. Also the path where the
-   *                 library will be writen i readOnly is False.
-   * @param readOnly Set if the libray path could be overwritten latter with
-   *                 updated content.
-   * @return True if file has been properly parsed.
-   */
-  bool readFile(const std::string& nativePath,
-                const std::string& UTF8Path,
-                const bool readOnly = true);
+  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
   /**
    * Load a library content store in the string.
@@ -101,7 +87,8 @@ class Manager
    */
   bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const std::string& libraryPath = "");
+               const std::string& libraryPath = "",
+               bool trustLibrary = true);
 
   /**
    * Load a library content stored in a OPDS stream.
@@ -168,8 +155,9 @@ class Manager
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   const bool readOnly,
-                   const std::string& libraryPath);
+                   bool readOnly,
+                   const std::string& libraryPath,
+                   bool trustLibrary);
   bool parseOpdsDom(const pugi::xml_document& doc,
                     const std::string& urlHost);
 

include/reader.h

@@ -43,6 +43,8 @@ namespace kiwix
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
  */
+
+using SuggestionsList_t = std::vector<std::vector<std::string>>;
 class Reader
 {
  public:
@@ -158,31 +160,7 @@ class Reader
    * @param[out] value The value will be set to the content of the metadata.
    * @return True if it was possible to get the content of the metadata.
    */
-  bool getMetatag(const string& name, string& value) const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
+  bool getMetadata(const string& name, string& value) const;
 
   /**
    * Get the name of the zim file.
@@ -192,18 +170,13 @@ class Reader
   string getName() const;
 
   /**
-   * Get the tags of the zim file.
+   * Get the title of the zim file.
    *
-   * @return The tags of the zim file as specified in the zim metadata.
+   * @return The title of zim file as specified in the zim metadata.
+   *         If no title has been set, return a title computed from the
+   *         file path.
    */
-  string getTags() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
+  string getTitle() const;
 
   /**
    * Get the creator of the zim file.
@@ -219,6 +192,100 @@ class Reader
    */
   string getPublisher() const;
 
+  /**
+   * Get the date of the zim file.
+   *
+   * @return The date of the zim file as specified in the zim metadata.
+   */
+  string getDate() const;
+
+  /**
+   * Get the description of the zim file.
+   *
+   * @return The description of the zim file as specified in the zim metadata.
+   *         If no description has been set, return the subtitle.
+   */
+  string getDescription() const;
+
+  /**
+   * Get the long description of the zim file.
+   *
+   * @return The long description of the zim file as specifed in the zim metadata.
+   */
+  string getLongDescription() const;
+
+  /**
+   * Get the language of the zim file.
+   *
+   * @return The language of the zim file as specified in the zim metadata.
+   */
+  string getLanguage() const;
+
+  /**
+   * Get the license of the zim file.
+   *
+   * @return The license of the zim file as specified in the zim metadata.
+   */
+  string getLicense() const;
+
+  /**
+   * Get the tags of the zim file.
+   *
+   * @param original If true, return the original tags as specified in the zim metadata.
+   *                 Else, try to convert it to the new 'normalized' format.
+   * @return The tags of the zim file.
+   */
+  string getTags(bool original=false) const;
+
+  /**
+   * Get the value (as a string) of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   */
+  string getTagStr(const std::string& tagName) const;
+
+  /**
+   * Get the boolean value of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The boolean value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   *         std::domain_error if the value of the tag cannot be convert to bool.
+   */
+  bool getTagBool(const std::string& tagName) const;
+
+  /**
+   * Get the relations of the zim file.
+   *
+   * @return The relation of the zim file as specified in the zim metadata.
+   */
+  string getRelation() const;
+
+  /**
+   * Get the flavour of the zim file.
+   *
+   * @return The flavour of the zim file as specified in the zim metadata.
+   */
+  string getFlavour() const;
+
+  /**
+   * Get the source of the zim file.
+   *
+   * @return The source of the zim file as specified in the zim metadata.
+   */
+  string getSource() const;
+
+  /**
+   * Get the scraper of the zim file.
+   *
+   * @return The scraper of the zim file as specified in the zim metadata.
+   */
+  string getScraper() const;
+
   /**
    * Get the origId of the zim file.
    *
@@ -354,6 +421,10 @@ class Reader
    *
    * Suggestions are stored in an internal vector and can be retrieved using
    * `getNextSuggestion` method.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestions(const string& prefix,
+   *                        unsigned int suggestionsCount,
+   *                        SuggestionsList_t& results);
    *
    * @param prefix The prefix to search.
    * @param suggestionsCount How many suggestions to search for.
@@ -361,12 +432,49 @@ class Reader
    *              If false, add suggestions to the internal vector
    *              (until internal vector size is suggestionCount (or no more
    *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
+   * @return True if some suggestions have been added to the internal vector.
    */
-  bool searchSuggestions(const string& prefix,
+  DEPRECATED bool searchSuggestions(const string& prefix,
                          unsigned int suggestionsCount,
                          const bool reset = true);
 
+  /**
+   * Search for entries with title starting with prefix (case sensitive).
+   *
+   * Suggestions are added to the `result` vector.
+   *
+   * @param prefix The prefix to search.
+   * @param suggestionsCount How many suggestions to search for.
+   * @param result The vector where to store the suggestions.
+   * @return True if some suggestions have been added to the vector.
+   */
+
+  bool searchSuggestions(const string& prefix,
+                         unsigned int suggestionsCount,
+                         SuggestionsList_t& resuls);
+
+  /**
+   * Search for entries for the given prefix.
+   *
+   * If the zim file has a internal fulltext index, the suggestions will be
+   * searched using it.
+   * Else the suggestions will be search using `searchSuggestions` while trying
+   * to be smart about case sensitivity (using `getTitleVariants`).
+   *
+   * In any case, suggestions are stored in an internal vector and can be
+   * retrieved using `getNextSuggestion` method.
+   * The internal vector will be reset.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestionsSmart(const string& prefix,
+   *                             unsigned int suggestionsCount,
+   *                             SuggestionsList_t& results);
+   *
+   * @param prefix The prefix to search for.
+   * @param suggestionsCount How many suggestions to search for.
+   */
+  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount);
+
   /**
    * Search for entries for the given prefix.
    *
@@ -381,9 +489,13 @@ class Reader
    *
    * @param prefix The prefix to search for.
    * @param suggestionsCount How many suggestions to search for.
+   * @param results The vector where to store the suggestions
+   * @return True if some suggestions have been added to the results.
    */
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
+   bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount,
+                              SuggestionsList_t& results);
+
 
   /**
    * Check if the url exists in the zim file.
@@ -425,7 +537,7 @@ class Reader
    * @param[out] title the title of the suggestion.
    * @return True if title has been set.
    */
-  bool getNextSuggestion(string& title);
+  DEPRECATED bool getNextSuggestion(string& title);
 
   /**
    * Get the next suggestion title and url.
@@ -434,7 +546,7 @@ class Reader
    * @param[out] url the url of the suggestion.
    * @return True if title and url have been set.
    */
-  bool getNextSuggestion(string& title, string& url);
+  DEPRECATED bool getNextSuggestion(string& title, string& url);
 
   /**
    * Get if we can check zim file integrity (has a checksum).
@@ -494,8 +606,8 @@ class Reader
   zim::size_type nsICount;
   std::string zimFilePath;
 
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
+  SuggestionsList_t suggestions;
+  SuggestionsList_t::iterator suggestionsOffset;
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;

include/search_renderer.h

@@ -60,6 +60,13 @@ class SearchRenderer
    */
   void setSearchProtocolPrefix(const std::string& prefix);
 
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
   /**
    * Generate the html page with the resutls of the search.
    */
@@ -73,10 +80,9 @@ class SearchRenderer
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
+  unsigned int pageLength;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
-  unsigned int resultEnd;
 };
 
 

include/searcher.h

@@ -31,7 +31,6 @@
 #include <vector>
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
-#include "kiwix_config.h"
 
 using namespace std;
 
@@ -89,7 +88,7 @@ class Searcher
    * @param resultEnd the end offset of the search results (used for pagination).
    * @param verbose print some info on stdout if true.
    */
-  void search(std::string& search,
+  void search(const std::string& search,
               unsigned int resultStart,
               unsigned int resultEnd,
               const bool verbose = false);

include/server.h

@@ -56,7 +56,9 @@ namespace kiwix
        void setNbThreads(int threads) { m_nbThreads = threads; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
-         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
 
      protected:
        Library* mp_library;
@@ -68,6 +70,7 @@ namespace kiwix
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

include/tools/otherTools.h

@@ -21,6 +21,9 @@
 #define KIWIX_OTHERTOOLS_H
 
 #include <string>
+#include <vector>
+#include <map>
+#include <zim/zim.h>
 
 namespace pugi {
   class xml_node;
@@ -28,9 +31,20 @@ namespace pugi {
 
 namespace kiwix
 {
-void sleep(unsigned int milliseconds);
-std::string nodeToString(const pugi::xml_node& node);
-std::string converta2toa3(const std::string& a2code);
+  void sleep(unsigned int milliseconds);
+  std::string nodeToString(const pugi::xml_node& node);
+  std::string converta2toa3(const std::string& a2code);
+
+  /*
+   * Convert all format tag string to new format
+   */
+  std::vector<std::string> convertTags(const std::string& tags_str);
+  std::string getTagValueFromTagList(const std::vector<std::string>& tagList,
+                                     const std::string& tagName);
+  bool convertStrToBool(const std::string& value);
+
+  using MimeCounterType = std::map<const std::string, zim::article_index_type>;
+  MimeCounterType parseMimetypeCounter(const std::string& counterData);
 }
 
 #endif

include/tools/pathTools.h

@@ -22,12 +22,14 @@
 
 #include <string>
 
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr);
+std::wstring Utf8ToWide(const std::string& str);
+#endif
 bool isRelativePath(const std::string& path);
 std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
-std::string removeLastPathElement(const std::string& path,
-                                  const bool removePreSeparator = false,
-                                  const bool removePostSeparator = false);
+std::string removeLastPathElement(const std::string& path);
 std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
 
 unsigned int getFileSize(const std::string& path);
@@ -38,7 +40,7 @@ bool makeDirectory(const std::string& path);
 std::string makeTmpDirectory();
 bool copyFile(const std::string& sourcePath, const std::string& destPath);
 std::string getLastPathElement(const std::string& path);
-std::string getExecutablePath();
+std::string getExecutablePath(bool realPathOnly = false);
 std::string getCurrentDirectory();
 std::string getDataDirectory();
 bool writeTextFile(const std::string& path, const std::string& content);

include/tools/stringTools.h

@@ -43,10 +43,8 @@ void loadICUExternalTables();
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string&, const std::string&);
-std::vector<std::string> split(const char*, const char*);
-std::vector<std::string> split(const std::string&, const char*);
-std::vector<std::string> split(const char*, const std::string&);
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
+std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);
 std::string lcAll(const std::string& word);

meson.build

@@ -1,25 +1,41 @@
 project('kiwix-lib', 'cpp',
-  version : '6.0.1',
-  license : 'GPL',
+  version : '9.4.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
-static_deps = get_option('android') or get_option('default_library') == 'static'
-if get_option('android')
+wrapper = get_option('wrapper')
+
+static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
+if wrapper.contains('android')
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
+if wrapper.contains('java')
+  add_languages('java')
+endif
+
+# See https://github.com/kiwix/kiwix-lib/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
+  extra_libs += '-latomic'
+endif
+
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.3.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+zlib_dep = dependency('zlib', static:static_deps)
 
-if not compiler.has_header('mustache.hpp')
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
   error('Cannot found header mustache.hpp')
 endif
 
@@ -29,9 +45,9 @@ if target_machine.system() == 'windows' and static_deps
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep]
 
-inc = include_directories('include')
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
@@ -48,7 +64,7 @@ subdir('static')
 subdir('src')
 subdir('test')
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd']
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,2 +1,2 @@
-option('android', type : 'boolean', value : false,
-  description : 'Do we make a kiwix-lib for android')
+option('wrapper', type:'array', choices:['java', 'android'], value:[],
+  description: 'The wrapper to generate.')

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,11 +95,11 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
+
 
 
 master_c_template = """//This file is automaically generated. Do not modify it.
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/meson.build

@@ -2,3 +2,5 @@
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')

src/android/kiwixlibrary.cpp

@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <android/log.h>
-#include "org_kiwix_kiwixlib_JNIKiwixLibrary.h"
-
-#include "library.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixLibrary_getNativeLibrary(
-    JNIEnv* env, jobject obj)
-{
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create library");
-  Lock l;
-  try {
-    kiwix::Library* library = new kiwix::Library();
-    return reinterpret_cast<jlong>(new Handle<kiwix::Library>(library));
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating ZIM library");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixLibrary_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Library>::dispose(env, obj);
-}
-
-#define LIBRARY (Handle<kiwix::Library>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixLibrary_addBook(JNIEnv* env, jobject obj, jstring path)
-{
-  std::string cPath = jni2c(path, env);
-  bool ret;
-
-  try {
-    kiwix::Reader reader(cPath);
-    kiwix::Book book;
-    book.update(reader);
-    ret = LIBRARY->addBook(book);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to add the book");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    ret = false;
-  }
-  return ret;
-}

src/android/meson.build

@@ -1,32 +0,0 @@
-
-kiwix_jni = custom_target('jni',
-  input: ['org/kiwix/kiwixlib/JNIICU.java',
-          'org/kiwix/kiwixlib/JNIKiwixReader.java',
-          'org/kiwix/kiwixlib/JNIKiwixLibrary.java',
-          'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-          'org/kiwix/kiwixlib/JNIKiwixServer.java',
-          'org/kiwix/kiwixlib/JNIKiwixInt.java',
-          'org/kiwix/kiwixlib/JNIKiwixString.java',
-          'org/kiwix/kiwixlib/JNIKiwixBool.java',
-          'org/kiwix/kiwixlib/JNIKiwixException.java',
-          'org/kiwix/kiwixlib/Pair.java'],
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_JNIKiwixLibrary.h',
-           'org_kiwix_kiwixlib_JNIKiwixServer.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-kiwix_sources += [
-    'android/kiwixicu.cpp',
-    'android/kiwixreader.cpp',
-    'android/kiwixlibrary.cpp',
-    'android/kiwixsearcher.cpp',
-    'android/kiwixserver.cpp',
-    kiwix_jni]
-
-install_subdir('org', install_dir: 'kiwix-lib/java')
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/aria2.cpp

@@ -19,6 +19,11 @@
 #endif
 
 
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+  }
 
 namespace kiwix {
 
@@ -26,8 +31,8 @@ Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
   m_secret("kiwixariarpc"),
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
+  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
+  mp_curl(nullptr)
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -49,7 +54,7 @@ Aria2::Aria2():
   m_secret = "token:"+m_secret;
 
   std::string aria2cmd = appendToDirectory(
-    removeLastPathElement(getExecutablePath(), true, true),
+    removeLastPathElement(getExecutablePath(true)),
     ARIA2_CMD);
   if (fileExists(aria2cmd)) {
     // A local aria2c exe exists (packaged with kiwix-desktop), use it.
@@ -78,28 +83,38 @@ Aria2::Aria2():
   callCmd.push_back("--max-concurrent-downloads=42");
   callCmd.push_back("--rpc-max-request-size=6M");
   callCmd.push_back("--file-allocation=none");
+  std::string launchCmd;
+  for (auto &cmd : callCmd) {
+      launchCmd.append(cmd).append(" ");
+  }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
+
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
+    m_curlErrorBuffer[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
+    } else if (watchdog == 1) {
+      LOG_ARIA_ERROR();
     }
   }
   if (!watchdog) {
     curl_easy_cleanup(mp_curl);
-    throw std::runtime_error("Cannot connect to aria2c rpc");
+    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
 Aria2::~Aria2()
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   curl_easy_cleanup(mp_curl);
 }
 
@@ -111,47 +126,56 @@ void Aria2::close()
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
+  auto outStream = static_cast<std::stringstream*>(userdata);
+  outStream->write(ptr, nmemb);
   return nmemb;
 }
 
 std::string Aria2::doRequest(const MethodCall& methodCall)
 {
-  pthread_mutex_lock(&m_lock);
   auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
+  std::stringstream outStream;
   CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
-  res = curl_easy_perform(mp_curl);
-  if (res == CURLE_OK) {
-    long response_code;
+  long response_code;
+  {
+    std::unique_lock<std::mutex> lock(m_lock);
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
+    m_curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(mp_curl);
+    if (res != CURLE_OK) {
+      LOG_ARIA_ERROR();
+      throw std::runtime_error("Cannot perform request");
+    }
     curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
-    pthread_mutex_unlock(&m_lock);
-    if (response_code != 200) {
-      throw std::runtime_error("Invalid return code from aria");
-    }
-    auto responseContent = stringstream.str();
-    MethodResponse response(responseContent);
-    if (response.isFault()) {
-      throw AriaError(response.getFault().getFaultString());
-    }
-    return responseContent;
   }
-  pthread_mutex_unlock(&m_lock);
-  throw std::runtime_error("Cannot perform request");
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
 }
 
-std::string Aria2::addUri(const std::vector<std::string>& uris)
+std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
 {
   MethodCall methodCall("aria2.addUri", m_secret);
   auto uriParams = methodCall.newParamValue().getArray();
   for (auto& uri : uris) {
     uriParams.addValue().set(uri);
   }
+  for (auto& option : options) {
+    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
+  }
   auto ret = doRequest(methodCall);
   MethodResponse response(ret);
   return response.getParamValue(0).getAsS();

src/aria2.h

@@ -12,8 +12,8 @@
 #include "xmlrpc.h"
 
 #include <memory>
+#include <mutex>
 #include <curl/curl.h>
-#include <pthread.h>
 
 namespace kiwix {
 
@@ -24,8 +24,9 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
+    std::unique_ptr<char[]> m_curlErrorBuffer;
     CURL* mp_curl;
-    pthread_mutex_t m_lock;
+    std::mutex m_lock;
 
     std::string doRequest(const MethodCall& methodCall);
 
@@ -34,7 +35,7 @@ class Aria2
     virtual ~Aria2();
     void close();
 
-    std::string addUri(const std::vector<std::string>& uri);
+    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
     std::vector<std::string> tellActive();
     std::vector<std::string> tellWaiting();

src/book.cpp

@@ -23,6 +23,7 @@
 #include "tools/base64.h"
 #include "tools/regexTools.h"
 #include "tools/networkTools.h"
+#include "tools/otherTools.h"
 
 #include <pugixml.hpp>
 
@@ -44,43 +45,48 @@ bool Book::update(const kiwix::Book& other)
   if (m_readOnly)
     return false;
 
+  if (m_id != other.m_id)
+    return false;
+
   m_readOnly = other.m_readOnly;
+  m_path = other.m_path;
+  m_pathValid = other.m_pathValid;
+  m_title = other.m_title;
+  m_description = other.m_description;
+  m_language = other.m_language;
+  m_creator = other.m_creator;
+  m_publisher = other.m_publisher;
+  m_date = other.m_date;
+  m_url = other.m_url;
+  m_name = other.m_name;
+  m_flavour = other.m_flavour;
+  m_tags = other.m_tags;
+  m_origId = other.m_origId;
+  m_articleCount = other.m_articleCount;
+  m_mediaCount = other.m_mediaCount;
+  m_size = other.m_size;
+  m_favicon = other.m_favicon;
+  m_faviconMimeType = other.m_faviconMimeType;
+  m_faviconUrl = other.m_faviconUrl;
 
-  if (m_path.empty()) {
-    m_path = other.m_path;
-    m_pathValid = other.m_pathValid;
-  }
+  m_downloadId = other.m_downloadId;
 
-  if (m_url.empty()) {
-    m_url = other.m_url;
-  }
-
-  if (m_tags.empty()) {
-    m_tags = other.m_tags;
-  }
-
-  if (m_name.empty()) {
-    m_name = other.m_name;
-  }
-
-  if (m_faviconMimeType.empty()) {
-    m_favicon = other.m_favicon;
-    m_faviconMimeType = other.m_faviconMimeType;
-  }
   return true;
 }
 
 void Book::update(const kiwix::Reader& reader)
 {
   m_path = reader.getZimFilePath();
+  m_pathValid = true;
   m_id = reader.getId();
+  m_title = reader.getTitle();
   m_description = reader.getDescription();
   m_language = reader.getLanguage();
-  m_date = reader.getDate();
   m_creator = reader.getCreator();
   m_publisher = reader.getPublisher();
-  m_title = reader.getTitle();
+  m_date = reader.getDate();
   m_name = reader.getName();
+  m_flavour = reader.getFlavour();
   m_tags = reader.getTags();
   m_origId = reader.getOrigId();
   m_articleCount = reader.getArticleCount();
@@ -100,21 +106,24 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
+  m_pathValid = fileExists(path);
   m_title = ATTR("title");
-  m_name = ATTR("name");
-  m_tags = ATTR("tags");
   m_description = ATTR("description");
   m_language = ATTR("language");
-  m_date = ATTR("date");
   m_creator = ATTR("creator");
   m_publisher = ATTR("publisher");
+  m_date = ATTR("date");
   m_url = ATTR("url");
+  m_name = ATTR("name");
+  m_flavour = ATTR("flavour");
+  m_tags = ATTR("tags");
   m_origId = ATTR("origId");
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
   m_favicon = base64_decode(ATTR("favicon"));
   m_faviconMimeType = ATTR("faviconMimeType");
+  m_faviconUrl = ATTR("faviconUrl");
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -137,12 +146,18 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   if (!m_id.compare(0, 9, "urn:uuid:")) {
     m_id.erase(0, 9);
   }
+  // No path on opds.
   m_title = VALUE("title");
-  m_description = VALUE("description");
+  m_description = VALUE("summary");
   m_language = VALUE("language");
-  m_date = fromOpdsDate(VALUE("updated"));
   m_creator = node.child("author").child("name").child_value();
+  m_publisher = node.child("publisher").child("name").child_value();
+  m_date = fromOpdsDate(VALUE("updated"));
+  m_name = VALUE("name");
+  m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();
@@ -197,4 +212,12 @@ const std::string& Book::getFavicon() const {
   return m_favicon;
 }
 
+std::string Book::getTagStr(const std::string& tagName) const {
+  return getTagValueFromTagList(convertTags(m_tags), tagName);
+}
+
+bool Book::getTagBool(const std::string& tagName) const {
+  return convertStrToBool(getTagStr(tagName));
+}
+
 }

src/downloader.cpp

@@ -127,17 +127,24 @@ void Download::cancelDownload()
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  for (auto gid : mp_aria->tellActive()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
   }
-  for (auto gid : mp_aria->tellWaiting()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellWaiting()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
 }
 
-
 /* Destructor */
 Downloader::~Downloader()
 {
@@ -156,7 +163,7 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri)
+Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
@@ -165,7 +172,7 @@ Download* Downloader::startDownload(const std::string& uri)
       return d.get();
   }
   std::vector<std::string> uris = {uri};
-  auto gid = mp_aria->addUri(uris);
+  auto gid = mp_aria->addUri(uris, options);
   m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
   return m_knownDownloads[gid].get();
 }

src/kiwixserve.cpp

@@ -14,7 +14,9 @@
 
 namespace kiwix {
 
-KiwixServe::KiwixServe(int port) : m_port(port)
+KiwixServe::KiwixServe(const std::string& libraryPath, int port)
+  : m_port(port),
+    m_libraryPath(libraryPath)
 {
 }
 
@@ -34,7 +36,7 @@ void KiwixServe::run()
 
     std::vector<const char*> callCmd;
     std::string kiwixServeCmd = appendToDirectory(
-        removeLastPathElement(getExecutablePath(), true, true),
+        removeLastPathElement(getExecutablePath(true)),
         KIWIXSERVE_CMD);
     if (fileExists(kiwixServeCmd)) {
         // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
@@ -43,13 +45,12 @@ void KiwixServe::run()
         // Try to use a potential installed kiwix-serve.
         callCmd.push_back(KIWIXSERVE_CMD);
     }
-    std::string libraryPath = getDataDirectory() + "/library.xml";
     std::string attachProcessOpt = "-a" + to_string(pid);
     std::string portOpt = "-p" + to_string(m_port);
     callCmd.push_back(attachProcessOpt.c_str());
     callCmd.push_back(portOpt.c_str());
     callCmd.push_back("-l");
-    callCmd.push_back(libraryPath.c_str());
+    callCmd.push_back(m_libraryPath.c_str());
     mp_kiwixServe = Subprocess::run(callCmd);
 }
 
@@ -67,4 +68,14 @@ bool KiwixServe::isRunning()
     return false;
 }
 
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
 }

src/library.cpp

@@ -85,6 +85,18 @@ Book& Library::getBookById(const std::string& id)
   return m_books.at(id);
 }
 
+Book& Library::getBookByPath(const std::string& path)
+{
+  for(auto& it: m_books) {
+    auto& book = it.second;
+    if (book.getPath() == path)
+      return book;
+  }
+  std::ostringstream ss;
+  ss << "No book with path " << path << " in the library." << std::endl;
+  throw std::out_of_range(ss.str());
+}
+
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {
@@ -115,7 +127,7 @@ unsigned int Library::getBookCount(const bool localBooks,
 
 bool Library::writeToFile(const std::string& path)
 {
-  auto baseDir = removeLastPathElement(path, true, false);
+  auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
   dumper.setBaseDir(baseDir);
   return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
@@ -184,6 +196,21 @@ std::vector<std::string> Library::getBooksPublishers()
   return booksPublishers;
 }
 
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
+{
+  if (!onlyValidBookmarks) {
+    return m_bookmarks;
+  }
+  std::vector<kiwix::Bookmark> validBookmarks;
+  auto booksId = getBooksIds();
+  for(auto& bookmark:m_bookmarks) {
+    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
+      validBookmarks.push_back(bookmark);
+    }
+  }
+  return validBookmarks;
+}
+
 std::vector<std::string> Library::getBooksIds()
 {
   std::vector<std::string> bookIds;
@@ -363,6 +390,7 @@ enum filterTypes {
   _CREATOR = FLAG(10),
   MAXSIZE = FLAG(11),
   QUERY = FLAG(12),
+  NAME = FLAG(13),
 };
 
 Filter& Filter::local(bool accept)
@@ -450,24 +478,35 @@ Filter& Filter::query(std::string query)
   return *this;
 }
 
+Filter& Filter::name(std::string name)
+{
+  _name = name;
+  activeFilters |= NAME;
+  return *this;
+}
+
 #define ACTIVE(X) (activeFilters & (X))
+#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
 bool Filter::accept(const Book& book) const
 {
   auto local = !book.getPath().empty();
-  if (ACTIVE(_LOCAL) && !local)
-    return false;
-  if (ACTIVE(_NOLOCAL) && local)
-    return false;
+  FILTER(_LOCAL, local)
+  FILTER(_NOLOCAL, !local)
+
   auto valid = book.isPathValid();
-  if (ACTIVE(_VALID) && !valid)
-    return false;
-  if (ACTIVE(_NOVALID) && valid)
-    return false;
+  FILTER(_VALID, valid)
+  FILTER(_NOVALID, !valid)
+
   auto remote = !book.getUrl().empty();
-  if (ACTIVE(_REMOTE) && !remote)
-    return false;
-  if (ACTIVE(_NOREMOTE) && remote)
-    return false;
+  FILTER(_REMOTE, remote)
+  FILTER(_NOREMOTE, !remote)
+
+  FILTER(MAXSIZE, book.getSize() <= _maxSize)
+  FILTER(LANG, book.getLanguage() == _lang)
+  FILTER(_PUBLISHER, book.getPublisher() == _publisher)
+  FILTER(_CREATOR, book.getCreator() == _creator)
+  FILTER(NAME, book.getName() == _name)
+
   if (ACTIVE(ACCEPTTAGS)) {
     if (!_acceptTags.empty()) {
       auto vBookTags = split(book.getTags(), ";");
@@ -490,18 +529,6 @@ bool Filter::accept(const Book& book) const
       }
     }
   }
-  if (ACTIVE(MAXSIZE) && book.getSize() > _maxSize)
-    return false;
-
-  if (ACTIVE(LANG) && book.getLanguage() != _lang)
-    return false;
-
-  if (ACTIVE(_PUBLISHER) && book.getPublisher() != _publisher)
-    return false;
-
-  if (ACTIVE(_CREATOR) && book.getCreator() != _creator)
-    return false;
-
   if ( ACTIVE(QUERY)
     && !(matchRegex(book.getTitle(), "\\Q" + _query + "\\E")
         || matchRegex(book.getDescription(), "\\Q" + _query + "\\E")))

src/libxml_dumper.cpp

@@ -53,13 +53,15 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
 
   if (book.getOrigId().empty()) {
     ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
-    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
-    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
     ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
     ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
     ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
     ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
     ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
+    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
     if (!book.getFavicon().empty())
       ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
   } else {

src/manager.cpp

@@ -19,6 +19,8 @@
 
 #include "manager.h"
 
+#include "tools/pathTools.h"
+
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -46,8 +48,9 @@ Manager::~Manager()
   }
 }
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          const bool readOnly,
-                          const std::string& libraryPath)
+                          bool readOnly,
+                          const std::string& libraryPath,
+                          bool trustLibrary)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
@@ -59,14 +62,10 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 
     book.setReadOnly(readOnly);
     book.updateFromXml(bookNode,
-                       removeLastPathElement(libraryPath, true, false));
+                       removeLastPathElement(libraryPath));
 
-    /* Update the book properties with the new importer */
-    if (libraryVersion.empty()
-        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.getPath().empty()) {
-        this->readBookFromPath(book.getPath(), &book);
-      }
+    if (!trustLibrary && !book.getPath().empty()) {
+      this->readBookFromPath(book.getPath(), &book);
     }
     manipulator->addBookToLibrary(book);
   }
@@ -75,15 +74,16 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 }
 
 bool Manager::readXml(const std::string& xml,
-                      const bool readOnly,
-                      const std::string& libraryPath)
+                      bool readOnly,
+                      const std::string& libraryPath,
+                      bool trustLibrary)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
       = doc.load_buffer_inplace((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath);
+    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
   }
 
   return true;
@@ -134,21 +134,22 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
   return false;
 }
 
-bool Manager::readFile(const std::string& path, const bool readOnly)
-{
-  return this->readFile(path, path, readOnly);
-}
-
-bool Manager::readFile(const std::string& nativePath,
-                       const std::string& UTF8Path,
-                       const bool readOnly)
+bool Manager::readFile(
+  const std::string& path,
+  bool readOnly,
+  bool trustLibrary)
 {
   bool retVal = true;
   pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
+
+#ifdef _WIN32
+  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
+#else
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+#endif
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, UTF8Path);
+    this->parseXmlDom(doc, readOnly, path, trustLibrary);
   } else {
     retVal = false;
   }
@@ -157,7 +158,7 @@ bool Manager::readFile(const std::string& nativePath,
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = UTF8Path;
+    this->writableLibraryPath = path;
   }
 
   return retVal;
@@ -174,10 +175,10 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (pathToSave != pathToOpen) {
+    if (!pathToSave.empty() && pathToSave != pathToOpen) {
       book.setPath(isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
+                      removeLastPathElement(writableLibraryPath),
                       pathToSave)
                 : pathToSave);
     }

src/meson.build

@@ -21,8 +21,11 @@ kiwix_sources = [
   'tools/otherTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
+  'server/byte_range.cpp',
+  'server/etag.cpp',
   'server/request_context.cpp',
-  'server/response.cpp'
+  'server/response.cpp',
+  'server/internalServer.cpp'
 ]
 kiwix_sources += lib_resources
 
@@ -32,13 +35,16 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if get_option('android')
-  subdir('android')
+if wrapper.contains('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
+if wrapper.contains('android') or wrapper.contains('java')
+  subdir('wrapper/java')
+endif
+
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
                input : 'config.h.in')
@@ -48,6 +54,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
                    install_dir: install_dir)

src/microhttpd_wrapper.h

@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -18,21 +17,8 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#include <microhttpd.h>
 
-import org.kiwix.kiwixlib.JNIKiwixException;
-
-public class JNIKiwixLibrary
-{
-  public native boolean addBook(String path) throws JNIKiwixException;
-
-  public JNIKiwixLibrary()
-  {
-    nativeHandle = getNativeLibrary();
-  }
-
-  public native void dispose();
-
-  private native long getNativeLibrary();
-  private long nativeHandle;
-}
+#if MHD_VERSION < 0x00097002
+typedef int MHD_Result;
+#endif

src/opds_dumper.cpp

@@ -70,12 +70,17 @@ void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 
 pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
+  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
+  ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage());
+  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
+  ADD_TEXT_ENTRY(entry_node, "name", book.getName());
+  ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
   ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
+  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
+  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
+  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
 
   auto content_node = entry_node.append_child("link");
   content_node.append_attribute("type") = "text/html";
@@ -84,6 +89,9 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto author_node = entry_node.append_child("author");
   ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
 
+  auto publisher_node = entry_node.append_child("publisher");
+  ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher());
+
   if (! book.getUrl().empty()) {
     auto acquisition_link = entry_node.append_child("link");
     acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";

src/reader.cpp

@@ -22,6 +22,8 @@
 
 #include <zim/search.h>
 
+#include "tools/otherTools.h"
+
 inline char hi(char v)
 {
   char hex[] = "0123456789abcdef";
@@ -101,29 +103,16 @@ zim::File* Reader::getZimFileHandler() const
 {
   return this->zimFileHandler;
 }
-std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
-{
-  std::map<const std::string, unsigned int> counters;
-  string mimeType, item, counterString;
-  unsigned int counter;
 
+MimeCounterType Reader::parseCounterMetadata() const
+{
   zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
 
   if (article.good()) {
-    stringstream ssContent(article.getData());
-
-    while (getline(ssContent, item, ';')) {
-      stringstream ssItem(item);
-      getline(ssItem, mimeType, '=');
-      getline(ssItem, counterString, '=');
-      if (!counterString.empty() && !mimeType.empty()) {
-        sscanf(counterString.c_str(), "%u", &counter);
-        counters.insert(pair<string, int>(mimeType, counter));
-      }
-    }
+    return parseMimetypeCounter(article.getData());
   }
 
-  return counters;
+  return MimeCounterType();
 }
 
 /* Get the count of articles which can be indexed/displayed */
@@ -136,9 +125,10 @@ unsigned int Reader::getArticleCount() const
   if (counterMap.empty()) {
     counter = this->nsACount;
   } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
+    for(auto &pair:counterMap) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
     }
   }
 
@@ -278,7 +268,7 @@ string Reader::getZimFilePath() const
   return this->zimFilePath;
 }
 /* Return a metatag value */
-bool Reader::getMetatag(const string& name, string& value) const
+bool Reader::getMetadata(const string& name, string& value) const
 {
   try {
     auto entry = getEntryFromPath("M/"+name);
@@ -289,10 +279,17 @@ bool Reader::getMetatag(const string& name, string& value) const
   }
 }
 
+#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
+
+string Reader::getName() const
+{
+  METADATA("Name")
+}
+
 string Reader::getTitle() const
 {
   string value;
-  this->getMetatag("Title", value);
+  this->getMetadata("Title", value);
   if (value.empty()) {
     value = getLastPathElement(zimFileHandler->getFilename());
     std::replace(value.begin(), value.end(), '_', ' ');
@@ -302,65 +299,98 @@ string Reader::getTitle() const
   return value;
 }
 
-string Reader::getName() const
+string Reader::getCreator() const
 {
-  string value;
-  this->getMetatag("Name", value);
-  return value;
+  METADATA("Creator")
 }
 
-string Reader::getTags() const
+string Reader::getPublisher() const
 {
-  string value;
-  this->getMetatag("Tags", value);
-  return value;
+  METADATA("Publisher")
+}
+
+string Reader::getDate() const
+{
+  METADATA("Date")
 }
 
 string Reader::getDescription() const
 {
   string value;
-  this->getMetatag("Description", value);
+  this->getMetadata("Description", value);
 
   /* Mediawiki Collection tends to use the "Subtitle" name */
   if (value.empty()) {
-    this->getMetatag("Subtitle", value);
+    this->getMetadata("Subtitle", value);
   }
 
   return value;
 }
 
+string Reader::getLongDescription() const
+{
+  METADATA("LongDescription")
+}
+
 string Reader::getLanguage() const
 {
-  string value;
-  this->getMetatag("Language", value);
-  return value;
+  METADATA("Language")
 }
 
-string Reader::getDate() const
+string Reader::getLicense() const
 {
-  string value;
-  this->getMetatag("Date", value);
-  return value;
+  METADATA("License")
 }
 
-string Reader::getCreator() const
+string Reader::getTags(bool original) const
 {
-  string value;
-  this->getMetatag("Creator", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  if (original) {
+    return tags_str;
+  }
+  auto tags = convertTags(tags_str);
+  return join(tags, ";");
 }
 
-string Reader::getPublisher() const
+
+string Reader::getTagStr(const std::string& tagName) const
 {
-  string value;
-  this->getMetatag("Publisher", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  return getTagValueFromTagList(convertTags(tags_str), tagName);
 }
 
+bool Reader::getTagBool(const std::string& tagName) const
+{
+  return convertStrToBool(getTagStr(tagName));
+}
+
+string Reader::getRelation() const
+{
+  METADATA("Relation")
+}
+
+string Reader::getFlavour() const
+{
+  METADATA("Flavour")
+}
+
+string Reader::getSource() const
+{
+  METADATA("Source")
+}
+
+string Reader::getScraper() const
+{
+  METADATA("Scraper")
+}
+#undef METADATA
+
 string Reader::getOrigId() const
 {
   string value;
-  this->getMetatag("startfileuid", value);
+  this->getMetadata("startfileuid", value);
   if (value.empty()) {
     return "";
   }
@@ -667,13 +697,11 @@ bool Reader::hasFulltextIndex() const
 }
 
 /* Search titles by prefix */
+
 bool Reader::searchSuggestions(const string& prefix,
                                unsigned int suggestionsCount,
                                const bool reset)
 {
-  bool retVal = false;
-  zim::File::const_iterator articleItr;
-
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
   if (reset) {
@@ -685,15 +713,30 @@ bool Reader::searchSuggestions(const string& prefix,
     }
   }
 
+  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
+
+  /* Set the cursor to the begining */
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+
+bool Reader::searchSuggestions(const string& prefix,
+                               unsigned int suggestionsCount,
+                               SuggestionsList_t& results)
+{
+  bool retVal = false;
+
   /* Return if no prefix */
   if (prefix.size() == 0) {
     return false;
   }
 
-  for (articleItr = zimFileHandler->findByTitle('A', prefix);
+  for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
+       && results.size() < suggestionsCount;
        ++articleItr) {
     /* Extract the interesting part of article title & url */
     std::string normalizedArticleTitle
@@ -713,8 +756,8 @@ bool Reader::searchSuggestions(const string& prefix,
        title) */
     bool insert = true;
     std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
+    for (suggestionItr = results.begin();
+         suggestionItr != results.end();
          suggestionItr++) {
       int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
       if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
@@ -731,16 +774,13 @@ bool Reader::searchSuggestions(const string& prefix,
       suggestion.push_back(articleItr->getTitle());
       suggestion.push_back(articleFinalUrl);
       suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
+      results.insert(suggestionItr, suggestion);
     }
 
     /* Suggestions where found */
     retVal = true;
   }
 
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
   return retVal;
 }
 
@@ -755,15 +795,28 @@ std::vector<std::string> Reader::getTitleVariants(
   return variants;
 }
 
-/* Try also a few variations of the prefix to have better results */
+
 bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
+{
+  this->suggestions.clear();
+  this->suggestionsOffset = this->suggestions.begin();
+
+  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
+
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+/* Try also a few variations of the prefix to have better results */
+bool Reader::searchSuggestionsSmart(const string& prefix,
+                                    unsigned int suggestionsCount,
+                                    SuggestionsList_t& results)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
   bool retVal = false;
 
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
   const auto suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
@@ -779,15 +832,14 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
       suggestion.push_back(current->getTitle());
       suggestion.push_back("/A/" + current->getUrl());
       suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
+      results.push_back(suggestion);
     }
-    this->suggestionsOffset = this->suggestions.begin();
     retVal = true;
   } else {
     for (std::vector<std::string>::iterator variantsItr = variants.begin();
          variantsItr != variants.end();
          variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
+      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
                || retVal;
     }
   }

src/search_renderer.cpp

@@ -93,30 +93,32 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
   auto resultStart = mp_searcher->getResultStart();
-  auto resultEnd = mp_searcher->getResultEnd();
-  auto resultCountPerPage = resultEnd - resultStart;
+  auto resultEnd = 0U;
   auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
-
-  unsigned int pageStart
-      = resultStart / resultCountPerPage >= 5
-            ? resultStart / resultCountPerPage - 4
-            : 0;
-  unsigned int pageCount
-      = estimatedResultCount / resultCountPerPage + 1 - pageStart;
-
-  if (pageCount > 10) {
-    pageCount = 10;
-  } else if (pageCount == 1) {
-    pageCount = 0;
+  auto currentPage = 0U;
+  auto pageStart = 0U;
+  auto pageEnd = 0U;
+  auto lastPageStart = 0U;
+  if (pageLength) {
+    currentPage = resultStart/pageLength;
+    pageStart = currentPage > 4 ? currentPage-4 : 0;
+    pageEnd = currentPage + 5;
+    if (pageEnd > estimatedResultCount / pageLength) {
+      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
+    }
+    if (estimatedResultCount > pageLength) {
+      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
+    }
   }
 
-  for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
+  resultEnd = resultStart+pageLength; //setting result end
+
+  for (unsigned int i = pageStart; i < pageEnd; i++) {
     kainjow::mustache::data page;
     page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * resultCountPerPage));
-    page.set("end", to_string((i + 1) * resultCountPerPage));
+    page.set("start", to_string(i * pageLength));
 
-    if (i * resultCountPerPage == resultStart) {
+    if (i == currentPage) {
       page.set("selected", true);
     }
     pages.push_back(page);
@@ -128,17 +130,15 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data allData;
   allData.set("results", results);
   allData.set("pages", pages);
-  allData.set("hasResult", estimatedResultCount != 0);
+  allData.set("hasResults", estimatedResultCount != 0);
+  allData.set("hasPages", pageStart != pageEnd);
   allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
   allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
   allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
   allData.set("resultStart", to_string(resultStart + 1));
   allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("resultRange", to_string(resultCountPerPage));
-  allData.set("resultLastPageStart", to_string(estimatedResultCount > resultCountPerPage
-             ? round(estimatedResultCount / resultCountPerPage) * resultCountPerPage
-             : 0));
-  allData.set("lastResult", to_string(estimatedResultCount));
+  allData.set("pageLength", to_string(pageLength));
+  allData.set("resultLastPageStart", to_string(lastPageStart));
   allData.set("protocolPrefix", this->protocolPrefix);
   allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
   allData.set("contentId", this->searchContent);
@@ -148,4 +148,4 @@ std::string SearchRenderer::getHtml()
   return ss.str();
 }
 
-}
+}

src/searcher.cpp

@@ -97,7 +97,7 @@ Reader* Searcher::get_reader(int readerIndex)
 }
 
 /* Search strings in the database */
-void Searcher::search(std::string& search,
+void Searcher::search(const std::string& search,
                       unsigned int resultStart,
                       unsigned int resultEnd,
                       const bool verbose)
@@ -108,12 +108,12 @@ void Searcher::search(std::string& search,
     cout << "Performing query `" << search << "'" << endl;
   }
 
+  this->searchPattern = search;
+  this->resultStart = resultStart;
+  this->resultEnd = resultEnd;
   /* Try to find results */
   if (resultStart != resultEnd) {
     /* Perform the search */
-    this->searchPattern = search;
-    this->resultStart = resultStart;
-    this->resultEnd = resultEnd;
     string unaccentedSearch = removeAccents(search);
     std::vector<const zim::File*> zims;
     for (auto current = this->readers.begin(); current != this->readers.end();
@@ -146,11 +146,6 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
   }
 
-  /* Try to find results */
-  if (resultStart == resultEnd) {
-    return;
-  }
-
   /* Perform the search */
   std::ostringstream oss;
   oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
@@ -158,6 +153,11 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
   this->resultStart = resultStart;
   this->resultEnd = resultEnd;
 
+  /* Try to find results */
+  if (resultStart == resultEnd) {
+    return;
+  }
+
   std::vector<const zim::File*> zims;
   for (auto current = this->readers.begin(); current != this->readers.end();
        current++) {

src/server.cpp

@@ -19,125 +19,15 @@
 
 #include "server.h"
 
-#ifdef _WIN32
-# if !defined(__MINGW32__) && (_MSC_VER < 1600)
-#   include "stdint4win.h"
-# endif
-# include <winsock2.h>
-# include <ws2tcpip.h>
-# ifdef __GNUC__
-  // inet_pton is not declared in mingw, even if the function exists.
-  extern "C" {
-    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
-  }
-# endif
-  typedef UINT64 uint64_t;
-  typedef UINT16 uint16_t;
-#endif
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
 #include "library.h"
 #include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
-#include "search_renderer.h"
-#include "opds_dumper.h"
 
-#include <zim/uuid.h>
-
-#include <mustache.hpp>
-
-#include <pthread.h>
-#include <atomic>
 #include <string>
-#include <vector>
-#include <chrono>
-#include "kiwixlib-resources.h"
 
-#ifndef _WIN32
-# include <arpa/inet.h>
-#endif
-
-#include "server/request_context.h"
-#include "server/response.h"
-
-#define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#include "server/internalServer.h"
 
 namespace kiwix {
 
-static IdNameMapper defaultNameMapper;
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls);
-
-
-class InternalServer {
-  public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
-                   int port,
-                   std::string root,
-                   int nbThreads,
-                   bool verbose,
-                   bool withTaskbar,
-                   bool withLibraryButton);
-    virtual ~InternalServer() = default;
-
-    int handlerCallback(struct MHD_Connection* connection,
-                        const char* url,
-                        const char* method,
-                        const char* version,
-                        const char* upload_data,
-                        size_t* upload_data_size,
-                        void** cont_cls);
-    bool start();
-    void stop();
-
-  private:
-    Response handle_request(const RequestContext& request);
-    Response build_500(const std::string& msg);
-    Response build_404(const RequestContext& request, const std::string& zimName);
-    Response build_homepage(const RequestContext& request);
-    Response handle_skin(const RequestContext& request);
-    Response handle_catalog(const RequestContext& request);
-    Response handle_meta(const RequestContext& request);
-    Response handle_search(const RequestContext& request);
-    Response handle_suggest(const RequestContext& request);
-    Response handle_random(const RequestContext& request);
-    Response handle_content(const RequestContext& request);
-
-    kainjow::mustache::data get_default_data();
-    Response get_default_response();
-
-    std::string m_addr;
-    int m_port;
-    std::string m_root;
-    int m_nbThreads;
-    std::atomic_bool m_verbose;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    struct MHD_Daemon* mp_daemon;
-
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
-};
-
-
 Server::Server(Library* library, NameMapper* nameMapper) :
   mp_library(library),
   mp_nameMapper(nameMapper),
@@ -157,13 +47,16 @@ bool Server::start() {
     m_nbThreads,
     m_verbose,
     m_withTaskbar,
-    m_withLibraryButton));
+    m_withLibraryButton,
+    m_blockExternalLinks));
   return mp_server->start();
 }
 
 void Server::stop() {
-  mp_server->stop();
-  mp_server.reset(nullptr);
+  if (mp_server) {
+    mp_server->stop();
+    mp_server.reset(nullptr);
+  }
 }
 
 void Server::setRoot(const std::string& root)
@@ -177,735 +70,4 @@ void Server::setRoot(const std::string& root)
   }
 }
 
-
-InternalServer::InternalServer(Library* library,
-                               NameMapper* nameMapper,
-                               std::string addr,
-                               int port,
-                               std::string root,
-                               int nbThreads,
-                               bool verbose,
-                               bool withTaskbar,
-                               bool withLibraryButton) :
-  m_addr(addr),
-  m_port(port),
-  m_root(root),
-  m_nbThreads(nbThreads),
-  m_verbose(verbose),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  mp_daemon(nullptr),
-  mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
-{}
-
-bool InternalServer::start() {
-#ifdef _WIN32
-  int flags = MHD_USE_SELECT_INTERNALLY;
-#else
-  int flags = MHD_USE_POLL_INTERNALLY;
-#endif
-  if (m_verbose.load())
-    flags |= MHD_USE_DEBUG;
-
-
-  struct sockaddr_in sockAddr;
-  memset(&sockAddr, 0, sizeof(sockAddr));
-  sockAddr.sin_family = AF_INET;
-  sockAddr.sin_port = htons(m_port);
-  if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
-      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-  } else {
-    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
-      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
-      return false;
-    }
-  }
-
-  mp_daemon = MHD_start_daemon(flags,
-                            m_port,
-                            NULL,
-                            NULL,
-                            &staticHandlerCallback,
-                            this,
-                            MHD_OPTION_SOCK_ADDR, &sockAddr,
-                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
-                            MHD_OPTION_END);
-  if (mp_daemon == nullptr) {
-    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
-              << " is maybe already occupied or need more permissions to be open. "
-                 "Please try as root or with a port number higher or equal to 1024."
-              << std::endl;
-    return false;
-  }
-  return true;
-}
-
-void InternalServer::stop()
-{
-  MHD_stop_daemon(mp_daemon);
-}
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls)
-{
-  InternalServer* _this = static_cast<InternalServer*>(cls);
-
-  return _this->handlerCallback(connection,
-                                url,
-                                method,
-                                version,
-                                upload_data,
-                                upload_data_size,
-                                cont_cls);
-}
-
-int InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                    const char* url,
-                                    const char* method,
-                                    const char* version,
-                                    const char* upload_data,
-                                    size_t* upload_data_size,
-                                    void** cont_cls)
-{
-  auto start_time = std::chrono::steady_clock::now();
-  if (m_verbose.load() ) {
-    printf("======================\n");
-    printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
-  }
-  RequestContext request(connection, m_root, url, method, version);
-
-  if (m_verbose.load() ) {
-    request.print_debug_info();
-  }
-  /* Unexpected method */
-  if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST) {
-    printf("Reject request because of unhandled request method.\n");
-    printf("----------------------\n");
-    return MHD_NO;
-  }
-
-  auto response = handle_request(request);
-
-  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
-    printf("========== INTERNAL ERROR !! ============\n");
-    if (!m_verbose.load()) {
-      printf("Requesting : \n");
-      printf("full_url : %s\n", url);
-      request.print_debug_info();
-    }
-  }
-
-  auto ret = response.send(request, connection);
-  auto end_time = std::chrono::steady_clock::now();
-  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
-  if (m_verbose.load()) {
-    printf("Request time : %fs\n", time_span.count());
-    printf("----------------------\n");
-  }
-  return ret;
-}
-
-
-Response InternalServer::handle_request(const RequestContext& request)
-{
-  try {
-    if (! request.is_valid_url())
-      return build_404(request, "");
-
-    if (kiwix::startsWith(request.get_url(), "/skin/"))
-      return handle_skin(request);
-
-    if (startsWith(request.get_url(), "/catalog"))
-      return handle_catalog(request);
-
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
-
-    if (request.get_url() == "/search")
-      return handle_search(request);
-
-    if (request.get_url() == "/suggest")
-     return handle_suggest(request);
-
-    if (request.get_url() == "/random")
-      return handle_random(request);
-
-    return handle_content(request);
-  } catch (std::exception& e) {
-    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return build_500(e.what());
-  } catch (...) {
-    fprintf(stderr, "===== Unhandled unknown error\n");
-    return build_500("Unknown error");
-  }
-}
-
-kainjow::mustache::data InternalServer::get_default_data()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  return data;
-}
-
-Response InternalServer::get_default_response()
-{
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton);
-}
-
-
-Response InternalServer::build_404(const RequestContext& request,
-                                   const std::string& bookName)
-{
-  kainjow::mustache::data results;
-  results.set("url", request.get_full_url());
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::_404_html, results);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_NOT_FOUND);
-  response.set_compress(true);
-  response.set_taskbar(bookName, "");
-
-  return response;
-}
-
-Response InternalServer::build_500(const std::string& msg)
-{
-  kainjow::mustache::data data;
-  data.set("error", msg);
-  Response response(m_root, true, false, false);
-  response.set_template(RESOURCE::templates::_500_html, data);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
-Response InternalServer::build_homepage(const RequestContext& request)
-{
-  auto data = get_default_data();
-
-  kainjow::mustache::data books{kainjow::mustache::data::type::list};
-  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = mp_library->getBookById(bookId);
-
-    kainjow::mustache::data book;
-    book.set("name", mp_nameMapper->getNameForId(bookId));
-    book.set("title", currentBook.getTitle());
-    book.set("description", currentBook.getDescription());
-    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
-    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
-    books.push_back(book);
-  }
-
-  data.set("books", books);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, data);
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_meta(const RequestContext& request)
-{
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
-  } else {
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-  response.set_content(content);
-  response.set_mimeType(mimeType);
-  response.set_compress(false);
-  response.set_cache(true);
-  return response;
-}
-
-Response InternalServer::handle_suggest(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_suggest\n");
-  }
-
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-  std::string suggestion;
-
-  std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
-  }
-
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
-
-  bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    reader->searchSuggestionsSmart(term, maxSuggestionCount);
-    while (reader->getNextSuggestion(suggestion)) {
-      kainjow::mustache::data result;
-      result.set("label", suggestion);
-      result.set("value", suggestion);
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
-    }
-  }
-
-  /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
-    kainjow::mustache::data result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term);
-    result.set("first", first);
-    results.push_back(result);
-  }
-
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::suggestion_json, data);
-  response.set_mimeType("application/json; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_skin(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_skin\n");
-  }
-
-  auto response = get_default_response();
-  auto resourceName = request.get_url().substr(1);
-  try {
-    response.set_content(getResource(resourceName));
-  } catch (const ResourceNotFound& e) {
-    return build_404(request, "");
-  }
-  response.set_mimeType(getMimeTypeForFile(resourceName));
-  response.set_compress(true);
-  response.set_cache(true);
-  return response;
-}
-
-Response InternalServer::handle_search(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_search\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
-
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
-
-  std::shared_ptr<Reader> reader(nullptr);
-  try {
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Try first to load directly the article */
-  if (reader != nullptr && !patternString.empty()) {
-    std::string patternCorrespondingUrl;
-    auto variants = reader->getTitleVariants(patternString);
-    auto variantsItr = variants.begin();
-
-    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
-      try {
-        auto entry = reader->getEntryFromTitle(*variantsItr);
-        entry = entry.getFinalEntry();
-        patternCorrespondingUrl = entry.getPath();
-        break;
-      } catch(kiwix::NoEntry& e) {
-        variantsItr++;
-      }
-    }
-
-    /* If article found then redirect directly to it */
-    if (!patternCorrespondingUrl.empty()) {
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
-      return response;
-    }
-  }
-
-  /* Make the search */
-  auto response = get_default_response();
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
-  response.set_compress(true);
-
-  if ( (!reader && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    response.set_template(RESOURCE::templates::no_search_result_html, data);
-    response.set_code(MHD_HTTP_NOT_FOUND);
-    return response;
-  }
-
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-  auto end = 25;
-  try {
-    end = request.get_argument<unsigned int>("end");
-  } catch (const std::exception&) {}
-  if (start>end) {
-    auto tmp = start;
-    start = end;
-    end = tmp;
-  }
-  if (end > start + MAX_SEARCH_LEN) {
-    end = start + MAX_SEARCH_LEN;
-  }
-
-  /* Get the results */
-  try {
-    if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
-    } else {
-      searcher.search(patternString,
-                       start, end, m_verbose.load());
-    }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
-    response.set_content(renderer.getHtml());
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-  return response;
-}
-
-Response InternalServer::handle_random(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_random\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  try {
-    auto entry = reader->getRandomPage();
-    entry = entry.getFinalEntry();
-    auto response = get_default_response();
-    response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
-    return response;
-  } catch(kiwix::NoEntry& e) {
-    return build_404(request, bookName);
-  }
-}
-
-Response InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return build_404(request, "");
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return build_404(request, "");
-  }
-
-  auto response = get_default_response();
-  response.set_compress(true);
-  if (url == "searchdescription.xml") {
-    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
-    response.set_mimeType("application/opensearchdescription+xml");
-    return response;
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper;
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setId(kiwix::to_string(uuid));
-  opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8");
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    opdsDumper.setTitle("All zims");
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    std::string query;
-    std::string language;
-    std::vector<std::string> tags;
-    std::vector<std::string> noTags;
-    size_t count(10);
-    size_t startIndex(0);
-    try {
-      query = request.get_argument("q");
-    } catch (const std::out_of_range&) {}
-    try {
-      language = request.get_argument("lang");
-    } catch (const std::out_of_range&) {}
-    try {
-      count = extractFromString<unsigned long>(request.get_argument("count"));
-    } catch (...) {}
-    try {
-      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
-    } catch (...) {}
-    try {
-      tags = kiwix::split(request.get_argument("notag"), ";");
-    } catch (...) {}
-    try {
-      noTags = kiwix::split(request.get_argument("notag"), ";");
-    } catch (...) {}
-    opdsDumper.setTitle("Search result for " + query);
-    uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(
-      kiwix::Filter().valid(true).local(true).remote(true)
-        .query(query)
-        .lang(language)
-        .acceptTags(tags)
-        .rejectTags(noTags)
-      );
-    auto totalResults = bookIdsToDump.size();
-    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
-    if (count>0 && bookIdsToDump.size() > count) {
-      bookIdsToDump.resize(count);
-    }
-    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
-  }
-
-  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
-  return response;
-}
-
-Response InternalServer::handle_content(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_content\n");
-  }
-
-  std::string baseUrl;
-  std::string content;
-  std::string mimeType;
-
-  kiwix::Entry entry;
-
-  std::string bookName;
-  try {
-    bookName = request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return build_homepage(request);
-  }
-  if (bookName.empty())
-    return build_homepage(request);
-
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  auto urlStr = request.get_url().substr(bookName.size()+1);
-  if (urlStr[0] == '/') {
-    urlStr = urlStr.substr(1);
-  }
-
-  try {
-    entry = reader->getEntryFromPath(urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
-      entry = entry.getFinalEntry();
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" +
-        kiwix::urlEncode(entry.getPath()));
-      return response;
-    }
-  } catch(kiwix::NoEntry& e) {
-    if (m_verbose.load())
-      printf("Failed to find %s\n", urlStr.c_str());
-
-    return build_404(request, bookName);
-  }
-
-  try {
-    mimeType = entry.getMimetype();
-  } catch (exception& e) {
-    mimeType = "application/octet-stream";
-  }
-
-  if (m_verbose.load()) {
-    printf("Found %s\n", urlStr.c_str());
-    printf("mimeType: %s\n", mimeType.c_str());
-  }
-
-  if (mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/json") != string::npos) {
-    zim::Blob raw_content = entry.getBlob();
-    content = string(raw_content.data(), raw_content.size());
-    auto response = get_default_response();
-    response.set_mimeType(mimeType);
-
-    /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
-     * /A/Kiwix */
-    if (mimeType.find("text/html") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(href|src)(=[\"|\']{0,1})/([A-Z|\\-])/");
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(@import[ ]+)([\"|\']{0,1})/([A-Z|\\-])/");
-      response.set_taskbar(bookName, reader->getTitle());
-    } else if (mimeType.find("text/css") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(url|URL)(\\([\"|\']{0,1})/([A-Z|\\-])/");
-    }
-    response.set_content(content);
-    response.set_compress(true);
-    response.set_cache(true);
-    return response;
-  } else {
-    int range_len;
-    if (request.get_range().second == -1) {
-       range_len = entry.getSize() - request.get_range().first;
-    } else {
-       range_len = request.get_range().second - request.get_range().first;
-    }
-    auto response = get_default_response();
-    response.set_entry(entry);
-    response.set_mimeType(mimeType);
-    response.set_range_first(request.get_range().first);
-    response.set_range_len(range_len);
-    response.set_cache(true);
-    return response;
-  }
-}
-
 }

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/etag.cpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "etag.h"
+
+#include "tools/stringTools.h"
+
+#include <algorithm>
+#include <sstream>
+
+namespace kiwix {
+
+namespace {
+
+// Characters in the options part of the ETag could in principle be picked up
+// from the latin alphabet in natural order (the character corresponding to
+// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
+// this file). However it is better to have some mnemonics in the option names,
+// hence below variable: all_options[opt] corresponds to the character going
+// into the ETag for ETag::Option opt.
+// IMPORTANT: The characters in all_options must come in sorted order (so that
+// IMPORTANT: isValidOptionsString() works correctly).
+const char all_options[] = "cz";
+
+static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
+
+bool isValidServerId(const std::string& s)
+{
+  return !s.empty() && s.find_first_of("\"/") == std::string::npos;
+}
+
+bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
+{
+  std::string::size_type i = 0;
+  for ( const char c : s )
+  {
+    const std::string::size_type j = sortedString.find(c, i);
+    if ( j == std::string::npos )
+      return false;
+    i = j+1;
+  }
+  return true;
+}
+
+bool isValidOptionsString(const std::string& s)
+{
+  return isSubsequenceOf(s, all_options);
+}
+
+} // namespace
+
+
+void ETag::set_option(Option opt)
+{
+  if ( ! get_option(opt) )
+  {
+    m_options.push_back(all_options[opt]);
+    std::sort(m_options.begin(), m_options.end());
+  }
+}
+
+bool ETag::get_option(Option opt) const
+{
+  return m_options.find(all_options[opt]) != std::string::npos;
+}
+
+std::string ETag::get_etag() const
+{
+  if ( m_serverId.empty() )
+    return std::string();
+
+  return "\"" + m_serverId + "/" + m_options + "\"";
+}
+
+ETag::ETag(const std::string& serverId, const std::string& options)
+{
+  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  {
+    m_serverId = serverId;
+    m_options = options;
+  }
+}
+
+ETag ETag::parse(std::string s)
+{
+  if ( kiwix::startsWith("W/", s) )
+    s = s.substr(2);
+
+  if ( s.front() != '"' || s.back() != '"' )
+    return ETag();
+
+  s = s.substr(1, s.size()-2);
+
+  const std::string::size_type i = s.find('/');
+  if ( i == std::string::npos )
+    return ETag();
+
+  return ETag(s.substr(0, i), s.substr(i+1));
+}
+
+ETag ETag::match(const std::string& etags, const std::string& server_id)
+{
+  std::istringstream ss(etags);
+  std::string etag_str;
+  while ( ss >> etag_str )
+  {
+    if ( etag_str.back() == ',' )
+      etag_str.pop_back();
+
+    const ETag etag = parse(etag_str);
+    if ( etag && etag.m_serverId == server_id )
+      return etag;
+  }
+
+  return ETag();
+}
+
+} // namespace kiwix

src/server/etag.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_ETAG_H
+#define KIWIXLIB_SERVER_ETAG_H
+
+#include <string>
+
+namespace kiwix {
+
+// The ETag string used by Kiwix server (more precisely, its value inside the
+// double quotes) consists of two parts:
+//
+// 1. ServerId - The string obtained on server start up
+//
+// 2. Options -  Zero or more characters encoding the values of some of the
+//               headers of the response
+//
+// The two parts are separated with a slash (/) symbol (which is always present,
+// even when the the options part is empty). Neither portion of a Kiwix ETag
+// may contain the slash symbol.
+// Examples of valid Kiwix server ETags (including the double quotes):
+//
+//   "abcdefghijklmn/"
+//   "1234567890/z"
+//   "1234567890/cz"
+//
+// The options part of the Kiwix ETag allows to correctly set the required
+// headers when responding to a conditional If-None-Match request with a 304
+// (Not Modified) response without following the full code path that would
+// discover the necessary options.
+
+class ETag
+{
+  public: // types
+    enum Option {
+      CACHEABLE_ENTITY,
+      COMPRESSED_CONTENT,
+      OPTION_COUNT
+    };
+
+  public: // functions
+    ETag() {}
+
+    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_option(Option opt);
+
+    explicit operator bool() const { return !m_serverId.empty(); }
+
+    bool get_option(Option opt) const;
+    std::string get_etag() const;
+
+
+    static ETag match(const std::string& etags, const std::string& server_id);
+
+  private: // functions
+    ETag(const std::string& serverId, const std::string& options);
+
+    static ETag parse(std::string s);
+
+  private: // data
+    std::string m_serverId;
+    std::string m_options;
+};
+
+} // namespace kiwix
+
+#endif // KIWIXLIB_SERVER_ETAG_H

src/server/internalServer.cpp

@@ -0,0 +1,798 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "internalServer.h"
+
+#ifdef _WIN32
+# if !defined(__MINGW32__) && (_MSC_VER < 1600)
+#   include "stdint4win.h"
+# endif
+# include <winsock2.h>
+# include <ws2tcpip.h>
+# ifdef __GNUC__
+  // inet_pton is not declared in mingw, even if the function exists.
+  extern "C" {
+    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
+  }
+# endif
+  typedef UINT64 uint64_t;
+  typedef UINT16 uint16_t;
+#endif
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+#include "library.h"
+#include "name_mapper.h"
+#include "entry.h"
+#include "searcher.h"
+#include "search_renderer.h"
+#include "opds_dumper.h"
+
+#include <zim/uuid.h>
+
+#include <mustache.hpp>
+
+#include <pthread.h>
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+#include "kiwixlib-resources.h"
+
+#ifndef _WIN32
+# include <arpa/inet.h>
+#endif
+
+#include "request_context.h"
+#include "response.h"
+
+#define MAX_SEARCH_LEN 140
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+static IdNameMapper defaultNameMapper;
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls);
+
+
+InternalServer::InternalServer(Library* library,
+                               NameMapper* nameMapper,
+                               std::string addr,
+                               int port,
+                               std::string root,
+                               int nbThreads,
+                               bool verbose,
+                               bool withTaskbar,
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
+  m_addr(addr),
+  m_port(port),
+  m_root(root),
+  m_nbThreads(nbThreads),
+  m_verbose(verbose),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  mp_daemon(nullptr),
+  mp_library(library),
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+{}
+
+bool InternalServer::start() {
+#ifdef _WIN32
+  int flags = MHD_USE_SELECT_INTERNALLY;
+#else
+  int flags = MHD_USE_POLL_INTERNALLY;
+#endif
+  if (m_verbose.load())
+    flags |= MHD_USE_DEBUG;
+
+
+  struct sockaddr_in sockAddr;
+  memset(&sockAddr, 0, sizeof(sockAddr));
+  sockAddr.sin_family = AF_INET;
+  sockAddr.sin_port = htons(m_port);
+  if (m_addr.empty()) {
+    if (0 != INADDR_ANY)
+      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+  } else {
+    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
+      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
+      return false;
+    }
+  }
+
+  mp_daemon = MHD_start_daemon(flags,
+                            m_port,
+                            NULL,
+                            NULL,
+                            &staticHandlerCallback,
+                            this,
+                            MHD_OPTION_SOCK_ADDR, &sockAddr,
+                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_END);
+  if (mp_daemon == nullptr) {
+    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
+              << " is maybe already occupied or need more permissions to be open. "
+                 "Please try as root or with a port number higher or equal to 1024."
+              << std::endl;
+    return false;
+  }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
+  return true;
+}
+
+void InternalServer::stop()
+{
+  MHD_stop_daemon(mp_daemon);
+}
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls)
+{
+  InternalServer* _this = static_cast<InternalServer*>(cls);
+
+  return _this->handlerCallback(connection,
+                                url,
+                                method,
+                                version,
+                                upload_data,
+                                upload_data_size,
+                                cont_cls);
+}
+
+MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                           const char* url,
+                                           const char* method,
+                                           const char* version,
+                                           const char* upload_data,
+                                           size_t* upload_data_size,
+                                           void** cont_cls)
+{
+  auto start_time = std::chrono::steady_clock::now();
+  if (m_verbose.load() ) {
+    printf("======================\n");
+    printf("Requesting : \n");
+    printf("full_url  : %s\n", url);
+  }
+  RequestContext request(connection, m_root, url, method, version);
+
+  if (m_verbose.load() ) {
+    request.print_debug_info();
+  }
+  /* Unexpected method */
+  if (request.get_method() != RequestMethod::GET
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
+    printf("Reject request because of unhandled request method.\n");
+    printf("----------------------\n");
+    return MHD_NO;
+  }
+
+  auto response = handle_request(request);
+
+  if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
+    printf("========== INTERNAL ERROR !! ============\n");
+    if (!m_verbose.load()) {
+      printf("Requesting : \n");
+      printf("full_url : %s\n", url);
+      request.print_debug_info();
+    }
+  }
+
+  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response->set_server_id(m_server_id);
+
+  auto ret = response->send(request, connection);
+  auto end_time = std::chrono::steady_clock::now();
+  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
+  if (m_verbose.load()) {
+    printf("Request time : %fs\n", time_span.count());
+    printf("----------------------\n");
+  }
+  return ret;
+}
+
+std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
+{
+  try {
+    if (! request.is_valid_url())
+      return Response::build_404(*this, request, "");
+
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return Response::build_304(*this, etag);
+
+    if (kiwix::startsWith(request.get_url(), "/skin/"))
+      return handle_skin(request);
+
+    if (startsWith(request.get_url(), "/catalog"))
+      return handle_catalog(request);
+
+    if (request.get_url() == "/meta")
+      return handle_meta(request);
+
+    if (request.get_url() == "/search")
+      return handle_search(request);
+
+    if (request.get_url() == "/suggest")
+     return handle_suggest(request);
+
+    if (request.get_url() == "/random")
+      return handle_random(request);
+
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
+    return handle_content(request);
+  } catch (std::exception& e) {
+    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
+    return Response::build_500(*this, e.what());
+  } catch (...) {
+    fprintf(stderr, "===== Unhandled unknown error\n");
+    return Response::build_500(*this, "Unknown error");
+  }
+}
+
+MustacheData InternalServer::get_default_data() const
+{
+  MustacheData data;
+  data.set("root", m_root);
+  return data;
+}
+
+MustacheData InternalServer::homepage_data() const
+{
+  auto data = get_default_data();
+
+  MustacheData books{MustacheData::type::list};
+  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = mp_library->getBookById(bookId);
+
+    MustacheData book;
+    book.set("name", mp_nameMapper->getNameForId(bookId));
+    book.set("title", currentBook.getTitle());
+    book.set("description", currentBook.getDescription());
+    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
+    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
+    books.push_back(book);
+  }
+
+  data.set("books", books);
+  return data;
+}
+
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
+{
+  return ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+{
+  std::string bookName;
+  std::string bookId;
+  std::string meta_name;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    meta_name = request.get_argument("name");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = ContentResponse::build(*this, content, mimeType);
+  response->set_cacheable();
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_suggest\n");
+  }
+
+  std::string content;
+  std::string mimeType;
+  unsigned int maxSuggestionCount = 10;
+  unsigned int suggestionCount = 0;
+
+  std::string bookName;
+  std::string bookId;
+  std::string term;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    term = request.get_argument("term");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (m_verbose.load()) {
+    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+  }
+
+  MustacheData results{MustacheData::type::list};
+
+  bool first = true;
+  if (reader != nullptr) {
+    /* Get the suggestions */
+    SuggestionsList_t suggestions;
+    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
+    for(auto& suggestion:suggestions) {
+      MustacheData result;
+      result.set("label", suggestion[0]);
+      result.set("value", suggestion[0]);
+      result.set("first", first);
+      first = false;
+      results.push_back(result);
+      suggestionCount++;
+    }
+  }
+
+  /* Propose the fulltext search if possible */
+  if (reader->hasFulltextIndex()) {
+    MustacheData result;
+    result.set("label", "containing '" + term + "'...");
+    result.set("value", term + " ");
+    result.set("first", first);
+    results.push_back(result);
+  }
+
+  auto data = get_default_data();
+  data.set("suggestions", results);
+
+  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_skin\n");
+  }
+
+  auto resourceName = request.get_url().substr(1);
+  try {
+    auto response = ContentResponse::build(
+        *this,
+        getResource(resourceName),
+        getMimeTypeForFile(resourceName));
+    response->set_cacheable();
+    return std::move(response);
+  } catch (const ResourceNotFound& e) {
+    return Response::build_404(*this, request, "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_search\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+  } catch (const std::out_of_range&) {}
+
+  std::string patternString;
+  try {
+    patternString = request.get_argument("pattern");
+  } catch (const std::out_of_range&) {}
+
+  /* Retrive geo search */
+  bool has_geo_query = false;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
+  try {
+    latitude = request.get_argument<float>("latitude");
+    longitude = request.get_argument<float>("longitude");
+    distance = request.get_argument<float>("distance");
+    has_geo_query = true;
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  std::shared_ptr<Reader> reader(nullptr);
+  try {
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {}
+
+  /* Try first to load directly the article */
+  if (reader != nullptr && !patternString.empty()) {
+    std::string patternCorrespondingUrl;
+    auto variants = reader->getTitleVariants(patternString);
+    auto variantsItr = variants.begin();
+
+    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
+      try {
+        auto entry = reader->getEntryFromTitle(*variantsItr);
+        entry = entry.getFinalEntry();
+        patternCorrespondingUrl = entry.getPath();
+        break;
+      } catch(kiwix::NoEntry& e) {
+        variantsItr++;
+      }
+    }
+
+    /* If article found then redirect directly to it */
+    if (!patternCorrespondingUrl.empty()) {
+      auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl;
+      return Response::build_redirect(*this, redirectUrl);
+    }
+  }
+
+  /* Make the search */
+  if ( (!reader && !bookName.empty())
+    || (patternString.empty() && ! has_geo_query) ) {
+    auto data = get_default_data();
+    data.set("pattern", encodeDiples(patternString));
+    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_code(MHD_HTTP_NOT_FOUND);
+    return std::move(response);
+  }
+
+  Searcher searcher;
+  if (reader) {
+    searcher.add_reader(reader.get());
+  } else {
+    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+      auto currentReader = mp_library->getReaderById(bookId);
+      if (currentReader) {
+        searcher.add_reader(currentReader.get());
+      }
+    }
+  }
+
+  auto start = 0;
+  try {
+    start = request.get_argument<unsigned int>("start");
+  } catch (const std::exception&) {}
+
+  auto pageLength = 25;
+  try {
+    pageLength = request.get_argument<unsigned int>("pageLength");
+  } catch (const std::exception&) {}
+  if (pageLength > MAX_SEARCH_LEN) {
+    pageLength = MAX_SEARCH_LEN;
+  }
+  if (pageLength == 0) {
+    pageLength = 25;
+  }
+
+  auto end = start + pageLength;
+
+  /* Get the results */
+  try {
+    if (patternString.empty()) {
+      searcher.geo_search(latitude, longitude, distance,
+                           start, end, m_verbose.load());
+    } else {
+      searcher.search(patternString,
+                       start, end, m_verbose.load());
+    }
+    SearchRenderer renderer(&searcher, mp_nameMapper);
+    renderer.setSearchPattern(patternString);
+    renderer.setSearchContent(bookName);
+    renderer.setProtocolPrefix(m_root + "/");
+    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setPageLength(pageLength);
+    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    //changing status code if no result obtained
+    if(searcher.getEstimatedResultCount() == 0)
+    {
+      response->set_code(MHD_HTTP_NO_CONTENT);
+    }
+
+    return std::move(response);
+  } catch (const std::exception& e) {
+    std::cerr << e.what() << std::endl;
+    return Response::build_500(*this, e.what());
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_random\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  try {
+    auto entry = reader->getRandomPage();
+    return build_redirect(bookName, entry.getFinalEntry());
+  } catch(kiwix::NoEntry& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return Response::build_404(*this, request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper;
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+  opdsDumper.setLibrary(mp_library);
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    opdsDumper.setTitle("All zims");
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
+    string query("<Empty query>");
+    size_t count(10);
+    size_t startIndex(0);
+    try {
+      query = request.get_argument("q");
+      filter.query(query);
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      count = extractFromString<unsigned long>(request.get_argument("count"));
+    } catch (...) {}
+    try {
+      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
+    } catch (...) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
+    } catch (...) {}
+    opdsDumper.setTitle("Search result for " + query);
+    uuid = zim::Uuid::generate();
+    bookIdsToDump = mp_library->filter(filter);
+    auto totalResults = bookIdsToDump.size();
+    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
+    if (count>0 && bookIdsToDump.size() > count) {
+      bookIdsToDump.resize(count);
+    }
+    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
+  }
+
+  opdsDumper.setId(kiwix::to_string(uuid));
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump),
+      "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
+  return std::move(response);
+}
+
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+std::unique_ptr<Response>
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
+  return Response::build_redirect(*this, redirectUrl);
+}
+
+std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_content\n");
+  }
+
+  const std::string bookName = get_book_name(request);
+  if (bookName.empty())
+    return build_homepage(request);
+
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto urlStr = request.get_url().substr(bookName.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
+
+  kiwix::Entry entry;
+
+  try {
+    entry = reader->getEntryFromPath(urlStr);
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      return build_redirect(bookName, entry.getFinalEntry());
+    }
+  } catch(kiwix::NoEntry& e) {
+    if (m_verbose.load())
+      printf("Failed to find %s\n", urlStr.c_str());
+
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = EntryResponse::build(*this, request, entry);
+  try {
+    dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
+  } catch (std::bad_cast& e) {}
+
+  if (m_verbose.load()) {
+    printf("Found %s\n", entry.getPath().c_str());
+    printf("mimeType: %s\n", entry.getMimetype().c_str());
+  }
+
+  return response;
+}
+
+}

src/server/internalServer.h

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H
+#define KIWIXLIB_SERVER_INTERNALSERVER_H
+
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "library.h"
+#include "name_mapper.h"
+
+#include <mustache.hpp>
+
+#include <atomic>
+#include <string>
+
+#include "server/request_context.h"
+#include "server/response.h"
+
+namespace kiwix {
+
+typedef kainjow::mustache::data MustacheData;
+
+class Entry;
+
+class InternalServer {
+  public:
+    InternalServer(Library* library,
+                   NameMapper* nameMapper,
+                   std::string addr,
+                   int port,
+                   std::string root,
+                   int nbThreads,
+                   bool verbose,
+                   bool withTaskbar,
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
+    virtual ~InternalServer() = default;
+
+    MHD_Result handlerCallback(struct MHD_Connection* connection,
+                               const char* url,
+                               const char* method,
+                               const char* version,
+                               const char* upload_data,
+                               size_t* upload_data_size,
+                               void** cont_cls);
+    bool start();
+    void stop();
+
+  private: // functions
+    std::unique_ptr<Response> handle_request(const RequestContext& request);
+    std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_skin(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog(const RequestContext& request);
+    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_suggest(const RequestContext& request);
+    std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
+    std::unique_ptr<Response> handle_content(const RequestContext& request);
+
+    MustacheData get_default_data() const;
+    MustacheData homepage_data() const;
+
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
+    std::string m_addr;
+    int m_port;
+    std::string m_root;
+    int m_nbThreads;
+    std::atomic_bool m_verbose;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    struct MHD_Daemon* mp_daemon;
+
+    Library* mp_library;
+    NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
+
+    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
+
+};
+
+}
+
+#endif //KIWIXLIB_SERVER_INTERNALSERVER_H

src/server/request_context.cpp

@@ -26,106 +26,84 @@
 #include <cstdio>
 #include <atomic>
 
+#include "tools/stringTools.h"
+
 namespace kiwix {
 
 static std::atomic_ullong s_requestIndex(0);
 
+namespace {
+
+RequestMethod str2RequestMethod(const std::string& method) {
+  if      (method == "GET")     return RequestMethod::GET;
+  else if (method == "HEAD")    return RequestMethod::HEAD;
+  else if (method == "POST")    return RequestMethod::POST;
+  else if (method == "PUT")     return RequestMethod::PUT;
+  else if (method == "DELETE")  return RequestMethod::DELETE_;
+  else if (method == "CONNECT") return RequestMethod::CONNECT;
+  else if (method == "OPTIONS") return RequestMethod::OPTIONS;
+  else if (method == "TRACE")   return RequestMethod::TRACE;
+  else if (method == "PATCH")   return RequestMethod::PATCH;
+  else                          return RequestMethod::OTHER;
+}
+
+std::string
+fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
+{
+  if (rootLocation.empty()) {
+    // nothing special to handle.
+    return full_url;
+  } else if (full_url == rootLocation) {
+    return "/";
+  } else if (full_url.size() > rootLocation.size() &&
+             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
+    return full_url.substr(rootLocation.size());
+  } else {
+    return "";
+  }
+}
+
+} // unnamed namespace
+
 RequestContext::RequestContext(struct MHD_Connection* connection,
                                std::string rootLocation,
                                const std::string& _url,
-                               const std::string& method,
+                               const std::string& _method,
                                const std::string& version) :
   full_url(_url),
-  url(_url),
-  valid_url(true),
+  url(fullURL2LocalURL(_url, rootLocation)),
+  method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingDeflate(false),
-  accept_range(false),
-  range_pair(0, -1)
+  byteRange_()
 {
-  if (method == "GET") {
-    this->method = RequestMethod::GET;
-  } else if (method == "HEAD") {
-    this->method = RequestMethod::HEAD;
-  } else if (method == "POST") {
-    this->method = RequestMethod::POST;
-  } else if (method == "PUT") {
-    this->method = RequestMethod::PUT;
-  } else if (method == "DELETE") {
-    this->method = RequestMethod::DELETE_;
-  } else if (method == "CONNECT") {
-    this->method = RequestMethod::CONNECT;
-  } else if (method == "OPTIONS") {
-    this->method = RequestMethod::OPTIONS;
-  } else if (method == "TRACE") {
-    this->method = RequestMethod::TRACE;
-  } else if (method == "PATCH") {
-    this->method = RequestMethod::PATCH;
-  } else {
-    this->method = RequestMethod::OTHER;
-  }
-
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
 
-  valid_url = true;
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    url = full_url;
-  } else {
-    if (full_url == rootLocation) {
-      url = "/";
-    } else if (full_url.size() > rootLocation.size() &&
-               full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-      url = full_url.substr(rootLocation.size());
-    } else {
-      valid_url = false;
-    }
-  }
-
   try {
     acceptEncodingDeflate =
         (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
-  /*Check if range is requested. */
   try {
-    auto range = get_header(MHD_HTTP_HEADER_RANGE);
-    int start = 0;
-    int end = -1;
-    std::istringstream iss(range);
-    char c;
-
-    iss >> start >> c;
-    if (iss.good() && c=='-') {
-      iss >> end;
-      if (iss.fail()) {
-        // Something went wrong will extracting.
-        end = -1;
-      }
-      if (iss.eof()) {
-        accept_range = true;
-        range_pair = std::pair<int, int>(start, end);
-      }
-    }
+    byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
 }
 
 RequestContext::~RequestContext()
 {}
 
-
-int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                 const char *key, const char *value)
+MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
+                                       const char *key, const char *value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[key] = value;
+  _this->headers[lcAll(key)] = value;
   return MHD_YES;
 }
 
-int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                   const char *key, const char* value)
+MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key] = value == nullptr ? "" : value;
@@ -147,10 +125,11 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("Parsed : \n");
+  printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
-  printf("has_range : %d\n", accept_range);
-  printf("is_valid_url : %d\n", valid_url);
+  printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
+  printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
 }
 
@@ -188,15 +167,11 @@ std::string RequestContext::get_full_url() const {
 }
 
 bool RequestContext::is_valid_url() const {
-  return valid_url;
+  return !url.empty();
 }
 
-bool RequestContext::has_range() const {
-  return accept_range;
-}
-
-std::pair<int, int> RequestContext::get_range() const {
-  return range_pair;
+ByteRange RequestContext::get_range() const {
+  return byteRange_;
 }
 
 template<>
@@ -205,7 +180,7 @@ std::string RequestContext::get_argument(const std::string& name) const {
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
-  return headers.at(name);
+  return headers.at(lcAll(name));
 }
 
 }

src/server/request_context.h

@@ -27,8 +27,10 @@
 #include <map>
 #include <stdexcept>
 
+#include "byte_range.h"
+
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
@@ -51,7 +53,7 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
-  public:
+  public: // functions
     RequestContext(struct MHD_Connection* connection,
                    std::string rootLocation,
                    const std::string& url,
@@ -78,28 +80,26 @@ class RequestContext {
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
 
-    bool has_range() const;
-    std::pair<int, int> get_range() const;
+    ByteRange get_range() const;
 
     bool can_compress() const { return acceptEncodingDeflate; }
 
-  private:
+  private: // data
     std::string full_url;
     std::string url;
-    bool valid_url;
     RequestMethod method;
     std::string version;
     unsigned long long requestIndex;
 
     bool acceptEncodingDeflate;
 
-    bool accept_range;
-    std::pair<int, int> range_pair;
+    ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::string> arguments;
 
-    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+  private: // functions
+    static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 
 template<> std::string RequestContext::get_argument(const std::string& name) const;

src/server/response.cpp

@@ -3,6 +3,7 @@
 
 #include "response.h"
 #include "request_context.h"
+#include "internalServer.h"
 #include "kiwixlib-resources.h"
 
 #include "tools/regexTools.h"
@@ -17,25 +18,112 @@
 
 namespace kiwix {
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton)
-  : m_verbose(verbose),
-    m_root(root),
-    m_content(""),
-    m_mimeType(""),
-    m_returnCode(MHD_HTTP_OK),
-    m_withTaskbar(withTaskbar),
-    m_withLibraryButton(withLibraryButton),
-    m_useCache(false),
-    m_addTaskbar(false),
-    m_bookName(""),
-    m_startRange(0),
-    m_lenRange(0)
+namespace
 {
+// some utilities
+
+std::string render_template(const std::string& template_str, kainjow::mustache::data data)
+{
+  kainjow::mustache::mustache tmpl(template_str);
+  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
+                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
+  data.set("urlencoded", urlencode);
+  std::stringstream ss;
+  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}
+
+std::string get_mime_type(const kiwix::Entry& entry)
+{
+  try {
+    return entry.getMimetype();
+  } catch (exception& e) {
+    return "application/octet-stream";
+  }
+}
+
+bool is_compressible_mime_type(const std::string& mimeType)
+{
+  return mimeType.find("text/") != string::npos
+      || mimeType.find("application/javascript") != string::npos
+      || mimeType.find("application/atom") != string::npos
+      || mimeType.find("application/opensearchdescription") != string::npos
+      || mimeType.find("application/json") != string::npos;
 }
 
 
-static int print_key_value (void *cls, enum MHD_ValueKind kind,
-                            const char *key, const char *value)
+} // unnamed namespace
+
+Response::Response(bool verbose)
+  : m_verbose(verbose),
+    m_returnCode(MHD_HTTP_OK)
+{
+  add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+}
+
+std::unique_ptr<Response> Response::build(const InternalServer& server)
+{
+  return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
+}
+
+std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
+{
+  auto response = Response::build(server);
+  response->set_code(MHD_HTTP_NOT_MODIFIED);
+  response->m_etag = etag;
+  if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName)
+{
+  MustacheData results;
+  results.set("url", request.get_full_url());
+
+  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
+  response->set_code(MHD_HTTP_NOT_FOUND);
+  response->set_taskbar(bookName, "");
+
+  return std::move(response);
+}
+
+std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
+{
+  auto response = Response::build(server);
+// [FIXME] (compile with recent enough version of libmicrohttpd)
+//  response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
+  response->set_code(416);
+  std::ostringstream oss;
+  oss << "bytes */" << resourceLength;
+  response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str());
+
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
+{
+  MustacheData data;
+  data.set("error", msg);
+  auto content = render_template(RESOURCE::templates::_500_html, data);
+  std::unique_ptr<Response> response (
+      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
+  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
+  return response;
+}
+
+
+std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
+{
+  auto response = Response::build(server);
+  response->m_returnCode = MHD_HTTP_FOUND;
+  response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
+  return response;
+}
+
+static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
+                                   const char *key, const char *value)
 {
   printf (" - %s: '%s'\n", key, value);
   return MHD_YES;
@@ -90,23 +178,10 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-std::string render_template(const std::string& template_str, kainjow::mustache::data data)
-{
-  kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
-  std::stringstream ss;
-  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
 
 
-void Response::introduce_taskbar()
+void ContentResponse::introduce_taskbar()
 {
-  if (! m_withTaskbar)
-    // Taskbar is globally disabled.
-    return;
   kainjow::mustache::data data;
   data.set("root", m_root);
   data.set("content", m_bookName);
@@ -127,88 +202,98 @@ void Response::introduce_taskbar()
 }
 
 
-
-int Response::send(const RequestContext& request, MHD_Connection* connection)
+void ContentResponse::inject_externallinks_blocker()
 {
-  MHD_Response* response = nullptr;
-  switch (m_mode) {
-    case ResponseMode::RAW_CONTENT : {
-      if (m_addTaskbar) {
-        introduce_taskbar();
-      }
+  kainjow::mustache::data data;
+  data.set("root", m_root);
+  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<head>",
+    script_tag);
+}
 
-      bool shouldCompress = m_compress && request.can_compress();
-      shouldCompress &= m_mimeType.find("text/") != string::npos
-                     || m_mimeType.find("application/javascript") != string::npos
-                     || m_mimeType.find("application/json") != string::npos; 
+bool
+ContentResponse::can_compress(const RequestContext& request) const
+{
+  return request.can_compress()
+      && is_compressible_mime_type(m_mimeType)
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+}
 
-      shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+bool
+ContentResponse::contentDecorationAllowed() const
+{
+    return (startsWith(m_mimeType, "text/html")
+         && m_mimeType.find(";raw=true") == std::string::npos);
+}
 
-      if (shouldCompress) {
-        std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-        uLongf comprLen = compr_buffer.capacity();
-        int err = compress(&compr_buffer[0],
-                           &comprLen,
-                           (const Bytef*)(m_content.data()),
-                           m_content.size());
-        if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-          /* /!\ Internet Explorer has a bug with deflate compression.
-             It can not handle the first two bytes (compression headers)
-             We need to chunk them off (move the content 2bytes)
-             It has no incidence on other browsers
-             See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-          m_content = string((char*)&compr_buffer[2], comprLen - 2);
-        } else {
-          shouldCompress = false;
-        }
-      }
+MHD_Response*
+Response::create_mhd_response(const RequestContext& request)
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT);
+  return response;
+}
 
-      response = MHD_create_response_from_buffer(
-        m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
-
-      if (shouldCompress) {
-        MHD_add_response_header(
-            response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-        MHD_add_response_header(
-            response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
-      }
-      MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-      break;
+MHD_Response*
+ContentResponse::create_mhd_response(const RequestContext& request)
+{
+  if (contentDecorationAllowed()) {
+    if (m_withTaskbar) {
+      introduce_taskbar();
     }
-
-    case ResponseMode::REDIRECTION : {
-      response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-      MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-      break;
-    }
-
-    case ResponseMode::ENTRY : {
-      response = MHD_create_response_from_callback(m_entry.getSize(),
-                                                   16384,
-                                                   callback_reader_from_entry,
-                                                   new RunningResponse(m_entry, m_startRange),
-                                                   callback_free_response);
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-      std::ostringstream oss;
-      oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-          << "/" << m_entry.getSize();
-
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
-      break;
+    if (m_blockExternalLinks) {
+      inject_externallinks_blocker();
     }
   }
 
-  MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_useCache ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+  bool shouldCompress = can_compress(request);
+  if (shouldCompress) {
+    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
+    uLongf comprLen = compr_buffer.capacity();
+    int err = compress(&compr_buffer[0],
+                       &comprLen,
+                       (const Bytef*)(m_content.data()),
+                       m_content.size());
+    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
+      /* /!\ Internet Explorer has a bug with deflate compression.
+         It can not handle the first two bytes (compression headers)
+         We need to chunk them off (move the content 2bytes)
+         It has no incidence on other browsers
+         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
+      m_content = string((char*)&compr_buffer[2], comprLen - 2);
+      m_etag.set_option(ETag::COMPRESSED_CONTENT);
+    } else {
+      shouldCompress = false;
+    }
+  }
 
-  if (m_returnCode == MHD_HTTP_OK && request.has_range())
+  MHD_Response* response = MHD_create_response_from_buffer(
+    m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
+
+  if (shouldCompress) {
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+  }
+  return response;
+}
+
+MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
+{
+  MHD_Response* response = create_mhd_response(request);
+
+  MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
+    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+  const std::string etag = m_etag.get_etag();
+  if ( ! etag.empty() )
+    MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  for(auto& p: m_customHeaders) {
+    MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
+  }
+
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
   if (m_verbose)
@@ -219,32 +304,105 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
   return ret;
 }
 
-void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
-  set_content(render_template(template_str, data));
-}
-
-void Response::set_content(const std::string& content) {
-  m_content = content;
-  m_mode = ResponseMode::RAW_CONTENT;
-}
-
-void Response::set_redirection(const std::string& url) {
-  m_content = url;
-  m_mode = ResponseMode::REDIRECTION;
-  m_returnCode = MHD_HTTP_FOUND;
-}
-
-void Response::set_entry(const Entry& entry) {
-  m_entry = entry;
-  m_mode = ResponseMode::ENTRY;
-}
-
-void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
 {
-  m_addTaskbar = true;
   m_bookName = bookName;
   m_bookTitle = bookTitle;
 }
 
 
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+  Response(verbose),
+  m_root(root),
+  m_content(content),
+  m_mimeType(mimetype),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  m_bookName(""),
+  m_bookTitle("")
+{
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype)
+{
+   return std::unique_ptr<ContentResponse>(new ContentResponse(
+        server.m_root,
+        server.m_verbose.load(),
+        server.m_withTaskbar,
+        server.m_withLibraryButton,
+        server.m_blockExternalLinks,
+        content,
+        mimetype));
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) {
+  auto content = render_template(template_str, data);
+  return ContentResponse::build(server, content, mimetype);
+}
+
+EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) :
+  Response(verbose),
+  m_entry(entry),
+  m_mimeType(mimetype)
+{
+  m_byteRange = byterange;
+  set_cacheable();
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry)
+{
+  const std::string mimetype = get_mime_type(entry);
+  auto byteRange = request.get_range().resolve(entry.getSize());
+  const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if (noRange && is_compressible_mime_type(mimetype)) {
+    // Return a contentResponse
+    zim::Blob raw_content = entry.getBlob();
+    const std::string content = string(raw_content.data(), raw_content.size());
+    auto response = ContentResponse::build(server, content, mimetype);
+    response->set_cacheable();
+    response->m_byteRange = byteRange;
+    return std::move(response);
+  }
+
+  if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
+    auto response = Response::build_416(server, entry.getSize());
+    response->set_cacheable();
+    return response;
+  }
+
+  return std::unique_ptr<Response>(new EntryResponse(
+        server.m_verbose.load(),
+        entry,
+        mimetype,
+        byteRange));
+}
+
+MHD_Response*
+EntryResponse::create_mhd_response(const RequestContext& request)
+{
+  const auto content_length = m_byteRange.length();
+  MHD_Response* response = MHD_create_response_from_callback(content_length,
+                                               16384,
+                                               callback_reader_from_entry,
+                                               new RunningResponse(m_entry, m_byteRange.first()),
+                                               callback_free_response);
+  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
+  if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
+    std::ostringstream oss;
+    oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
+        << "/" << m_entry.getSize();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
+  return response;
+}
+
+
 }

src/server/response.h

@@ -22,66 +22,98 @@
 #define KIWIXLIB_SERVER_RESPONSE_H
 
 #include <string>
+#include <map>
 
 #include <mustache.hpp>
+#include "byte_range.h"
 #include "entry.h"
+#include "etag.h"
 
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
 
-enum class ResponseMode {
-  RAW_CONTENT,
-  REDIRECTION,
-  ENTRY
-};
-
+class InternalServer;
 class RequestContext;
 
+class EntryResponse;
+
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton);
-    ~Response() = default;
+    Response(bool verbose);
+    virtual ~Response() = default;
 
-    int send(const RequestContext& request, MHD_Connection* connection);
+    static std::unique_ptr<Response> build(const InternalServer& server);
+    static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
+    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName);
+    static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
+    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
+    static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
-    void set_template(const std::string& template_str, kainjow::mustache::data data);
-    void set_content(const std::string& content);
-    void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry);
+    MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
-
-    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
     void set_code(int code) { m_returnCode = code; }
-    void set_cache(bool cache) { m_useCache = cache; }
-    void set_compress(bool compress) { m_compress = compress; }
+    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
+    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
+
+    int getReturnCode() const { return m_returnCode; }
+
+  private: // functions
+    virtual MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
+
+  protected: // data
+    bool m_verbose;
+    int m_returnCode;
+    ByteRange m_byteRange;
+    ETag m_etag;
+    std::map<std::string, std::string> m_customHeaders;
+
+    friend class EntryResponse; // temporary to allow the builder to change m_mode
+};
+
+
+class ContentResponse : public Response {
+  public:
+    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype);
+
     void set_taskbar(const std::string& bookName, const std::string& bookTitle);
-    void set_range_first(uint64_t start) { m_startRange = start; }
-    void set_range_len(uint64_t len) { m_lenRange = len; }
-
-    int getReturnCode() { return m_returnCode; }
-
-    void introduce_taskbar();
 
   private:
-    bool m_verbose;
-    ResponseMode m_mode;
+    MHD_Response* create_mhd_response(const RequestContext& request);
+
+    void introduce_taskbar();
+    void inject_externallinks_blocker();
+    bool can_compress(const RequestContext& request) const;
+    bool contentDecorationAllowed() const;
+
+
+  private:
     std::string m_root;
     std::string m_content;
-    Entry m_entry;
     std::string m_mimeType;
-    int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
-    bool m_useCache;
-    bool m_compress;
-    bool m_addTaskbar;
+    bool m_blockExternalLinks;
     std::string m_bookName;
     std::string m_bookTitle;
-    uint64_t m_startRange;
-    uint64_t m_lenRange;
+ };
+
+class EntryResponse : public Response {
+  public:
+    EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
+
+    Entry m_entry;
+    std::string m_mimeType;
 };
 
 }

src/subprocess_unix.cpp

@@ -26,6 +26,7 @@ UnixImpl::~UnixImpl()
 #else
   pthread_cancel(m_waitingThread);
 #endif
+  pthread_join(m_waitingThread, nullptr);
 }
 
 #ifdef __ANDROID__

src/subprocess_windows.cpp

@@ -4,13 +4,15 @@
 
 #include <windows.h>
 #include <winbase.h>
+#include <shlwapi.h>
 #include <iostream>
 #include <sstream>
 
 WinImpl::WinImpl():
   m_pid(0),
   m_running(false),
-  m_handle(INVALID_HANDLE_VALUE)
+  m_subprocessHandle(INVALID_HANDLE_VALUE),
+  m_waitingThreadHandle(INVALID_HANDLE_VALUE)
 {
   InitializeCriticalSection(&m_criticalSection);
 }
@@ -18,14 +20,15 @@ WinImpl::WinImpl():
 WinImpl::~WinImpl()
 {
   kill();
-  CloseHandle(m_handle);
+  WaitForSingleObject(m_waitingThreadHandle, INFINITE);
+  CloseHandle(m_subprocessHandle);
   DeleteCriticalSection(&m_criticalSection);
 }
 
 DWORD WINAPI WinImpl::waitForPID(void* _self)
 {
   WinImpl* self = static_cast<WinImpl*>(_self);
-  WaitForSingleObject(self->m_handle, INFINITE);
+  WaitForSingleObject(self->m_subprocessHandle, INFINITE);
 
   EnterCriticalSection(&self->m_criticalSection);
   self->m_running = false;
@@ -34,11 +37,11 @@ DWORD WINAPI WinImpl::waitForPID(void* _self)
   return 0;
 }
 
-std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
+std::unique_ptr<wchar_t[]> toWideChar(const std::string& value, size_t min_size = 0)
 {
-  auto size = MultiByteToWideChar(CP_UTF8, 0,
+  size_t size = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, nullptr, 0);
-  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size]);
+  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size>min_size?size:min_size]);
   auto ret = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, wdata.get(), size);
   if (0 == ret) {
@@ -46,6 +49,9 @@ std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
     oss << "Cannot convert to wchar : " << GetLastError();
     throw std::runtime_error(oss.str());
   }
+  if (size < min_size) {
+    memset(wdata.get() + size, 0, min_size-size);
+  }
   return wdata;
 }
 
@@ -55,14 +61,16 @@ void WinImpl::run(commandLine_t& commandLine)
   STARTUPINFOW startInfo = {0};
   PROCESS_INFORMATION procInfo;
   startInfo.cb = sizeof(startInfo);
-  std::ostringstream oss;
+  std::wostringstream oss;
   for(auto& item: commandLine) {
-    oss << item << " ";
+    auto witem = toWideChar(item, MAX_PATH);
+    PathQuoteSpacesW(witem.get());
+    oss << witem.get() << " ";
   }
-  auto wCommandLine = toWideChar(oss.str());
+  auto wCommandLine = oss.str();
   if (CreateProcessW(
     NULL,
-    wCommandLine.get(),
+    const_cast<wchar_t*>(wCommandLine.c_str()),
     NULL,
     NULL,
     false,
@@ -70,18 +78,19 @@ void WinImpl::run(commandLine_t& commandLine)
     NULL,
     NULL,
     &startInfo,
-    &procInfo)) {
+    &procInfo))
+  {
     m_pid = procInfo.dwProcessId;
-    m_handle = procInfo.hProcess;
+    m_subprocessHandle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);
     m_running = true;
-    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
+    m_waitingThreadHandle = CreateThread(NULL, 0, &waitForPID, this, 0, NULL);
   }
 }
 
 bool WinImpl::kill()
 {
-  return TerminateProcess(m_handle, 0);
+  return TerminateProcess(m_subprocessHandle, 0);
 }
 
 bool WinImpl::isRunning()

src/subprocess_windows.h

@@ -11,7 +11,8 @@ class WinImpl : public SubprocessImpl
   private:
     int m_pid;
     bool m_running;
-    HANDLE m_handle;
+    HANDLE m_subprocessHandle;
+    HANDLE  m_waitingThreadHandle;
     CRITICAL_SECTION m_criticalSection;
 
   public:

src/tools/base64.cpp

@@ -1,4 +1,4 @@
-/* 
+/*
    base64.cpp and base64.h
 
    Copyright (C) 2004-2008 René Nyffenegger
@@ -27,7 +27,7 @@
 #include <tools/base64.h>
 #include <iostream>
 
-static const std::string base64_chars = 
+static const std::string base64_chars =
              "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
              "abcdefghijklmnopqrstuvwxyz"
   "0123456789+/";

src/tools/otherTools.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "tools/otherTools.h"
+#include <algorithm>
 
 #ifdef _WIN32
 #include <windows.h>
@@ -25,7 +26,10 @@
 #include <unistd.h>
 #endif
 
+#include "tools/stringTools.h"
+
 #include <map>
+#include <sstream>
 #include <pugixml.hpp>
 
 
@@ -204,3 +208,136 @@ std::string kiwix::nodeToString(const pugi::xml_node& node)
 std::string kiwix::converta2toa3(const std::string& a2code){
   return codeisomapping.at(a2code);
 }
+
+std::vector<std::string> kiwix::convertTags(const std::string& tags_str)
+{
+  auto tags = kiwix::split(tags_str, ";");
+  std::vector<std::string> tagsList;
+  bool picSeen(false), vidSeen(false), detSeen(false), indexSeen(false);
+  for (auto tag: tags) {
+    picSeen |= (tag == "nopic" || startsWith(tag, "_pictures:"));
+    vidSeen |= (tag == "novid" || startsWith(tag, "_videos:"));
+    detSeen |= (tag == "nodet" || startsWith(tag, "_details:"));
+    indexSeen |= kiwix::startsWith(tag, "_ftindex");
+    if (tag == "nopic") {
+      tagsList.push_back("_pictures:no");
+    } else if (tag == "novid") {
+      tagsList.push_back("_videos:no");
+    } else if (tag == "nodet") {
+      tagsList.push_back("_details:no");
+    } else if (tag == "_ftindex") {
+      tagsList.push_back("_ftindex:yes");
+    } else {
+      tagsList.push_back(tag);
+    }
+  }
+  if (!indexSeen) {
+    tagsList.push_back("_ftindex:no");
+  }
+  if (!picSeen) {
+    tagsList.push_back("_pictures:yes");
+  }
+  if (!vidSeen) {
+    tagsList.push_back("_videos:yes");
+  }
+  if (!detSeen) {
+    tagsList.push_back("_details:yes");
+  }
+  return tagsList;
+}
+
+std::string kiwix::getTagValueFromTagList(
+  const std::vector<std::string>& tagList, const std::string& tagName)
+{
+  for (auto tag: tagList) {
+    if (tag[0] == '_') {
+      auto delimPos = tag.find(':');
+      if (delimPos == std::string::npos) {
+        // No delimiter... what to do ?
+        continue;
+      }
+      auto cTagName = tag.substr(1, delimPos-1);
+      auto cTagValue = tag.substr(delimPos+1);
+      if (cTagName == tagName) {
+        return cTagValue;
+      }
+    }
+  }
+  std::stringstream ss;
+  ss << tagName << " cannot be found";
+  throw std::out_of_range(ss.str());
+}
+
+bool kiwix::convertStrToBool(const std::string& value)
+{
+  if (value == "yes") {
+    return true;
+  } else if (value == "no") {
+    return false;
+  }
+
+  std::stringstream ss;
+  ss << "Tag value '" << value << "' cannot be converted to bool.";
+  throw std::domain_error(ss.str());
+}
+
+namespace
+{
+// The counter metadata format is a list of item separated by a `;` :
+// item0;item1;item2
+// Each item is a "tuple" mimetype=number.
+// However, the mimetype may contains parameters:
+// text/html;raw=true;foo=bar
+// So the final format may be complex to parse:
+// key0=value0;key1;foo=bar=value1;key2=value2
+
+typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
+
+std::string readFullMimetypeAndCounterString(std::istream& in)
+{
+  std::string mtcStr, params;
+  getline(in, mtcStr, ';');
+  if ( mtcStr.find('=') == std::string::npos )
+  {
+    do
+    {
+      if ( !getline(in, params, ';' ) )
+        return std::string();
+      mtcStr += ";" + params;
+    }
+    while ( std::count(params.begin(), params.end(), '=') != 2 );
+  }
+  return mtcStr;
+}
+
+MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
+{
+  const std::string::size_type k = s.find_last_of("=");
+  if ( k != std::string::npos )
+  {
+    const std::string mimeType = s.substr(0, k);
+    std::istringstream counterSS(s.substr(k+1));
+    unsigned int counter;
+    if (counterSS >> counter && counterSS.eof())
+      return MimetypeAndCounter{mimeType, counter};
+  }
+  return MimetypeAndCounter{"", 0};
+}
+
+} // unnamed namespace
+
+kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
+{
+  kiwix::MimeCounterType counters;
+  std::istringstream ss(counterData);
+
+  while (ss)
+  {
+    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
+    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
+    if ( !mtc.first.empty() )
+      counters.insert(mtc);
+  }
+
+  return counters;
+}

src/tools/pathTools.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "tools/pathTools.h"
+#include <stdexcept>
 
 #ifdef __APPLE__
 #include <limits.h>
@@ -39,34 +40,145 @@
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <algorithm>
 
 #ifdef _WIN32
-const std::string SEPARATOR("\\");
+# define SEPARATOR "\\"
+# include <io.h>
 #else
-const std::string SEPARATOR("/");
-#include <unistd.h>
+# define SEPARATOR "/"
+# include <unistd.h>
+# include <sys/stat.h>
 #endif
 
+#include <fcntl.h>
+
 #include <stdlib.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX 1024
 #endif
 
+
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr)
+{
+  auto needed_size = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), NULL, 0, NULL, NULL);
+  std::string ret(needed_size, 0);
+  WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), &ret[0], needed_size, NULL, NULL);
+  return ret;
+}
+
+std::wstring Utf8ToWide(const std::string& str)
+{
+  auto needed_size = MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), NULL, 0);
+  std::wstring ret(needed_size, 0);
+  MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), &ret[0], needed_size);
+  return ret;
+}
+#endif
+
 bool isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+  if (path.size() < 3 ) {
+    return true;
+  }
+  if (path.substr(1, 2) == ":\\" || path.substr(0, 2) == "\\\\") {
+    return false;
+  }
+  return true;
 #else
   return path.empty() || path.substr(0, 1) == "/" ? false : true;
 #endif
 }
 
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute)
+{
+  std::vector<std::string> ret;
+#ifdef _WIN32
+  //Special case if we have a drive directory not at first.
+  //Starts from there.
+  auto it = find_if(parts.rbegin(), parts.rend(),
+                    [](const std::string& p) ->bool
+                      { return ((p.length() == 2 && p[1] == ':')
+                             || (p.length() > 2 && p[0] == '\\' && p[1] == '\\')); });
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+  }
+  // Special case for samba mount point starting with two "\\" ("\\\\samba\\foo")
+  if (parts.size() > 2  && parts[0].empty() && parts[1].empty()) {
+    parts.erase(parts.begin(), parts.begin()+2);
+    parts[0] = "\\\\" + parts[0];
+  }
+  // Special case if we have a samba drive not at first.
+  // Path is "..\\\\sambdadrive\\..\\.." So we will have an empty part.
+  auto previous_empty = false;
+  for (it = parts.rbegin(); it!=parts.rend(); it++) {
+    if(it->empty()) {
+      if (previous_empty) {
+        it++;
+        break;
+      } else {
+        previous_empty = true;
+      }
+    } else {
+      previous_empty = false;
+    }
+  }
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+    parts[0] = "\\\\" + parts[0];
+  }
+#endif
+
+  size_t index = 0;
+  for (auto& part: parts) {
+    index++;
+    if (part == "..") {
+      if (absolute) {
+        // We try to remove as far as possible.
+        if (ret.size() > 1) {
+          ret.pop_back();
+        }
+      } else {
+        // We remove only if we can remove it.
+        // Else we add it.
+        if (!ret.empty() && ret.back() != "..") {
+          ret.pop_back();
+        } else {
+          ret.push_back("..");
+        }
+      }
+      continue;
+    }
+    if (part == "") {
+#ifndef _WIN32
+      if (ret.empty() && (absolute || index<parts.size())) {
+        ret.push_back("");
+      }
+#endif
+      continue;
+    }
+    if (part == ".") {
+      continue;
+    }
+    ret.push_back(part);
+  }
+#ifndef _WIN32
+  if (absolute && ret.size() == 1 && ret.back() == "") {
+    ret.push_back("");
+  }
+#endif
+  return ret;
+}
+
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
-  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
-  std::vector<std::string> absolutePathParts
-      = kiwix::split(absolutePath, SEPARATOR);
+  auto parts = kiwix::split(path, SEPARATOR, false);
+  auto pathParts = normalizeParts(parts, false);
+  parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absolutePathParts = normalizeParts(parts, true);
 
   unsigned int commonCount = 0;
   while (commonCount < pathParts.size()
@@ -75,108 +187,64 @@ std::string computeRelativePath(const std::string& path, const std::string& abso
       commonCount++;
   }
 
-  std::string relativePath;
-#ifdef _WIN32
-  /* On Windows you have a token more because the root is represented
-     by a letter */
-  if (commonCount == 0) {
-    relativePath = ".." + SEPARATOR;
-  }
-#endif
-
+  std::vector<std::string> relativeParts;
   for (unsigned int i = commonCount; i < pathParts.size(); i++) {
-    relativePath += ".." + SEPARATOR;
+    relativeParts.push_back("..");
   }
   for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
-    relativePath += absolutePathParts[i];
-    relativePath += i + 1 < absolutePathParts.size() ? SEPARATOR : "";
+    relativeParts.push_back(absolutePathParts[i]);
   }
-  return relativePath;
+  auto ret = kiwix::join(normalizeParts(relativeParts, false), SEPARATOR);
+  return ret;
 }
 
-#ifdef _WIN32
-# define STRTOK strtok_s
-#else
-# define STRTOK strtok_r
-#endif
-
-/* Warning: the relative path must be with slashes */
 std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
 {
-  std::string absolutePath;
-
+  std::string absolutePath = path;
   if (path.empty()) {
-    char* path = NULL;
-    size_t size = 0;
-
-#ifdef _WIN32
-    path = _getcwd(path, size);
-#else
-    path = getcwd(path, size);
-#endif
-
-    absolutePath = std::string(path) + SEPARATOR;
-  } else {
-    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
-                       ? path
-                       : path + SEPARATOR;
+    absolutePath = getCurrentDirectory();
   }
 
-#if _WIN32
-  char* cRelativePath = _strdup(relativePath.c_str());
-#else
-  char* cRelativePath = strdup(relativePath.c_str());
-#endif
-  char* saveptr = nullptr;
-  char* token = STRTOK(cRelativePath, "/", &saveptr);
+  auto parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absoluteParts = normalizeParts(parts, true);
+  parts = kiwix::split(relativePath, SEPARATOR, false);
+  auto relativeParts = normalizeParts(parts, false);
 
-  while (token != NULL) {
-    if (std::string(token) == "..") {
-      absolutePath = removeLastPathElement(absolutePath, true, false);
-      token = STRTOK(NULL, "/", &saveptr);
-    } else if (strcmp(token, ".") && strcmp(token, "")) {
-      absolutePath += std::string(token);
-      token = STRTOK(NULL, "/", &saveptr);
-      if (token != NULL) {
-        absolutePath += SEPARATOR;
-      }
-    } else {
-      token = STRTOK(NULL, "/", &saveptr);
-    }
-  }
-  free(cRelativePath);
-
-  return absolutePath;
+  absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
+  auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
+  return ret;
 }
 
-std::string removeLastPathElement(const std::string& path,
-                                  const bool removePreSeparator,
-                                  const bool removePostSeparator)
+std::string removeLastPathElement(const std::string& path)
 {
-  std::string newPath = path;
-  size_t offset = newPath.find_last_of(SEPARATOR);
-  if (removePreSeparator &&
-#ifndef _WIN32
-      offset != newPath.find_first_of(SEPARATOR) &&
-#endif
-      offset == newPath.length() - 1) {
-    newPath = newPath.substr(0, offset);
-    offset = newPath.find_last_of(SEPARATOR);
+  auto parts_ = kiwix::split(path, SEPARATOR, false);
+  auto parts = normalizeParts(parts_, false);
+  if (!parts.empty()) {
+    parts.pop_back();
   }
-  newPath = removePostSeparator ? newPath.substr(0, offset)
-                                : newPath.substr(0, offset + 1);
-  return newPath;
+  auto ret = kiwix::join(parts, SEPARATOR);
+  return ret;
 }
 
 std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
 {
-  std::string newPath = directoryPath + SEPARATOR + filename;
+  std::string newPath = directoryPath;
+  if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
+    newPath += SEPARATOR;
+  }
+  newPath += filename;
   return newPath;
 }
 
 std::string getLastPathElement(const std::string& path)
 {
-  return path.substr(path.find_last_of(SEPARATOR) + 1);
+  auto parts_ = kiwix::split(path, SEPARATOR);
+  auto parts = normalizeParts(parts_, false);
+  if (parts.empty()) {
+    return "";
+  }
+  auto ret = parts.back();
+  return ret;
 }
 
 unsigned int getFileSize(const std::string& path)
@@ -201,14 +269,33 @@ std::string getFileSizeAsString(const std::string& path)
 
 std::string getFileContent(const std::string& path)
 {
-  std::ifstream f(path, std::ios::in|std::ios::ate);
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_RDONLY | _O_BINARY);
+#else
+  auto fd = open(path.c_str(), O_RDONLY);
+#endif
   std::string content;
-  if (f.is_open()) {
-    auto size = f.tellg();
-    content.reserve(size);
-    f.seekg(0, std::ios::beg);
-    content.assign((std::istreambuf_iterator<char>(f)),
-                    std::istreambuf_iterator<char>());
+  if (fd != -1) {
+#ifdef _WIN32
+    auto size = _lseeki64(fd, 0, SEEK_END);
+#else
+    auto size = lseek(fd, 0, SEEK_END);
+#endif
+    content.resize(size);
+#ifdef _WIN32
+    _lseeki64(fd, 0, SEEK_SET);
+#else
+    lseek(fd, 0, SEEK_SET);
+#endif
+    auto p = (char*)content.data();
+    while (size) {
+      auto readsize = size > 2048 ? 2048 : size;
+      readsize = ::read(fd, p, readsize);
+      p += readsize;
+      size -= readsize;
+    }
+    close(fd);
   }
   return content;
 }
@@ -216,7 +303,7 @@ std::string getFileContent(const std::string& path)
 bool fileExists(const std::string& path)
 {
 #ifdef _WIN32
-  return PathFileExists(path.c_str());
+  return PathFileExistsW(Utf8ToWide(path).c_str());
 #else
   bool flag = false;
   std::fstream fin;
@@ -232,7 +319,7 @@ bool fileExists(const std::string& path)
 bool makeDirectory(const std::string& path)
 {
 #ifdef _WIN32
-  int status = _mkdir(path.c_str());
+  int status = _wmkdir(Utf8ToWide(path).c_str());
 #else
   int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 #endif
@@ -242,15 +329,15 @@ bool makeDirectory(const std::string& path)
 std::string makeTmpDirectory()
 {
 #ifdef _WIN32
-  char cbase[MAX_PATH];
-  char ctmp[MAX_PATH];
-  GetTempPath(MAX_PATH-14, cbase);
+  wchar_t cbase[MAX_PATH];
+  wchar_t ctmp[MAX_PATH];
+  GetTempPathW(MAX_PATH-14, cbase);
   // This create a file for us, ensure it is unique.
   // So we need to delete it and create the directory using the same name.
-  GetTempFileName(cbase, "kiwix", 0, ctmp);
-  DeleteFile(ctmp);
-  _mkdir(ctmp);
-  return std::string(ctmp);
+  GetTempFileNameW(cbase, L"kiwix", 0, ctmp);
+  DeleteFileW(ctmp);
+  _wmkdir(ctmp);
+  return WideToUtf8(ctmp);
 #else
   char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
   std::string dir = mkdtemp(_template_array);
@@ -261,86 +348,136 @@ std::string makeTmpDirectory()
 /* Try to create a link and if does not work then make a copy */
 bool copyFile(const std::string& sourcePath, const std::string& destPath)
 {
+#ifdef _WIN32
+  return CopyFileW(Utf8ToWide(sourcePath).c_str(), Utf8ToWide(destPath).c_str(), 1);
+#else
   try {
-#ifndef _WIN32
     if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
-#endif
       std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
       std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
       outfile << infile.rdbuf();
-#ifndef _WIN32
     }
-#endif
   } catch (std::exception& e) {
     std::cerr << e.what() << std::endl;
     return false;
   }
 
   return true;
+#endif
 }
 
-std::string getExecutablePath()
+std::string getExecutablePath(bool realPathOnly)
 {
-  char binRootPath[PATH_MAX];
+  if (!realPathOnly) {
+    char* cAppImage = ::getenv("APPIMAGE");
+    if (cAppImage) {
+      char* cArgv0 = ::getenv("ARGV0");
+      char* cOwd = ::getenv("OWD");
+      if (cArgv0 && cOwd) {
+        auto ret = appendToDirectory(cOwd, cArgv0);
+        return ret;
+      }
+    }
+  }
 
 #ifdef _WIN32
-  GetModuleFileName(NULL, binRootPath, PATH_MAX);
-  return std::string(binRootPath);
+  std::wstring binRootPath(PATH_MAX, 0);
+  GetModuleFileNameW(NULL, &binRootPath[0], PATH_MAX);
+  std::string ret = WideToUtf8(binRootPath);
+  return ret;
 #elif __APPLE__
+  char binRootPath[PATH_MAX];
   uint32_t max = (uint32_t)PATH_MAX;
   _NSGetExecutablePath(binRootPath, &max);
   return std::string(binRootPath);
 #else
+  char binRootPath[PATH_MAX];
   ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
   if (size != -1) {
     return std::string(binRootPath, size);
   }
-#endif
-
   return "";
+#endif
 }
 
 bool writeTextFile(const std::string& path, const std::string& content)
 {
-  std::ofstream file;
-  file.open(path.c_str());
-  file << content;
-  file.close();
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC, S_IWRITE);
+#else
+  auto fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC,
+                               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+#endif
+  if (fd == -1)
+    return false;
+
+  if (write(fd, content.c_str(), content.size()) != (long)content.size()) {
+    close(fd);
+    return false;
+  }
+  close(fd);
   return true;
 }
 
 std::string getCurrentDirectory()
 {
-  char* a_cwd = getcwd(NULL, 0);
-  std::string s_cwd(a_cwd);
+#ifdef _WIN32
+  wchar_t* a_cwd = _wgetcwd(NULL, 0);
+  std::string ret = WideToUtf8(a_cwd);
   free(a_cwd);
-  return s_cwd;
+#else
+  char* a_cwd = getcwd(NULL, 0);
+  std::string ret(a_cwd);
+  free(a_cwd);
+#endif
+  return ret;
 }
 
 std::string getDataDirectory()
 {
+// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
 #ifdef _WIN32
-  char* cDataDir = ::getenv("APPDATA");
+  wchar_t* cDataDir = ::_wgetenv(L"KIWIX_DATA_DIR");
+  if (cDataDir != nullptr) {
+    return WideToUtf8(cDataDir);
+  }
 #else
   char* cDataDir = ::getenv("KIWIX_DATA_DIR");
-#endif
-  std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (!dataDir.empty())
-    return dataDir;
-#ifdef _WIN32
-  cDataDir = ::getenv("USERPROFILE");
-  dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-#else
-  cDataDir = ::getenv("XDG_DATA_HOME");
-  dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (dataDir.empty()) {
-    cDataDir = ::getenv("HOME");
-    dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-    dataDir = appendToDirectory(dataDir, ".local");
-    dataDir = appendToDirectory(dataDir, "share");
+  if (cDataDir != nullptr) {
+    return cDataDir;
   }
 #endif
-  return appendToDirectory(dataDir, "kiwix");
+
+// Compute the dataDir from the user directory.
+  std::string dataDir;
+#ifdef _WIN32
+  cDataDir = ::_wgetenv(L"APPDATA");
+  if (cDataDir == nullptr)
+    cDataDir = ::_wgetenv(L"USERPROFILE");
+  if (cDataDir != nullptr)
+    dataDir = WideToUtf8(cDataDir);
+#else
+  cDataDir = ::getenv("XDG_DATA_HOME");
+  if (cDataDir != nullptr) {
+    dataDir = cDataDir;
+  } else {
+    cDataDir = ::getenv("HOME");
+    if (cDataDir != nullptr) {
+      dataDir = cDataDir;
+      dataDir = appendToDirectory(dataDir, ".local");
+      dataDir = appendToDirectory(dataDir, "share");
+    }
+  }
+#endif
+  if (!dataDir.empty()) {
+    dataDir = appendToDirectory(dataDir, "kiwix");
+    makeDirectory(dataDir);
+    return dataDir;
+  }
+
+// Let's use the currentDirectory
+  return getCurrentDirectory();
 }
 
 static std::map<std::string, std::string> extMimeTypes = {

src/tools/regexTools.cpp

@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <map>
+#include <stdexcept>
 #include <pthread.h>
 
 std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;

src/tools/stringTools.cpp

@@ -267,36 +267,46 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
-                                      const std::string& delims = " *-")
+                                      const std::string& delims,
+                                      bool trimEmpty,
+                                      bool keepDelim)
 {
-  std::string::size_type lastPos = str.find_first_not_of(delims, 0);
-  std::string::size_type pos = str.find_first_of(delims, lastPos);
+  std::string::size_type lastPos = 0;
+  std::string::size_type pos = 0;
   std::vector<std::string> tokens;
-
-  while (std::string::npos != pos || std::string::npos != lastPos) {
-    tokens.push_back(str.substr(lastPos, pos - lastPos));
-    lastPos = str.find_first_not_of(delims, pos);
-    pos = str.find_first_of(delims, lastPos);
+  while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
+  {
+    auto token = str.substr(lastPos, pos - lastPos);
+    if (!trimEmpty || !token.empty()) {
+      tokens.push_back(token);
+    }
+    if (keepDelim) {
+      tokens.push_back(str.substr(pos, 1));
+    }
+    lastPos = pos + 1;
   }
 
+  auto token = str.substr(lastPos);
+  if (!trimEmpty || !token.empty()) {
+    tokens.push_back(token);
+  }
   return tokens;
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
+std::string kiwix::join(const std::vector<std::string>& list, const std::string& sep)
 {
-  const std::string m1(lhs), m2(rhs);
-  return split(m1, m2);
+  std::stringstream ss;
+  bool first = true;
+  for (auto& s:list) {
+    if (!first) {
+      ss << sep;
+    }
+    first = false;
+    ss << s;
+  }
+  return ss.str();
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
-{
-  return split(lhs, rhs.c_str());
-}
-
-std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
-{
-  return split(lhs.c_str(), rhs);
-}
 
 std::string kiwix::ucFirst(const std::string& word)
 {

src/wrapper/java/book.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Book.h"
+
+#include "utils.h"
+#include "book.h"
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_allocate(
+  JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Book>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Book>(env, thisObj);
+}
+
+#define BOOK (getPtr<kiwix::Book>(env, thisObj))
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
+{
+  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
+}
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
+{
+  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
+}
+
+#define GETTER(retType, name) JNIEXPORT retType JNICALL \
+Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
+{ \
+  auto cRet = BOOK->name(); \
+  retType ret = c2jni(cRet, env); \
+  return ret; \
+}
+
+GETTER(jstring, getId)
+GETTER(jstring, getPath)
+GETTER(jboolean, isPathValid)
+GETTER(jstring, getTitle)
+GETTER(jstring, getDescription)
+GETTER(jstring, getLanguage)
+GETTER(jstring, getCreator)
+GETTER(jstring, getPublisher)
+GETTER(jstring, getDate)
+GETTER(jstring, getUrl)
+GETTER(jstring, getName)
+GETTER(jstring, getFlavour)
+GETTER(jstring, getTags)
+GETTER(jlong, getArticleCount)
+GETTER(jlong, getMediaCount)
+GETTER(jlong, getSize)
+GETTER(jstring, getFavicon)
+GETTER(jstring, getFaviconUrl)
+GETTER(jstring, getFaviconMimeType)
+
+METHOD(jstring, Book, getTagStr, jstring tagName) try {
+  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
+  return c2jni(cRet, env);
+} catch(...) {
+  return c2jni<std::string>("", env);
+}
+
+#undef GETTER

src/wrapper/java/filter.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Filter.h"
+
+#include "library.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+METHOD0(void, Filter, allocate) {
+  allocate<kiwix::Filter>(env, thisObj);
+}
+
+METHOD0(void, Filter, dispose) {
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
+
+#define FORWARD(name, args_type) \
+METHOD(jobject, Filter, name, args_type value) { \
+  FILTER->name(jni2c(value, env)); \
+  return thisObj; \
+}
+
+#define FORWARDA(name, args_type) \
+METHOD(jobject, Filter, name, jobjectArray value) { \
+  FILTER->name(jni2c<args_type>(value, env)); \
+  return thisObj; \
+}
+
+
+
+FORWARD(local, jboolean)
+FORWARD(remote, jboolean)
+FORWARD(valid, jboolean)
+FORWARDA(acceptTags, jstring)
+FORWARDA(rejectTags, jstring)
+FORWARD(lang, jstring)
+FORWARD(publisher, jstring)
+FORWARD(creator, jstring)
+FORWARD(maxSize, jlong)
+FORWARD(query, jstring)
+
+

src/wrapper/java/kiwixicu.cpp

@@ -28,7 +28,11 @@
 
 #include "utils.h"
 
+#if __ANDROID__
 pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER;
+#else
+pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+#endif
 
 JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
     JNIEnv* env, jclass kclass, jstring dirStr)

src/wrapper/java/kiwixreader.cpp

@@ -21,7 +21,7 @@
 
 #include <jni.h>
 #include <zim/file.h>
-#include <android/log.h>
+#include <exception>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
 #include "tools/base64.h"
@@ -34,14 +34,14 @@ JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
 {
   std::string cPath = jni2c(filename, env);
 
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create reader with: %s", cPath.c_str());
+  LOG("Attempting to create reader with: %s", cPath.c_str());
   Lock l;
   try {
     kiwix::Reader* reader = new kiwix::Reader(cPath);
     return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error opening ZIM file");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Error opening ZIM file");
+    LOG(e.what());
     return 0;
   }
 }
@@ -64,8 +64,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
     std::string cUrl = READER->getMainPage().getPath();
     url = c2jni(cUrl, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM main page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM main page");
+    LOG(e.what());
     url = NULL;
   }
   return url;
@@ -80,8 +80,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
     std::string cId = READER->getId();
     id = c2jni(cId, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM id");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
     id = NULL;
   }
 
@@ -95,10 +95,10 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
 
   try {
     int cSize = READER->getFileSize();
-    size = c2jni(cSize);
+    size = c2jni(cSize, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM file size");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM file size");
+    LOG(e.what());
   }
 
   return size;
@@ -113,8 +113,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
     std::string cCreator = READER->getCreator();
     creator = c2jni(cCreator, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM creator");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM creator");
+    LOG(e.what());
     creator = NULL;
   }
 
@@ -130,8 +130,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
     std::string cPublisher = READER->getPublisher();
     publisher = c2jni(cPublisher, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM publish");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM publish");
+    LOG(e.what());
     publisher = NULL;
   }
   return publisher;
@@ -146,8 +146,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
     std::string cName = READER->getName();
     name = c2jni(cName, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM name");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM name");
+    LOG(e.what());
     name = NULL;
   }
   return name;
@@ -166,8 +166,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
         base64_encode(cContent),
         env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM favicon");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM favicon");
+    LOG(e.what());
     favicon = NULL;
   }
   return favicon;
@@ -182,8 +182,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
     std::string cDate = READER->getDate();
     date = c2jni(cDate, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM date");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM date");
+    LOG(e.what());
     date = NULL;
   }
   return date;
@@ -198,8 +198,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
     std::string cLanguage = READER->getLanguage();
     language = c2jni(cLanguage, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM language");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM language");
+    LOG(e.what());
     language = NULL;
   }
 
@@ -217,8 +217,8 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
     auto cMimeType = entry.getMimetype();
     mimeType = c2jni(cMimeType, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get mime-type for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
+    LOG(e.what());
     mimeType = NULL;
   }
   return mimeType;
@@ -234,7 +234,7 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
     entry = entry.getFinalEntry();
     finalUrl = c2jni(entry.getPath(), env);
   } catch (std::exception& e) {
-    finalUrl = c2jni("", env);
+    finalUrl = c2jni(std::string(), env);
   }
   return finalUrl;
 }
@@ -268,8 +268,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
           data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get content for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get content for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
 
   return data;
@@ -284,8 +284,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
   /* Default values */
   /* Retrieve the content */
   std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset);
-  unsigned int cLen = jni2c(len);
+  unsigned int cOffset = jni2c(offset, env);
+  unsigned int cLen = jni2c(len, env);
   try {
     auto entry = READER->getEntryFromEncodedPath(cUrl);
     entry = entry.getFinalEntry();
@@ -300,12 +300,28 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
       setIntObjValue(cLen, sizeObj, env);
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
+    LOG(e.what());
   }
   return data;
 }
 
+JNIEXPORT jlong JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
+    JNIEnv* env, jobject obj, jstring url)
+{
+  std::string cUrl = jni2c(url, env);
+  try {
+    auto entry = READER->getEntryFromEncodedPath(cUrl);
+    entry = entry.getFinalEntry();
+    return c2jni(entry.getSize(), env);
+  } catch(std::exception& e) {
+    LOG("Unable to get size for url : %s", cUrl.c_str());
+    LOG(e.what());
+  }
+  return c2jni(0, env);
+}
+
 JNIEXPORT jobject JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     JNIEnv* env, jobject obj, jstring url)
@@ -322,8 +338,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     auto part_info = entry.getDirectAccessInfo();
     setPairObjValue(part_info.first, part_info.second, pair, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get direct access info for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
   return pair;
 }
@@ -336,15 +352,18 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 {
   jboolean retVal = JNI_FALSE;
   std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count);
+  unsigned int cCount = jni2c(count, env);
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get search results for pattern: %s", cPrefix.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
+    LOG(e.what());
   }
 
   return retVal;
@@ -353,19 +372,25 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
                                                          jobject obj,
-                                                         jobject titleObj)
+                                                         jobject titleObj,
+                                                         jobject urlObj)
 {
   jboolean retVal = JNI_FALSE;
   std::string cTitle;
+  std::string cUrl;
 
   try {
-    if (READER->getNextSuggestion(cTitle)) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+    if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
+      setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get next suggestion");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get next suggestion");
+    LOG(e.what());
   }
 
   return retVal;
@@ -385,8 +410,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
     setStringObjValue(entry.getPath(), urlObj, env);
     return JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get url for title %s: ", cTitle.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get url for title %s: ", cTitle.c_str());
+    LOG(e.what());
   }
 
   return JNI_FALSE;
@@ -401,8 +426,8 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
     std::string cTitle = READER->getTitle();
     title = c2jni(cTitle, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim title");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim title");
+    LOG(e.what());
     title = NULL;
   }
   return title;
@@ -417,13 +442,42 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
     std::string cDescription = READER->getDescription();
     description = c2jni(cDescription, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim description");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim description");
+    LOG(e.what());
     description = NULL;
   }
   return description;
 }
 
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
+{
+  jint articleCount = 0;
+  try {
+    auto cArticleCount = READER->getArticleCount();
+    articleCount = c2jni(cArticleCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get article count.");
+    LOG(e.what());
+  }
+  return articleCount;
+}
+
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
+{
+  jint mediaCount = 0;
+  try {
+    auto cMediaCount = READER->getMediaCount();
+    mediaCount = c2jni(cMediaCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get media count.");
+    LOG(e.what());
+  }
+  return mediaCount;
+}
+
+
 JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     JNIEnv* env, jobject obj, jobject urlObj)
 {
@@ -435,8 +489,8 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     setStringObjValue(cUrl, urlObj, env);
     retVal = JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get random page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get random page");
+    LOG(e.what());
   }
   return retVal;
 }

src/wrapper/java/kiwixsearcher.cpp

@@ -59,7 +59,7 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
     JNIEnv* env, jobject obj, jstring query, jint count)
 {
   std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count);
+  unsigned int ccount = jni2c(count, env);
 
   SEARCHER->search(cquery, 0, ccount);
 }

src/wrapper/java/kiwixserver.cpp

@@ -21,7 +21,6 @@
 
 #include <jni.h>
 #include <zim/file.h>
-#include <android/log.h>
 #include "org_kiwix_kiwixlib_JNIKiwixServer.h"
 
 #include "tools/base64.h"
@@ -32,15 +31,15 @@
 JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
     JNIEnv* env, jobject obj, jobject jLibrary)
 {
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create server");
+  LOG("Attempting to create server");
   Lock l;
   try {
-    auto library = Handle<kiwix::Library>::getHandle(env, jLibrary);
-    kiwix::Server* server = new kiwix::Server(*library);
+    auto library = getPtr<kiwix::Library>(env, jLibrary);
+    kiwix::Server* server = new kiwix::Server(library);
     return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating the server");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Error creating the server");
+    LOG(e.what());
     return 0;
   }
 }
@@ -86,6 +85,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboo
   SERVER->setTaskbar(withTaskbar, withLibraryButton);
 }
 
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
+{
+  SERVER->setBlockExternalLinks(blockExternalLinks);
+}
+
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
 {

src/wrapper/java/library.cpp

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Library.h"
+
+#include "library.h"
+#include "reader.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_allocate(
+    JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Library>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
+
+/* Kiwix library functions */
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Library_addBook(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    kiwix::Reader reader(cPath);
+    kiwix::Book book;
+    book.update(reader);
+    return LIBRARY->addBook(book);
+  } catch (std::exception& e) {
+    LOG("Unable to add the book");
+    LOG(e.what()); }
+  return false;
+}
+
+METHOD(jobject, Library, getBookById, jstring id) {
+  auto cId = jni2c(id, env);
+  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
+  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
+  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
+  jobject book = env->NewObject(cls, constructorId);
+  setPtr(env, book, cBook);
+  return book;
+}
+
+METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
+  return LIBRARY->getBookCount(localBooks, remoteBooks);
+}
+
+METHOD0(jobjectArray, Library, getBooksIds) {
+  return c2jni(LIBRARY->getBooksIds(), env);
+}
+
+METHOD(jobjectArray, Library, filter, jobject filterObj) {
+  auto filter = getPtr<kiwix::Filter>(env, filterObj);
+  return c2jni(LIBRARY->filter(*filter), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksLanguages) {
+  return c2jni(LIBRARY->getBooksLanguages(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksCreators) {
+  return c2jni(LIBRARY->getBooksCreators(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksPublisher) {
+  return c2jni(LIBRARY->getBooksPublishers(), env);
+}
+

src/wrapper/java/manager.cpp

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include <zim/file.h>
+#include "org_kiwix_kiwixlib_Manager.h"
+
+#include "manager.h"
+#include "utils.h"
+
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Manager_allocate(
+  JNIEnv* env, jobject thisObj, jobject libraryObj)
+{
+  auto lib = getPtr<kiwix::Library>(env, libraryObj);
+  allocate<kiwix::Manager>(env, thisObj, lib);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Manager_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Manager>(env, thisObj);
+}
+
+#define MANAGER (getPtr<kiwix::Manager>(env, thisObj))
+
+/* Kiwix manager functions */
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readFile(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    return MANAGER->readFile(cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get readFile");
+    LOG(e.what());
+  }
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readXml(
+  JNIEnv* env, jobject thisObj, jstring content, jstring libraryPath)
+{
+  auto cContent = jni2c(content, env);
+  auto cPath = jni2c(libraryPath, env);
+
+  try {
+    return MANAGER->readXml(cContent, false, cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readOpds(
+  JNIEnv* env, jobject thisObj, jstring content, jstring urlHost)
+{
+  auto cContent = jni2c(content, env);
+  auto cUrl = jni2c(urlHost, env);
+
+  try {
+    return MANAGER->readOpds(cContent, cUrl);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readBookmarkFile(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    return MANAGER->readBookmarkFile(cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_Manager_addBookFromPath(
+  JNIEnv* env, jobject thisObj,
+  jstring pathToOpen, jstring pathToSave, jstring url, jboolean checkMetaData)
+{
+  auto cPathToOpen = jni2c(pathToOpen, env);
+  auto cPathToSave = jni2c(pathToSave, env);
+  auto cUrl = jni2c(url, env);
+  jstring id = NULL;
+
+  try {
+    auto cId = MANAGER->addBookFromPathAndGetId(cPathToOpen, cPathToSave, cUrl, checkMetaData);
+    if ( !cId.empty() ) {
+      id = c2jni(cId, env);
+    }
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM file size");
+    LOG(e.what());
+  }
+
+  return id;
+}

src/wrapper/java/meson.build

@@ -0,0 +1,55 @@
+
+java_sources = files([
+  'org/kiwix/kiwixlib/JNIICU.java',
+  'org/kiwix/kiwixlib/Book.java',
+  'org/kiwix/kiwixlib/JNIKiwixReader.java',
+  'org/kiwix/kiwixlib/Library.java',
+  'org/kiwix/kiwixlib/Manager.java',
+  'org/kiwix/kiwixlib/Filter.java',
+  'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
+  'org/kiwix/kiwixlib/JNIKiwixServer.java',
+  'org/kiwix/kiwixlib/JNIKiwixInt.java',
+  'org/kiwix/kiwixlib/JNIKiwixString.java',
+  'org/kiwix/kiwixlib/JNIKiwixBool.java',
+  'org/kiwix/kiwixlib/JNIKiwixException.java',
+  'org/kiwix/kiwixlib/Pair.java'
+])
+
+kiwix_jni = custom_target('jni',
+  input: java_sources,
+  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
+           'org_kiwix_kiwixlib_Book.h',
+           'org_kiwix_kiwixlib_JNIKiwixReader.h',
+           'org_kiwix_kiwixlib_Library.h',
+           'org_kiwix_kiwixlib_Manager.h',
+           'org_kiwix_kiwixlib_Filter.h',
+           'org_kiwix_kiwixlib_JNIKiwixServer.h',
+           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
+           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
+  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
+)
+
+jni_sources = files([
+     'kiwixicu.cpp',
+     'book.cpp',
+     'kiwixreader.cpp',
+     'library.cpp',
+     'manager.cpp',
+     'filter.cpp',
+     'kiwixsearcher.cpp',
+     'kiwixserver.cpp',
+])
+
+kiwix_sources += jni_sources + [kiwix_jni]
+
+if 'java' in wrapper
+  kiwix_jar = jar('kiwixlib', java_sources)
+  #junit_jar = files('org/kiwix/testing/junit-4.13.jar')
+  #test_jar = jar('testing', 'org/kiwix/testing/test.java',
+  #               link_with: [kiwix_jar, junit_jar])
+  #test('javatest', test_jar)
+endif
+
+install_subdir('org', install_dir: 'kiwix-lib/java', exclude_directories: ['kiwix/testing'])
+install_subdir('res', install_dir: 'kiwix-lib')
+install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -0,0 +1,47 @@
+
+package org.kiwix.kiwixlib;
+
+public class Book
+{
+  public Book() { allocate(); }
+
+
+  public native void update(Book book);
+  public native void update(JNIKiwixReader reader);
+
+  @Override
+  protected void finalize() { dispose();  }
+
+  public native String getId();
+  public native String getPath();
+  public native boolean isPathValid();
+  public native String getTitle();
+  public native String getDescription();
+  public native String getLanguage();
+  public native String getCreator();
+  public native String getPublisher();
+  public native String getDate();
+  public native String getUrl();
+  public native String getName();
+  public native String getFlavour();
+  public native String getTags();
+  /**
+   * Return the value associated to the tag tagName
+   *
+   * @param tagName the tag name to search for.
+   * @return The value of the tag. If the tag is not found, return empty string.
+   */
+  public native String getTagStr(String tagName);
+
+  public native long getArticleCount();
+  public native long getMediaCount();
+  public native long getSize();
+
+  public native String getFavicon();
+  public native String getFaviconUrl();
+  public native String getFaviconMimeType();
+
+  private native void allocate();
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/Filter.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+public class Filter
+{
+
+  public native Filter local(boolean accept);
+  public native Filter remote(boolean accept);
+  public native Filter valid(boolean accept);
+  public native Filter acceptTags(String[] tags);
+  public native Filter rejectTags(String[] tags);
+  public native Filter lang(String lang);
+  public native Filter publisher(String publisher);
+  public native Filter creator(String creator);
+  public native Filter maxSize(long size);
+  public native Filter query(String query);
+
+
+  public Filter() { allocate(); }
+
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate();
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -82,6 +82,16 @@ public class JNIKiwixReader
                                       int len,
                                       JNIKiwixInt size);
 
+  /**
+   *
+   * Get the size of an article.
+   *
+   * @param   url The url of the article.
+   * @return  The size of the final (redirections are resolved) article (in byte).
+   *          Return 0 if the article is not found.
+   */
+  public native long getArticleSize(String url);
+
   /**
    * getDirectAccessInformation.
    *
@@ -102,7 +112,7 @@ public class JNIKiwixReader
 
   public native boolean searchSuggestions(String prefix, int count);
 
-  public native boolean getNextSuggestion(JNIKiwixString title);
+  public native boolean getNextSuggestion(JNIKiwixString title, JNIKiwixString url);
 
   public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
 

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixServer.java

@@ -20,7 +20,7 @@
 package org.kiwix.kiwixlib;
 
 import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.JNIKiwixLibrary;
+import org.kiwix.kiwixlib.Library;
 
 public class JNIKiwixServer
 {
@@ -34,15 +34,17 @@ public class JNIKiwixServer
 
   public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
 
+  public native void setBlockExternalLinks(boolean blockExternalLinks);
+
   public native boolean start();
 
   public native void stop();
 
-  public JNIKiwixServer(JNIKiwixLibrary library)
+  public JNIKiwixServer(Library library)
   {
     nativeHandle = getNativeServer(library);
   }
 
-  private native long getNativeServer(JNIKiwixLibrary library);
+  private native long getNativeServer(Library library);
   private long nativeHandle;
 }

src/wrapper/java/org/kiwix/kiwixlib/Library.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+import org.kiwix.kiwixlib.Book;
+import org.kiwix.kiwixlib.JNIKiwixException;
+
+public class Library
+{
+  public native boolean addBook(String path) throws JNIKiwixException;
+
+  public native Book getBookById(String id);
+  public native int  getBookCount(boolean localBooks, boolean remoteBooks);
+
+  public native String[] getBooksIds();
+  public native String[] filter(Filter filter);
+
+  public native String[] getBooksLanguages();
+  public native String[] getBooksCreators();
+  public native String[] getBooksPublishers();
+
+  public Library()
+  {
+    allocate();
+  }
+
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate();
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/Manager.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+import org.kiwix.kiwixlib.Library;
+
+public class Manager
+{
+  /**
+   * Read a `library.xml` file and add books in the library.
+   *
+   * @param path The (utf8) path to the `library.xml` file.
+   * @return True if the file has been properly parsed.
+   */
+  public native boolean readFile(String path);
+
+  /**
+   * Load a library content stored in a string (at `library.xml` format).
+   *
+   * @param content The content corresponding of the library xml.
+   * @param libraryPath The library path (used to resolve relative paths)
+   * @return True if the content has been properly parsed.
+   */
+  public native boolean readXml(String content, String libraryPath);
+
+  /**
+   * Load a library content stored in a string (at OPDS stream format)
+   *
+   * @param content the content of the OPDS stream.
+   * @param urlHost the url of the stream (used to resolve relative url)
+   * @return True if the content has been properly parsed.
+   */
+  public native boolean readOpds(String content, String urlHost);
+
+  /**
+   * Load a bookmark file
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed
+   */
+  public native boolean readBookmarkFile(String path);
+
+  /**
+   * Add a book to the library.
+   *
+   * @param pathToOpen    The path of the zim file to add.
+   * @param pathToSave    The path to store in the library in place of
+   *                      pathToOpen.
+   * @param url           The url of the book to store in the library
+   *                      (useful for kiiwix-serve catalog)
+   * @param checkMetaData Tell if we check metadata before adding a book to the
+   *                      library.
+   * @return The id of te book if the book has been added to the library.
+   *         Empty string if not.
+   */
+  public native String addBookFromPath(String pathToOpen,
+                                       String pathToSave,
+                                       String url,
+                                       boolean checkMetaData);
+
+  public Manager(Library library) {
+    allocate(library);
+    _library = library;
+  }
+
+  private Library _library;
+
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate(Library library);
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/Pair.java

@@ -22,5 +22,5 @@ package org.kiwix.kiwixlib;
 public class Pair
 {
   public String filename;
-  public int offset;
+  public long offset;
 }