Release 9.2.3

Pass -latomic for architectures that need it

.codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+
+ignore:
+  - "test"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/

.github/move.yml

@@ -0,0 +1,27 @@
+# Configuration for Move Issues - https://github.com/dessant/move-issues
+
+# Delete the command comment when it contains no other content
+deleteCommand: true
+
+# Close the source issue after moving
+closeSourceIssue: true
+
+# Lock the source issue after moving
+lockSourceIssue: false
+
+# Mention issue and comment authors
+mentionAuthors: true
+
+# Preserve mentions in the issue content
+keepContentMentions: true
+
+# Move labels that also exist on the target repository
+moveLabels: true
+
+# Set custom aliases for targets
+# aliases:
+#   r: repo
+#   or: owner/repo
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

.github/workflows/ci.yml

@@ -0,0 +1,163 @@
+name: CI
+
+on: [push]
+
+jobs:
+  Macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v1
+      - name: Setup python 3.5
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.5'
+      - name: Install packages
+        uses: mstksg/get-package@v1
+        with:
+          brew: gcovr pkg-config ninja
+      - name: Install python modules
+        run: pip3 install meson==0.49.2 pytest
+      - name: Install deps
+        shell: bash
+        run: |
+          ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
+          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
+      - name: Compile
+        shell: bash
+        run: |
+          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
+          meson . build --default-library=shared -Db_coverage=true
+          cd build
+          ninja
+      - name: Test
+        shell: bash
+        run: |
+          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
+          cd build
+          meson test --verbose
+          ninja coverage
+        env:
+          SKIP_BIG_MEMORY_TEST: 1
+      - name: Publish coverage
+        shell: bash
+        run: |
+          curl https://codecov.io/bash -o codecov.sh
+          bash codecov.sh -n osx_native_dyn -Z
+          rm codecov.sh
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - native_static
+          - native_dyn
+          - native_dyn_bionic
+          - android_arm
+          - android_arm64
+          - win32_static
+          - win32_dyn
+        include:
+          - name: native_static
+            target: native_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: native_dyn
+            target: native_dyn
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: native_dyn_bionic
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm64
+            target: android_arm64
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: win32_static
+            target: win32_static
+            image_variant: f31
+            lib_postfix: '64'
+          - name: win32_dyn
+            target: win32_dyn
+            image_variant: f31
+            lib_postfix: '64'
+    env:
+      HOME: /home/runner
+    runs-on: ubuntu-latest
+    container:
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
+    steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+    - name: Checkout code
+      shell: python
+      run: |
+        from subprocess import check_call
+        from os import environ
+        command = [
+          'git', 'clone',
+          'https://github.com/${{github.repository}}',
+          '--depth=1',
+          '--branch', '${{steps.extract_branch.outputs.branch}}'
+        ]
+        check_call(command, cwd=environ['HOME'])
+    - name: Install deps
+      shell: bash
+      run: |
+        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
+        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
+    - name: Compile
+      shell: bash
+      run: |
+        meson --version
+        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
+          MESON_OPTION="--default-library=shared"
+        else
+          MESON_OPTION="--default-library=static"
+        fi
+        if [[ "${{matrix.target}}" =~ native_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
+        else
+          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
+        fi
+        if [[ "${{matrix.target}}" =~ android_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+        fi
+        cd $HOME/kiwix-lib
+        meson . build ${MESON_OPTION}
+        cd build
+        ninja
+      env:
+        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
+    - name: Test
+      if: startsWith(matrix.target, 'native_')
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib/build
+        meson test --verbose
+        ninja coverage
+      env:
+        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
+        SKIP_BIG_MEMORY_TEST: 1
+    - name: Publish coverage
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib
+        curl https://codecov.io/bash -o codecov.sh
+        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
+        rm codecov.sh
+      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -0,0 +1,4 @@
+.idea/
+*.swp
+subprojects/googletest-release*
+*.class

.travis.yml

@@ -1,36 +0,0 @@
-language: cpp
-dist: trusty
-sudo: false
-cache: ccache
-before_install:
-- PATH=$PATH:$HOME/bin
-install: travis/install_deps.sh
-script: travis/compile.sh
-env:
-  matrix:
-    - PLATFORM="native_static"
-    - PLATFORM="native_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-addons:
-  apt:
-    packages:
-      - cmake
-      - python3-pip
-      - libbz2-dev
-      - ccache
-      - zlib1g-dev
-      - uuid-dev
-      - libctpp2-dev
-      - ctpp2-utils
-      - libmicrohttpd-dev
-      - g++-mingw-w64-i686
-      - gcc-mingw-w64-i686
-      - gcc-mingw-w64-base
-      - mingw-w64-tools
-matrix:
-  include:
-    - env: PLATFORM="native_dyn"
-      os: osx

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,235 @@
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
+kiwix-lib 9.2
+=============
+
+ * Add tests
+ * Refactoring server code.
+ * [SERVER] Add HEAD, Etag and If-None-Match support.
+ * [SERVER] Compress opds catalog answers.
+
+kiwix-lib 9.1.2
+===============
+
+ * Do not use the pathToSave if it is empty.
+
+kiwix-lib 9.1.1
+===============
+
+ * Fix the detection of the dataDirectory on windows.
+
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
+kiwix-lib 9.0.1
+===============
+
+ * [JAVA] Use a long to store the offset of an article in the zim file instead
+    of an int.
+
+kiwix-lib 9.0.0
+===============
+
+ * [OPDS] Correctly set the id of the OPDS stream.
+ * [OPDS] Do not try to filter the catalog if no filter field is given in the
+   request.
+ * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
+ * [LIBRARY] Remove the function the read file using a native path.
+   All path must be utf8, no need to pass a native path along the utf8 path.
+ * [TEST] Fix tests using the main function of gtest instead of custom one.
+ * [CI] Move to github CI instead of Travis.
+ * The `Book::update` method always update the book's fields. Even if they are
+   not empty.
+ * [JAVA] Add wrapping around the library manager (opds parsing)
+ * [ARIA2] Add api option to start download with option (destination folder)
+ * [OPDS] Fixes about opds parsing, generation (missing attributes)
+   and requesting (server)
+ * Add methods on `Book` to get specific tag values (was on `Reader` only)
+ * Add flavour attribute to `Book`
+ * Fix opensearch description.
+ * Trust the given library.xml (by default) instead of reading the value from
+   the zim files.
+ * [OPDS] Be able to filter the content by name or size.
+ * [WINDOWS] Fix launching subcommand when there is spaces in the path.
+
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
+kiwix-lib 8.1.0
+===============
+
+ * Fix pathTools manipulation.
+ * Add missing implementation of getArticleCount and getMediaCount on android.
+ * Correctly convert windows path to utf8.
+ * Add code coverage in the CI
+
+kiwix-lib 8.0.1
+===============
+
+ * Fix join function
+
+kiwix-lib 8.0.0
+===============
+
+ * Add new methods to get all (and new) metadata from the zim file.
+ * Add methods to get the value of a specific tag.
+ * [API Change] Convert tags value to the new convention.
+ * [API Change] Rename `getMatatag` method to `getMetadata`
+ * [ABI Change] Correctly detect executable path in appimage.
+
+kiwix-lib 7.0.0
+===============
+
+ * [API break] Add a argument to kiwix-serve to specify the library to use.
+
+kiwix-lib 6.0.4
+===============
+
+ * Fix HTML rendering of the search result if there is no result.
+ * Do not crash at html rendering if request ask for 0 results (start == end)
+ * Correctly find the executable path if we are using AppImage
+
+kiwix-lib 6.0.3
+===============
+
+ * force one column suggestion in kiwix-serve suggestions
+ * fix fulltext search link in suggestions
+ * UI fixes in kiwix-serve rendering
+
+kiwix-lib 6.0.2
+===============
+
+ * Correctly set the groupId in the pom file.
+
+kiwix-lib 6.0.1
+===============
+
+ * Generate the pom file for android/maven
+
+kiwix-lib 6.0.0
+===============
+
+ * Move the server code in kiwix-lib (from kiwix-serve).
+ * Add unit test on regex functions.
+ * Fix computerAbsolutePath (thread safe, memory leak).
+ * Correctly set the book's path as valid if we construct the book from a
+   reader.
+ * [JNI] Add a method to know if a article is a redirection.
+ * Do not embed the gtest dependency.
+ * [JNI] Add a constructor to JNIKiwixString.
+ * Change order of search of the favicon urls.
+ * Clean a lot of unecessary includes in headers. (potential "API break")
+
+kiwix-lib 5.2.0
+===============
+
+ * kiwix-serve integration (as a seperated process).
+ * Fix crash in the suggestion search.
+ * Better API to filter the library books.
+ * New kiwix-lib application for android. (.aar)
+ * Use ReLinker to link with libkiwix.so in android.
+ * Correctly set the verbosity of zim search.
+
+kiwix-lib 5.1.0
+===============
+
+ * Add function to pause, resume and stop downloads.
+ * Add zim's tags in the opds stream.
+ * Addapt to new libzim 5.0.0 API.
+
+kiwix-lib 5.0.0
+===============
+
+ * Remove error message when trying to open an wrong zim file.
+ * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
+ * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
+ * Remove rpath
+ * Detect infinite (and too long) redirect loops.
+
+kiwix-lib 4.1.0
+===============
+
+ * Allow the library to be filtered by tags.
+ * Fix language mapping.
+ * Update README about mustache dependency.
+
+kiwix-lib 4.0.1
+===============
+
+ * Fix "maybe uninitialize variable" issue.
+ * Ensure path are stored correctly (absolute path) in the library.
+ * [CI] Use the new deps archive xz
+
+kiwix-lib 4.0.0
+===============
+
+ * [API break] Remove support for external index.
+ * Move to the mustache templating system instead of ctpp2.
+ * Make meson.build works for meson>=0.43.0
+ * [API break] Move the basic tools from the `common` directory to `tools`.
+
+kiwix-lib 3.1.1
+===============
+
+ * The OPDS feed book's date must be the date of the book, not the date of the
+ feed generation.
+ * Convert the standard opds date to our format (YYYY-MM-DD)
+ * Remove duplicate language attribute in the libxml dumper.
+ * Create the datadirectory to not fail to write a file in a non-existent
+   directory
+
+kiwix-lib 3.1.0
+===============
+
+ * Add a method to get the favicon url of book (if available).
+ * Move dump code of library.xml in a specific class.
+ * Add a first support to bookmarks
+
+kiwix-lib 3.0.3
+===============
+
+ * Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
+ ('eng').
+ * Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
+ * Correctly fill the book size for the zim file size.
+ * Fix launch of aria2c.
+
 kiwix-lib 3.0.2
 ===============
 

README.md

@@ -1,8 +1,16 @@
 Kiwix library
 =============
 
-The Kiwix library provides the Kiwix software core. It contains the
-code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
+The Kiwix library provides the [Kiwix](https://kiwix.org) software
+suite core. It contains the code shared by all Kiwix ports (Windows,
+GNU/Linux, macOS, Android, iOS, ...).
+
+[![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
+[![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
+[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
+[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
+[![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
 ----------
@@ -27,18 +35,13 @@ The Kiwix library relies on many third parts software libraries. They
 are prerequisites to the Kiwix library compilation. Following
 libraries need to be available:
 
-* ICU ................................... http://site.icu-project.org/
-(package libicu-dev on Ubuntu)
-* ZIM ........................................ http://www.openzim.org/
-(package libzim-dev on Ubuntu)
-* Pugixml ........................................ http://pugixml.org/
-(package libpugixml-dev on Ubuntu)
-* ctpp2 ........................................ http://ctpp.havoc.ru/
-(package libctpp2-dev on Ubuntu)
-* Xapian ......................................... https://xapian.org/
-(package libxapian-dev on Ubuntu)
-* libaria2 .................................. https://aria2.github.io/
-(no package on Ubuntu)
+* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
+* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
+* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
+* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
+header `mustache.hpp` somewhere it can be found by the compiler and/or
+set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -47,33 +50,30 @@ In the worse case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
-kiwix-lib directory as install prefix.
-
-If you compile ctpp2 from source and want to compile the Kiwix library
-statically then you will probably need to rename ctpp2 static library
-from ctpp2-st.a to ctpp2.a.
+`kiwix-lib` directory as install prefix.
 
 Environment
 -------------
 
-The Kiwix library builds using [Meson](http://mesonbuild.com/) version
-0.39 or higher. Meson relies itself on Ninja, pkg-config and few other
+The Kiwix library builds using [Meson](https://mesonbuild.com/) version
+0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
-* Meson
-* Ninja
-* Pkg-config
+* [Meson](https://mesonbuild.com/)
+* [Ninja](https://ninja-build.org/)
+* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
 
 These tools should be packaged if you use a cutting edge operating
-system. If not, have a look to the "Troubleshooting" section.
+system. If not, have a look to the [Troubleshooting](#Troubleshooting)
+section.
 
 Compilation
 -----------
 
 Once all dependencies are installed, you can compile the Kiwix library
 with:
-```
+```bash
 meson . build
 ninja -C build
 ```
@@ -85,37 +85,44 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+Testing
+-------
+
+To run the automated tests:
+```bash
+cd build
+meson test
+```
+
 Installation
 ------------
 
 If you want to install the Kiwix library and the headers you just have
 compiled on your system, here we go:
-
-```
+```bash
 ninja -C build install
 ```
 
-You might need to run the command as root (or using 'sudo'), depending
+You might need to run the command as root (or using `sudo`), depending
 where you want to install the libraries. After the installation
-succeeded, you may need to run ldconfig (as root).
+succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
 ------------
 
 If you want to uninstall the Kiwix library:
-
-```
+```bash
 ninja -C build uninstall
 ```
 
-Like for the installation, you might need to run the command as root
-(or using 'sudo').
+Like for the installation, you might need to run the command as `root`
+(or using `sudo`).
 
 Troubleshooting
 ---------------
 
 If you need to install Meson "manually":
-```
+```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
 pip3 install meson       # Install Meson
@@ -123,7 +130,7 @@ hash -r                  # Refresh bash paths
 ```
 
 If you need to install Ninja "manually":
-```
+```bash
 git clone git://github.com/ninja-build/ninja.git
 cd ninja
 git checkout release
@@ -142,4 +149,5 @@ repository.
 License
 -------
 
-GPLv3 or later, see COPYING for more details.
+[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
+[COPYING](COPYING) for more details.

android-kiwix-lib-publisher/.gitignore

@@ -0,0 +1,13 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -0,0 +1,25 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    repositories {
+        google()
+        jcenter()
+
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.4.1'
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

android-kiwix-lib-publisher/gradle.properties

@@ -0,0 +1,15 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx1536m
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed Jun 19 15:28:39 BST 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -0,0 +1 @@
+/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -0,0 +1,64 @@
+apply plugin: 'com.android.library'
+apply plugin: 'maven'
+
+android {
+    compileSdkVersion 28
+    defaultConfig {
+        minSdkVersion 15
+        targetSdkVersion 28
+        versionCode 1
+        versionName "1.0"
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+dependencies {
+    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
+}
+
+task writePom {
+  pom {
+    project {
+        groupId 'org.kiwix.kiwixlib'
+        artifactId 'kiwixlib'
+        version '9.2.3' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        packaging 'aar'
+        name 'kiwixlib'
+        url 'https://github.com/kiwix/kiwix-lib'
+        licenses {
+            license {
+                name 'GPLv3'
+                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
+            }
+        }
+      developers {
+          developer {
+              id 'kiwix'
+              name 'kiwix'
+              email 'contact@kiwix.org'
+          }
+      }
+      scm {
+          connection 'https://github.com/kiwix/kiwix-lib.git'
+          developerConnection 'https://github.com/kiwix/kiwix-lib.git'
+          url 'https://github.com/kiwix/kiwix-lib'
+      }
+    }
+  }.withXml {
+    def dependenciesNode = asNode().appendNode('dependencies')
+
+    //Iterate over the implementation dependencies, adding a <dependency> node for each
+    configurations.implementation.allDependencies.each {
+      def dependencyNode = dependenciesNode.appendNode('dependency')
+      dependencyNode.appendNode('groupId', it.group)
+      dependencyNode.appendNode('artifactId', it.name)
+      dependencyNode.appendNode('version', it.version)
+    }
+  }.writeTo("$buildDir/pom.xml")
+}
+

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -0,0 +1,10 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.kiwix.kiwixlib">
+
+    <application
+        android:allowBackup="true"
+        android:supportsRtl="true">
+
+    </application>
+
+</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -0,0 +1 @@
+include ':kiwixLibAndroid'

include/book.h

@@ -28,7 +28,6 @@ class xml_node;
 
 namespace kiwix
 {
-enum supportedIndexType { UNKNOWN, XAPIAN };
 
 class OPDSDumper;
 class Reader;
@@ -46,14 +45,12 @@ class Book
   void update(const Reader& reader);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
-  std::string getHumanReadableIdFromPath();
+  std::string getHumanReadableIdFromPath() const;
 
   bool readOnly() const { return m_readOnly; }
   const std::string& getId() const { return m_id; }
   const std::string& getPath() const { return m_path; }
   bool isPathValid() const { return m_pathValid; }
-  const std::string& getIndexPath() const { return m_indexPath; }
-  const supportedIndexType& getIndexType() const { return m_indexType; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
   const std::string& getLanguage() const { return m_language; }
@@ -63,11 +60,15 @@ class Book
   const std::string& getUrl() const { return m_url; }
   const std::string& getName() const { return m_name; }
   const std::string& getTags() const { return m_tags; }
+  std::string getTagStr(const std::string& tagName) const;
+  bool getTagBool(const std::string& tagName) const;
+  const std::string& getFlavour() const { return m_flavour; }
   const std::string& getOrigId() const { return m_origId; }
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
   const std::string& getFavicon() const;
+  const std::string& getFaviconUrl() const { return m_faviconUrl; }
   const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
   const std::string& getDownloadId() const { return m_downloadId; }
 
@@ -75,8 +76,6 @@ class Book
   void setId(const std::string& id) { m_id = id; }
   void setPath(const std::string& path);
   void setPathValid(bool valid) { m_pathValid = valid; }
-  void setIndexPath(const std::string& indexPath);
-  void setIndexType(supportedIndexType indexType) { m_indexType = indexType;}
   void setTitle(const std::string& title) { m_title = title; }
   void setDescription(const std::string& description) { m_description = description; }
   void setLanguage(const std::string& language) { m_language = language; }
@@ -85,6 +84,7 @@ class Book
   void setDate(const std::string& date) { m_date = date; }
   void setUrl(const std::string& url) { m_url = url; }
   void setName(const std::string& name) { m_name = name; }
+  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
   void setTags(const std::string& tags) { m_tags = tags; }
   void setOrigId(const std::string& origId) { m_origId = origId; }
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
@@ -98,9 +98,7 @@ class Book
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
-  bool m_pathValid;
-  std::string m_indexPath;
-  supportedIndexType m_indexType;
+  bool m_pathValid = false;
   std::string m_title;
   std::string m_description;
   std::string m_language;
@@ -109,12 +107,13 @@ class Book
   std::string m_date;
   std::string m_url;
   std::string m_name;
+  std::string m_flavour;
   std::string m_tags;
   std::string m_origId;
-  uint64_t m_articleCount;
-  uint64_t m_mediaCount;
-  bool m_readOnly;
-  uint64_t m_size;
+  uint64_t m_articleCount = 0;
+  uint64_t m_mediaCount = 0;
+  bool m_readOnly = false;
+  uint64_t m_size = 0;
   mutable std::string m_favicon;
   std::string m_faviconUrl;
   std::string m_faviconMimeType;

include/bookmark.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOKMARK_H
+#define KIWIX_BOOKMARK_H
+
+#include <string>
+
+namespace pugi {
+class xml_node;
+}
+
+namespace kiwix
+{
+
+/**
+ * A class to store information about a bookmark (an article in a book)
+ */
+class Bookmark
+{
+ public:
+  Bookmark();
+  ~Bookmark();
+
+  void updateFromXml(const pugi::xml_node& node);
+
+  const std::string& getBookId() const { return m_bookId; }
+  const std::string& getBookTitle()  const { return m_bookTitle; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getDate() const { return m_date; }
+
+  void setBookId(const std::string& bookId) { m_bookId = bookId; }
+  void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setTitle(const std::string& title) { m_title = title; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setDate(const std::string& date) { m_date = date; }
+
+ protected:
+  std::string m_bookId;
+  std::string m_bookTitle;
+  std::string m_url;
+  std::string m_title;
+  std::string m_language;
+  std::string m_date;
+};
+
+}
+
+#endif

include/common/pathTools.h

@@ -1,64 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_PATHTOOLS_H
-#define KIWIX_PATHTOOLS_H
-
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <fstream>
-#include <ios>
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#ifdef _WIN32
-#include <direct.h>
-#endif
-
-#include "stringTools.h"
-
-using namespace std;
-
-bool isRelativePath(const string& path);
-string computeAbsolutePath(const string path, const string relativePath);
-string computeRelativePath(const string path, const string absolutePath);
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator = false,
-                             const bool removePostSeparator = false);
-string appendToDirectory(const string& directoryPath, const string& filename);
-
-unsigned int getFileSize(const string& path);
-string getFileSizeAsString(const string& path);
-string getFileContent(const string& path);
-bool fileExists(const string& path);
-bool makeDirectory(const string& path);
-string makeTmpDirectory();
-bool copyFile(const string& sourcePath, const string& destPath);
-string getLastPathElement(const string& path);
-string getExecutablePath();
-string getCurrentDirectory();
-string getDataDirectory();
-bool writeTextFile(const string& path, const string& content);
-#endif

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -1,79 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef _CTPP2_VM_STRING_LOADER_HPP__
-#define _CTPP2_VM_STRING_LOADER_HPP__ 1
-
-#include <ctpp2/CTPP2VMLoader.hpp>
-#include <ctpp2/CTPP2Util.hpp>
-#include <ctpp2/CTPP2Exception.hpp>
-#include <ctpp2/CTPP2VMExecutable.hpp>
-#include <ctpp2/CTPP2VMInstruction.hpp>
-#include <ctpp2/CTPP2VMMemoryCore.hpp>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <iostream>
-#include <string>
-
-/**
-  @file VMStringLoader.hpp
-  @brief Load program core from file
-*/
-
-namespace CTPP // C++ Template Engine
-{
-// FWD
-struct VMExecutable;
-
-/**
-  @class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
-  @brief Load program core from file
-*/
-class CTPP2DECL VMStringLoader:
-  public VMLoader
-{
-public:
-	/**
-	*/
-	VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
-	/**
-	  @brief Get ready-to-run program
-	*/
-	const VMMemoryCore * GetCore() const;
-
-	/**
-	  @brief A destructor
-	*/
-	~VMStringLoader() throw();
-private:
-	/** Program core             */
-	VMExecutable  * oCore;
-	/** Ready-to-run program     */
-	VMMemoryCore  * pVMMemoryCore;
-};
-
-} // namespace CTPP
-#endif // _CTPP2_VM_STRING_LOADER_HPP__
-// End.

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <pthread.h>
 #include <memory>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -54,6 +55,9 @@ class Download {
       m_status(K_UNKNOWN),
       m_did(did) {};
   void updateStatus(bool follow=false);
+  void pauseDownload();
+  void resumeDownload();
+  void cancelDownload();
   StatusResult getStatus()          { return m_status; }
   std::string  getDid()             { return m_did; }
   std::string  getFollowedBy()      { return m_followedBy; }
@@ -89,7 +93,7 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri);
+  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
   Download* getDownload(const std::string& did);
 
   size_t getNbDownload() { return m_knownDownloads.size(); }

include/entry.h

@@ -24,6 +24,7 @@
 #include <zim/article.h>
 #include <exception>
 #include <string>
+
 #include "common.h"
 
 using namespace std;
@@ -63,14 +64,14 @@ class Entry
      * @return the path of the entry.
      */
     std::string getPath() const;
-    
+
     /**
      * Get the title of the entry.
      *
      * @return the title of the entry.
      */
     std::string getTitle() const;
-        
+
     /**
      * Get the content of the entry.
      *
@@ -80,7 +81,7 @@ class Entry
      * @return the content of the entry.
      */
     std::string getContent() const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -90,7 +91,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset = 0) const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -101,7 +102,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset, size_type size) const;
-        
+
     /**
      * Get the info for direct access to the content of the entry.
      *
@@ -117,7 +118,7 @@ class Entry
      *         Return <"",0> if is not possible to read directly.
      */
     std::pair<std::string, offset_type> getDirectAccessInfo() const;
-        
+
     /**
      * Get the size of the entry.
      *
@@ -131,8 +132,8 @@ class Entry
      * @return the mime_type of the entry.
      */
     std::string getMimetype() const;
-    
-    
+
+
     /**
      * Get if the entry is a redirect entry.
      *

include/kiwixserve.h

@@ -0,0 +1,30 @@
+#ifndef KIWIXLIB_KIWIX_SERVE_H_
+#define KIWIXLIB_KIWIX_SERVE_H_
+
+#include <memory>
+#include <string>
+
+class Subprocess;
+namespace kiwix {
+
+class KiwixServe
+{
+  public:
+    KiwixServe(const std::string& libraryPath, int port = 8181);
+    ~KiwixServe();
+
+    void run();
+    void shutDown();
+    bool isRunning();
+    int getPort() { return m_port; }
+    int setPort(int port);
+
+  private:
+  std::unique_ptr<Subprocess> mp_kiwixServe;
+  int m_port;
+  std::string m_libraryPath;
+};
+
+}; //end namespace kiwix
+
+#endif // KIWIXLIB_KIWIX_SERVE_H_

include/library.h

@@ -23,13 +23,17 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <memory>
+
+#include "book.h"
+#include "bookmark.h"
+#include "common.h"
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
 namespace kiwix
 {
 
-class Book;
 class OPDSDumper;
 
 enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
@@ -42,17 +46,80 @@ enum supportedListMode {
   VALID = 1 << 4,
   NOVALID = 1 << 5
 };
+
+class Filter {
+  private:
+    uint64_t activeFilters;
+    std::vector<std::string> _acceptTags;
+    std::vector<std::string> _rejectTags;
+    std::string _lang;
+    std::string _publisher;
+    std::string _creator;
+    size_t _maxSize;
+    std::string _query;
+    std::string _name;
+
+  public:
+    Filter();
+    ~Filter() = default;
+
+    /**
+     *  Set the filter to check local.
+     *
+     *  A local book is a book with a path.
+     *  If accept is true, only local book are accepted.
+     *  If accept is false, only non local book are accepted.
+     */
+    Filter& local(bool accept);
+
+    /**
+     *  Set the filter to check remote.
+     *
+     *  A remote book is a book with a url.
+     *  If accept is true, only remote book are accepted.
+     *  If accept is false, only non remote book are accepted.
+     */
+    Filter& remote(bool accept);
+
+    /**
+     *  Set the filter to check validity.
+     *
+     *  A valid book is a book with a path pointing to a existing zim file.
+     *  If accept is true, only valid book are accepted.
+     *  If accept is false, only non valid book are accepted.
+     */
+    Filter& valid(bool accept);
+
+    /**
+     * Set the filter to only accept book with corresponding tag.
+     */
+    Filter& acceptTags(std::vector<std::string> tags);
+    Filter& rejectTags(std::vector<std::string> tags);
+
+    Filter& lang(std::string lang);
+    Filter& publisher(std::string publisher);
+    Filter& creator(std::string creator);
+    Filter& maxSize(size_t size);
+    Filter& query(std::string query);
+    Filter& name(std::string name);
+
+    bool accept(const Book& book) const;
+};
+
+
 /**
  * A Library store several books.
  */
 class Library
 {
-  std::map<std::string, kiwix::Book> books;
+  std::map<std::string, kiwix::Book> m_books;
+  std::map<std::string, std::shared_ptr<Reader>> m_readers;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+
  public:
   Library();
   ~Library();
 
-  std::string version;
   /**
    * Add a book to the library.
    *
@@ -65,7 +132,25 @@ class Library
    */
   bool addBook(const Book& book);
 
+  /**
+   * Add a bookmark to the library.
+   *
+   * @param bookmark the book to add.
+   */
+  void addBookmark(const Bookmark& bookmark);
+
+  /**
+   * Remove a bookmarkk
+   *
+   * @param zimId The zimId of the bookmark.
+   * @param url The url of the bookmark.
+   * @return True if the bookmark has been removed.
+   */
+  bool removeBookmark(const std::string& zimId, const std::string& url);
+
   Book& getBookById(const std::string& id);
+  Book& getBookByPath(const std::string& path);
+  std::shared_ptr<Reader> getReaderById(const std::string& id);
 
   /**
    * Remove a book from the library.
@@ -79,10 +164,18 @@ class Library
    * Write the library to a file.
    *
    * @param path the path of the file to write to.
-   * @return True if the library has been correctly save.
+   * @return True if the library has been correctly saved.
    */
   bool writeToFile(const std::string& path);
 
+  /**
+   * Write the library bookmarks to a file.
+   *
+   * @param path the path of the file to write to.
+   * @return True if the library has been correctly saved.
+   */
+  bool writeBookmarksToFile(const std::string& path);
+
   /**
    * Get the number of book in the library.
    *
@@ -113,6 +206,13 @@ class Library
    */
   std::vector<std::string> getBooksPublishers();
 
+  /**
+   * Get all bookmarks.
+   *
+   * @return A list of bookmarks
+   */
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
+
   /**
    * Get all book ids of the books in the library.
    *
@@ -128,9 +228,27 @@ class Library
    * @param search List only books with search in the title or description.
    * @return The list of bookIds corresponding to the query.
    */
-  std::vector<std::string> filter(const std::string& search);
+  DEPRECATED std::vector<std::string> filter(const std::string& search);
 
 
+  /**
+   * Filter the library and return the id of the keep elements.
+   *
+   * @param filter The filter to use.
+   * @return The list of bookIds corresponding to the filter.
+   */
+  std::vector<std::string> filter(const Filter& filter);
+
+
+  /**
+   * Sort (in place) bookIds using the given comparator.
+   *
+   * @param bookIds the list of book Ids to sort
+   * @param comparator how to sort the books
+   * @return The sorted list of books
+   */
+  void sort(std::vector<std::string>& bookIds, supportedListSortBy sortBy, bool ascending);
+
   /**
    * List books in the library.
    *
@@ -153,16 +271,18 @@ class Library
    *                Set to 0 to cancel this filter.
    * @return The list of bookIds corresponding to the query.
    */
-  std::vector<std::string> listBooksIds(
+  DEPRECATED std::vector<std::string> listBooksIds(
     int supportedListMode = ALL,
     supportedListSortBy sortBy = UNSORTED,
     const std::string& search = "",
     const std::string& language = "",
     const std::string& creator = "",
     const std::string& publisher = "",
+    const std::vector<std::string>& tags = {},
     size_t maxSize = 0);
 
   friend class OPDSDumper;
+  friend class libXMLDumper;
 };
 }
 

include/libxml_dumper.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBXML_DUMPER_H
+#define KIWIX_LIBXML_DUMPER_H
+
+#include <string>
+#include <vector>
+
+#include <pugixml.hpp>
+
+#include "library.h"
+
+namespace kiwix
+{
+
+/**
+ * A tool to dump a `Library` into a basic library.xml
+ *
+ */
+class LibXMLDumper
+{
+ public:
+  LibXMLDumper() = default;
+  LibXMLDumper(Library* library);
+  ~LibXMLDumper();
+
+  /**
+   * Dump the library.xml
+   *
+   * @param id The id of the library.
+   * @return The library.xml content.
+   */
+  std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
+
+
+  /**
+   * Dump the bookmark of the library.
+   *
+   * @return The bookmark.xml content.
+   */
+  std::string dumpLibXMLBookmark();
+
+  /**
+   * Set the base directory used.
+   *
+   * @param baseDir the base directory to use.
+   */
+  void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
+
+  /**
+   * Set the library to dump.
+   *
+   * @param library The library to dump.
+   */
+  void setLibrary(Library* library) { this->library = library; }
+
+ protected:
+  kiwix::Library* library;
+  std::string baseDir;
+ private:
+  void handleBook(Book book, pugi::xml_node root_node);
+  void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
+};
+}
+
+#endif // KIWIX_OPDS_DUMPER_H

include/manager.h

@@ -38,6 +38,7 @@ class LibraryManipulator {
  public:
   virtual ~LibraryManipulator() {}
   virtual bool addBookToLibrary(Book book) = 0;
+  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
 };
 
 class DefaultLibraryManipulator : public LibraryManipulator {
@@ -48,15 +49,15 @@ class DefaultLibraryManipulator : public LibraryManipulator {
   bool addBookToLibrary(Book book) {
     return library->addBook(book);
   }
+  void addBookmarkToLibrary(Bookmark bookmark) {
+    library->addBookmark(bookmark);
+  }
  private:
    kiwix::Library* library;
 };
 
 /**
  * A tool to manage a `Library`.
- *
- * A `Manager` handle a internal `Library`.
- * This `Library` can be retrived with `cloneLibrary` method.
  */
 class Manager
 {
@@ -68,26 +69,12 @@ class Manager
   /**
    * Read a `library.xml` and add book in the file to the library.
    *
-   * @param path The path to the `library.xml`.
+   * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const std::string& path, const bool readOnly = true);
-
-  /**
-   * Read a `library.xml` and add book in the file to the library.
-   *
-   * @param nativePath The path of the `library.xml`
-   * @param UTF8Path The utf8 version (?) of the path. Also the path where the
-   *                 library will be writen i readOnly is False.
-   * @param readOnly Set if the libray path could be overwritten latter with
-   *                 updated content.
-   * @return True if file has been properly parsed.
-   */
-  bool readFile(const std::string& nativePath,
-                const std::string& UTF8Path,
-                const bool readOnly = true);
+  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
   /**
    * Load a library content store in the string.
@@ -100,7 +87,8 @@ class Manager
    */
   bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const std::string& libraryPath = "");
+               const std::string& libraryPath = "",
+               bool trustLibrary = true);
 
   /**
    * Load a library content stored in a OPDS stream.
@@ -113,6 +101,15 @@ class Manager
    */
   bool readOpds(const std::string& content, const std::string& urlHost);
 
+
+  /**
+   * Load a bookmark file.
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed.
+   */
+  bool readBookmarkFile(const std::string& path);
+
   /**
    * Add a book to the library.
    *
@@ -145,55 +142,6 @@ class Manager
                        const std::string& url = "",
                        const bool checkMetaData = false);
 
-  /**
-   * Get the book corresponding to an id.
-   *
-   * @param[in] id The id of the book
-   * @param[out] book The book corresponding to the id.
-   * @return True if the book has been found.
-   */
-  bool getBookById(const std::string& id, Book& book);
-
-  /**
-   * Update the "last open date" of a book
-   *
-   * @param id the id of the book.
-   * @return True if the book is in the library.
-   */
-  bool updateBookLastOpenDateById(const std::string& id);
-
-  /**
-   * Remove (set to empty) paths of all books in the library.
-   */
-  void removeBookPaths();
-
-  /**
-   * List books in the library.
-   *
-   * The books list will be available in public vector member `bookIdList`.
-   *
-   * @param mode The mode of listing :
-   *             - LASTOPEN sort by last opened book.
-   *             - LOCAL list only local file.
-   *             - REMOTE list only remote file.
-   * @param sortBy Attribute to sort by the book list.
-   * @param maxSize Do not list book bigger than maxSize MiB.
-   *                Set to 0 to cancel this filter.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param search List only books with search in the title, description or
-   *               language.
-   * @return True
-   */
-  bool listBooks(const supportedListMode mode,
-                 const supportedListSortBy sortBy,
-                 const unsigned int maxSize,
-                 const std::string& language,
-                 const std::string& creator,
-                 const std::string& publisher,
-                 const std::string& search);
-
   std::string writableLibraryPath;
 
   bool m_hasSearchResult = false;
@@ -207,13 +155,12 @@ class Manager
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   const bool readOnly,
-                   const std::string& libraryPath);
+                   bool readOnly,
+                   const std::string& libraryPath,
+                   bool trustLibrary);
   bool parseOpdsDom(const pugi::xml_document& doc,
                     const std::string& urlHost);
 
- private:
-  void checkAndCleanBookPaths(Book& book, const std::string& libraryPath);
 };
 }
 

include/meson.build

@@ -1,35 +1,31 @@
 headers = [
   'book.h',
+  'bookmark.h',
   'common.h',
   'library.h',
   'manager.h',
+  'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
   'reader.h',
   'entry.h',
-  'searcher.h'
+  'searcher.h',
+  'search_renderer.h',
+  'server.h',
+  'kiwixserve.h',
+  'name_mapper.h'
 ]
 
-if xapian_dep.found()
-  headers += ['xapianSearcher.h']
-endif
-
 install_headers(headers, subdir:'kiwix')
 
 install_headers(
-  'common/base64.h',
-  'common/networkTools.h',
-  'common/otherTools.h',
-  'common/pathTools.h',
-  'common/regexTools.h',
-  'common/stringTools.h',
-  subdir:'kiwix/common'
+  'tools/base64.h',
+  'tools/networkTools.h',
+  'tools/otherTools.h',
+  'tools/pathTools.h',
+  'tools/regexTools.h',
+  'tools/stringTools.h',
+  'tools/lock.h',
+  subdir:'kiwix/tools'
 )
 
-if has_ctpp2_dep
-  install_headers(
-    'ctpp2/CTPP2VMStringLoader.hpp',
-    subdir:'kiwix/ctpp2'
-  )
-endif
-

include/name_mapper.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_NAMEMAPPER_H
+#define KIWIX_NAMEMAPPER_H
+
+#include <string>
+#include <map>
+
+namespace kiwix
+{
+
+class Library;
+
+class NameMapper {
+  public:
+    virtual ~NameMapper() = default;
+    virtual std::string getNameForId(const std::string& id) = 0;
+    virtual std::string getIdForName(const std::string& name) = 0;
+};
+
+
+class IdNameMapper : public NameMapper {
+  public:
+    virtual std::string getNameForId(const std::string& id) { return id; };
+    virtual std::string getIdForName(const std::string& name) { return name; };
+};
+
+class HumanReadableNameMapper : public NameMapper {
+  private:
+    std::map<std::string, std::string> m_idToName;
+    std::map<std::string, std::string> m_nameToId;
+
+  public:
+    HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
+    virtual ~HumanReadableNameMapper() = default;
+    virtual std::string getNameForId(const std::string& id);
+    virtual std::string getIdForName(const std::string& name);
+};
+
+
+
+}
+
+#endif

include/opds_dumper.h

@@ -26,9 +26,9 @@
 
 #include <pugixml.hpp>
 
-#include "common/base64.h"
-#include "common/pathTools.h"
-#include "common/regexTools.h"
+#include "tools/base64.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
 #include "library.h"
 #include "reader.h"
 

include/reader.h

@@ -31,8 +31,8 @@
 #include <string>
 #include "common.h"
 #include "entry.h"
-#include "common/pathTools.h"
-#include "common/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 
 using namespace std;
 
@@ -158,31 +158,7 @@ class Reader
    * @param[out] value The value will be set to the content of the metadata.
    * @return True if it was possible to get the content of the metadata.
    */
-  bool getMetatag(const string& name, string& value) const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
+  bool getMetadata(const string& name, string& value) const;
 
   /**
    * Get the name of the zim file.
@@ -192,18 +168,13 @@ class Reader
   string getName() const;
 
   /**
-   * Get the tags of the zim file.
+   * Get the title of the zim file.
    *
-   * @return The tags of the zim file as specified in the zim metadata.
+   * @return The title of zim file as specified in the zim metadata.
+   *         If no title has been set, return a title computed from the
+   *         file path.
    */
-  string getTags() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
+  string getTitle() const;
 
   /**
    * Get the creator of the zim file.
@@ -219,6 +190,100 @@ class Reader
    */
   string getPublisher() const;
 
+  /**
+   * Get the date of the zim file.
+   *
+   * @return The date of the zim file as specified in the zim metadata.
+   */
+  string getDate() const;
+
+  /**
+   * Get the description of the zim file.
+   *
+   * @return The description of the zim file as specified in the zim metadata.
+   *         If no description has been set, return the subtitle.
+   */
+  string getDescription() const;
+
+  /**
+   * Get the long description of the zim file.
+   *
+   * @return The long description of the zim file as specifed in the zim metadata.
+   */
+  string getLongDescription() const;
+
+  /**
+   * Get the language of the zim file.
+   *
+   * @return The language of the zim file as specified in the zim metadata.
+   */
+  string getLanguage() const;
+
+  /**
+   * Get the license of the zim file.
+   *
+   * @return The license of the zim file as specified in the zim metadata.
+   */
+  string getLicense() const;
+
+  /**
+   * Get the tags of the zim file.
+   *
+   * @param original If true, return the original tags as specified in the zim metadata.
+   *                 Else, try to convert it to the new 'normalized' format.
+   * @return The tags of the zim file.
+   */
+  string getTags(bool original=false) const;
+
+  /**
+   * Get the value (as a string) of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   */
+  string getTagStr(const std::string& tagName) const;
+
+  /**
+   * Get the boolean value of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The boolean value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   *         std::domain_error if the value of the tag cannot be convert to bool.
+   */
+  bool getTagBool(const std::string& tagName) const;
+
+  /**
+   * Get the relations of the zim file.
+   *
+   * @return The relation of the zim file as specified in the zim metadata.
+   */
+  string getRelation() const;
+
+  /**
+   * Get the flavour of the zim file.
+   *
+   * @return The flavour of the zim file as specified in the zim metadata.
+   */
+  string getFlavour() const;
+
+  /**
+   * Get the source of the zim file.
+   *
+   * @return The source of the zim file as specified in the zim metadata.
+   */
+  string getSource() const;
+
+  /**
+   * Get the scraper of the zim file.
+   *
+   * @return The scraper of the zim file as specified in the zim metadata.
+   */
+  string getScraper() const;
+
   /**
    * Get the origId of the zim file.
    *

include/search_renderer.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SEARCH_RENDERER_H
+#define KIWIX_SEARCH_RENDERER_H
+
+#include <string>
+
+namespace kiwix
+{
+
+class Searcher;
+class NameMapper;
+/**
+ * The SearcherRenderer class is used to render a search result to a html page.
+ */
+class SearchRenderer
+{
+ public:
+  /**
+   * The default constructor.
+   *
+   * @param humanReadableName The global zim's humanReadableName.
+   *                          Used to generate pagination links.
+   */
+  SearchRenderer(Searcher* searcher, NameMapper* mapper);
+
+  ~SearchRenderer();
+
+  void setSearchPattern(const std::string& pattern);
+
+  /**
+   * Set the search content id.
+   */
+  void setSearchContent(const std::string& name);
+
+  /**
+   * Set protocol prefix.
+   */
+  void setProtocolPrefix(const std::string& prefix);
+
+  /**
+   * Set search protocol prefix.
+   */
+  void setSearchProtocolPrefix(const std::string& prefix);
+
+  /**
+   * Generate the html page with the resutls of the search.
+   */
+  std::string getHtml();
+
+ protected:
+  std::string beautifyInteger(const unsigned int number);
+  Searcher* mp_searcher;
+  NameMapper* mp_nameMapper;
+  std::string searchContent;
+  std::string searchPattern;
+  std::string protocolPrefix;
+  std::string searchProtocolPrefix;
+  unsigned int resultCountPerPage;
+  unsigned int estimatedResultCount;
+  unsigned int resultStart;
+  unsigned int resultEnd;
+};
+
+
+}
+
+#endif

include/searcher.h

@@ -29,8 +29,8 @@
 #include <string>
 #include <vector>
 #include <vector>
-#include "common/pathTools.h"
-#include "common/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 #include "kiwix_config.h"
 
 using namespace std;
@@ -56,65 +56,28 @@ struct SearcherInternal;
 /**
  * The Searcher class is reponsible to do different kind of search using the
  * fulltext index.
- *
- * Historically, there are two kind of fulltext index :
- *  - The legacy one, is the external fulltext index. A directory stored outside
- *    of the zim file.
- *  - The new one, a embedded fulltext index in the zim file.
- *
- * Legacy external fulltext index has to be considered as obsolet format with
- * less functionnalities:
- *  - No multi zim search ;
- *  - No geo_search ;
- *  - No suggestions search ;
- *
- * To reflect this, there is two Search creation "API":
- *  - One for the external fulltext index, using the constructor taking a
- *    xapianDirectoryPath) ;
- *  - One for the embedded fulltext index, using a "empty" constructor and the
- *  `add_reader` method".
- *
- *  On top of that, the Searcher may (if compiled with ctpp2) be used to
- *  generate a html page for the search result. This use a template that need a
- *  humanReaderName. This feature is only used by kiwix-serve and this should be
- *  move outside of Searcher (and with a better API). If you don't use the html
- *  rendering (getHtml method), you better should simply ignore the different
- *  humanReadeableName attributes (or give an empty string).
  */
 class Searcher
 {
  public:
   /**
    * The default constructor.
-   *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
    */
-  Searcher(const string& humanReadableName = "");
+  Searcher();
 
-  /**
-   * The constructor for legacy external fulltext index.
-   *
-   * @param xapianDirectoryPath The path to the external index directory.
-   * @param reader The reader associated to the external index.
-   *               It will be used retrive the article content or generate
-   *               the snippet.
-   * @param humanReadableName The humanReadableName for the zim.
-   */
-  Searcher(const string& xapianDirectoryPath,
-           Reader* reader,
-           const string& humanReadableName);
   ~Searcher();
 
   /**
    * Add a reader (containing embedded fulltext index) to the search.
    *
    * @param reader The Reader for the zim containing the fulltext index.
-   * @param humanReaderName The human readable name of the reader.
    * @return true if the reader has been added.
    *         false if the reader cannot be added (no embedded fulltext index present)
    */
-  bool add_reader(Reader* reader, const std::string& humanReaderName);
+  bool add_reader(Reader* reader);
+
+
+  Reader* get_reader(int index);
 
   /**
    * Start a search on the zim associated to the Searcher.
@@ -126,7 +89,7 @@ class Searcher
    * @param resultEnd the end offset of the search results (used for pagination).
    * @param verbose print some info on stdout if true.
    */
-  void search(std::string& search,
+  void search(const std::string& search,
               unsigned int resultStart,
               unsigned int resultEnd,
               const bool verbose = false);
@@ -180,24 +143,8 @@ class Searcher
    */
   unsigned int getEstimatedResultCount();
 
-  /**
-   * Set protocol prefix.
-   * Only used by getHtml.
-   */
-  bool setProtocolPrefix(const std::string prefix);
-
-  /**
-   * Set search protocol prefix.
-   * Only used by getHtml.
-   */
-  bool setSearchProtocolPrefix(const std::string prefix);
-
-#ifdef ENABLE_CTPP2
-  /**
-   * Generate the html page with the resutls of the search.
-   */
-  string getHtml();
-#endif
+  unsigned int getResultStart() { return resultStart; }
+  unsigned int getResultEnd() { return resultEnd; }
 
  protected:
   std::string beautifyInteger(const unsigned int number);
@@ -208,16 +155,11 @@ class Searcher
                      const bool verbose = false);
 
   std::vector<Reader*> readers;
-  std::vector<std::string> humanReaderNames;
   SearcherInternal* internal;
   std::string searchPattern;
-  std::string protocolPrefix;
-  std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
   unsigned int resultEnd;
-  std::string contentHumanReadableId;
 
  private:
   void reset();

include/server.h

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_H
+#define KIWIX_SERVER_H
+
+#include <string>
+#include <memory>
+
+namespace kiwix
+{
+  class Library;
+  class NameMapper;
+  class InternalServer;
+
+  class Server {
+     public:
+       /**
+        * The default constructor.
+        *
+        * @param library The library to serve.
+        */
+       Server(Library* library, NameMapper* nameMapper=nullptr);
+
+       virtual ~Server();
+
+       /**
+        * Serve the content.
+        */
+       bool start();
+
+       /**
+        * Stop the daemon.
+        */
+       void stop();
+
+       void setRoot(const std::string& root);
+       void setAddress(const std::string& addr) { m_addr = addr; }
+       void setPort(int port) { m_port = port; }
+       void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setVerbose(bool verbose) { m_verbose = verbose; }
+       void setTaskbar(bool withTaskbar, bool withLibraryButton)
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
+
+     protected:
+       Library* mp_library;
+       NameMapper* mp_nameMapper;
+       std::string m_root = "";
+       std::string m_addr = "";
+       int m_port = 80;
+       int m_nbThreads = 1;
+       bool m_verbose = false;
+       bool m_withTaskbar = true;
+       bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
+       std::unique_ptr<InternalServer> mp_server;
+  };
+}
+
+#endif

include/tools/lock.h

@@ -0,0 +1,46 @@
+
+
+#ifndef KIWIXLIB_TOOL_LOCK_H
+#define KIWIXLIB_TOOL_LOCK_H
+
+#include <pthread.h>
+
+namespace kiwix {
+
+class Lock
+{
+  public:
+    explicit Lock(pthread_mutex_t* mutex) :
+      mp_mutex(mutex)
+    {
+      pthread_mutex_lock(mp_mutex);
+    }
+    ~Lock() {
+      if (mp_mutex != nullptr) {
+        pthread_mutex_unlock(mp_mutex);
+      }
+    }
+    Lock(Lock && other) :
+      mp_mutex(other.mp_mutex)
+    {
+      other.mp_mutex = nullptr;
+    }
+    Lock & operator=(Lock && other)
+    {
+      mp_mutex = other.mp_mutex;
+      other.mp_mutex = nullptr;
+      return *this;
+    }
+
+
+  private:
+    pthread_mutex_t* mp_mutex;
+
+    Lock(Lock const &) = delete;
+    Lock & operator=(Lock const &) = delete;
+};
+
+
+}
+
+#endif //KIWIXLIB_TOOL_LOCK_H

include/tools/networkTools.h

@@ -20,13 +20,10 @@
 #ifndef KIWIX_NETWORKTOOLS_H
 #define KIWIX_NETWORKTOOLS_H
 
-#include <map>
 #include <string>
 
 namespace kiwix
 {
-std::map<std::string, std::string> getNetworkInterfaces();
-std::string getBestPublicIp();
 std::string download(const std::string& url);
 }
 

include/tools/otherTools.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_OTHERTOOLS_H
+#define KIWIX_OTHERTOOLS_H
+
+#include <string>
+#include <vector>
+
+namespace pugi {
+  class xml_node;
+}
+
+namespace kiwix
+{
+  void sleep(unsigned int milliseconds);
+  std::string nodeToString(const pugi::xml_node& node);
+  std::string converta2toa3(const std::string& a2code);
+
+  /*
+   * Convert all format tag string to new format
+   */
+  std::vector<std::string> convertTags(const std::string& tags_str);
+  std::string getTagValueFromTagList(const std::vector<std::string>& tagList,
+                                     const std::string& tagName);
+  bool convertStrToBool(const std::string& value);
+
+}
+
+#endif

include/tools/pathTools.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_PATHTOOLS_H
+#define KIWIX_PATHTOOLS_H
+
+#include <string>
+
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr);
+std::wstring Utf8ToWide(const std::string& str);
+#endif
+bool isRelativePath(const std::string& path);
+std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
+std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
+std::string removeLastPathElement(const std::string& path);
+std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
+
+unsigned int getFileSize(const std::string& path);
+std::string getFileSizeAsString(const std::string& path);
+std::string getFileContent(const std::string& path);
+bool fileExists(const std::string& path);
+bool makeDirectory(const std::string& path);
+std::string makeTmpDirectory();
+bool copyFile(const std::string& sourcePath, const std::string& destPath);
+std::string getLastPathElement(const std::string& path);
+std::string getExecutablePath(bool realPathOnly = false);
+std::string getCurrentDirectory();
+std::string getDataDirectory();
+bool writeTextFile(const std::string& path, const std::string& content);
+std::string getMimeTypeForFile(const std::string& filename);
+#endif

include/tools/regexTools.h

@@ -20,9 +20,6 @@
 #ifndef KIWIX_REGEXTOOLS_H
 #define KIWIX_REGEXTOOLS_H
 
-#include <unicode/regex.h>
-#include <unicode/ucnv.h>
-#include <map>
 #include <string>
 
 bool matchRegex(const std::string& content, const std::string& regex);
@@ -30,7 +27,7 @@ std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
 std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string regex,
+                                   const std::string& regex,
                                    const std::string& replacement);
 
 #endif

include/tools/stringTools.h

@@ -22,19 +22,12 @@
 
 #include <unicode/unistr.h>
 
-#include <fstream>
-#include <iomanip>
-#include <iostream>
-#include <sstream>
 #include <string>
 #include <vector>
-
-#include "pathTools.h"
+#include <sstream>
 
 namespace kiwix
 {
-#ifndef __ANDROID__
-
 std::string beautifyInteger(uint64_t number);
 std::string beautifyFileSize(uint64_t number);
 void printStringInHexadecimal(const char* s);
@@ -44,18 +37,14 @@ void stringReplacement(std::string& str,
                        const std::string& newStr);
 std::string encodeDiples(const std::string& str);
 
-#endif
-
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
 
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string&, const std::string&);
-std::vector<std::string> split(const char*, const char*);
-std::vector<std::string> split(const std::string&, const char*);
-std::vector<std::string> split(const char*, const std::string&);
+std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
+std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);
 std::string lcAll(const std::string& word);
@@ -79,5 +68,7 @@ T extractFromString(const std::string& str) {
     iss >> ret;
     return ret;
 }
+
+bool startsWith(const std::string& base, const std::string& start);
 } //namespace kiwix
 #endif

include/xapianSearcher.h

@@ -1,98 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_XAPIAN_SEARCHER_H
-#define KIWIX_XAPIAN_SEARCHER_H
-
-#include <xapian.h>
-#include "reader.h"
-#include "searcher.h"
-
-#include <map>
-#include <string>
-
-using namespace std;
-
-namespace kiwix
-{
-class XapianSearcher;
-
-class XapianResult : public Result
-{
- public:
-  XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
-  virtual ~XapianResult(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual int get_readerIndex() { return 0; };
-
- private:
-  XapianSearcher* searcher;
-  Xapian::MSetIterator iterator;
-  Xapian::Document document;
-};
-
-class NoXapianIndexInZim : public exception
-{
-  virtual const char* what() const throw()
-  {
-    return "There is no fulltext index in the zim file";
-  }
-};
-
-class XapianSearcher
-{
-  friend class XapianResult;
-
- public:
-  XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
-  virtual ~XapianSearcher(){};
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-  virtual Result* getNextResult();
-  void restart_search();
-
-  Xapian::MSet results;
-
- protected:
-  void closeIndex();
-  void openIndex(const string& xapianDirectoryPath);
-  void setup_queryParser();
-
-  Reader* reader;
-  Xapian::Database readableDatabase;
-  std::string language;
-  std::string stopwords;
-  Xapian::QueryParser queryParser;
-  Xapian::Stem stemmer;
-  Xapian::SimpleStopper stopper;
-  Xapian::MSetIterator current_result;
-  std::map<std::string, int> valuesmap;
-};
-}
-
-#endif

meson.build

@@ -1,23 +1,42 @@
 project('kiwix-lib', 'cpp',
-  version : '3.0.2',
+  version : '9.2.3', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
-find_library_in_compiler = meson.version().version_compare('>=0.31.0')
 
-static_deps = get_option('android') or get_option('default_library') == 'static'
-if get_option('android')
+wrapper = get_option('wrapper')
+
+static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
+if 'android' in wrapper
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
+if 'java' in wrapper
+  add_languages('java')
+endif
+
+# See https://github.com/kiwix/kiwix-lib/issues/371
+if target_machine.cpu_family() in ['arm', 'mips', 'm68k', 'ppc']
+  extra_libs += '-latomic'
+endif
+
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
+microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
+  error('Cannot found header mustache.hpp')
+endif
 
 extra_cflags = ''
 if target_machine.system() == 'windows' and static_deps
@@ -25,77 +44,12 @@ if target_machine.system() == 'windows' and static_deps
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
 
-
-ctpp2_include_path = ''
-has_ctpp2_dep = false
-ctpp2_prefix_install = get_option('ctpp2-install-prefix')
-ctpp2_link_args = []
-if ctpp2_prefix_install == ''
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp')
-    if find_library_in_compiler
-      ctpp2_lib = compiler.find_library('ctpp2')
-    else
-      ctpp2_lib = find_library('ctpp2')
-    endif
-    ctpp2_link_args = ['-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      if find_library_in_compiler
-        iconv_lib = compiler.find_library('iconv', required:false)
-      else
-        iconv_lib = find_library('iconv', required:false)
-      endif
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
-else
-  if not find_library_in_compiler
-    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
-  endif
-  ctpp2_include_path = ctpp2_prefix_install + '/include'
-  ctpp2_include_args = ['-I'+ctpp2_include_path]
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
-    ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
-    ctpp2_lib_path = join_paths(ctpp2_prefix_install, get_option('libdir'))
-    message(ctpp2_lib_path)
-    ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path, required:false)
-    if not ctpp2_lib.found()
-      ctpp2_lib_path = join_paths(ctpp2_prefix_install, 'lib')
-      message(ctpp2_lib_path)
-      ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
-    endif
-    ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      iconv_lib = compiler.find_library('iconv', required:false)
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
-endif
-
-xapian_dep = dependency('xapian-core', required:false, static:static_deps)
-
-all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libcurl_dep]
-if has_ctpp2_dep
-  all_deps += [ctpp2_dep]
-endif
-
-inc = include_directories('include')
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
-conf.set('ENABLE_CTPP2', has_ctpp2_dep)
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -110,16 +64,6 @@ subdir('src')
 subdir('test')
 
 pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
-if xapian_dep.found()
-    pkg_requires += ['xapian-core']
-endif
-
-if has_ctpp2_dep
-  extra_libs += ctpp2_link_args
-  if ctpp2_include_path != ''
-    extra_cflags = ' -I'+ctpp2_include_path
-  endif
-endif
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,4 +1,2 @@
-option('ctpp2-install-prefix', type : 'string', value : '',
-  description : 'Prefix where ctpp libs has been installed')
-option('android', type : 'boolean', value : false,
-  description : 'Do we make a kiwix-lib for android')
+option('wrapper', type:'array', choices:['java', 'android'], value:[],
+  description: 'The wrapper to generate.')

scripts/ctpp2c.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-
-ctpp2c=$1
-SOURCE=$(pwd)/$2
-DEST=$3
-
-$ctpp2c $SOURCE $DEST

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,11 +95,11 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
+
 
 
 master_c_template = """//This file is automaically generated. Do not modify it.
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

scripts/meson.build

@@ -1,5 +1,4 @@
 
 res_compiler = find_program('kiwix-compile-resources')
-intermediate_ctpp2c = find_program('ctpp2c.sh')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))

src/android/meson.build

@@ -1,26 +0,0 @@
-
-kiwix_jni = custom_target('jni',
-  input: ['org/kiwix/kiwixlib/JNIKiwix.java',
-          'org/kiwix/kiwixlib/JNIKiwixReader.java',
-          'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-          'org/kiwix/kiwixlib/JNIKiwixInt.java',
-          'org/kiwix/kiwixlib/JNIKiwixString.java',
-          'org/kiwix/kiwixlib/JNIKiwixBool.java',
-          'org/kiwix/kiwixlib/JNIKiwixException.java',
-          'org/kiwix/kiwixlib/Pair.java'],
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-kiwix_sources += [
-    'android/kiwix.cpp',
-    'android/kiwixreader.cpp',
-    'android/kiwixsearcher.cpp',
-    kiwix_jni]
-
-install_subdir('org', install_dir: 'kiwix-lib/java')
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/android/utils.h

@@ -1,150 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef _ANDROID_JNI_UTILS_H
-#define _ANDROID_JNI_UTILS_H
-
-#include <jni.h>
-
-#include <pthread.h>
-#include <string>
-
-extern pthread_mutex_t globalLock;
-
-inline jfieldID getHandleField(JNIEnv* env, jobject obj)
-{
-  jclass c = env->GetObjectClass(obj);
-  // J is the type signature for long:
-  return env->GetFieldID(c, "nativeHandle", "J");
-}
-
-class Lock
-{
- protected:
-  pthread_mutex_t* lock;
-
- public:
-  Lock() : lock(&globalLock) { pthread_mutex_lock(lock); }
-  Lock(const Lock&) = delete;
-  Lock& operator=(const Lock&) = delete;
-  Lock(Lock&& other) : lock(&globalLock) { other.lock = nullptr; }
-  virtual ~Lock()
-  {
-    if (lock) {
-      pthread_mutex_unlock(lock);
-    }
-  }
-};
-
-template <class T>
-class LockedHandle;
-
-template <class T>
-class Handle
-{
- protected:
-  T* h;
-
- public:
-  Handle(T* h) : h(h){};
-
-  // No destructor. This must and will be handled by dispose method.
-
-  static LockedHandle<T> getHandle(JNIEnv* env, jobject obj)
-  {
-    jlong handle = env->GetLongField(obj, getHandleField(env, obj));
-    return LockedHandle<T>(reinterpret_cast<Handle<T>*>(handle));
-  }
-
-  static void dispose(JNIEnv* env, jobject obj)
-  {
-    auto lHandle = getHandle(env, obj);
-    auto handle = lHandle.h;
-    delete handle->h;
-    delete handle;
-  }
-  friend class LockedHandle<T>;
-};
-
-template <class T>
-struct LockedHandle : public Lock {
-  Handle<T>* h;
-  LockedHandle(Handle<T>* h) : h(h) {}
-  T* operator->() { return h->h; }
-  T* operator*() { return h->h; }
-  operator bool() const { return (h->h != nullptr); }
-};
-
-/* c2jni type conversion functions */
-inline jboolean c2jni(const bool& val) { return val ? JNI_TRUE : JNI_FALSE; }
-inline jstring c2jni(const std::string& val, JNIEnv* env)
-{
-  return env->NewStringUTF(val.c_str());
-}
-
-inline jint c2jni(const int val) { return (jint)val; }
-inline jint c2jni(const unsigned val) { return (unsigned)val; }
-/* jni2c type conversion functions */
-inline bool jni2c(const jboolean& val) { return val == JNI_TRUE; }
-inline std::string jni2c(const jstring& val, JNIEnv* env)
-{
-  const char* chars = env->GetStringUTFChars(val, 0);
-  std::string ret(chars);
-  env->ReleaseStringUTFChars(val, chars);
-  return ret;
-}
-
-inline int jni2c(const jint val) { return (int)val; }
-/* Method to deal with variable passed by reference */
-inline void setStringObjValue(const std::string& value,
-                              const jobject obj,
-                              JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
-  env->SetObjectField(obj, objFid, c2jni(value, env));
-}
-
-inline void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "I");
-  env->SetIntField(obj, objFid, value);
-}
-
-inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
-  env->SetIntField(obj, objFid, c2jni(value));
-}
-
-inline void setPairObjValue(const std::string& filename, const int offset,
-                            const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID filenameFid = env->GetFieldID(objClass, "filename", "Ljava/lang/String;");
-  env->SetObjectField(obj, filenameFid, c2jni(filename, env));
-  jfieldID offsetFid = env->GetFieldID(objClass, "offset", "I");
-  env->SetIntField(obj, offsetFid, offset);
-}
-
-#endif // _ANDROID_JNI_UTILS_H

src/aria2.cpp

@@ -2,13 +2,24 @@
 
 #include "aria2.h"
 #include "xmlrpc.h"
+#include <iostream>
 #include <sstream>
 #include <thread>
 #include <chrono>
-#include <common/otherTools.h>
-#include <common/pathTools.h>
+#include <tools/otherTools.h>
+#include <tools/pathTools.h>
+#include <tools/stringTools.h>
 #include <downloader.h> // For AriaError
 
+#ifdef _WIN32
+# define ARIA2_CMD "aria2c.exe"
+#else
+# define ARIA2_CMD "aria2c"
+# include <unistd.h>
+#endif
+
+
+
 namespace kiwix {
 
 Aria2::Aria2():
@@ -19,6 +30,7 @@ Aria2::Aria2():
   m_lock(PTHREAD_MUTEX_INITIALIZER)
 {
   m_downloadDir = getDataDirectory();
+  makeDirectory(m_downloadDir);
   std::vector<const char*> callCmd;
 
   std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
@@ -36,11 +48,16 @@ Aria2::Aria2():
   std::string rpc_secret = "--rpc-secret=" + m_secret;
   m_secret = "token:"+m_secret;
 
-#ifdef _WIN32
-  callCmd.push_back("aria2c.exe");
-#else
-  callCmd.push_back("aria2c");
-#endif
+  std::string aria2cmd = appendToDirectory(
+    removeLastPathElement(getExecutablePath(true)),
+    ARIA2_CMD);
+  if (fileExists(aria2cmd)) {
+    // A local aria2c exe exists (packaged with kiwix-desktop), use it.
+    callCmd.push_back(aria2cmd.c_str());
+  } else {
+    // Try to use a potential installed aria2c.
+    callCmd.push_back(ARIA2_CMD);
+  }
   callCmd.push_back("--enable-rpc");
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
@@ -61,23 +78,38 @@ Aria2::Aria2():
   callCmd.push_back("--max-concurrent-downloads=42");
   callCmd.push_back("--rpc-max-request-size=6M");
   callCmd.push_back("--file-allocation=none");
+  std::string launchCmd;
+  for (auto &cmd : callCmd) {
+      launchCmd.append(cmd).append(" ");
+  }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
+  char errbuf[CURL_ERROR_SIZE];
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
 
   int watchdog = 50;
   while(--watchdog) {
-    std::this_thread::sleep_for(std::chrono::microseconds(100));
+    sleep(10);
+    errbuf[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
+    } else if (watchdog == 1) {
+      std::cerr <<" curl_easy_perform() failed." << std::endl;
+      fprintf(stderr, "\nlibcurl: (%d) ", res);
+      if (errbuf[0] != 0) {
+        std::cerr << errbuf << std::endl;
+      } else {
+        std::cerr << curl_easy_strerror(res) << std::endl;
+      }
     }
   }
   if (!watchdog) {
     curl_easy_cleanup(mp_curl);
-    throw std::runtime_error("Cannot connect to aria2c rpc");
+    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
@@ -128,13 +160,16 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
   throw std::runtime_error("Cannot perform request");
 }
 
-std::string Aria2::addUri(const std::vector<std::string>& uris)
+std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
 {
   MethodCall methodCall("aria2.addUri", m_secret);
   auto uriParams = methodCall.newParamValue().getArray();
   for (auto& uri : uris) {
     uriParams.addValue().set(uri);
   }
+  for (auto& option : options) {
+    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
+  }
   auto ret = doRequest(methodCall);
   MethodResponse response(ret);
   return response.getParamValue(0).getAsS();
@@ -158,7 +193,6 @@ std::vector<std::string> Aria2::tellActive()
   MethodCall methodCall("aria2.tellActive", m_secret);
   auto statusArray = methodCall.newParamValue().getArray();
   statusArray.addValue().set(std::string("gid"));
-  statusArray.addValue().set(std::string("following"));
   auto responseContent = doRequest(methodCall);
   MethodResponse response(responseContent);
   std::vector<std::string> activeGID;
@@ -173,6 +207,27 @@ std::vector<std::string> Aria2::tellActive()
   return activeGID;
 }
 
+std::vector<std::string> Aria2::tellWaiting()
+{
+  MethodCall methodCall("aria2.tellWaiting", m_secret);
+  methodCall.newParamValue().set(0);
+  methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> waitingGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      waitingGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return waitingGID;
+}
+
 void Aria2::saveSession()
 {
   MethodCall methodCall("aria2.saveSession", m_secret);
@@ -186,5 +241,25 @@ void Aria2::shutdown()
   doRequest(methodCall);
 }
 
+void Aria2::pause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.pause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::unpause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.unpause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::remove(const std::string& gid)
+{
+    MethodCall methodCall("aria2.remove", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
 
 } // end namespace kiwix

src/aria2.h

@@ -34,11 +34,15 @@ class Aria2
     virtual ~Aria2();
     void close();
 
-    std::string addUri(const std::vector<std::string>& uri);
+    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
     std::vector<std::string> tellActive();
+    std::vector<std::string> tellWaiting();
     void saveSession();
     void shutdown();
+    void pause(const std::string& gid);
+    void unpause(const std::string& gid);
+    void remove(const std::string& gid);
 };
 
 }; //end namespace kiwix

src/book.cpp

@@ -20,16 +20,19 @@
 #include "book.h"
 #include "reader.h"
 
-#include "common/base64.h"
-#include "common/regexTools.h"
-#include "common/networkTools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/networkTools.h"
+#include "tools/otherTools.h"
 
 #include <pugixml.hpp>
 
 namespace kiwix
 {
 /* Constructor */
-Book::Book() : m_readOnly(false)
+Book::Book() :
+  m_pathValid(false),
+  m_readOnly(false)
 {
 }
 /* Destructor */
@@ -42,52 +45,54 @@ bool Book::update(const kiwix::Book& other)
   if (m_readOnly)
     return false;
 
+  if (m_id != other.m_id)
+    return false;
+
   m_readOnly = other.m_readOnly;
+  m_path = other.m_path;
+  m_pathValid = other.m_pathValid;
+  m_title = other.m_title;
+  m_description = other.m_description;
+  m_language = other.m_language;
+  m_creator = other.m_creator;
+  m_publisher = other.m_publisher;
+  m_date = other.m_date;
+  m_url = other.m_url;
+  m_name = other.m_name;
+  m_flavour = other.m_flavour;
+  m_tags = other.m_tags;
+  m_origId = other.m_origId;
+  m_articleCount = other.m_articleCount;
+  m_mediaCount = other.m_mediaCount;
+  m_size = other.m_size;
+  m_favicon = other.m_favicon;
+  m_faviconMimeType = other.m_faviconMimeType;
+  m_faviconUrl = other.m_faviconUrl;
 
-  if (m_path.empty()) {
-    m_path = other.m_path;
-  }
+  m_downloadId = other.m_downloadId;
 
-  if (m_url.empty()) {
-    m_url = other.m_url;
-  }
-
-  if (m_tags.empty()) {
-    m_tags = other.m_tags;
-  }
-
-  if (m_name.empty()) {
-    m_name = other.m_name;
-  }
-
-  if (m_indexPath.empty()) {
-    m_indexPath = other.m_indexPath;
-    m_indexType = other.m_indexType;
-  }
-
-  if (m_faviconMimeType.empty()) {
-    m_favicon = other.m_favicon;
-    m_faviconMimeType = other.m_faviconMimeType;
-  }
   return true;
 }
 
 void Book::update(const kiwix::Reader& reader)
 {
   m_path = reader.getZimFilePath();
+  m_pathValid = true;
   m_id = reader.getId();
+  m_title = reader.getTitle();
   m_description = reader.getDescription();
   m_language = reader.getLanguage();
-  m_date = reader.getDate();
   m_creator = reader.getCreator();
   m_publisher = reader.getPublisher();
-  m_title = reader.getTitle();
+  m_date = reader.getDate();
   m_name = reader.getName();
+  m_flavour = reader.getFlavour();
   m_tags = reader.getTags();
   m_origId = reader.getOrigId();
   m_articleCount = reader.getArticleCount();
   m_mediaCount = reader.getMediaCount();
   m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
+  m_pathValid = true;
 
   reader.getFavicon(m_favicon, m_faviconMimeType);
 }
@@ -101,29 +106,24 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  path = ATTR("indexPath");
-  if (!path.empty()) {
-    if (isRelativePath(path)) {
-      path = computeAbsolutePath(baseDir, path);
-    }
-    m_indexPath = path;
-    m_indexType = XAPIAN;
-  }
+  m_pathValid = fileExists(path);
   m_title = ATTR("title");
-  m_name = ATTR("name");
-  m_tags = ATTR("tags");
   m_description = ATTR("description");
   m_language = ATTR("language");
-  m_date = ATTR("date");
   m_creator = ATTR("creator");
   m_publisher = ATTR("publisher");
+  m_date = ATTR("date");
   m_url = ATTR("url");
+  m_name = ATTR("name");
+  m_flavour = ATTR("flavour");
+  m_tags = ATTR("tags");
   m_origId = ATTR("origId");
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
   m_favicon = base64_decode(ATTR("favicon"));
   m_faviconMimeType = ATTR("faviconMimeType");
+  m_faviconUrl = ATTR("faviconUrl");
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -131,6 +131,14 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
 #undef ATTR
 
 
+static std::string fromOpdsDate(const std::string& date)
+{
+  //The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
+  //and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
+  return date.substr(0, 10);
+}
+
+
 #define VALUE(name) node.child(name).child_value()
 void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
 {
@@ -138,11 +146,18 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   if (!m_id.compare(0, 9, "urn:uuid:")) {
     m_id.erase(0, 9);
   }
+  // No path on opds.
   m_title = VALUE("title");
-  m_description = VALUE("description");
+  m_description = VALUE("summary");
   m_language = VALUE("language");
-  m_date = VALUE("updated");
   m_creator = node.child("author").child("name").child_value();
+  m_publisher = node.child("publisher").child("name").child_value();
+  m_date = fromOpdsDate(VALUE("updated"));
+  m_name = VALUE("name");
+  m_flavour = VALUE("flavour");
+  m_tags = VALUE("tags");
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();
@@ -160,7 +175,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
 }
 #undef VALUE
 
-std::string Book::getHumanReadableIdFromPath()
+std::string Book::getHumanReadableIdFromPath() const
 {
   std::string id = m_path;
   if (!id.empty()) {
@@ -186,13 +201,6 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-void Book::setIndexPath(const std::string& indexPath)
-{
-  m_indexPath = isRelativePath(indexPath)
-    ? computeAbsolutePath(getCurrentDirectory(), indexPath)
-    : indexPath;
-}
-
 const std::string& Book::getFavicon() const {
   if (m_favicon.empty() && !m_faviconUrl.empty()) {
     try {
@@ -204,4 +212,12 @@ const std::string& Book::getFavicon() const {
   return m_favicon;
 }
 
+std::string Book::getTagStr(const std::string& tagName) const {
+  return getTagValueFromTagList(convertTags(m_tags), tagName);
+}
+
+bool Book::getTagBool(const std::string& tagName) const {
+  return convertStrToBool(getTagStr(tagName));
+}
+
 }

src/bookmark.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "bookmark.h"
+
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Bookmark::Bookmark()
+{
+}
+
+/* Destructor */
+Bookmark::~Bookmark()
+{
+}
+
+void Bookmark::updateFromXml(const pugi::xml_node& node)
+{
+  auto bookNode = node.child("book");
+  m_bookId = bookNode.child("id").child_value();
+  m_bookTitle = bookNode.child("title").child_value();
+  m_language = bookNode.child("language").child_value();
+  m_date = bookNode.child("date").child_value();
+  m_title = node.child("title").child_value();
+  m_url = node.child("url").child_value();
+}
+
+}

src/common/networkTools.cpp

@@ -1,209 +0,0 @@
-/*
- * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/networkTools.h>
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#else
-#include <net/if.h>
-#include <netdb.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-
-#include <curl/curl.h>
-
-#include <sstream>
-#include <iostream>
-
-
-std::map<std::string, std::string> kiwix::getNetworkInterfaces()
-{
-  std::map<std::string, std::string> interfaces;
-
-#ifdef _WIN32
-  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
-  if (sd == (SOCKET)SOCKET_ERROR) {
-    std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  INTERFACE_INFO InterfaceList[20];
-  unsigned long nBytesReturned;
-  if (WSAIoctl(sd,
-               SIO_GET_INTERFACE_LIST,
-               0,
-               0,
-               &InterfaceList,
-               sizeof(InterfaceList),
-               &nBytesReturned,
-               0,
-               0)
-      == SOCKET_ERROR) {
-    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
-  for (int i = 0; i < nNumInterfaces; ++i) {
-    sockaddr_in* pAddress;
-    pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
-
-    /* Add to the map */
-    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
-    std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
-    interfaces.insert(
-        std::pair<std::string, std::string>(interfaceName, interfaceIp));
-  }
-#else
-  /* Get Network interfaces information */
-  char buf[16384];
-  struct ifconf ifconf;
-  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
-  ifconf.ifc_len = sizeof buf;
-  ifconf.ifc_buf = buf;
-  if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
-    perror("ioctl(SIOCGIFCONF)");
-    exit(EXIT_FAILURE);
-  }
-
-  /* Go through each interface */
-  int i;
-  size_t len;
-  struct ifreq* ifreq;
-  ifreq = ifconf.ifc_req;
-  for (i = 0; i < ifconf.ifc_len;) {
-    if (ifreq->ifr_addr.sa_family == AF_INET) {
-      /* Get the network interface ip */
-      char host[128] = {0};
-      const int error = getnameinfo(&(ifreq->ifr_addr),
-                                    sizeof ifreq->ifr_addr,
-                                    host,
-                                    sizeof host,
-                                    0,
-                                    0,
-                                    NI_NUMERICHOST);
-      if (!error) {
-        std::string interfaceName = std::string(ifreq->ifr_name);
-        std::string interfaceIp = std::string(host);
-        /* Add to the map */
-        interfaces.insert(
-            std::pair<std::string, std::string>(interfaceName, interfaceIp));
-      } else {
-        perror("getnameinfo()");
-      }
-    }
-
-/* some systems have ifr_addr.sa_len and adjust the length that
- * way, but not mine. weird */
-#ifndef __linux__
-    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
-#else
-    len = sizeof *ifreq;
-#endif
-    ifreq = (struct ifreq*)((char*)ifreq + len);
-    i += len;
-  }
-#endif
-  return interfaces;
-}
-
-std::string kiwix::getBestPublicIp()
-{
-  std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
-
-#ifndef _WIN32
-  const char* const prioritizedNames[]
-      = {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
-  const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
-  for (int i = 0; i < count; ++i) {
-    std::map<std::string, std::string>::const_iterator it
-        = interfaces.find(prioritizedNames[i]);
-    if (it != interfaces.end()) {
-      return it->second;
-    }
-  }
-#endif
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
-      return interfaceIp;
-    }
-  }
-
-  return "127.0.0.1";
-}
-
-size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
-{
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
-  return nmemb;
-}
-
-std::string kiwix::download(const std::string& url) {
-  auto curl = curl_easy_init();
-  std::stringstream ss;
-  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-  curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
-  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);
-  auto res = curl_easy_perform(curl);
-  if (res != CURLE_OK) {
-    curl_easy_cleanup(curl);
-    throw std::runtime_error("Cannot perform request");
-  }
-  long response_code;
-  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-  curl_easy_cleanup(curl);
-  if (response_code != 200) {
-    throw std::runtime_error("Invalid return code from server");
-  }
-  return ss.str();
-}

src/common/otherTools.cpp

@@ -1,326 +0,0 @@
-/*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/otherTools.h>
-#include <map>
-
-static std::map<std::string, std::string> codeisomapping {
-//a
-  { "ad", "and" },
-  { "ae", "are" },
-  { "af", "afg" },
-  { "ag", "atg" },
-  { "ai", "aia" },
-  { "al", "alb" },
-  { "am", "arm" },
-  { "an", "ant" },
-  { "ao", "ago" },
-  { "aq", "ata" },
-  { "ar", "arg" },
-  { "as", "asm" },
-  { "at", "aut" },
-  { "au", "aus" },
-  { "aw", "abw" },
-  { "ax", "ala" },
-  { "az", "aze" },
-//b
-  { "ba", "bih" },
-  { "bb", "brb" },
-  { "bd", "bgd" },
-  { "be", "bel" },
-  { "bf", "bfa" },
-  { "bg", "bgr" },
-  { "bh", "bhr" },
-  { "bi", "bdi" },
-  { "bj", "ben" },
-  { "bl", "blm" },
-  { "bn", "brn" },
-  { "bm", "bmu" },
-  { "bo", "bol" },
-  { "br", "bra" },
-  { "bs", "bhs" },
-  { "bt", "btn" },
-  { "bv", "bvt" },
-  { "bw", "bwa" },
-  { "by", "blr" },
-  { "bz", "blz" },
-//c
-  { "ca", "can" },
-  { "cc", "cck" },
-  { "cd", "cod" },
-  { "cf", "caf" },
-  { "cg", "cog" },
-  { "ch", "che" },
-  { "ci", "civ" },
-  { "ck", "cok" },
-  { "cl", "chl" },
-  { "cm", "cmr" },
-  { "cn", "chn" },
-  { "co", "col" },
-  { "cr", "cri" },
-  { "cu", "cub" },
-  { "cv", "cpv" },
-  { "cx", "cxr" },
-  { "cy", "cyp" },
-  { "cz", "cze" },
-//d
-  { "de", "deu" },
-  { "dj", "dji" },
-  { "dk", "dnk" },
-  { "dm", "dma" },
-  { "do", "dom" },
-  { "dz", "dza" },
-//e
-  { "ec", "ecu" },
-  { "ee", "est" },
-  { "eg", "egy" },
-  { "eh", "esh" },
-  { "en", "eng" },
-  { "er", "eri" },
-  { "es", "esp" },
-  { "et", "eth" },
-//f
-  { "fi", "fin" },
-  { "fj", "fji" },
-  { "fk", "flk" },
-  { "fm", "fsm" },
-  { "fo", "fro" },
-  { "fr", "fra" },
-//g
-  { "ga", "gab" },
-  { "gb", "gbr" },
-  { "gd", "grd" },
-  { "ge", "geo" },
-  { "gf", "guf" },
-  { "gg", "ggy" },
-  { "gh", "gha" },
-  { "gi", "gib" },
-  { "gl", "grl" },
-  { "gm", "gmb" },
-  { "gn", "gin" },
-  { "gp", "glp" },
-  { "gq", "gnq" },
-  { "gr", "grc" },
-  { "gs", "sgs" },
-  { "gt", "gtm" },
-  { "gu", "gum" },
-  { "gw", "gnb" },
-  { "gy", "guy" },
-//h
-  { "hk", "hkg" },
-  { "hm", "hmd" },
-  { "hn", "hnd" },
-  { "hr", "hrv" },
-  { "ht", "hti" },
-  { "hu", "hun" },
-//i
-  { "id", "idn" },
-  { "ie", "irl" },
-  { "il", "isr" },
-  { "im", "imn" },
-  { "in", "ind" },
-  { "io", "iot" },
-  { "iq", "irq" },
-  { "ir", "irn" },
-  { "is", "isl" },
-  { "it", "ita" },
-//j
-  { "je", "jey" },
-  { "jm", "jam" },
-  { "jo", "jor" },
-  { "jp", "jpn" },
-//k
-  { "ke", "ken" },
-  { "kg", "kgz" },
-  { "kh", "khm" },
-  { "ki", "kir" },
-  { "km", "com" },
-  { "kn", "kna" },
-  { "kp", "prk" },
-  { "kr", "kor" },
-  { "kw", "kwt" },
-  { "ky", "cym" },
-  { "kz", "kaz" },
-//l
-  { "la", "lao" },
-  { "lb", "lbn" },
-  { "lc", "lca" },
-  { "li", "lie" },
-  { "lk", "lka" },
-  { "lr", "lbr" },
-  { "ls", "lso" },
-  { "lt", "ltu" },
-  { "lu", "lux" },
-  { "lv", "lva" },
-  { "ly", "lby" },
-//m
-  { "ma", "mar" },
-  { "mc", "mco" },
-  { "md", "mda" },
-  { "me", "mne" },
-  { "mf", "maf" },
-  { "mg", "mdg" },
-  { "mh", "mhl" },
-  { "mk", "mkd" },
-  { "ml", "mli" },
-  { "mm", "mmr" },
-  { "mn", "mng" },
-  { "mo", "mac" },
-  { "mp", "mnp" },
-  { "mq", "mtq" },
-  { "mr", "mrt" },
-  { "ms", "msr" },
-  { "mt", "mlt" },
-  { "mu", "mus" },
-  { "mv", "mdv" },
-  { "mw", "mwi" },
-  { "mx", "mex" },
-  { "my", "mys" },
-  { "mz", "moz" },
-//n
-  { "na", "nam" },
-  { "nc", "ncl" },
-  { "ne", "ner" },
-  { "nf", "nfk" },
-  { "ng", "nga" },
-  { "ni", "nic" },
-  { "nl", "nld" },
-  { "no", "nor" },
-  { "np", "npl" },
-  { "nr", "nru" },
-  { "nu", "niu" },
-  { "nz", "nzl" },
-//o
-  { "om", "omn" },
-//p
-  { "pa", "pan" },
-  { "pe", "per" },
-  { "pf", "pyf" },
-  { "pg", "png" },
-  { "ph", "phl" },
-  { "pk", "pak" },
-  { "pl", "pol" },
-  { "pm", "spm" },
-  { "pn", "pcn" },
-  { "pr", "pri" },
-  { "ps", "pse" },
-  { "pt", "prt" },
-  { "pw", "plw" },
-  { "py", "pry" },
-//q
-  { "qa", "qat" },
-//r
-  { "re", "reu" },
-  { "ro", "rou" },
-  { "rs", "srb" },
-  { "ru", "rus" },
-  { "rw", "rwa" },
-//s
-  { "sa", "sau" },
-  { "sb", "slb" },
-  { "sc", "syc" },
-  { "sd", "sdn" },
-  { "se", "swe" },
-  { "sg", "sgp" },
-  { "sh", "shn" },
-  { "si", "svn" },
-  { "sj", "sjm" },
-  { "sk", "svk" },
-  { "sl", "sle" },
-  { "sm", "smr" },
-  { "sn", "sen" },
-  { "so", "som" },
-  { "sr", "sur" },
-  { "ss", "ssd" },
-  { "st", "stp" },
-  { "sv", "slv" },
-  { "sy", "syr" },
-  { "sz", "swz" },
-//t
-  { "tc", "tca" },
-  { "td", "tcd" },
-  { "tf", "atf" },
-  { "tg", "tgo" },
-  { "th", "tha" },
-  { "tj", "tjk" },
-  { "tk", "tkl" },
-  { "tl", "tls" },
-  { "tm", "tkm" },
-  { "tn", "tun" },
-  { "to", "ton" },
-  { "tr", "tur" },
-  { "tt", "tto" },
-  { "tv", "tuv" },
-  { "tw", "twn" },
-  { "tz", "tza" },
-//u
-  { "ua", "ukr" },
-  { "ug", "uga" },
-  { "um", "umi" },
-  { "us", "usa" },
-  { "uy", "ury" },
-  { "uz", "uzb" },
-//v
-  { "va", "vat" },
-  { "vc", "vct" },
-  { "ve", "ven" },
-  { "vg", "vgb" },
-  { "vi", "vir" },
-  { "vn", "vnm" },
-  { "vu", "vut" },
-//w
-  { "wf", "wlf" },
-  { "ws", "wsm" },
-//y
-  { "ye", "yem" },
-  { "yt", "myt" },
-// z
-  { "za", "zaf" },
-  { "zm", "zmb" },
-  { "zw", "zwe" }
-};
-
-void kiwix::sleep(unsigned int milliseconds)
-{
-#ifdef _WIN32
-  Sleep(milliseconds);
-#else
-  usleep(1000 * milliseconds);
-#endif
-}
-
-
-struct XmlStringWriter: pugi::xml_writer
-{
-  std::string result;
-  virtual void write(const void* data, size_t size){
-    result.append(static_cast<const char*>(data), size);
-  }
-};
-
-std::string kiwix::nodeToString(pugi::xml_node node)
-{
-  XmlStringWriter writer;
-  node.print(writer, "  ");
-  return writer.result;
-}
-
-std::string kiwix::converta2toa3(const std::string& a2code){
-  return codeisomapping.at(a2code);
-}

src/common/pathTools.cpp

@@ -1,335 +0,0 @@
-/*
- * Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/pathTools.h>
-
-#ifdef __APPLE__
-#include <limits.h>
-#include <mach-o/dyld.h>
-#elif _WIN32
-#include <direct.h>
-#include <windows.h>
-#include "shlwapi.h"
-#define getcwd _getcwd  // stupid MSFT "deprecation" warning
-#endif
-
-#ifdef _WIN32
-#else
-#include <unistd.h>
-#endif
-
-#ifdef _WIN32
-const std::string SEPARATOR("\\");
-#else
-const std::string SEPARATOR("/");
-#include <unistd.h>
-#endif
-
-#include <stdlib.h>
-
-#ifndef PATH_MAX
-#define PATH_MAX 1024
-#endif
-
-bool isRelativePath(const string& path)
-{
-#ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
-#else
-  return path.empty() || path.substr(0, 1) == "/" ? false : true;
-#endif
-}
-
-string computeRelativePath(const string path, const string absolutePath)
-{
-  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
-  std::vector<std::string> absolutePathParts
-      = kiwix::split(absolutePath, SEPARATOR);
-
-  unsigned int commonCount = 0;
-  while (commonCount < pathParts.size()
-         && commonCount < absolutePathParts.size()
-         && pathParts[commonCount] == absolutePathParts[commonCount]) {
-      commonCount++;
-  }
-
-  string relativePath;
-#ifdef _WIN32
-  /* On Windows you have a token more because the root is represented
-     by a letter */
-  if (commonCount == 0) {
-    relativePath = ".." + SEPARATOR;
-  }
-#endif
-
-  for (unsigned int i = commonCount; i < pathParts.size(); i++) {
-    relativePath += ".." + SEPARATOR;
-  }
-  for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
-    relativePath += absolutePathParts[i];
-    relativePath += i + 1 < absolutePathParts.size() ? SEPARATOR : "";
-  }
-  return relativePath;
-}
-
-/* Warning: the relative path must be with slashes */
-string computeAbsolutePath(const string path, const string relativePath)
-{
-  string absolutePath;
-
-  if (path.empty()) {
-    char* path = NULL;
-    size_t size = 0;
-
-#ifdef _WIN32
-    path = _getcwd(path, size);
-#else
-    path = getcwd(path, size);
-#endif
-
-    absolutePath = string(path) + SEPARATOR;
-  } else {
-    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
-                       ? path
-                       : path + SEPARATOR;
-  }
-
-#if _WIN32
-  char* cRelativePath = _strdup(relativePath.c_str());
-#else
-  char* cRelativePath = strdup(relativePath.c_str());
-#endif
-  char* token = strtok(cRelativePath, "/");
-
-  while (token != NULL) {
-    if (string(token) == "..") {
-      absolutePath = removeLastPathElement(absolutePath, true, false);
-      token = strtok(NULL, "/");
-    } else if (strcmp(token, ".") && strcmp(token, "")) {
-      absolutePath += string(token);
-      token = strtok(NULL, "/");
-      if (token != NULL) {
-        absolutePath += SEPARATOR;
-      }
-    } else {
-      token = strtok(NULL, "/");
-    }
-  }
-
-  return absolutePath;
-}
-
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator,
-                             const bool removePostSeparator)
-{
-  string newPath = path;
-  size_t offset = newPath.find_last_of(SEPARATOR);
-  if (removePreSeparator &&
-#ifndef _WIN32
-      offset != newPath.find_first_of(SEPARATOR) &&
-#endif
-      offset == newPath.length() - 1) {
-    newPath = newPath.substr(0, offset);
-    offset = newPath.find_last_of(SEPARATOR);
-  }
-  newPath = removePostSeparator ? newPath.substr(0, offset)
-                                : newPath.substr(0, offset + 1);
-  return newPath;
-}
-
-string appendToDirectory(const string& directoryPath, const string& filename)
-{
-  string newPath = directoryPath + SEPARATOR + filename;
-  return newPath;
-}
-
-string getLastPathElement(const string& path)
-{
-  return path.substr(path.find_last_of(SEPARATOR) + 1);
-}
-
-unsigned int getFileSize(const string& path)
-{
-#ifdef _WIN32
-  struct _stat filestatus;
-  _stat(path.c_str(), &filestatus);
-#else
-  struct stat filestatus;
-  stat(path.c_str(), &filestatus);
-#endif
-
-  return filestatus.st_size / 1024;
-}
-
-string getFileSizeAsString(const string& path)
-{
-  ostringstream convert;
-  convert << getFileSize(path);
-  return convert.str();
-}
-
-string getFileContent(const string& path)
-{
-  std::ifstream f(path, std::ios::in|std::ios::ate);
-  std::string content;
-  if (f.is_open()) {
-    auto size = f.tellg();
-    content.reserve(size);
-    f.seekg(0, std::ios::beg);
-    content.assign((std::istreambuf_iterator<char>(f)),
-                    std::istreambuf_iterator<char>());
-  }
-  return content;
-}
-
-bool fileExists(const string& path)
-{
-#ifdef _WIN32
-  return PathFileExists(path.c_str());
-#else
-  bool flag = false;
-  fstream fin;
-  fin.open(path.c_str(), ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
-#endif
-}
-
-bool makeDirectory(const string& path)
-{
-#ifdef _WIN32
-  int status = _mkdir(path.c_str());
-#else
-  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-#endif
-  return status == 0;
-}
-
-string makeTmpDirectory()
-{
-#ifdef _WIN32
-  char cbase[MAX_PATH+1];
-  int base_len = GetTempPath(MAX_PATH+1, cbase);
-  UUID uuid;
-  UuidCreate(&uuid);
-  char* dir_name;
-  UuidToString(&uuid, reinterpret_cast<unsigned char**>(&dir_name));
-  string dir(cbase, base_len);
-  dir += dir_name;
-  _mkdir(dir.c_str());
-  RpcStringFree(reinterpret_cast<unsigned char**>(&dir_name));
-#else
-  string base = "/tmp";
-  auto _template = base + "/kiwix-lib_XXXXXX";
-  char* _template_array = new char[_template.size()+1];
-  memcpy(_template_array, _template.c_str(), _template.size());
-  string dir = mkdtemp(_template_array);
-  delete[] _template_array;
-#endif
-  return dir;
-}
-
-/* Try to create a link and if does not work then make a copy */
-bool copyFile(const string& sourcePath, const string& destPath)
-{
-  try {
-#ifndef _WIN32
-    if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
-#endif
-      std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
-      std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
-      outfile << infile.rdbuf();
-#ifndef _WIN32
-    }
-#endif
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return false;
-  }
-
-  return true;
-}
-
-string getExecutablePath()
-{
-  char binRootPath[PATH_MAX];
-
-#ifdef _WIN32
-  GetModuleFileName(NULL, binRootPath, PATH_MAX);
-  return std::string(binRootPath);
-#elif __APPLE__
-  uint32_t max = (uint32_t)PATH_MAX;
-  _NSGetExecutablePath(binRootPath, &max);
-  return std::string(binRootPath);
-#else
-  ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
-  if (size != -1) {
-    return std::string(binRootPath, size);
-  }
-#endif
-
-  return "";
-}
-
-bool writeTextFile(const string& path, const string& content)
-{
-  std::ofstream file;
-  file.open(path.c_str());
-  file << content;
-  file.close();
-  return true;
-}
-
-string getCurrentDirectory()
-{
-  char* a_cwd = getcwd(NULL, 0);
-  string s_cwd(a_cwd);
-  free(a_cwd);
-  return s_cwd;
-}
-
-string getDataDirectory()
-{
-#ifdef _WIN32
-  char* cDataDir = ::getenv("APPDATA");
-#else
-  char* cDataDir = ::getenv("KIWIX_DATA_DIR");
-#endif
-  std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (!dataDir.empty())
-    return dataDir;
-#ifdef _WIN32
-  cDataDir = ::getenv("USERPROFILE");
-  dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-#else
-  cDataDir = ::getenv("XDG_DATA_HOME");
-  dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (dataDir.empty()) {
-    cDataDir = ::getenv("HOME");
-    dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-    dataDir = appendToDirectory(dataDir, ".local");
-    dataDir = appendToDirectory(dataDir, "share");
-  }
-#endif
-  return appendToDirectory(dataDir, "kiwix");
-}

src/config.h.in

@@ -1,4 +1,3 @@
 
 #mesondefine VERSION
 
-#mesondefine ENABLE_CTPP2

src/ctpp2/CTPP2VMStringLoader.cpp

@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <ctpp2/CTPP2VMStringLoader.hpp>
-
-namespace CTPP // C++ Template Engine
-{
-
-//
-// Convert byte order
-//
-static void ConvertExecutable(VMExecutable * oCore)
-{
-	// Code entry point
-	oCore -> entry_point = Swap32(oCore -> entry_point);
-	// Offset of code segment
-	oCore -> code_offset = Swap32(oCore -> code_offset);
-
-	// Code segment size
-	oCore -> code_size   = Swap32(oCore -> code_size);
-
-	// Offset of static text segment
-	oCore -> syscalls_offset    = Swap32(oCore -> syscalls_offset);
-	// Static text segment size
-	oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
-
-	// Offset of static text index segment
-	oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
-	// Static text index segment size
-	oCore -> syscalls_index_size   = Swap32(oCore -> syscalls_index_size);
-
-	// Offset of static data segment
-	oCore -> static_data_offset    = Swap32(oCore -> static_data_offset);
-
-	// Static data segment size
-	oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
-
-	// Offset of static text segment
-	oCore -> static_text_offset    = Swap32(oCore -> static_text_offset);
-	// Static text segment size
-	oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
-
-	// Offset of static text index segment
-	oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
-	// Static text index segment size
-	oCore -> static_text_index_size   = Swap32(oCore -> static_text_index_size);
-
-	// Version 2.2+
-	// Offset of static data bit index
-	oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
-	/// Offset of static data bit index
-	oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
-
-	// Platform
-	oCore -> platform      = Swap64(oCore -> platform);
-
-	// Ugly-jolly hack!
-	// ... dereferencing type-punned pointer will break strict-aliasing rules ...
-	UINT_64 iTMP;
-	memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
-	iTMP = Swap64(iTMP);
-	memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
-
-	// Cyclic Redundancy Check
-	oCore -> crc = 0;
-
-	// Convert data structures
-
-	// Convert code segment
-	VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
-	UINT_32 iI = 0;
-	UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pInstructions -> instruction = Swap32(pInstructions -> instruction);
-		pInstructions -> argument    = Swap32(pInstructions -> argument);
-		pInstructions -> reserved    = Swap64(pInstructions -> reserved);
-		++pInstructions;
-	}
-
-	// Convert syscalls index
-	TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
-	iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static text index
-	pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
-	iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static data
-	StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
-	iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
-		++pStaticDataVar;
-	}
-}
-
-//
-// Constructor
-//
-VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
-{
-    oCore = (VMExecutable *)malloc(rawContentSize + 1);
-    memcpy(oCore, rawContent, rawContentSize);
-
-	if (oCore -> magic[0] == 'C' &&
-	    oCore -> magic[1] == 'T' &&
-	    oCore -> magic[2] == 'P' &&
-	    oCore -> magic[3] == 'P')
-	{
-		// Check version
-		if (oCore -> version[0] >= 1)
-		{
-			// Platform-dependent data (byte order)
-			if (oCore -> platform == 0x4142434445464748ull)
-			{
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
-#endif
-
-				// Nothing to do, only check crc
-				UINT_32 iCRC = oCore -> crc;
-				oCore -> crc = 0;
-
-				// Calculate CRC of file
-                // KELSON: next line used to refer to oStat.st_size
-                // changed it to rawContentSize
-				if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
-				{
-					free(oCore);
-					throw CTPPLogicError("CRC checksum invalid");
-				}
-			}
-			// Platform-dependent data (byte order)
-			else if (oCore -> platform == 0x4847464544434241ull)
-			{
-				// Need to reconvert data
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
-#endif
-				ConvertExecutable(oCore);
-			}
-			else
-			{
-				free(oCore);
-				throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
-			}
-
-			// Check IEEE 754 format
-			if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
-			{
-				free(oCore);
-				throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
-			}
-		}
-
-		pVMMemoryCore = new VMMemoryCore(oCore);
-	}
-	else
-	{
-		free(oCore);
-		throw CTPPLogicError("Not an CTPP bytecode file.");
-	}
-}
-
-//
-// Get ready-to-run program
-//
-const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
-
-//
-// A destructor
-//
-VMStringLoader::~VMStringLoader() throw()
-{
-	delete pVMMemoryCore;
-	free(oCore);
-}
-
-} // namespace CTPP
-// End.

src/downloader.cpp

@@ -18,7 +18,8 @@
  */
 
 #include "downloader.h"
-#include "common/pathTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 
 #include <algorithm>
 #include <thread>
@@ -28,7 +29,7 @@
 
 #include "aria2.h"
 #include "xmlrpc.h"
-#include "common/otherTools.h"
+#include "tools/otherTools.h"
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -36,6 +37,8 @@ namespace kiwix
 
 void Download::updateStatus(bool follow)
 {
+  if (m_status == Download::K_REMOVED)
+    return;
   static std::vector<std::string> statusKey = {"status", "files", "totalLength",
                                                "completedLength", "followedBy",
                                                "downloadSpeed", "verifiedLength"};
@@ -93,17 +96,55 @@ void Download::updateStatus(bool follow)
   }
 }
 
+void Download::resumeDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->unpause(m_followedBy);
+    else
+        mp_aria->unpause(m_did);
+    updateStatus(true);
+}
+
+void Download::pauseDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->pause(m_followedBy);
+    else
+        mp_aria->pause(m_did);
+    updateStatus(true);
+}
+
+void Download::cancelDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->remove(m_followedBy);
+    else
+        mp_aria->remove(m_did);
+    m_status = Download::K_REMOVED;
+}
+
 /* Constructor */
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  for (auto gid : mp_aria->tellActive()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what();
+  }
+  try {
+    for (auto gid : mp_aria->tellWaiting()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what();
   }
 }
 
-
 /* Destructor */
 Downloader::~Downloader()
 {
@@ -122,7 +163,7 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri)
+Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
@@ -131,7 +172,7 @@ Download* Downloader::startDownload(const std::string& uri)
       return d.get();
   }
   std::vector<std::string> uris = {uri};
-  auto gid = mp_aria->addUri(uris);
+  auto gid = mp_aria->addUri(uris, options);
   m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
   return m_knownDownloads[gid].get();
 }
@@ -139,14 +180,23 @@ Download* Downloader::startDownload(const std::string& uri)
 Download* Downloader::getDownload(const std::string& did)
 {
   try {
+    m_knownDownloads.at(did).get()->updateStatus(true);
     return m_knownDownloads.at(did).get();
-  } catch(exception& e) {
+  } catch(std::exception& e) {
     for (auto gid : mp_aria->tellActive()) {
       if (gid == did) {
         m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
         return m_knownDownloads[gid].get();
       }
     }
+    for (auto gid : mp_aria->tellWaiting()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
+        return m_knownDownloads[gid].get();
+      }
+  }
     throw e;
   }
 }

src/entry.cpp

@@ -122,7 +122,6 @@ Entry Entry::getFinalEntry() const
   if (final_article.good()) {
     return final_article;
   }
-
   int loopCounter = 42;
   final_article = article;
   while (final_article.isRedirect() && loopCounter--) {
@@ -131,7 +130,10 @@ Entry Entry::getFinalEntry() const
       throw NoEntry();
     }
   }
-
+  // Prevent infinite loops.
+  if (final_article.isRedirect()) {
+    throw NoEntry();
+  }
   return final_article;
 }
 

src/kiwixserve.cpp

@@ -0,0 +1,81 @@
+#include "kiwixserve.h"
+#include "subprocess.h"
+
+#ifdef _WIN32
+    # define KIWIXSERVE_CMD "kiwix-serve.exe"
+    # include <windows.h>
+#else
+    # define KIWIXSERVE_CMD "kiwix-serve"
+    # include <unistd.h>
+#endif
+
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+
+namespace kiwix {
+
+KiwixServe::KiwixServe(const std::string& libraryPath, int port)
+  : m_port(port),
+    m_libraryPath(libraryPath)
+{
+}
+
+KiwixServe::~KiwixServe()
+{
+    shutDown();
+}
+
+void KiwixServe::run()
+{
+    #ifdef _WIN32
+        int pid = GetCurrentProcessId();
+    #else
+        pid_t pid = getpid();
+
+    #endif
+
+    std::vector<const char*> callCmd;
+    std::string kiwixServeCmd = appendToDirectory(
+        removeLastPathElement(getExecutablePath(true)),
+        KIWIXSERVE_CMD);
+    if (fileExists(kiwixServeCmd)) {
+        // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
+        callCmd.push_back(kiwixServeCmd.c_str());
+    } else {
+        // Try to use a potential installed kiwix-serve.
+        callCmd.push_back(KIWIXSERVE_CMD);
+    }
+    std::string attachProcessOpt = "-a" + to_string(pid);
+    std::string portOpt = "-p" + to_string(m_port);
+    callCmd.push_back(attachProcessOpt.c_str());
+    callCmd.push_back(portOpt.c_str());
+    callCmd.push_back("-l");
+    callCmd.push_back(m_libraryPath.c_str());
+    mp_kiwixServe = Subprocess::run(callCmd);
+}
+
+void KiwixServe::shutDown()
+{
+    if (mp_kiwixServe)
+        mp_kiwixServe->kill();
+}
+
+bool KiwixServe::isRunning()
+{
+    if (mp_kiwixServe) {
+        return (mp_kiwixServe->isRunning());
+    }
+    return false;
+}
+
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
+}

src/library.cpp

@@ -19,18 +19,23 @@
 
 #include "library.h"
 #include "book.h"
+#include "reader.h"
+#include "libxml_dumper.h"
 
-#include "common/base64.h"
-#include "common/regexTools.h"
-#include "common/pathTools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
+#include <set>
 
 namespace kiwix
 {
+
 /* Constructor */
-Library::Library() : version(KIWIX_LIBRARY_VERSION)
+Library::Library()
 {
 }
 /* Destructor */
@@ -43,31 +48,74 @@ bool Library::addBook(const Book& book)
 {
   /* Try to find it */
   try {
-    auto& oldbook = books.at(book.getId());
+    auto& oldbook = m_books.at(book.getId());
     oldbook.update(book);
     return false;
   } catch (std::out_of_range&) {
-    books[book.getId()] = book;
+    m_books[book.getId()] = book;
     return true;
   }
 }
 
+void Library::addBookmark(const Bookmark& bookmark)
+{
+  m_bookmarks.push_back(bookmark);
+}
+
+bool Library::removeBookmark(const std::string& zimId, const std::string& url)
+{
+  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
+    if (it->getBookId() == zimId && it->getUrl() == url) {
+      m_bookmarks.erase(it);
+      return true;
+    }
+  }
+  return false;
+}
+
+
 
 bool Library::removeBookById(const std::string& id)
 {
-  return books.erase(id) == 1;
+  return m_books.erase(id) == 1;
 }
 
 Book& Library::getBookById(const std::string& id)
 {
-  return books.at(id);
+  return m_books.at(id);
+}
+
+Book& Library::getBookByPath(const std::string& path)
+{
+  for(auto& it: m_books) {
+    auto& book = it.second;
+    if (book.getPath() == path)
+      return book;
+  }
+  std::ostringstream ss;
+  ss << "No book with path " << path << " in the library." << std::endl;
+  throw std::out_of_range(ss.str());
+}
+
+std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
+{
+  try {
+    return m_readers.at(id);
+  } catch (std::out_of_range& e) {}
+
+  auto book = getBookById(id);
+  if (!book.isPathValid())
+    return nullptr;
+  auto sptr = make_shared<Reader>(book.getPath());
+  m_readers[id] = sptr;
+  return sptr;
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks)
 {
   unsigned int result = 0;
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     if ((!book.getPath().empty() && localBooks)
         || (book.getPath().empty() && remoteBooks)) {
@@ -77,92 +125,18 @@ unsigned int Library::getBookCount(const bool localBooks,
   return result;
 }
 
-bool Library::writeToFile(const std::string& path) {
-  pugi::xml_document doc;
-  auto baseDir = removeLastPathElement(path, true, false);
+bool Library::writeToFile(const std::string& path)
+{
+  auto baseDir = removeLastPathElement(path);
+  LibXMLDumper dumper(this);
+  dumper.setBaseDir(baseDir);
+  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
+}
 
-  /* Add the library node */
-  pugi::xml_node libraryNode = doc.append_child("library");
-
-  if (!version.empty())
-    libraryNode.append_attribute("version") = version.c_str();
-
-  /* Add each book */
-  for (auto& pair: books) {
-    auto& book = pair.second;
-    if (!book.readOnly()) {
-      pugi::xml_node bookNode = libraryNode.append_child("book");
-      bookNode.append_attribute("id") = book.getId().c_str();
-
-      if (!book.getPath().empty()) {
-        bookNode.append_attribute("path") = computeRelativePath(
-            baseDir, book.getPath()).c_str();
-      }
-
-      if (!book.getIndexPath().empty()) {
-        bookNode.append_attribute("indexPath") = computeRelativePath(
-            baseDir, book.getIndexPath()).c_str();
-        bookNode.append_attribute("indexType") = "xapian";
-      }
-
-      if (book.getOrigId().empty()) {
-        if (!book.getTitle().empty())
-          bookNode.append_attribute("title") = book.getTitle().c_str();
-
-        if (!book.getName().empty())
-          bookNode.append_attribute("name") = book.getName().c_str();
-
-        if (!book.getTags().empty())
-          bookNode.append_attribute("tags") = book.getTags().c_str();
-
-        if (!book.getDescription().empty())
-          bookNode.append_attribute("description") = book.getDescription().c_str();
-
-        if (!book.getLanguage().empty())
-          bookNode.append_attribute("language") = book.getLanguage().c_str();
-
-        if (!book.getCreator().empty())
-          bookNode.append_attribute("creator") = book.getCreator().c_str();
-
-        if (!book.getPublisher().empty())
-          bookNode.append_attribute("publisher") = book.getPublisher().c_str();
-
-        if (!book.getFavicon().empty())
-          bookNode.append_attribute("favicon") = base64_encode(book.getFavicon()).c_str();
-
-        if (!book.getFaviconMimeType().empty())
-          bookNode.append_attribute("faviconMimeType")
-              = book.getFaviconMimeType().c_str();
-      } else {
-        bookNode.append_attribute("origId") = book.getOrigId().c_str();
-      }
-
-      if (!book.getDate().empty()) {
-        bookNode.append_attribute("date") = book.getDate().c_str();
-      }
-
-      if (!book.getUrl().empty()) {
-        bookNode.append_attribute("url") = book.getUrl().c_str();
-      }
-
-      if (book.getArticleCount())
-        bookNode.append_attribute("articleCount") = to_string(book.getArticleCount()).c_str();
-
-      if (book.getMediaCount())
-        bookNode.append_attribute("mediaCount") = to_string(book.getMediaCount()).c_str();
-
-      if (book.getSize()) {
-        bookNode.append_attribute("size") = to_string(book.getSize()>>10).c_str();
-      }
-
-      if (!book.getDownloadId().empty()) {
-        bookNode.append_attribute("downloadId") = book.getDownloadId().c_str();
-      }
-    }
-  }
-
-  /* saving file */
-  return doc.save_file(path.c_str());
+bool Library::writeBookmarksToFile(const std::string& path)
+{
+  LibXMLDumper dumper(this);
+  return writeTextFile(path, dumper.dumpLibXMLBookmark());
 }
 
 std::vector<std::string> Library::getBooksLanguages()
@@ -170,7 +144,7 @@ std::vector<std::string> Library::getBooksLanguages()
   std::vector<std::string> booksLanguages;
   std::map<std::string, bool> booksLanguagesMap;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     auto& language = book.getLanguage();
     if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
@@ -189,7 +163,7 @@ std::vector<std::string> Library::getBooksCreators()
   std::vector<std::string> booksCreators;
   std::map<std::string, bool> booksCreatorsMap;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     auto& book = pair.second;
     auto& creator = book.getCreator();
     if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
@@ -208,7 +182,7 @@ std::vector<std::string> Library::getBooksPublishers()
   std::vector<std::string> booksPublishers;
   std::map<std::string, bool> booksPublishersMap;
 
-  for (auto& pair:books) {
+  for (auto& pair:m_books) {
     auto& book = pair.second;
     auto& publisher = book.getPublisher();
     if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
@@ -222,11 +196,26 @@ std::vector<std::string> Library::getBooksPublishers()
   return booksPublishers;
 }
 
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
+{
+  if (!onlyValidBookmarks) {
+    return m_bookmarks;
+  }
+  std::vector<kiwix::Bookmark> validBookmarks;
+  auto booksId = getBooksIds();
+  for(auto& bookmark:m_bookmarks) {
+    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
+      validBookmarks.push_back(bookmark);
+    }
+  }
+  return validBookmarks;
+}
+
 std::vector<std::string> Library::getBooksIds()
 {
   std::vector<std::string> bookIds;
 
-  for (auto& pair: books) {
+  for (auto& pair: m_books) {
     bookIds.push_back(pair.first);
   }
 
@@ -239,67 +228,104 @@ std::vector<std::string> Library::filter(const std::string& search)
     return getBooksIds();
   }
 
-  std::vector<std::string> bookIds;
-  for(auto& pair:books) {
-     auto& book = pair.second;
-     if (matchRegex(book.getTitle(), "\\Q" + search + "\\E")
-         || matchRegex(book.getDescription(), "\\Q" + search + "\\E")) {
-       bookIds.push_back(pair.first);
-     }
-  }
+  return filter(Filter().query(search));
+}
 
+
+std::vector<std::string> Library::filter(const Filter& filter)
+{
+  std::vector<std::string> bookIds;
+  for(auto& pair:m_books) {
+    auto book = pair.second;
+    if(filter.accept(book)) {
+      bookIds.push_back(pair.first);
+    }
+  }
   return bookIds;
 }
 
-template<supportedListSortBy sort>
-struct Comparator {
-  Library* lib;
-  Comparator(Library* lib) : lib(lib) {}
-
-  bool operator() (const std::string& id1, const std::string& id2) {
-    return get_keys(id1) < get_keys(id2);
-  }
-
-  std::string get_keys(const std::string& id);
-  unsigned int get_keyi(const std::string& id);
+template<supportedListSortBy SORT>
+struct KEY_TYPE {
+  typedef std::string TYPE;
 };
 
 template<>
-std::string Comparator<TITLE>::get_keys(const std::string& id)
+struct KEY_TYPE<SIZE> {
+  typedef size_t TYPE;
+};
+
+template<supportedListSortBy sort>
+class Comparator {
+  private:
+    Library* lib;
+    bool     ascending;
+
+    inline typename KEY_TYPE<sort>::TYPE get_key(const std::string& id);
+
+  public:
+    Comparator(Library* lib, bool ascending) : lib(lib), ascending(ascending) {}
+    inline bool operator() (const std::string& id1, const std::string& id2) {
+      if (ascending) {
+        return get_key(id1) < get_key(id2);
+      } else {
+        return get_key(id2) < get_key(id1);
+      }
+    }
+};
+
+template<>
+std::string Comparator<TITLE>::get_key(const std::string& id)
 {
   return lib->getBookById(id).getTitle();
 }
 
 template<>
-unsigned int Comparator<SIZE>::get_keyi(const std::string& id)
+size_t Comparator<SIZE>::get_key(const std::string& id)
 {
   return lib->getBookById(id).getSize();
 }
 
 template<>
-bool Comparator<SIZE>::operator() (const std::string& id1, const std::string& id2)
-{
-  return get_keyi(id1) < get_keyi(id2);
-}
-
-template<>
-std::string Comparator<DATE>::get_keys(const std::string& id)
+std::string Comparator<DATE>::get_key(const std::string& id)
 {
   return lib->getBookById(id).getDate();
 }
 
 template<>
-std::string Comparator<CREATOR>::get_keys(const std::string& id)
+std::string Comparator<CREATOR>::get_key(const std::string& id)
 {
   return lib->getBookById(id).getCreator();
 }
 
 template<>
-std::string Comparator<PUBLISHER>::get_keys(const std::string& id)
+std::string Comparator<PUBLISHER>::get_key(const std::string& id)
 {
   return lib->getBookById(id).getPublisher();
 }
 
+void Library::sort(std::vector<std::string>& bookIds, supportedListSortBy sort, bool ascending)
+{
+  switch(sort) {
+    case TITLE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
+      break;
+    case SIZE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<SIZE>(this, ascending));
+      break;
+    case DATE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<DATE>(this, ascending));
+      break;
+    case CREATOR:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<CREATOR>(this, ascending));
+      break;
+    case PUBLISHER:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<PUBLISHER>(this, ascending));
+      break;
+    default:
+      break;
+  }
+}
+
 
 std::vector<std::string> Library::listBooksIds(
     int mode,
@@ -308,59 +334,208 @@ std::vector<std::string> Library::listBooksIds(
     const std::string& language,
     const std::string& creator,
     const std::string& publisher,
+    const std::vector<std::string>& tags,
     size_t maxSize) {
 
-  std::vector<std::string> bookIds;
-  for(auto& pair:books) {
-    auto& book = pair.second;
-    auto local = !book.getPath().empty();
-    if (mode & LOCAL && !local)
-      continue;
-    if (mode & NOLOCAL && local)
-      continue;
-    auto valid = book.isPathValid();
-    if (mode & VALID && !valid)
-      continue;
-    if (mode & NOVALID && valid)
-      continue;
-    auto remote = !book.getUrl().empty();
-    if (mode & REMOTE && !remote)
-      continue;
-    if (mode & NOREMOTE && remote)
-      continue;
-    if (maxSize != 0 && book.getSize() > maxSize)
-      continue;
-    if (!language.empty() && book.getLanguage() != language)
-      continue;
-    if (!publisher.empty() && book.getPublisher() != publisher)
-      continue;
-    if (!creator.empty() && book.getCreator() != creator)
-      continue;
-    if (!search.empty() && !(matchRegex(book.getTitle(), "\\Q" + search + "\\E")
-                          || matchRegex(book.getDescription(), "\\Q" + search + "\\E")))
-      continue;
-    bookIds.push_back(pair.first);
-  }
+  Filter _filter;
+  if (mode & LOCAL)
+    _filter.local(true);
+  if (mode & NOLOCAL)
+    _filter.local(false);
+  if (mode & VALID)
+    _filter.valid(true);
+  if (mode & NOVALID)
+    _filter.valid(false);
+  if (mode & REMOTE)
+    _filter.remote(true);
+  if (mode & NOREMOTE)
+    _filter.remote(false);
+  if (!tags.empty())
+    _filter.acceptTags(tags);
+  if (maxSize != 0)
+    _filter.maxSize(maxSize);
+  if (!language.empty())
+    _filter.lang(language);
+  if (!publisher.empty())
+    _filter.publisher(publisher);
+  if (!creator.empty())
+    _filter.creator(creator);
+  if (!search.empty())
+    _filter.query(search);
 
-  switch(sortBy) {
-    case TITLE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this));
-      break;
-    case SIZE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<SIZE>(this));
-      break;
-    case DATE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<DATE>(this));
-      break;
-    case CREATOR:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<CREATOR>(this));
-      break;
-    case PUBLISHER:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<PUBLISHER>(this));
-      break;
-    default:
-      break;
-  }
+  auto bookIds = filter(_filter);
+
+  sort(bookIds, sortBy, true);
   return bookIds;
 }
+
+Filter::Filter()
+  : activeFilters(0),
+    _maxSize(0)
+{};
+
+#define FLAG(x) (1 << x)
+enum filterTypes {
+  NONE = 0,
+  _LOCAL = FLAG(0),
+  _REMOTE = FLAG(1),
+  _NOLOCAL = FLAG(2),
+  _NOREMOTE = FLAG(3),
+  _VALID = FLAG(4),
+  _NOVALID = FLAG(5),
+  ACCEPTTAGS = FLAG(6),
+  REJECTTAGS = FLAG(7),
+  LANG = FLAG(8),
+  _PUBLISHER = FLAG(9),
+  _CREATOR = FLAG(10),
+  MAXSIZE = FLAG(11),
+  QUERY = FLAG(12),
+  NAME = FLAG(13),
+};
+
+Filter& Filter::local(bool accept)
+{
+  if (accept) {
+    activeFilters |= _LOCAL;
+    activeFilters &= ~_NOLOCAL;
+  } else {
+    activeFilters |= _NOLOCAL;
+    activeFilters &= ~_LOCAL;
+  }
+  return *this;
+}
+
+Filter& Filter::remote(bool accept)
+{
+  if (accept) {
+    activeFilters |= _REMOTE;
+    activeFilters &= ~_NOREMOTE;
+  } else {
+    activeFilters |= _NOREMOTE;
+    activeFilters &= ~_REMOTE;
+  }
+  return *this;
+}
+
+Filter& Filter::valid(bool accept)
+{
+  if (accept) {
+    activeFilters |= _VALID;
+    activeFilters &= ~_NOVALID;
+  } else {
+    activeFilters |= _NOVALID;
+    activeFilters &= ~_VALID;
+  }
+  return *this;
+}
+
+Filter& Filter::acceptTags(std::vector<std::string> tags)
+{
+  _acceptTags = tags;
+  activeFilters |= ACCEPTTAGS;
+  return *this;
+}
+
+Filter& Filter::rejectTags(std::vector<std::string> tags)
+{
+  _rejectTags = tags;
+  activeFilters |= REJECTTAGS;
+  return *this;
+}
+
+Filter& Filter::lang(std::string lang)
+{
+  _lang = lang;
+  activeFilters |= LANG;
+  return *this;
+}
+
+Filter& Filter::publisher(std::string publisher)
+{
+  _publisher = publisher;
+  activeFilters |= _PUBLISHER;
+  return *this;
+}
+
+Filter& Filter::creator(std::string creator)
+{
+  _creator = creator;
+  activeFilters |= _CREATOR;
+  return *this;
+}
+
+Filter& Filter::maxSize(size_t maxSize)
+{
+  _maxSize = maxSize;
+  activeFilters |= MAXSIZE;
+  return *this;
+}
+
+Filter& Filter::query(std::string query)
+{
+  _query = query;
+  activeFilters |= QUERY;
+  return *this;
+}
+
+Filter& Filter::name(std::string name)
+{
+  _name = name;
+  activeFilters |= NAME;
+  return *this;
+}
+
+#define ACTIVE(X) (activeFilters & (X))
+#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
+bool Filter::accept(const Book& book) const
+{
+  auto local = !book.getPath().empty();
+  FILTER(_LOCAL, local)
+  FILTER(_NOLOCAL, !local)
+
+  auto valid = book.isPathValid();
+  FILTER(_VALID, valid)
+  FILTER(_NOVALID, !valid)
+
+  auto remote = !book.getUrl().empty();
+  FILTER(_REMOTE, remote)
+  FILTER(_NOREMOTE, !remote)
+
+  FILTER(MAXSIZE, book.getSize() <= _maxSize)
+  FILTER(LANG, book.getLanguage() == _lang)
+  FILTER(_PUBLISHER, book.getPublisher() == _publisher)
+  FILTER(_CREATOR, book.getCreator() == _creator)
+  FILTER(NAME, book.getName() == _name)
+
+  if (ACTIVE(ACCEPTTAGS)) {
+    if (!_acceptTags.empty()) {
+      auto vBookTags = split(book.getTags(), ";");
+      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
+      for (auto& t: _acceptTags) {
+        if (sBookTags.find(t) == sBookTags.end()) {
+          return false;
+        }
+      }
+    }
+  }
+  if (ACTIVE(REJECTTAGS)) {
+    if (!_rejectTags.empty()) {
+      auto vBookTags = split(book.getTags(), ";");
+      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
+      for (auto& t: _rejectTags) {
+        if (sBookTags.find(t) != sBookTags.end()) {
+          return false;
+        }
+      }
+    }
+  }
+  if ( ACTIVE(QUERY)
+    && !(matchRegex(book.getTitle(), "\\Q" + _query + "\\E")
+        || matchRegex(book.getDescription(), "\\Q" + _query + "\\E")))
+    return false;
+
+  return true;
+
+}
+
 }

src/libxml_dumper.cpp

@@ -0,0 +1,142 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "libxml_dumper.h"
+#include "book.h"
+
+#include "tools/base64.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibXMLDumper::LibXMLDumper(Library* library)
+  : library(library)
+{
+}
+/* Destructor */
+LibXMLDumper::~LibXMLDumper()
+{
+}
+
+#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
+#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
+
+void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
+  if (book.readOnly())
+    return;
+
+  auto entry_node = root_node.append_child("book");
+  ADD_ATTRIBUTE(entry_node, "id", book.getId());
+
+  if (!book.getPath().empty()) {
+    ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
+  }
+
+  if (book.getOrigId().empty()) {
+    ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
+    ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
+    ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
+    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
+    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
+    if (!book.getFavicon().empty())
+      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
+  } else {
+    ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
+  }
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
+  ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
+
+  if (book.getArticleCount())
+    ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
+
+  if (book.getMediaCount())
+    ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
+
+  if (book.getSize())
+    ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
+}
+
+#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
+
+void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
+
+  auto entry_node = root_node.append_child("bookmark");
+  auto book_node = entry_node.append_child("book");
+
+  try {
+    auto book = library->getBookById(bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "id", book.getId());
+    ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
+    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", book.getDate());
+  } catch (...) {
+    ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
+    ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
+  }
+  ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
+  ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
+}
+
+
+std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node libraryNode = doc.append_child("library");
+
+  libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
+
+  if (library) {
+    for (auto& bookId: bookIds) {
+      handleBook(library->getBookById(bookId), libraryNode);
+    }
+  }
+  return nodeToString(libraryNode);
+}
+
+std::string LibXMLDumper::dumpLibXMLBookmark()
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
+
+  if (library) {
+    for (auto& bookmark: library->getBookmarks()) {
+      handleBookmark(bookmark, bookmarksNode);
+    }
+  }
+  return nodeToString(bookmarksNode);
+}
+
+}

src/manager.cpp

@@ -19,6 +19,8 @@
 
 #include "manager.h"
 
+#include "tools/pathTools.h"
+
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -46,8 +48,9 @@ Manager::~Manager()
   }
 }
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          const bool readOnly,
-                          const std::string& libraryPath)
+                          bool readOnly,
+                          const std::string& libraryPath,
+                          bool trustLibrary)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
@@ -59,14 +62,10 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 
     book.setReadOnly(readOnly);
     book.updateFromXml(bookNode,
-                       removeLastPathElement(libraryPath, true, false));
+                       removeLastPathElement(libraryPath));
 
-    /* Update the book properties with the new importer */
-    if (libraryVersion.empty()
-        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.getPath().empty()) {
-        this->readBookFromPath(book.getPath(), &book);
-      }
+    if (!trustLibrary && !book.getPath().empty()) {
+      this->readBookFromPath(book.getPath(), &book);
     }
     manipulator->addBookToLibrary(book);
   }
@@ -75,15 +74,16 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 }
 
 bool Manager::readXml(const std::string& xml,
-                      const bool readOnly,
-                      const std::string& libraryPath)
+                      bool readOnly,
+                      const std::string& libraryPath,
+                      bool trustLibrary)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
       = doc.load_buffer_inplace((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath);
+    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
   }
 
   return true;
@@ -134,21 +134,22 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
   return false;
 }
 
-bool Manager::readFile(const std::string& path, const bool readOnly)
-{
-  return this->readFile(path, path, readOnly);
-}
-
-bool Manager::readFile(const std::string& nativePath,
-                       const std::string& UTF8Path,
-                       const bool readOnly)
+bool Manager::readFile(
+  const std::string& path,
+  bool readOnly,
+  bool trustLibrary)
 {
   bool retVal = true;
   pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
+
+#ifdef _WIN32
+  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
+#else
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+#endif
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, UTF8Path);
+    this->parseXmlDom(doc, readOnly, path, trustLibrary);
   } else {
     retVal = false;
   }
@@ -157,7 +158,7 @@ bool Manager::readFile(const std::string& nativePath,
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = UTF8Path;
+    this->writableLibraryPath = path;
   }
 
   return retVal;
@@ -174,10 +175,10 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (pathToSave != pathToOpen) {
+    if (!pathToSave.empty() && pathToSave != pathToOpen) {
       book.setPath(isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
+                      removeLastPathElement(writableLibraryPath),
                       pathToSave)
                 : pathToSave);
     }
@@ -208,12 +209,15 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
 
 bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
+  std::string tmp_path = path;
+  if (isRelativePath(path)) {
+    tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
+  }
   try {
-    kiwix::Reader reader(path);
+    kiwix::Reader reader(tmp_path);
     book->update(reader);
     book->setPathValid(true);
   } catch (const std::exception& e) {
-    std::cerr << "Invalid " << path << " : " << e.what() << std::endl;
     book->setPathValid(false);
     return false;
   }
@@ -221,4 +225,27 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
   return true;
 }
 
+bool Manager::readBookmarkFile(const std::string& path)
+{
+  pugi::xml_document doc;
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+
+  if (!result) {
+    return false;
+  }
+
+  pugi::xml_node libraryNode = doc.child("bookmarks");
+
+  for (pugi::xml_node node = libraryNode.child("bookmark"); node;
+       node = node.next_sibling("bookmark")) {
+    kiwix::Bookmark bookmark;
+
+    bookmark.updateFromXml(node);
+
+    manipulator->addBookmarkToLibrary(bookmark);
+  }
+
+  return true;
+}
+
 }

src/meson.build

@@ -1,22 +1,30 @@
 kiwix_sources = [
   'book.cpp',
+  'bookmark.cpp',
   'library.cpp',
   'manager.cpp',
+  'libxml_dumper.cpp',
   'opds_dumper.cpp',
   'downloader.cpp',
   'reader.cpp',
   'entry.cpp',
+  'server.cpp',
   'searcher.cpp',
+  'search_renderer.cpp',
   'subprocess.cpp',
   'aria2.cpp',
-  'common/base64.cpp',
-  'common/pathTools.cpp',
-  'common/regexTools.cpp',
-  'common/stringTools.cpp',
-  'common/networkTools.cpp',
-  'common/otherTools.cpp',
-  'xapian/htmlparse.cc',
-  'xapian/myhtmlparse.cc'
+  'tools/base64.cpp',
+  'tools/pathTools.cpp',
+  'tools/regexTools.cpp',
+  'tools/stringTools.cpp',
+  'tools/networkTools.cpp',
+  'tools/otherTools.cpp',
+  'kiwixserve.cpp',
+  'name_mapper.cpp',
+  'server/byte_range.cpp',
+  'server/etag.cpp',
+  'server/request_context.cpp',
+  'server/response.cpp'
 ]
 kiwix_sources += lib_resources
 
@@ -26,20 +34,14 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if xapian_dep.found()
-  kiwix_sources += ['xapianSearcher.cpp']
-endif
-
-if get_option('android')
-  subdir('android')
+if 'android' in wrapper
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
-
-if has_ctpp2_dep
-  kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
+if 'android' in wrapper or 'java' in wrapper
+  subdir('wrapper/java')
 endif
 
 config_h = configure_file(output : 'kiwix_config.h',
@@ -51,7 +53,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
-                   install_dir: install_dir,
-                   install_rpath: '$ORIGIN')
+                   install_dir: install_dir)

src/name_mapper.cpp

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "name_mapper.h"
+#include "library.h"
+#include "tools/regexTools.h"
+#include <iostream>
+
+namespace kiwix {
+
+HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
+  for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = library.getBookById(bookId);
+    auto bookName = currentBook.getHumanReadableIdFromPath();
+    m_idToName[bookId] = bookName;
+    m_nameToId[bookName] = bookId;
+
+    if (!withAlias)
+      continue;
+
+    auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
+    if (aliasName == bookName) {
+      continue;
+    }
+    if (m_nameToId.find(aliasName) == m_nameToId.end()) {
+      m_nameToId[aliasName] = bookId;
+    } else {
+      auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
+      std::cerr << "Path collision: " << alreadyPresentPath
+                << " and " << currentBook.getPath()
+                << " can't share the same URL path '" << aliasName << "'."
+                << " Therefore, only " << alreadyPresentPath
+                << " will be served." << std::endl;
+    }
+  }
+}
+
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
+  return m_idToName.at(id);
+}
+
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
+  return m_nameToId.at(name);
+}
+
+}

src/opds_dumper.cpp

@@ -20,7 +20,8 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include <common/otherTools.h>
+#include "tools/otherTools.h"
+#include <iomanip>
 
 namespace kiwix
 {
@@ -50,6 +51,13 @@ std::string gen_date_str()
   return is.str();
 }
 
+static std::string gen_date_from_yyyy_mm_dd(const std::string& date)
+{
+  std::stringstream is;
+  is << date << "T00:00::00:Z";
+  return is.str();
+}
+
 void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 {
   m_totalResults = totalResults;
@@ -62,11 +70,17 @@ void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 
 pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-  ADD_TEXT_ENTRY(entry_node, "updated", date);
+  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
+  ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage());
+  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
+  ADD_TEXT_ENTRY(entry_node, "name", book.getName());
+  ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
+  ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
+  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
+  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
+  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
 
   auto content_node = entry_node.append_child("link");
   content_node.append_attribute("type") = "text/html";
@@ -75,6 +89,9 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto author_node = entry_node.append_child("author");
   ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
 
+  auto publisher_node = entry_node.append_child("publisher");
+  ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher());
+
   if (! book.getUrl().empty()) {
     auto acquisition_link = entry_node.append_child("link");
     acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";

src/reader.cpp

@@ -22,6 +22,8 @@
 
 #include <zim/search.h>
 
+#include "tools/otherTools.h"
+
 inline char hi(char v)
 {
   char hex[] = "0123456789abcdef";
@@ -241,8 +243,6 @@ Entry Reader::getMainPage() const
     throw NoEntry();
   }
 
-  string url = "";
-
   zim::Article article;
   if (this->zimFileHandler->getFileheader().hasMainPage())
   {
@@ -260,7 +260,7 @@ Entry Reader::getMainPage() const
 
 bool Reader::getFavicon(string& content, string& mimeType) const
 {
-  static const char* const paths[] = {"-/favicon.png", "I/favicon.png", "I/favicon", "-/favicon"};
+  static const char* const paths[] = {"-/favicon", "-/favicon.png", "I/favicon.png", "I/favicon"};
 
   for (auto &path: paths) {
     try {
@@ -280,7 +280,7 @@ string Reader::getZimFilePath() const
   return this->zimFilePath;
 }
 /* Return a metatag value */
-bool Reader::getMetatag(const string& name, string& value) const
+bool Reader::getMetadata(const string& name, string& value) const
 {
   try {
     auto entry = getEntryFromPath("M/"+name);
@@ -291,10 +291,17 @@ bool Reader::getMetatag(const string& name, string& value) const
   }
 }
 
+#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
+
+string Reader::getName() const
+{
+  METADATA("Name")
+}
+
 string Reader::getTitle() const
 {
   string value;
-  this->getMetatag("Title", value);
+  this->getMetadata("Title", value);
   if (value.empty()) {
     value = getLastPathElement(zimFileHandler->getFilename());
     std::replace(value.begin(), value.end(), '_', ' ');
@@ -304,65 +311,98 @@ string Reader::getTitle() const
   return value;
 }
 
-string Reader::getName() const
+string Reader::getCreator() const
 {
-  string value;
-  this->getMetatag("Name", value);
-  return value;
+  METADATA("Creator")
 }
 
-string Reader::getTags() const
+string Reader::getPublisher() const
 {
-  string value;
-  this->getMetatag("Tags", value);
-  return value;
+  METADATA("Publisher")
+}
+
+string Reader::getDate() const
+{
+  METADATA("Date")
 }
 
 string Reader::getDescription() const
 {
   string value;
-  this->getMetatag("Description", value);
+  this->getMetadata("Description", value);
 
   /* Mediawiki Collection tends to use the "Subtitle" name */
   if (value.empty()) {
-    this->getMetatag("Subtitle", value);
+    this->getMetadata("Subtitle", value);
   }
 
   return value;
 }
 
+string Reader::getLongDescription() const
+{
+  METADATA("LongDescription")
+}
+
 string Reader::getLanguage() const
 {
-  string value;
-  this->getMetatag("Language", value);
-  return value;
+  METADATA("Language")
 }
 
-string Reader::getDate() const
+string Reader::getLicense() const
 {
-  string value;
-  this->getMetatag("Date", value);
-  return value;
+  METADATA("License")
 }
 
-string Reader::getCreator() const
+string Reader::getTags(bool original) const
 {
-  string value;
-  this->getMetatag("Creator", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  if (original) {
+    return tags_str;
+  }
+  auto tags = convertTags(tags_str);
+  return join(tags, ";");
 }
 
-string Reader::getPublisher() const
+
+string Reader::getTagStr(const std::string& tagName) const
 {
-  string value;
-  this->getMetatag("Publisher", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  return getTagValueFromTagList(convertTags(tags_str), tagName);
 }
 
+bool Reader::getTagBool(const std::string& tagName) const
+{
+  return convertStrToBool(getTagStr(tagName));
+}
+
+string Reader::getRelation() const
+{
+  METADATA("Relation")
+}
+
+string Reader::getFlavour() const
+{
+  METADATA("Flavour")
+}
+
+string Reader::getSource() const
+{
+  METADATA("Source")
+}
+
+string Reader::getScraper() const
+{
+  METADATA("Scraper")
+}
+#undef METADATA
+
 string Reader::getOrigId() const
 {
   string value;
-  this->getMetatag("startfileuid", value);
+  this->getMetadata("startfileuid", value);
   if (value.empty()) {
     return "";
   }
@@ -674,7 +714,6 @@ bool Reader::searchSuggestions(const string& prefix,
                                const bool reset)
 {
   bool retVal = false;
-  zim::File::const_iterator articleItr;
 
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
@@ -692,7 +731,7 @@ bool Reader::searchSuggestions(const string& prefix,
     return false;
   }
 
-  for (articleItr = zimFileHandler->findByTitle('A', prefix);
+  for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
        && this->suggestions.size() < suggestionsCount;
@@ -762,18 +801,21 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal;
+  bool retVal = false;
 
   this->suggestions.clear();
   this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
-  const zim::Search* suggestionSearch
+  const auto suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
 
   if (suggestionSearch->get_matches_estimated()) {
     for (auto current = suggestionSearch->begin();
          current != suggestionSearch->end();
          current++) {
+      if (!current->good()) {
+          continue;
+      }
       std::vector<std::string> suggestion;
       suggestion.push_back(current->getTitle());
       suggestion.push_back("/A/" + current->getUrl());

src/search_renderer.cpp

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <cmath>
+
+#include "search_renderer.h"
+#include "searcher.h"
+#include "reader.h"
+#include "library.h"
+#include "name_mapper.h"
+
+#include <zim/search.h>
+
+#include <mustache.hpp>
+#include "kiwixlib-resources.h"
+
+
+namespace kiwix
+{
+
+/* Constructor */
+SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
+    : mp_searcher(searcher),
+      mp_nameMapper(mapper),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://?")
+{}
+
+/* Destructor */
+SearchRenderer::~SearchRenderer() = default;
+
+void SearchRenderer::setSearchPattern(const std::string& pattern)
+{
+  this->searchPattern = pattern;
+}
+
+void SearchRenderer::setSearchContent(const std::string& name)
+{
+  this->searchContent = name;
+}
+
+void SearchRenderer::setProtocolPrefix(const std::string& prefix)
+{
+  this->protocolPrefix = prefix;
+}
+
+void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
+{
+  this->searchProtocolPrefix = prefix;
+}
+
+std::string SearchRenderer::getHtml()
+{
+  kainjow::mustache::data results{kainjow::mustache::data::type::list};
+
+  mp_searcher->restart_search();
+  Result* p_result = NULL;
+  while ((p_result = mp_searcher->getNextResult())) {
+    kainjow::mustache::data result;
+    result.set("title", p_result->get_title());
+    result.set("url", p_result->get_url());
+    result.set("snippet", p_result->get_snippet());
+    auto readerIndex = p_result->get_readerIndex();
+    auto reader = mp_searcher->get_reader(readerIndex);
+    result.set("resultContentId", mp_nameMapper->getNameForId(reader->getId()));
+
+    if (p_result->get_wordCount() >= 0) {
+      result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
+    }
+
+    results.push_back(result);
+    delete p_result;
+  }
+
+  // pages
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  auto resultStart = mp_searcher->getResultStart();
+  auto resultEnd = mp_searcher->getResultEnd();
+  auto resultCountPerPage = resultEnd - resultStart;
+  auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
+  auto currentPage = 0U;
+  auto pageStart = 0U;
+  auto pageEnd = 0U;
+  auto lastPageStart = 0U;
+  if (resultCountPerPage) {
+    currentPage = resultStart/resultCountPerPage;
+    pageStart = currentPage > 4 ? currentPage-4 : 0;
+    pageEnd = currentPage + 5;
+    if (pageEnd > estimatedResultCount / resultCountPerPage) {
+      pageEnd = estimatedResultCount / resultCountPerPage;
+    }
+    if (estimatedResultCount > resultCountPerPage) {
+      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
+    }
+  }
+
+  for (unsigned int i = pageStart; i < pageEnd; i++) {
+    kainjow::mustache::data page;
+    page.set("label", to_string(i + 1));
+    page.set("start", to_string(i * resultCountPerPage));
+    page.set("end", to_string((i + 1) * resultCountPerPage));
+
+    if (i == currentPage) {
+      page.set("selected", true);
+    }
+    pages.push_back(page);
+  }
+
+  std::string template_str = RESOURCE::templates::search_result_html;
+  kainjow::mustache::mustache tmpl(template_str);
+
+  kainjow::mustache::data allData;
+  allData.set("results", results);
+  allData.set("pages", pages);
+  allData.set("hasResults", estimatedResultCount != 0);
+  allData.set("hasPages", pageStart != pageEnd);
+  allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
+  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
+  allData.set("resultStart", to_string(resultStart + 1));
+  allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
+  allData.set("resultRange", to_string(resultCountPerPage));
+  allData.set("resultLastPageStart", to_string(lastPageStart));
+  allData.set("lastResult", to_string(estimatedResultCount));
+  allData.set("protocolPrefix", this->protocolPrefix);
+  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
+  allData.set("contentId", this->searchContent);
+
+  std::stringstream ss;
+  tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}
+
+}

src/searcher.cpp

@@ -22,20 +22,12 @@
 
 #include "searcher.h"
 #include "reader.h"
-#include "xapianSearcher.h"
 
 #include <zim/search.h>
 
-#ifdef ENABLE_CTPP2
-#include <ctpp2/CDT.hpp>
-#include <ctpp2/CTPP2FileLogger.hpp>
-#include <ctpp2/CTPP2SimpleVM.hpp>
-#include "ctpp2/CTPP2VMStringLoader.hpp"
+#include <mustache.hpp>
 #include "kiwixlib-resources.h"
 
-using namespace CTPP;
-#endif
-
 #define MAX_SEARCH_LEN 140
 
 namespace kiwix
@@ -61,52 +53,24 @@ class _Result : public Result
 
 struct SearcherInternal {
   const zim::Search* _search;
-  XapianSearcher* _xapianSearcher;
   zim::Search::iterator current_iterator;
 
-  SearcherInternal() : _search(NULL), _xapianSearcher(NULL) {}
+  SearcherInternal() : _search(NULL) {}
   ~SearcherInternal()
   {
     if (_search != NULL) {
       delete _search;
     }
-    if (_xapianSearcher != NULL) {
-      delete _xapianSearcher;
-    }
   }
 };
 
 /* Constructor */
-Searcher::Searcher(const string& xapianDirectoryPath,
-                   Reader* reader,
-                   const string& humanReadableName)
+Searcher::Searcher()
     : internal(new SearcherInternal()),
       searchPattern(""),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      resultCountPerPage(0),
       estimatedResultCount(0),
       resultStart(0),
-      resultEnd(0),
-      contentHumanReadableId(humanReadableName)
-{
-  loadICUExternalTables();
-  if (!reader || !reader->hasFulltextIndex()) {
-    internal->_xapianSearcher = new XapianSearcher(xapianDirectoryPath, reader);
-  }
-  this->humanReaderNames.push_back(humanReadableName);
-}
-
-Searcher::Searcher(const std::string& humanReadableName)
-    : internal(new SearcherInternal()),
-      searchPattern(""),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      resultCountPerPage(0),
-      estimatedResultCount(0),
-      resultStart(0),
-      resultEnd(0),
-      contentHumanReadableId(humanReadableName)
+      resultEnd(0)
 {
   loadICUExternalTables();
 }
@@ -117,18 +81,23 @@ Searcher::~Searcher()
   delete internal;
 }
 
-bool Searcher::add_reader(Reader* reader, const std::string& humanReadableName)
+bool Searcher::add_reader(Reader* reader)
 {
   if (!reader->hasFulltextIndex()) {
       return false;
   }
   this->readers.push_back(reader);
-  this->humanReaderNames.push_back(humanReadableName);
   return true;
 }
 
+
+Reader* Searcher::get_reader(int readerIndex)
+{
+  return readers.at(readerIndex);
+}
+
 /* Search strings in the database */
-void Searcher::search(std::string& search,
+void Searcher::search(const std::string& search,
                       unsigned int resultStart,
                       unsigned int resultEnd,
                       const bool verbose)
@@ -139,47 +108,27 @@ void Searcher::search(std::string& search,
     cout << "Performing query `" << search << "'" << endl;
   }
 
-  /* If resultEnd & resultStart inverted */
-  if (resultStart > resultEnd) {
-    resultEnd += resultStart;
-    resultStart = resultEnd - resultStart;
-    resultEnd -= resultStart;
-  }
-
+  this->searchPattern = search;
+  this->resultStart = resultStart;
+  this->resultEnd = resultEnd;
   /* Try to find results */
   if (resultStart != resultEnd) {
-    /* Avoid big researches */
-    this->resultCountPerPage = resultEnd - resultStart;
-    if (this->resultCountPerPage > MAX_SEARCH_LEN) {
-      resultEnd = resultStart + MAX_SEARCH_LEN;
-      this->resultCountPerPage = MAX_SEARCH_LEN;
-    }
-
     /* Perform the search */
-    this->searchPattern = search;
-    this->resultStart = resultStart;
-    this->resultEnd = resultEnd;
     string unaccentedSearch = removeAccents(search);
-    if (internal->_xapianSearcher) {
-      internal->_xapianSearcher->searchInIndex(
-          unaccentedSearch, resultStart, resultEnd, verbose);
-      this->estimatedResultCount
-          = internal->_xapianSearcher->results.get_matches_estimated();
-    } else {
-      std::vector<const zim::File*> zims;
-      for (auto current = this->readers.begin(); current != this->readers.end();
-           current++) {
-        if ( (*current)->hasFulltextIndex() ) {
-            zims.push_back((*current)->getZimFileHandler());
-        }
+    std::vector<const zim::File*> zims;
+    for (auto current = this->readers.begin(); current != this->readers.end();
+         current++) {
+      if ( (*current)->hasFulltextIndex() ) {
+          zims.push_back((*current)->getZimFileHandler());
       }
-      zim::Search* search = new zim::Search(zims);
-      search->set_query(unaccentedSearch);
-      search->set_range(resultStart, resultEnd);
-      internal->_search = search;
-      internal->current_iterator = internal->_search->begin();
-      this->estimatedResultCount = internal->_search->get_matches_estimated();
     }
+    zim::Search* search = new zim::Search(zims);
+    search->set_verbose(verbose);
+    search->set_query(unaccentedSearch);
+    search->set_range(resultStart, resultEnd);
+    internal->_search = search;
+    internal->current_iterator = internal->_search->begin();
+    this->estimatedResultCount = internal->_search->get_matches_estimated();
   }
 
   return;
@@ -197,29 +146,6 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
   }
 
-  /* If resultEnd & resultStart inverted */
-  if (resultStart > resultEnd) {
-    resultEnd += resultStart;
-    resultStart = resultEnd - resultStart;
-    resultEnd -= resultStart;
-  }
-
-  /* Try to find results */
-  if (resultStart == resultEnd) {
-    return;
-  }
-
-  if (internal->_xapianSearcher) {
-    return;
-  }
-
-  /* Avoid big researches */
-  this->resultCountPerPage = resultEnd - resultStart;
-  if (this->resultCountPerPage > MAX_SEARCH_LEN) {
-    resultEnd = resultStart + MAX_SEARCH_LEN;
-    this->resultCountPerPage = MAX_SEARCH_LEN;
-  }
-
   /* Perform the search */
   std::ostringstream oss;
   oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
@@ -227,12 +153,18 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
   this->resultStart = resultStart;
   this->resultEnd = resultEnd;
 
+  /* Try to find results */
+  if (resultStart == resultEnd) {
+    return;
+  }
+
   std::vector<const zim::File*> zims;
   for (auto current = this->readers.begin(); current != this->readers.end();
        current++) {
     zims.push_back((*current)->getZimFileHandler());
   }
   zim::Search* search = new zim::Search(zims);
+  search->set_verbose(verbose);
   search->set_query("");
   search->set_georange(latitude, longitude, distance);
   search->set_range(resultStart, resultEnd);
@@ -244,18 +176,14 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
 
 void Searcher::restart_search()
 {
-  if (internal->_xapianSearcher) {
-    internal->_xapianSearcher->restart_search();
-  } else if (internal->_search) {
+  if (internal->_search) {
     internal->current_iterator = internal->_search->begin();
   }
 }
 
 Result* Searcher::getNextResult()
 {
-  if (internal->_xapianSearcher) {
-    return internal->_xapianSearcher->getNextResult();
-  } else if (internal->_search &&
+  if (internal->_search &&
              internal->current_iterator != internal->_search->end()) {
     Result* result = new _Result(internal->current_iterator);
     internal->current_iterator++;
@@ -272,37 +200,32 @@ void Searcher::reset()
   return;
 }
 
-void Searcher::suggestions(std::string& search, const bool verbose)
+void Searcher::suggestions(std::string& searchPattern, const bool verbose)
 {
   this->reset();
 
   if (verbose == true) {
-    cout << "Performing suggestion query `" << search << "`" << endl;
+    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
   }
 
-  this->searchPattern = search;
+  this->searchPattern = searchPattern;
   this->resultStart = 0;
   this->resultEnd = 10;
-  string unaccentedSearch = removeAccents(search);
+  string unaccentedSearch = removeAccents(searchPattern);
 
-  if (internal->_xapianSearcher) {
-    /* [TODO] Suggestion on a external database ?
-     * We do not support that. */
-    this->estimatedResultCount = 0;
-  } else {
-    std::vector<const zim::File*> zims;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      zims.push_back((*current)->getZimFileHandler());
-    }
-    zim::Search* search = new zim::Search(zims);
-    search->set_query(unaccentedSearch);
-    search->set_range(resultStart, resultEnd);
-    search->set_suggestion_mode(true);
-    internal->_search = search;
-    internal->current_iterator = internal->_search->begin();
-    this->estimatedResultCount = internal->_search->get_matches_estimated();
+  std::vector<const zim::File*> zims;
+  for (auto current = this->readers.begin(); current != this->readers.end();
+       current++) {
+    zims.push_back((*current)->getZimFileHandler());
   }
+  zim::Search* search = new zim::Search(zims);
+  search->set_verbose(verbose);
+  search->set_query(unaccentedSearch);
+  search->set_range(resultStart, resultEnd);
+  search->set_suggestion_mode(true);
+  internal->_search = search;
+  internal->current_iterator = internal->_search->begin();
+  this->estimatedResultCount = internal->_search->get_matches_estimated();
 }
 
 /* Return the result count estimation */
@@ -311,18 +234,6 @@ unsigned int Searcher::getEstimatedResultCount()
   return this->estimatedResultCount;
 }
 
-bool Searcher::setProtocolPrefix(const std::string prefix)
-{
-  this->protocolPrefix = prefix;
-  return true;
-}
-
-bool Searcher::setSearchProtocolPrefix(const std::string prefix)
-{
-  this->searchProtocolPrefix = prefix;
-  return true;
-}
-
 _Result::_Result(zim::Search::iterator& iterator)
     : iterator(iterator)
 {
@@ -363,103 +274,6 @@ int _Result::get_readerIndex()
 {
   return iterator.get_fileIndex();
 }
-#ifdef ENABLE_CTPP2
 
-string Searcher::getHtml()
-{
-  SimpleVM oSimpleVM(
-      1024, //iIMaxFunctions (default value)
-      4096, //iIMaxArgStackSize (default value)
-      4096, //iIMaxCodeStackSize (default value)
-      10240 * 2 //iIMaxSteps (default*2)
-  );
 
-  // Fill data
-  CDT oData;
-  CDT resultsCDT(CDT::ARRAY_VAL);
-
-  this->restart_search();
-  Result* p_result = NULL;
-  while ((p_result = this->getNextResult())) {
-    CDT result;
-    result["title"] = p_result->get_title();
-    result["url"] = p_result->get_url();
-    result["snippet"] = p_result->get_snippet();
-    result["contentId"] = humanReaderNames[p_result->get_readerIndex()];
-
-    if (p_result->get_size() >= 0) {
-      result["size"] = kiwix::beautifyInteger(p_result->get_size());
-    }
-
-    if (p_result->get_wordCount() >= 0) {
-      result["wordCount"] = kiwix::beautifyInteger(p_result->get_wordCount());
-    }
-
-    resultsCDT.PushBack(result);
-    delete p_result;
-  }
-  this->restart_search();
-  oData["results"] = resultsCDT;
-
-  // pages
-  CDT pagesCDT(CDT::ARRAY_VAL);
-
-  unsigned int pageStart
-      = this->resultStart / this->resultCountPerPage >= 5
-            ? this->resultStart / this->resultCountPerPage - 4
-            : 0;
-  unsigned int pageCount
-      = this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
-
-  if (pageCount > 10) {
-    pageCount = 10;
-  } else if (pageCount == 1) {
-    pageCount = 0;
-  }
-
-  for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
-    CDT page;
-    page["label"] = i + 1;
-    page["start"] = i * this->resultCountPerPage;
-    page["end"] = (i + 1) * this->resultCountPerPage;
-
-    if (i * this->resultCountPerPage == this->resultStart) {
-      page["selected"] = true;
-    }
-
-    pagesCDT.PushBack(page);
-  }
-  oData["pages"] = pagesCDT;
-
-  oData["count"] = kiwix::beautifyInteger(this->estimatedResultCount);
-  oData["searchPattern"] = kiwix::encodeDiples(this->searchPattern);
-  oData["searchPatternEncoded"] = urlEncode(this->searchPattern);
-  oData["resultStart"] = this->resultStart + 1;
-  oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount
-                            ? this->estimatedResultCount
-                            : this->resultEnd);
-  oData["resultRange"] = this->resultCountPerPage;
-  oData["resultLastPageStart"]
-      = this->estimatedResultCount > this->resultCountPerPage
-            ? std::round(this->estimatedResultCount / this->resultCountPerPage) * this->resultCountPerPage
-            : 0;
-  oData["protocolPrefix"] = this->protocolPrefix;
-  oData["searchProtocolPrefix"] = this->searchProtocolPrefix;
-  oData["contentId"] = this->contentHumanReadableId;
-
-  std::string template_ct2 = RESOURCE::results_ct2;
-  VMStringLoader oLoader(template_ct2.c_str(), template_ct2.size());
-
-  FileLogger oLogger(stderr);
-
-  // DEBUG only (write output to stdout)
-  // oSimpleVM.Run(oData, oLoader, stdout, oLogger);
-
-  std::string sResult;
-  oSimpleVM.Run(oData, oLoader, sResult, oLogger);
-
-  return sResult;
-}
-
-#endif
 }

src/server.cpp

@@ -0,0 +1,965 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "server.h"
+
+#ifdef _WIN32
+# if !defined(__MINGW32__) && (_MSC_VER < 1600)
+#   include "stdint4win.h"
+# endif
+# include <winsock2.h>
+# include <ws2tcpip.h>
+# ifdef __GNUC__
+  // inet_pton is not declared in mingw, even if the function exists.
+  extern "C" {
+    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
+  }
+# endif
+  typedef UINT64 uint64_t;
+  typedef UINT16 uint16_t;
+#endif
+
+extern "C" {
+#include <microhttpd.h>
+}
+
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+#include "library.h"
+#include "name_mapper.h"
+#include "entry.h"
+#include "searcher.h"
+#include "search_renderer.h"
+#include "opds_dumper.h"
+
+#include <zim/uuid.h>
+
+#include <mustache.hpp>
+
+#include <pthread.h>
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+#include "kiwixlib-resources.h"
+
+#ifndef _WIN32
+# include <arpa/inet.h>
+#endif
+
+#include "server/request_context.h"
+#include "server/response.h"
+
+#define MAX_SEARCH_LEN 140
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+static IdNameMapper defaultNameMapper;
+
+typedef kainjow::mustache::data MustacheData;
+
+static int staticHandlerCallback(void* cls,
+                                 struct MHD_Connection* connection,
+                                 const char* url,
+                                 const char* method,
+                                 const char* version,
+                                 const char* upload_data,
+                                 size_t* upload_data_size,
+                                 void** cont_cls);
+
+
+class InternalServer {
+  public:
+    InternalServer(Library* library,
+                   NameMapper* nameMapper,
+                   std::string addr,
+                   int port,
+                   std::string root,
+                   int nbThreads,
+                   bool verbose,
+                   bool withTaskbar,
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
+    virtual ~InternalServer() = default;
+
+    int handlerCallback(struct MHD_Connection* connection,
+                        const char* url,
+                        const char* method,
+                        const char* version,
+                        const char* upload_data,
+                        size_t* upload_data_size,
+                        void** cont_cls);
+    bool start();
+    void stop();
+
+  private: // functions
+    Response handle_request(const RequestContext& request);
+    Response build_500(const std::string& msg);
+    Response build_404(const RequestContext& request, const std::string& zimName);
+    Response build_304(const RequestContext& request, const ETag& etag) const;
+    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    Response build_homepage(const RequestContext& request);
+    Response handle_skin(const RequestContext& request);
+    Response handle_catalog(const RequestContext& request);
+    Response handle_meta(const RequestContext& request);
+    Response handle_search(const RequestContext& request);
+    Response handle_suggest(const RequestContext& request);
+    Response handle_random(const RequestContext& request);
+    Response handle_captured_external(const RequestContext& request);
+    Response handle_content(const RequestContext& request);
+
+    MustacheData get_default_data() const;
+    MustacheData homepage_data() const;
+    Response get_default_response() const;
+
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
+    std::string m_addr;
+    int m_port;
+    std::string m_root;
+    int m_nbThreads;
+    std::atomic_bool m_verbose;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    struct MHD_Daemon* mp_daemon;
+
+    Library* mp_library;
+    NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
+};
+
+
+Server::Server(Library* library, NameMapper* nameMapper) :
+  mp_library(library),
+  mp_nameMapper(nameMapper),
+  mp_server(nullptr)
+{
+}
+
+Server::~Server() = default;
+
+bool Server::start() {
+  mp_server.reset(new InternalServer(
+    mp_library,
+    mp_nameMapper,
+    m_addr,
+    m_port,
+    m_root,
+    m_nbThreads,
+    m_verbose,
+    m_withTaskbar,
+    m_withLibraryButton,
+    m_blockExternalLinks));
+  return mp_server->start();
+}
+
+void Server::stop() {
+  mp_server->stop();
+  mp_server.reset(nullptr);
+}
+
+void Server::setRoot(const std::string& root)
+{
+  m_root = root;
+  if (m_root[0] != '/') {
+    m_root = "/" + m_root;
+  }
+  if (m_root.back() == '/') {
+    m_root.erase(m_root.size() - 1);
+  }
+}
+
+
+InternalServer::InternalServer(Library* library,
+                               NameMapper* nameMapper,
+                               std::string addr,
+                               int port,
+                               std::string root,
+                               int nbThreads,
+                               bool verbose,
+                               bool withTaskbar,
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
+  m_addr(addr),
+  m_port(port),
+  m_root(root),
+  m_nbThreads(nbThreads),
+  m_verbose(verbose),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  mp_daemon(nullptr),
+  mp_library(library),
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+{}
+
+bool InternalServer::start() {
+#ifdef _WIN32
+  int flags = MHD_USE_SELECT_INTERNALLY;
+#else
+  int flags = MHD_USE_POLL_INTERNALLY;
+#endif
+  if (m_verbose.load())
+    flags |= MHD_USE_DEBUG;
+
+
+  struct sockaddr_in sockAddr;
+  memset(&sockAddr, 0, sizeof(sockAddr));
+  sockAddr.sin_family = AF_INET;
+  sockAddr.sin_port = htons(m_port);
+  if (m_addr.empty()) {
+    if (0 != INADDR_ANY)
+      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+  } else {
+    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
+      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
+      return false;
+    }
+  }
+
+  mp_daemon = MHD_start_daemon(flags,
+                            m_port,
+                            NULL,
+                            NULL,
+                            &staticHandlerCallback,
+                            this,
+                            MHD_OPTION_SOCK_ADDR, &sockAddr,
+                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_END);
+  if (mp_daemon == nullptr) {
+    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
+              << " is maybe already occupied or need more permissions to be open. "
+                 "Please try as root or with a port number higher or equal to 1024."
+              << std::endl;
+    return false;
+  }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
+  return true;
+}
+
+void InternalServer::stop()
+{
+  MHD_stop_daemon(mp_daemon);
+}
+
+static int staticHandlerCallback(void* cls,
+                                 struct MHD_Connection* connection,
+                                 const char* url,
+                                 const char* method,
+                                 const char* version,
+                                 const char* upload_data,
+                                 size_t* upload_data_size,
+                                 void** cont_cls)
+{
+  InternalServer* _this = static_cast<InternalServer*>(cls);
+
+  return _this->handlerCallback(connection,
+                                url,
+                                method,
+                                version,
+                                upload_data,
+                                upload_data_size,
+                                cont_cls);
+}
+
+int InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                    const char* url,
+                                    const char* method,
+                                    const char* version,
+                                    const char* upload_data,
+                                    size_t* upload_data_size,
+                                    void** cont_cls)
+{
+  auto start_time = std::chrono::steady_clock::now();
+  if (m_verbose.load() ) {
+    printf("======================\n");
+    printf("Requesting : \n");
+    printf("full_url  : %s\n", url);
+  }
+  RequestContext request(connection, m_root, url, method, version);
+
+  if (m_verbose.load() ) {
+    request.print_debug_info();
+  }
+  /* Unexpected method */
+  if (request.get_method() != RequestMethod::GET
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
+    printf("Reject request because of unhandled request method.\n");
+    printf("----------------------\n");
+    return MHD_NO;
+  }
+
+  auto response = handle_request(request);
+
+  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
+    printf("========== INTERNAL ERROR !! ============\n");
+    if (!m_verbose.load()) {
+      printf("Requesting : \n");
+      printf("full_url : %s\n", url);
+      request.print_debug_info();
+    }
+  }
+
+  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response.set_server_id(m_server_id);
+
+  auto ret = response.send(request, connection);
+  auto end_time = std::chrono::steady_clock::now();
+  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
+  if (m_verbose.load()) {
+    printf("Request time : %fs\n", time_span.count());
+    printf("----------------------\n");
+  }
+  return ret;
+}
+
+Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
+{
+  auto response = get_default_response();
+  response.set_code(MHD_HTTP_NOT_MODIFIED);
+  response.set_etag(etag);
+  response.set_content("");
+  return response;
+}
+
+Response InternalServer::handle_request(const RequestContext& request)
+{
+  try {
+    if (! request.is_valid_url())
+      return build_404(request, "");
+
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return build_304(request, etag);
+
+    if (kiwix::startsWith(request.get_url(), "/skin/"))
+      return handle_skin(request);
+
+    if (startsWith(request.get_url(), "/catalog"))
+      return handle_catalog(request);
+
+    if (request.get_url() == "/meta")
+      return handle_meta(request);
+
+    if (request.get_url() == "/search")
+      return handle_search(request);
+
+    if (request.get_url() == "/suggest")
+     return handle_suggest(request);
+
+    if (request.get_url() == "/random")
+      return handle_random(request);
+
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
+    return handle_content(request);
+  } catch (std::exception& e) {
+    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
+    return build_500(e.what());
+  } catch (...) {
+    fprintf(stderr, "===== Unhandled unknown error\n");
+    return build_500("Unknown error");
+  }
+}
+
+MustacheData InternalServer::get_default_data() const
+{
+  MustacheData data;
+  data.set("root", m_root);
+  return data;
+}
+
+Response InternalServer::get_default_response() const
+{
+  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
+}
+
+
+Response InternalServer::build_404(const RequestContext& request,
+                                   const std::string& bookName)
+{
+  MustacheData results;
+  results.set("url", request.get_full_url());
+
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::_404_html, results);
+  response.set_mimeType("text/html");
+  response.set_code(MHD_HTTP_NOT_FOUND);
+  response.set_compress(true);
+  response.set_taskbar(bookName, "");
+
+  return response;
+}
+
+Response InternalServer::build_500(const std::string& msg)
+{
+  MustacheData data;
+  data.set("error", msg);
+  Response response(m_root, true, false, false, false);
+  response.set_template(RESOURCE::templates::_500_html, data);
+  response.set_mimeType("text/html");
+  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
+  return response;
+}
+
+MustacheData InternalServer::homepage_data() const
+{
+  auto data = get_default_data();
+
+  MustacheData books{MustacheData::type::list};
+  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = mp_library->getBookById(bookId);
+
+    MustacheData book;
+    book.set("name", mp_nameMapper->getNameForId(bookId));
+    book.set("title", currentBook.getTitle());
+    book.set("description", currentBook.getDescription());
+    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
+    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
+    books.push_back(book);
+  }
+
+  data.set("books", books);
+  return data;
+}
+
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+Response InternalServer::build_homepage(const RequestContext& request)
+{
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::index_html, homepage_data());
+  response.set_mimeType("text/html; charset=utf-8");
+  response.set_compress(true);
+  response.set_taskbar("", "");
+  return response;
+}
+
+Response InternalServer::handle_meta(const RequestContext& request)
+{
+  std::string bookName;
+  std::string bookId;
+  std::string meta_name;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    meta_name = request.get_argument("name");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+    return build_404(request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return build_404(request, bookName);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return build_404(request, bookName);
+  }
+
+  auto response = get_default_response();
+  response.set_content(content);
+  response.set_mimeType(mimeType);
+  response.set_compress(false);
+  response.set_cacheable();
+  return response;
+}
+
+Response InternalServer::handle_suggest(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_suggest\n");
+  }
+
+  std::string content;
+  std::string mimeType;
+  unsigned int maxSuggestionCount = 10;
+  unsigned int suggestionCount = 0;
+  std::string suggestion;
+
+  std::string bookName;
+  std::string bookId;
+  std::string term;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    term = request.get_argument("term");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return build_404(request, bookName);
+  }
+
+  if (m_verbose.load()) {
+    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+  }
+
+  MustacheData results{MustacheData::type::list};
+
+  bool first = true;
+  if (reader != nullptr) {
+    /* Get the suggestions */
+    reader->searchSuggestionsSmart(term, maxSuggestionCount);
+    while (reader->getNextSuggestion(suggestion)) {
+      MustacheData result;
+      result.set("label", suggestion);
+      result.set("value", suggestion);
+      result.set("first", first);
+      first = false;
+      results.push_back(result);
+      suggestionCount++;
+    }
+  }
+
+  /* Propose the fulltext search if possible */
+  if (reader->hasFulltextIndex()) {
+    MustacheData result;
+    result.set("label", "containing '" + term + "'...");
+    result.set("value", term + " ");
+    result.set("first", first);
+    results.push_back(result);
+  }
+
+  auto data = get_default_data();
+  data.set("suggestions", results);
+
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::suggestion_json, data);
+  response.set_mimeType("application/json; charset=utf-8");
+  response.set_compress(true);
+  return response;
+}
+
+Response InternalServer::handle_skin(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_skin\n");
+  }
+
+  auto response = get_default_response();
+  auto resourceName = request.get_url().substr(1);
+  try {
+    response.set_content(getResource(resourceName));
+  } catch (const ResourceNotFound& e) {
+    return build_404(request, "");
+  }
+  response.set_mimeType(getMimeTypeForFile(resourceName));
+  response.set_compress(true);
+  response.set_cacheable();
+  return response;
+}
+
+Response InternalServer::handle_search(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_search\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+  } catch (const std::out_of_range&) {}
+
+  std::string patternString;
+  try {
+    patternString = request.get_argument("pattern");
+  } catch (const std::out_of_range&) {}
+
+  /* Retrive geo search */
+  bool has_geo_query = false;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
+  try {
+    latitude = request.get_argument<float>("latitude");
+    longitude = request.get_argument<float>("longitude");
+    distance = request.get_argument<float>("distance");
+    has_geo_query = true;
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  std::shared_ptr<Reader> reader(nullptr);
+  try {
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {}
+
+  /* Try first to load directly the article */
+  if (reader != nullptr && !patternString.empty()) {
+    std::string patternCorrespondingUrl;
+    auto variants = reader->getTitleVariants(patternString);
+    auto variantsItr = variants.begin();
+
+    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
+      try {
+        auto entry = reader->getEntryFromTitle(*variantsItr);
+        entry = entry.getFinalEntry();
+        patternCorrespondingUrl = entry.getPath();
+        break;
+      } catch(kiwix::NoEntry& e) {
+        variantsItr++;
+      }
+    }
+
+    /* If article found then redirect directly to it */
+    if (!patternCorrespondingUrl.empty()) {
+      auto response = get_default_response();
+      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
+      return response;
+    }
+  }
+
+  /* Make the search */
+  auto response = get_default_response();
+  response.set_mimeType("text/html; charset=utf-8");
+  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
+  response.set_compress(true);
+
+  if ( (!reader && !bookName.empty())
+    || (patternString.empty() && ! has_geo_query) ) {
+    auto data = get_default_data();
+    data.set("pattern", encodeDiples(patternString));
+    response.set_template(RESOURCE::templates::no_search_result_html, data);
+    response.set_code(MHD_HTTP_NOT_FOUND);
+    return response;
+  }
+
+  Searcher searcher;
+  if (reader) {
+    searcher.add_reader(reader.get());
+  } else {
+    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+      auto currentReader = mp_library->getReaderById(bookId);
+      if (currentReader) {
+        searcher.add_reader(currentReader.get());
+      }
+    }
+  }
+
+  auto start = 0;
+  try {
+    start = request.get_argument<unsigned int>("start");
+  } catch (const std::exception&) {}
+  auto end = 25;
+  try {
+    end = request.get_argument<unsigned int>("end");
+  } catch (const std::exception&) {}
+  if (start>end) {
+    auto tmp = start;
+    start = end;
+    end = tmp;
+  }
+  if (end > start + MAX_SEARCH_LEN) {
+    end = start + MAX_SEARCH_LEN;
+  }
+
+  /* Get the results */
+  try {
+    if (patternString.empty()) {
+      searcher.geo_search(latitude, longitude, distance,
+                           start, end, m_verbose.load());
+    } else {
+      searcher.search(patternString,
+                       start, end, m_verbose.load());
+    }
+    SearchRenderer renderer(&searcher, mp_nameMapper);
+    renderer.setSearchPattern(patternString);
+    renderer.setSearchContent(bookName);
+    renderer.setProtocolPrefix(m_root + "/");
+    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    response.set_content(renderer.getHtml());
+  } catch (const std::exception& e) {
+    std::cerr << e.what() << std::endl;
+  }
+  return response;
+}
+
+Response InternalServer::handle_random(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_random\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return build_404(request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return build_404(request, bookName);
+  }
+
+  try {
+    auto entry = reader->getRandomPage();
+    return build_redirect(bookName, entry.getFinalEntry());
+  } catch(kiwix::NoEntry& e) {
+    return build_404(request, bookName);
+  }
+}
+
+Response InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return build_404(request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::captured_external_html, data);
+  response.set_mimeType("text/html; charset=utf-8");
+  response.set_compress(true);
+  response.set_taskbar("", "");
+  return response;
+}
+
+Response InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return build_404(request, "");
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return build_404(request, "");
+  }
+
+  auto response = get_default_response();
+  response.set_compress(true);
+  if (url == "searchdescription.xml") {
+    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
+    response.set_mimeType("application/opensearchdescription+xml");
+    return response;
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper;
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+  opdsDumper.setLibrary(mp_library);
+  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    opdsDumper.setTitle("All zims");
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
+    string query("<Empty query>");
+    size_t count(10);
+    size_t startIndex(0);
+    try {
+      query = request.get_argument("q");
+      filter.query(query);
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      count = extractFromString<unsigned long>(request.get_argument("count"));
+    } catch (...) {}
+    try {
+      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
+    } catch (...) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
+    } catch (...) {}
+    opdsDumper.setTitle("Search result for " + query);
+    uuid = zim::Uuid::generate();
+    bookIdsToDump = mp_library->filter(filter);
+    auto totalResults = bookIdsToDump.size();
+    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
+    if (count>0 && bookIdsToDump.size() > count) {
+      bookIdsToDump.resize(count);
+    }
+    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
+  }
+
+  opdsDumper.setId(kiwix::to_string(uuid));
+  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
+  return response;
+}
+
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+Response
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto response = get_default_response();
+  response.set_redirection(m_root + "/" + bookName + "/" +
+                           kiwix::urlEncode(entry.getPath()));
+  return response;
+}
+
+Response InternalServer::handle_content(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_content\n");
+  }
+
+  const std::string bookName = get_book_name(request);
+  if (bookName.empty())
+    return build_homepage(request);
+
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
+  if (reader == nullptr) {
+    return build_404(request, bookName);
+  }
+
+  auto urlStr = request.get_url().substr(bookName.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
+
+  kiwix::Entry entry;
+
+  try {
+    entry = reader->getEntryFromPath(urlStr);
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      return build_redirect(bookName, entry.getFinalEntry());
+    }
+  } catch(kiwix::NoEntry& e) {
+    if (m_verbose.load())
+      printf("Failed to find %s\n", urlStr.c_str());
+
+    return build_404(request, bookName);
+  }
+
+  auto response = get_default_response();
+
+  response.set_entry(entry, request);
+
+  if (m_verbose.load()) {
+    printf("Found %s\n", entry.getPath().c_str());
+    printf("mimeType: %s\n", response.get_mimeType().c_str());
+  }
+
+  if (response.get_mimeType().find("text/html") != string::npos)
+    response.set_taskbar(bookName, reader->getTitle());
+
+  return response;
+}
+
+}

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/etag.cpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "etag.h"
+
+#include "tools/stringTools.h"
+
+#include <algorithm>
+#include <sstream>
+
+namespace kiwix {
+
+namespace {
+
+// Characters in the options part of the ETag could in principle be picked up
+// from the latin alphabet in natural order (the character corresponding to
+// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
+// this file). However it is better to have some mnemonics in the option names,
+// hence below variable: all_options[opt] corresponds to the character going
+// into the ETag for ETag::Option opt.
+// IMPORTANT: The characters in all_options must come in sorted order (so that
+// IMPORTANT: isValidOptionsString() works correctly).
+const char all_options[] = "cz";
+
+static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
+
+bool isValidServerId(const std::string& s)
+{
+  return !s.empty() && s.find_first_of("\"/") == std::string::npos;
+}
+
+bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
+{
+  std::string::size_type i = 0;
+  for ( const char c : s )
+  {
+    const std::string::size_type j = sortedString.find(c, i);
+    if ( j == std::string::npos )
+      return false;
+    i = j+1;
+  }
+  return true;
+}
+
+bool isValidOptionsString(const std::string& s)
+{
+  return isSubsequenceOf(s, all_options);
+}
+
+} // namespace
+
+
+void ETag::set_option(Option opt)
+{
+  if ( ! get_option(opt) )
+  {
+    m_options.push_back(all_options[opt]);
+    std::sort(m_options.begin(), m_options.end());
+  }
+}
+
+bool ETag::get_option(Option opt) const
+{
+  return m_options.find(all_options[opt]) != std::string::npos;
+}
+
+std::string ETag::get_etag() const
+{
+  if ( m_serverId.empty() )
+    return std::string();
+
+  return "\"" + m_serverId + "/" + m_options + "\"";
+}
+
+ETag::ETag(const std::string& serverId, const std::string& options)
+{
+  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  {
+    m_serverId = serverId;
+    m_options = options;
+  }
+}
+
+ETag ETag::parse(std::string s)
+{
+  if ( kiwix::startsWith("W/", s) )
+    s = s.substr(2);
+
+  if ( s.front() != '"' || s.back() != '"' )
+    return ETag();
+
+  s = s.substr(1, s.size()-2);
+
+  const std::string::size_type i = s.find('/');
+  if ( i == std::string::npos )
+    return ETag();
+
+  return ETag(s.substr(0, i), s.substr(i+1));
+}
+
+ETag ETag::match(const std::string& etags, const std::string& server_id)
+{
+  std::istringstream ss(etags);
+  std::string etag_str;
+  while ( ss >> etag_str )
+  {
+    if ( etag_str.back() == ',' )
+      etag_str.pop_back();
+
+    const ETag etag = parse(etag_str);
+    if ( etag && etag.m_serverId == server_id )
+      return etag;
+  }
+
+  return ETag();
+}
+
+} // namespace kiwix

src/server/etag.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_ETAG_H
+#define KIWIXLIB_SERVER_ETAG_H
+
+#include <string>
+
+namespace kiwix {
+
+// The ETag string used by Kiwix server (more precisely, its value inside the
+// double quotes) consists of two parts:
+//
+// 1. ServerId - The string obtained on server start up
+//
+// 2. Options -  Zero or more characters encoding the values of some of the
+//               headers of the response
+//
+// The two parts are separated with a slash (/) symbol (which is always present,
+// even when the the options part is empty). Neither portion of a Kiwix ETag
+// may contain the slash symbol.
+// Examples of valid Kiwix server ETags (including the double quotes):
+//
+//   "abcdefghijklmn/"
+//   "1234567890/z"
+//   "1234567890/cz"
+//
+// The options part of the Kiwix ETag allows to correctly set the required
+// headers when responding to a conditional If-None-Match request with a 304
+// (Not Modified) response without following the full code path that would
+// discover the necessary options.
+
+class ETag
+{
+  public: // types
+    enum Option {
+      CACHEABLE_ENTITY,
+      COMPRESSED_CONTENT,
+      OPTION_COUNT
+    };
+
+  public: // functions
+    ETag() {}
+
+    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_option(Option opt);
+
+    explicit operator bool() const { return !m_serverId.empty(); }
+
+    bool get_option(Option opt) const;
+    std::string get_etag() const;
+
+
+    static ETag match(const std::string& etags, const std::string& server_id);
+
+  private: // functions
+    ETag(const std::string& serverId, const std::string& options);
+
+    static ETag parse(std::string s);
+
+  private: // data
+    std::string m_serverId;
+    std::string m_options;
+};
+
+} // namespace kiwix
+
+#endif // KIWIXLIB_SERVER_ETAG_H

src/server/request_context.cpp

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "request_context.h"
+#include <string.h>
+#include <stdexcept>
+#include <sstream>
+#include <cstdio>
+#include <atomic>
+
+namespace kiwix {
+
+static std::atomic_ullong s_requestIndex(0);
+
+namespace {
+
+RequestMethod str2RequestMethod(const std::string& method) {
+  if      (method == "GET")     return RequestMethod::GET;
+  else if (method == "HEAD")    return RequestMethod::HEAD;
+  else if (method == "POST")    return RequestMethod::POST;
+  else if (method == "PUT")     return RequestMethod::PUT;
+  else if (method == "DELETE")  return RequestMethod::DELETE_;
+  else if (method == "CONNECT") return RequestMethod::CONNECT;
+  else if (method == "OPTIONS") return RequestMethod::OPTIONS;
+  else if (method == "TRACE")   return RequestMethod::TRACE;
+  else if (method == "PATCH")   return RequestMethod::PATCH;
+  else                          return RequestMethod::OTHER;
+}
+
+std::string
+fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
+{
+  if (rootLocation.empty()) {
+    // nothing special to handle.
+    return full_url;
+  } else if (full_url == rootLocation) {
+    return "/";
+  } else if (full_url.size() > rootLocation.size() &&
+             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
+    return full_url.substr(rootLocation.size());
+  } else {
+    return "";
+  }
+}
+
+} // unnamed namespace
+
+RequestContext::RequestContext(struct MHD_Connection* connection,
+                               std::string rootLocation,
+                               const std::string& _url,
+                               const std::string& _method,
+                               const std::string& version) :
+  full_url(_url),
+  url(fullURL2LocalURL(_url, rootLocation)),
+  method(str2RequestMethod(_method)),
+  version(version),
+  requestIndex(s_requestIndex++),
+  acceptEncodingDeflate(false),
+  byteRange_()
+{
+  MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
+  MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
+
+  try {
+    acceptEncodingDeflate =
+        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
+  } catch (const std::out_of_range&) {}
+
+  try {
+    byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
+  } catch (const std::out_of_range&) {}
+}
+
+RequestContext::~RequestContext()
+{}
+
+int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
+                                 const char *key, const char *value)
+{
+  RequestContext *_this = static_cast<RequestContext*>(__this);
+  _this->headers[key] = value;
+  return MHD_YES;
+}
+
+int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
+                                   const char *key, const char* value)
+{
+  RequestContext *_this = static_cast<RequestContext*>(__this);
+  _this->arguments[key] = value == nullptr ? "" : value;
+  return MHD_YES;
+}
+
+void RequestContext::print_debug_info() const {
+  printf("method    : %s (%d)\n", method==RequestMethod::GET ? "GET" :
+                                  method==RequestMethod::POST ? "POST" :
+                                  "OTHER", (int)method);
+  printf("version   : %s\n", version.c_str());
+  printf("request#  : %lld\n", requestIndex);
+  printf("headers   :\n");
+  for (auto it=headers.begin(); it!=headers.end(); it++) {
+    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+  }
+  printf("arguments :\n");
+  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
+    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+  }
+  printf("Parsed : \n");
+  printf("full_url: %s\n", full_url.c_str());
+  printf("url   : %s\n", url.c_str());
+  printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
+  printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
+  printf("is_valid_url : %d\n", is_valid_url());
+  printf(".............\n");
+}
+
+
+RequestMethod RequestContext::get_method() const {
+  return method;
+}
+
+std::string RequestContext::get_url() const {
+  return url;
+}
+
+std::string RequestContext::get_url_part(int number) const {
+  size_t start = 1;
+  while(true) {
+    auto found = url.find('/', start);
+    if (number == 0) {
+      if (found == std::string::npos) {
+        return url.substr(start);
+      } else {
+        return url.substr(start, found-start);
+      }
+    } else {
+      if (found == std::string::npos) {
+        throw std::out_of_range("No parts");
+      }
+      start = found + 1;
+      number -= 1;
+    }
+  }
+}
+
+std::string RequestContext::get_full_url() const {
+  return full_url;
+}
+
+bool RequestContext::is_valid_url() const {
+  return !url.empty();
+}
+
+ByteRange RequestContext::get_range() const {
+  return byteRange_;
+}
+
+template<>
+std::string RequestContext::get_argument(const std::string& name) const {
+  return arguments.at(name);
+}
+
+std::string RequestContext::get_header(const std::string& name) const {
+  return headers.at(name);
+}
+
+}

src/server/request_context.h

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef REQUEST_CONTEXT_H
+#define REQUEST_CONTEXT_H
+
+#include <string>
+#include <sstream>
+#include <map>
+#include <stdexcept>
+
+#include "byte_range.h"
+
+extern "C" {
+#include <microhttpd.h>
+}
+
+namespace kiwix {
+
+enum class RequestMethod {
+    GET,
+    HEAD,
+    POST,
+    PUT,
+    DELETE_,
+    CONNECT,
+    OPTIONS,
+    TRACE,
+    PATCH,
+    OTHER
+};
+
+class KeyError : public std::runtime_error {};
+class IndexError: public std::runtime_error {};
+
+
+class RequestContext {
+  public: // functions
+    RequestContext(struct MHD_Connection* connection,
+                   std::string rootLocation,
+                   const std::string& url,
+                   const std::string& method,
+                   const std::string& version);
+    ~RequestContext();
+
+    void print_debug_info() const;
+
+    bool is_valid_url() const;
+
+    std::string get_header(const std::string& name) const;
+    template<typename T=std::string>
+    T get_argument(const std::string& name) const {
+        std::istringstream stream(arguments.at(name));
+        T v;
+        stream >> v;
+        return v;
+    }
+
+
+    RequestMethod get_method() const;
+    std::string get_url() const;
+    std::string get_url_part(int part) const;
+    std::string get_full_url() const;
+
+    ByteRange get_range() const;
+
+    bool can_compress() const { return acceptEncodingDeflate; }
+
+  private: // data
+    std::string full_url;
+    std::string url;
+    RequestMethod method;
+    std::string version;
+    unsigned long long requestIndex;
+
+    bool acceptEncodingDeflate;
+
+    ByteRange byteRange_;
+    std::map<std::string, std::string> headers;
+    std::map<std::string, std::string> arguments;
+
+  private: // functions
+    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+};
+
+template<> std::string RequestContext::get_argument(const std::string& name) const;
+
+}
+
+#endif //REQUEST_CONTEXT_H

src/server/response.cpp

@@ -0,0 +1,361 @@
+
+
+
+#include "response.h"
+#include "request_context.h"
+#include "kiwixlib-resources.h"
+
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+
+#include "string.h"
+#include <mustache.hpp>
+#include <zlib.h>
+
+
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+namespace
+{
+// some utilities
+
+std::string get_mime_type(const kiwix::Entry& entry)
+{
+  try {
+    return entry.getMimetype();
+  } catch (exception& e) {
+    return "application/octet-stream";
+  }
+}
+
+bool is_compressible_mime_type(const std::string& mimeType)
+{
+  return mimeType.find("text/") != string::npos
+      || mimeType.find("application/javascript") != string::npos
+      || mimeType.find("application/atom") != string::npos
+      || mimeType.find("application/opensearchdescription") != string::npos
+      || mimeType.find("application/json") != string::npos;
+}
+
+
+} // unnamed namespace
+
+Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
+  : m_verbose(verbose),
+    m_root(root),
+    m_content(""),
+    m_mimeType(""),
+    m_returnCode(MHD_HTTP_OK),
+    m_withTaskbar(withTaskbar),
+    m_withLibraryButton(withLibraryButton),
+    m_blockExternalLinks(blockExternalLinks),
+    m_addTaskbar(false),
+    m_bookName("")
+{
+}
+
+
+static int print_key_value (void *cls, enum MHD_ValueKind kind,
+                            const char *key, const char *value)
+{
+  printf (" - %s: '%s'\n", key, value);
+  return MHD_YES;
+}
+
+
+struct RunningResponse {
+   kiwix::Entry entry;
+   int range_start;
+
+   RunningResponse(kiwix::Entry entry,
+                   int range_start) :
+     entry(entry),
+     range_start(range_start)
+   {}
+};
+
+static ssize_t callback_reader_from_entry(void* cls,
+                                  uint64_t pos,
+                                  char* buf,
+                                  size_t max)
+{
+  RunningResponse* response = static_cast<RunningResponse*>(cls);
+
+  size_t max_size_to_set = min<size_t>(
+    max,
+    response->entry.getSize() - pos - response->range_start);
+
+  if (max_size_to_set <= 0) {
+    return MHD_CONTENT_READER_END_WITH_ERROR;
+  }
+
+  zim::Blob blob = response->entry.getBlob(response->range_start+pos, max_size_to_set);
+  memcpy(buf, blob.data(), max_size_to_set);
+  return max_size_to_set;
+}
+
+static void callback_free_response(void* cls)
+{
+  RunningResponse* response = static_cast<RunningResponse*>(cls);
+  delete response;
+}
+
+
+
+void print_response_info(int retCode, MHD_Response* response)
+{
+  printf("Response :\n");
+  printf("httpResponseCode : %d\n", retCode);
+  printf("headers :\n");
+  MHD_get_response_headers(response, print_key_value, nullptr);
+}
+
+
+std::string render_template(const std::string& template_str, kainjow::mustache::data data)
+{
+  kainjow::mustache::mustache tmpl(template_str);
+  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
+                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
+  data.set("urlencoded", urlencode);
+  std::stringstream ss;
+  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}
+
+
+void Response::introduce_taskbar()
+{
+  if (! m_withTaskbar)
+    // Taskbar is globally disabled.
+    return;
+  kainjow::mustache::data data;
+  data.set("root", m_root);
+  data.set("content", m_bookName);
+  data.set("hascontent", !m_bookName.empty());
+  data.set("title", m_bookTitle);
+  data.set("withlibrarybutton", m_withLibraryButton);
+  auto head_content = render_template(RESOURCE::templates::head_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<head>",
+    head_content);
+
+  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<body[^>]*>",
+    taskbar_part);
+}
+
+
+void Response::inject_externallinks_blocker()
+{
+  kainjow::mustache::data data;
+  data.set("root", m_root);
+  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<head>",
+    script_tag);
+}
+
+bool
+Response::can_compress(const RequestContext& request) const
+{
+  return request.can_compress()
+      && is_compressible_mime_type(m_mimeType)
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+}
+
+MHD_Response*
+Response::create_error_response(const RequestContext& request) const
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT);
+  if ( m_returnCode == 416 ) {
+    std::ostringstream oss;
+    oss << "bytes */" << m_byteRange.length();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+  return response;
+}
+
+MHD_Response*
+Response::create_raw_content_mhd_response(const RequestContext& request)
+{
+  if (m_addTaskbar) {
+    introduce_taskbar();
+  }
+  if ( m_blockExternalLinks ) {
+    inject_externallinks_blocker();
+  }
+
+  bool shouldCompress = m_compress && can_compress(request);
+  if (shouldCompress) {
+    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
+    uLongf comprLen = compr_buffer.capacity();
+    int err = compress(&compr_buffer[0],
+                       &comprLen,
+                       (const Bytef*)(m_content.data()),
+                       m_content.size());
+    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
+      /* /!\ Internet Explorer has a bug with deflate compression.
+         It can not handle the first two bytes (compression headers)
+         We need to chunk them off (move the content 2bytes)
+         It has no incidence on other browsers
+         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
+      m_content = string((char*)&compr_buffer[2], comprLen - 2);
+      m_etag.set_option(ETag::COMPRESSED_CONTENT);
+    } else {
+      shouldCompress = false;
+    }
+  }
+
+  MHD_Response* response = MHD_create_response_from_buffer(
+    m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
+
+  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
+  // shouldCompress can have different values. This can happen for a 304 (Not
+  // Modified) response generated while handling a conditional If-None-Match
+  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
+  // option) is obtained from the ETag list of the If-None-Match header and the
+  // response has no body (which shouldn't be compressed).
+  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  if (shouldCompress) {
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+  }
+  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_redirection_mhd_response() const
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
+  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_entry_mhd_response() const
+{
+  const auto content_length = m_byteRange.length();
+  MHD_Response* response = MHD_create_response_from_callback(content_length,
+                                               16384,
+                                               callback_reader_from_entry,
+                                               new RunningResponse(m_entry, m_byteRange.first()),
+                                               callback_free_response);
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
+  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
+  if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
+    std::ostringstream oss;
+    oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
+        << "/" << m_entry.getSize();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_mhd_response(const RequestContext& request)
+{
+  switch (m_mode) {
+    case ResponseMode::ERROR_RESPONSE:
+      return create_error_response(request);
+
+    case ResponseMode::RAW_CONTENT :
+      return create_raw_content_mhd_response(request);
+
+    case ResponseMode::REDIRECTION :
+      return create_redirection_mhd_response();
+
+    case ResponseMode::ENTRY :
+      return create_entry_mhd_response();
+  }
+  return nullptr;
+}
+
+int Response::send(const RequestContext& request, MHD_Connection* connection)
+{
+  MHD_Response* response = create_mhd_response(request);
+
+  if ( m_mode != ResponseMode::ERROR_RESPONSE ) {
+    MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
+    MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
+      m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+    const std::string etag = m_etag.get_etag();
+    if ( ! etag.empty() )
+        MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  }
+
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
+    m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
+
+  if (m_verbose)
+    print_response_info(m_returnCode, response);
+
+  auto ret = MHD_queue_response(connection, m_returnCode, response);
+  MHD_destroy_response(response);
+  return ret;
+}
+
+void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
+  set_content(render_template(template_str, data));
+}
+
+void Response::set_content(const std::string& content) {
+  m_content = content;
+  m_mode = ResponseMode::RAW_CONTENT;
+}
+
+void Response::set_redirection(const std::string& url) {
+  m_content = url;
+  m_mode = ResponseMode::REDIRECTION;
+  m_returnCode = MHD_HTTP_FOUND;
+}
+
+void Response::set_entry(const Entry& entry, const RequestContext& request) {
+  m_entry = entry;
+  m_mode = ResponseMode::ENTRY;
+
+  const std::string mimeType = get_mime_type(entry);
+  set_mimeType(mimeType);
+  set_cacheable();
+
+  m_byteRange = request.get_range().resolve(entry.getSize());
+  const bool noRange = m_byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if ( noRange && is_compressible_mime_type(mimeType) ) {
+    zim::Blob raw_content = entry.getBlob();
+    const std::string content = string(raw_content.data(), raw_content.size());
+
+    set_content(content);
+    set_compress(true);
+  } else if ( m_byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE ) {
+    set_code(416);
+    set_content("");
+    m_mode = ResponseMode::ERROR_RESPONSE;
+  }
+}
+
+void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+{
+  m_addTaskbar = true;
+  m_bookName = bookName;
+  m_bookTitle = bookTitle;
+}
+
+
+}

src/server/response.h

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2019 Matthieu Gautier<mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_RESPONSE_H
+#define KIWIXLIB_SERVER_RESPONSE_H
+
+#include <string>
+
+#include <mustache.hpp>
+#include "byte_range.h"
+#include "entry.h"
+#include "etag.h"
+
+extern "C" {
+#include <microhttpd.h>
+}
+
+namespace kiwix {
+
+enum class ResponseMode {
+  ERROR_RESPONSE,
+  RAW_CONTENT,
+  REDIRECTION,
+  ENTRY
+};
+
+class RequestContext;
+
+class Response {
+  public:
+    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
+    ~Response() = default;
+
+    int send(const RequestContext& request, MHD_Connection* connection);
+
+    void set_template(const std::string& template_str, kainjow::mustache::data data);
+    void set_content(const std::string& content);
+    void set_redirection(const std::string& url);
+    void set_entry(const Entry& entry, const RequestContext& request);
+
+
+    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
+    void set_code(int code) { m_returnCode = code; }
+    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
+    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_etag(const ETag& etag) { m_etag = etag; }
+    void set_compress(bool compress) { m_compress = compress; }
+    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+
+    int getReturnCode() const { return m_returnCode; }
+    std::string get_mimeType() const { return m_mimeType; }
+
+    void introduce_taskbar();
+    void inject_externallinks_blocker();
+
+    bool can_compress(const RequestContext& request) const;
+
+  private: // functions
+    MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
+    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
+    MHD_Response* create_redirection_mhd_response() const;
+    MHD_Response* create_entry_mhd_response() const;
+
+  private: // data
+    bool m_verbose;
+    ResponseMode m_mode;
+    std::string m_root;
+    std::string m_content;
+    Entry m_entry;
+    std::string m_mimeType;
+    int m_returnCode;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    bool m_compress;
+    bool m_addTaskbar;
+    std::string m_bookName;
+    std::string m_bookTitle;
+    ByteRange m_byteRange;
+    ETag m_etag;
+};
+
+}
+
+#endif //KIWIXLIB_SERVER_RESPONSE_H

src/subprocess_unix.cpp

@@ -46,7 +46,7 @@ void* UnixImpl::waitForPID(void* _self)
 #endif
 
   UnixImpl* self = static_cast<UnixImpl*>(_self);
-  waitpid(self->m_pid, NULL, WEXITED);
+  waitpid(self->m_pid, NULL, 0);
 
   pthread_mutex_lock(&self->m_mutex);
   self->m_running = false;
@@ -67,7 +67,7 @@ void UnixImpl::run(commandLine_t& commandLine)
       commandLine.push_back(NULL);
       if (execvp(binary, const_cast<char* const*>(commandLine.data()))) {
         perror("Cannot launch\n");
-        exit(-1);
+        _exit(-1);
       }
 
       break;

src/subprocess_windows.cpp

@@ -4,6 +4,7 @@
 
 #include <windows.h>
 #include <winbase.h>
+#include <shlwapi.h>
 #include <iostream>
 #include <sstream>
 
@@ -34,11 +35,11 @@ DWORD WINAPI WinImpl::waitForPID(void* _self)
   return 0;
 }
 
-std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
+std::unique_ptr<wchar_t[]> toWideChar(const std::string& value, size_t min_size = 0)
 {
-  auto size = MultiByteToWideChar(CP_UTF8, 0,
+  size_t size = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, nullptr, 0);
-  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size]);
+  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size>min_size?size:min_size]);
   auto ret = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, wdata.get(), size);
   if (0 == ret) {
@@ -46,6 +47,9 @@ std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
     oss << "Cannot convert to wchar : " << GetLastError();
     throw std::runtime_error(oss.str());
   }
+  if (size < min_size) {
+    memset(wdata.get() + size, 0, min_size-size);
+  }
   return wdata;
 }
 
@@ -55,14 +59,16 @@ void WinImpl::run(commandLine_t& commandLine)
   STARTUPINFOW startInfo = {0};
   PROCESS_INFORMATION procInfo;
   startInfo.cb = sizeof(startInfo);
-  std::ostringstream oss;
+  std::wostringstream oss;
   for(auto& item: commandLine) {
-    oss << item << " ";
+    auto witem = toWideChar(item, MAX_PATH);
+    PathQuoteSpacesW(witem.get());
+    oss << witem.get() << " ";
   }
-  auto wCommandLine = toWideChar(oss.str());
+  auto wCommandLine = oss.str();
   if (CreateProcessW(
     NULL,
-    wCommandLine.get(),
+    const_cast<wchar_t*>(wCommandLine.c_str()),
     NULL,
     NULL,
     false,
@@ -70,7 +76,8 @@ void WinImpl::run(commandLine_t& commandLine)
     NULL,
     NULL,
     &startInfo,
-    &procInfo)) {
+    &procInfo))
+  {
     m_pid = procInfo.dwProcessId;
     m_handle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);

src/tools/base64.cpp

@@ -1,4 +1,4 @@
-/* 
+/*
    base64.cpp and base64.h
 
    Copyright (C) 2004-2008 René Nyffenegger
@@ -24,10 +24,10 @@
    René Nyffenegger rene.nyffenegger@adp-gmbh.ch
 */
 
-#include <common/base64.h>
+#include <tools/base64.h>
 #include <iostream>
 
-static const std::string base64_chars = 
+static const std::string base64_chars =
              "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
              "abcdefghijklmnopqrstuvwxyz"
   "0123456789+/";

src/tools/networkTools.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <tools/networkTools.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <curl/curl.h>
+
+#include <sstream>
+#include <iostream>
+#include <stdexcept>
+
+
+size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+  auto str = static_cast<std::stringstream*>(userdata);
+  str->write(ptr, nmemb);
+  return nmemb;
+}
+
+std::string kiwix::download(const std::string& url) {
+  auto curl = curl_easy_init();
+  std::stringstream ss;
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ss);
+  auto res = curl_easy_perform(curl);
+  if (res != CURLE_OK) {
+    curl_easy_cleanup(curl);
+    throw std::runtime_error("Cannot perform request");
+  }
+  long response_code;
+  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(curl);
+  if (response_code != 200) {
+    throw std::runtime_error("Invalid return code from server");
+  }
+  return ss.str();
+}

src/tools/otherTools.cpp

@@ -0,0 +1,282 @@
+/*
+ * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "tools/otherTools.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+#include "tools/stringTools.h"
+
+#include <map>
+#include <sstream>
+#include <pugixml.hpp>
+
+
+static std::map<std::string, std::string> codeisomapping {
+{ "aa", "aar" },
+{ "af", "afr" },
+{ "ak", "aka" },
+{ "am", "amh" },
+{ "ar", "ara" },
+{ "as", "asm" },
+{ "az", "aze" },
+{ "ba", "bak" },
+{ "be", "bel" },
+{ "bg", "bul" },
+{ "bm", "bam" },
+{ "bn", "ben" },
+{ "bo", "bod" },
+{ "br", "bre" },
+{ "bs", "bos" },
+{ "ca", "cat" },
+{ "ce", "che" },
+{ "co", "cos" },
+{ "cs", "ces" },
+{ "cu", "chu" },
+{ "cv", "chv" },
+{ "cy", "cym" },
+{ "da", "dan" },
+{ "de", "deu" },
+{ "dv", "div" },
+{ "dz", "dzo" },
+{ "ee", "ewe" },
+{ "el", "ell" },
+{ "en", "eng" },
+{ "es", "spa" },
+{ "et", "est" },
+{ "eu", "eus" },
+{ "fa", "fas" },
+{ "ff", "ful" },
+{ "fi", "fin" },
+{ "fo", "fao" },
+{ "fr", "fra" },
+{ "fy", "fry" },
+{ "ga", "gle" },
+{ "gd", "gla" },
+{ "gl", "glg" },
+{ "gn", "grn" },
+{ "gu", "guj" },
+{ "gv", "glv" },
+{ "ha", "hau" },
+{ "he", "heb" },
+{ "hi", "hin" },
+{ "hr", "hrv" },
+{ "hu", "hun" },
+{ "hy", "hye" },
+{ "ia", "ina" },
+{ "id", "ind" },
+{ "ig", "ibo" },
+{ "is", "isl" },
+{ "it", "ita" },
+{ "iu", "iku" },
+{ "ja", "jpn" },
+{ "jv", "jav" },
+{ "ka", "kat" },
+{ "ki", "kik" },
+{ "kk", "kaz" },
+{ "kl", "kal" },
+{ "km", "khm" },
+{ "kn", "kan" },
+{ "ko", "kor" },
+{ "ks", "kas" },
+{ "ku", "kur" },
+{ "kw", "cor" },
+{ "ky", "kir" },
+{ "lb", "ltz" },
+{ "lg", "lug" },
+{ "ln", "lin" },
+{ "lo", "lao" },
+{ "lt", "lit" },
+{ "lv", "lav" },
+{ "mg", "mlg" },
+{ "mi", "mri" },
+{ "mi", "mri" },
+{ "mk", "mkd" },
+{ "ml", "mal" },
+{ "mn", "mon" },
+{ "mr", "mar" },
+{ "ms", "msa" },
+{ "mt", "mlt" },
+{ "my", "mya" },
+{ "nb", "nob" },
+{ "ne", "nep" },
+{ "nl", "nld" },
+{ "nn", "nno" },
+{ "no", "nor" },
+{ "ny", "nya" },
+{ "oc", "oci" },
+{ "om", "orm" },
+{ "or", "ori" },
+{ "os", "oss" },
+{ "pa", "pan" },
+{ "pl", "pol" },
+{ "ps", "pus" },
+{ "pt", "por" },
+{ "qu", "que" },
+{ "rm", "roh" },
+{ "rn", "run" },
+{ "ro", "ron" },
+{ "ru", "rus" },
+{ "rw", "kin" },
+{ "sa", "san" },
+{ "sd", "snd" },
+{ "se", "sme" },
+{ "sg", "sag" },
+{ "si", "sin" },
+{ "sk", "slk" },
+{ "sl", "slv" },
+{ "sn", "sna" },
+{ "so", "som" },
+{ "sq", "sqi" },
+{ "sr", "srp" },
+{ "ss", "ssw" },
+{ "st", "sot" },
+{ "sv", "swe" },
+{ "sw", "swa" },
+{ "ta", "tam" },
+{ "te", "tel" },
+{ "tg", "tgk" },
+{ "th", "tha" },
+{ "ti", "tir" },
+{ "tk", "tuk" },
+{ "tl", "tgl" },
+{ "tn", "tsn" },
+{ "to", "ton" },
+{ "tr", "tur" },
+{ "ts", "tso" },
+{ "tt", "tat" },
+{ "ug", "uig" },
+{ "uk", "ukr" },
+{ "ur", "urd" },
+{ "uz", "uzb" },
+{ "ve", "ven" },
+{ "vi", "vie" },
+{ "wa", "wln" },
+{ "wo", "wol" },
+{ "xh", "xho" },
+{ "yo", "yor" },
+{ "zh", "zho" },
+{ "zu", "zul" }
+};
+
+void kiwix::sleep(unsigned int milliseconds)
+{
+#ifdef _WIN32
+  Sleep(milliseconds);
+#else
+  usleep(1000 * milliseconds);
+#endif
+}
+
+
+struct XmlStringWriter: pugi::xml_writer
+{
+  std::string result;
+  virtual void write(const void* data, size_t size){
+    result.append(static_cast<const char*>(data), size);
+  }
+};
+
+std::string kiwix::nodeToString(const pugi::xml_node& node)
+{
+  XmlStringWriter writer;
+  node.print(writer, "  ");
+  return writer.result;
+}
+
+std::string kiwix::converta2toa3(const std::string& a2code){
+  return codeisomapping.at(a2code);
+}
+
+std::vector<std::string> kiwix::convertTags(const std::string& tags_str)
+{
+  auto tags = kiwix::split(tags_str, ";");
+  std::vector<std::string> tagsList;
+  bool picSeen(false), vidSeen(false), detSeen(false), indexSeen(false);
+  for (auto tag: tags) {
+    picSeen |= (tag == "nopic" || startsWith(tag, "_pictures:"));
+    vidSeen |= (tag == "novid" || startsWith(tag, "_videos:"));
+    detSeen |= (tag == "nodet" || startsWith(tag, "_details:"));
+    indexSeen |= kiwix::startsWith(tag, "_ftindex");
+    if (tag == "nopic") {
+      tagsList.push_back("_pictures:no");
+    } else if (tag == "novid") {
+      tagsList.push_back("_videos:no");
+    } else if (tag == "nodet") {
+      tagsList.push_back("_details:no");
+    } else if (tag == "_ftindex") {
+      tagsList.push_back("_ftindex:yes");
+    } else {
+      tagsList.push_back(tag);
+    }
+  }
+  if (!indexSeen) {
+    tagsList.push_back("_ftindex:no");
+  }
+  if (!picSeen) {
+    tagsList.push_back("_pictures:yes");
+  }
+  if (!vidSeen) {
+    tagsList.push_back("_videos:yes");
+  }
+  if (!detSeen) {
+    tagsList.push_back("_details:yes");
+  }
+  return tagsList;
+}
+
+std::string kiwix::getTagValueFromTagList(
+  const std::vector<std::string>& tagList, const std::string& tagName)
+{
+  for (auto tag: tagList) {
+    if (tag[0] == '_') {
+      auto delimPos = tag.find(':');
+      if (delimPos == std::string::npos) {
+        // No delimiter... what to do ?
+        continue;
+      }
+      auto cTagName = tag.substr(1, delimPos-1);
+      auto cTagValue = tag.substr(delimPos+1);
+      if (cTagName == tagName) {
+        return cTagValue;
+      }
+    }
+  }
+  std::stringstream ss;
+  ss << tagName << " cannot be found";
+  throw std::out_of_range(ss.str());
+}
+
+bool kiwix::convertStrToBool(const std::string& value)
+{
+  if (value == "yes") {
+    return true;
+  } else if (value == "no") {
+    return false;
+  }
+
+  std::stringstream ss;
+  ss << "Tag value '" << value << "' cannot be converted to bool.";
+  throw std::domain_error(ss.str());
+}
+

src/tools/pathTools.cpp

@@ -0,0 +1,490 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "tools/pathTools.h"
+#include <stdexcept>
+
+#ifdef __APPLE__
+#include <limits.h>
+#include <mach-o/dyld.h>
+#elif _WIN32
+#include <direct.h>
+#include <windows.h>
+#include "shlwapi.h"
+#define getcwd _getcwd  // stupid MSFT "deprecation" warning
+#endif
+
+#include "tools/stringTools.h"
+
+#include <string.h>
+#include <map>
+#include <vector>
+#include <sys/stat.h>
+#include <sstream>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <algorithm>
+
+#ifdef _WIN32
+# define SEPARATOR "\\"
+# include <io.h>
+#else
+# define SEPARATOR "/"
+# include <unistd.h>
+# include <sys/stat.h>
+#endif
+
+#include <fcntl.h>
+
+#include <stdlib.h>
+
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
+
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr)
+{
+  auto needed_size = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), NULL, 0, NULL, NULL);
+  std::string ret(needed_size, 0);
+  WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), &ret[0], needed_size, NULL, NULL);
+  return ret;
+}
+
+std::wstring Utf8ToWide(const std::string& str)
+{
+  auto needed_size = MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), NULL, 0);
+  std::wstring ret(needed_size, 0);
+  MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), &ret[0], needed_size);
+  return ret;
+}
+#endif
+
+bool isRelativePath(const std::string& path)
+{
+#ifdef _WIN32
+  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+#else
+  return path.empty() || path.substr(0, 1) == "/" ? false : true;
+#endif
+}
+
+std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
+{
+  std::vector<std::string> ret;
+#ifdef _WIN32
+  //Special case if we have a drive directory not at first.
+  //Starts from there.
+  auto it = find_if(parts.rbegin(), parts.rend(),
+                    [](const std::string& p) ->bool
+                      { return p.length() == 2 && p[1] == ':'; });
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+  }
+#endif
+
+  size_t index = 0;
+  for (auto& part: parts) {
+    index++;
+    if (part == "..") {
+      if (absolute) {
+        // We try to remove as far as possible.
+        if (ret.size() > 1) {
+          ret.pop_back();
+        }
+      } else {
+        // We remove only if we can remove it.
+        // Else we add it.
+        if (!ret.empty() && ret.back() != "..") {
+          ret.pop_back();
+        } else {
+          ret.push_back("..");
+        }
+      }
+      continue;
+    }
+    if (part == "") {
+#ifndef _WIN32
+      if (ret.empty() && (absolute || index<parts.size())) {
+        ret.push_back("");
+      }
+#endif
+      continue;
+    }
+    if (part == ".") {
+      continue;
+    }
+    ret.push_back(part);
+  }
+#ifndef _WIN32
+  if (absolute && ret.size() == 1 && ret.back() == "") {
+    ret.push_back("");
+  }
+#endif
+  return ret;
+}
+
+std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
+{
+  auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
+
+  unsigned int commonCount = 0;
+  while (commonCount < pathParts.size()
+         && commonCount < absolutePathParts.size()
+         && pathParts[commonCount] == absolutePathParts[commonCount]) {
+      commonCount++;
+  }
+
+  std::vector<std::string> relativeParts;
+  for (unsigned int i = commonCount; i < pathParts.size(); i++) {
+    relativeParts.push_back("..");
+  }
+  for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
+    relativeParts.push_back(absolutePathParts[i]);
+  }
+  auto ret = kiwix::join(normalizeParts(relativeParts, false), SEPARATOR);
+  return ret;
+}
+
+std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
+{
+  std::string absolutePath = path;
+  if (path.empty()) {
+    absolutePath = getCurrentDirectory();
+  }
+
+  auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
+  auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
+
+  absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
+  auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
+  return ret;
+}
+
+std::string removeLastPathElement(const std::string& path)
+{
+  auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  if (!parts.empty()) {
+    parts.pop_back();
+  }
+  auto ret = kiwix::join(parts, SEPARATOR);
+  return ret;
+}
+
+std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
+{
+  std::string newPath = directoryPath;
+  if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
+    newPath += SEPARATOR;
+  }
+  newPath += filename;
+  return newPath;
+}
+
+std::string getLastPathElement(const std::string& path)
+{
+  auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
+  if (parts.empty()) {
+    return "";
+  }
+  auto ret = parts.back();
+  return ret;
+}
+
+unsigned int getFileSize(const std::string& path)
+{
+#ifdef _WIN32
+  struct _stat filestatus;
+  _stat(path.c_str(), &filestatus);
+#else
+  struct stat filestatus;
+  stat(path.c_str(), &filestatus);
+#endif
+
+  return filestatus.st_size / 1024;
+}
+
+std::string getFileSizeAsString(const std::string& path)
+{
+  std::ostringstream convert;
+  convert << getFileSize(path);
+  return convert.str();
+}
+
+std::string getFileContent(const std::string& path)
+{
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_RDONLY | _O_BINARY);
+#else
+  auto fd = open(path.c_str(), O_RDONLY);
+#endif
+  std::string content;
+  if (fd != -1) {
+#ifdef _WIN32
+    auto size = _lseeki64(fd, 0, SEEK_END);
+#else
+    auto size = lseek(fd, 0, SEEK_END);
+#endif
+    content.resize(size);
+#ifdef _WIN32
+    _lseeki64(fd, 0, SEEK_SET);
+#else
+    lseek(fd, 0, SEEK_SET);
+#endif
+    auto p = (char*)content.data();
+    while (size) {
+      auto readsize = size > 2048 ? 2048 : size;
+      readsize = ::read(fd, p, readsize);
+      p += readsize;
+      size -= readsize;
+    }
+    close(fd);
+  }
+  return content;
+}
+
+bool fileExists(const std::string& path)
+{
+#ifdef _WIN32
+  return PathFileExistsW(Utf8ToWide(path).c_str());
+#else
+  bool flag = false;
+  std::fstream fin;
+  fin.open(path.c_str(), std::ios::in);
+  if (fin.is_open()) {
+    flag = true;
+  }
+  fin.close();
+  return flag;
+#endif
+}
+
+bool makeDirectory(const std::string& path)
+{
+#ifdef _WIN32
+  int status = _wmkdir(Utf8ToWide(path).c_str());
+#else
+  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+#endif
+  return status == 0;
+}
+
+std::string makeTmpDirectory()
+{
+#ifdef _WIN32
+  wchar_t cbase[MAX_PATH];
+  wchar_t ctmp[MAX_PATH];
+  GetTempPathW(MAX_PATH-14, cbase);
+  // This create a file for us, ensure it is unique.
+  // So we need to delete it and create the directory using the same name.
+  GetTempFileNameW(cbase, L"kiwix", 0, ctmp);
+  DeleteFileW(ctmp);
+  _wmkdir(ctmp);
+  return WideToUtf8(ctmp);
+#else
+  char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
+  std::string dir = mkdtemp(_template_array);
+  return dir;
+#endif
+}
+
+/* Try to create a link and if does not work then make a copy */
+bool copyFile(const std::string& sourcePath, const std::string& destPath)
+{
+#ifdef _WIN32
+  return CopyFileW(Utf8ToWide(sourcePath).c_str(), Utf8ToWide(destPath).c_str(), 1);
+#else
+  try {
+    if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
+      std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
+      std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
+      outfile << infile.rdbuf();
+    }
+  } catch (std::exception& e) {
+    std::cerr << e.what() << std::endl;
+    return false;
+  }
+
+  return true;
+#endif
+}
+
+std::string getExecutablePath(bool realPathOnly)
+{
+  if (!realPathOnly) {
+    char* cAppImage = ::getenv("APPIMAGE");
+    if (cAppImage) {
+      char* cArgv0 = ::getenv("ARGV0");
+      char* cOwd = ::getenv("OWD");
+      if (cArgv0 && cOwd) {
+        auto ret = appendToDirectory(cOwd, cArgv0);
+        return ret;
+      }
+    }
+  }
+
+#ifdef _WIN32
+  std::wstring binRootPath(PATH_MAX, 0);
+  GetModuleFileNameW(NULL, &binRootPath[0], PATH_MAX);
+  std::string ret = WideToUtf8(binRootPath);
+  return ret;
+#elif __APPLE__
+  char binRootPath[PATH_MAX];
+  uint32_t max = (uint32_t)PATH_MAX;
+  _NSGetExecutablePath(binRootPath, &max);
+  return std::string(binRootPath);
+#else
+  char binRootPath[PATH_MAX];
+  ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
+  if (size != -1) {
+    return std::string(binRootPath, size);
+  }
+  return "";
+#endif
+}
+
+bool writeTextFile(const std::string& path, const std::string& content)
+{
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC, S_IWRITE);
+#else
+  auto fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC,
+                               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+#endif
+  if (fd == -1)
+    return false;
+
+  if (write(fd, content.c_str(), content.size()) != (long)content.size()) {
+    close(fd);
+    return false;
+  }
+  close(fd);
+  return true;
+}
+
+std::string getCurrentDirectory()
+{
+#ifdef _WIN32
+  wchar_t* a_cwd = _wgetcwd(NULL, 0);
+  std::string ret = WideToUtf8(a_cwd);
+  free(a_cwd);
+#else
+  char* a_cwd = getcwd(NULL, 0);
+  std::string ret(a_cwd);
+  free(a_cwd);
+#endif
+  return ret;
+}
+
+std::string getDataDirectory()
+{
+// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
+#ifdef _WIN32
+  wchar_t* cDataDir = ::_wgetenv(L"KIWIX_DATA_DIR");
+  if (cDataDir != nullptr) {
+    return WideToUtf8(cDataDir);
+  }
+#else
+  char* cDataDir = ::getenv("KIWIX_DATA_DIR");
+  if (cDataDir != nullptr) {
+    return cDataDir;
+  }
+#endif
+
+// Compute the dataDir from the user directory.
+  std::string dataDir;
+#ifdef _WIN32
+  cDataDir = ::_wgetenv(L"APPDATA");
+  if (cDataDir == nullptr)
+    cDataDir = ::_wgetenv(L"USERPROFILE");
+  if (cDataDir != nullptr)
+    dataDir = WideToUtf8(cDataDir);
+#else
+  cDataDir = ::getenv("XDG_DATA_HOME");
+  if (cDataDir != nullptr) {
+    dataDir = cDataDir;
+  } else {
+    cDataDir = ::getenv("HOME");
+    if (cDataDir != nullptr) {
+      dataDir = cDataDir;
+      dataDir = appendToDirectory(dataDir, ".local");
+      dataDir = appendToDirectory(dataDir, "share");
+    }
+  }
+#endif
+  if (!dataDir.empty()) {
+    dataDir = appendToDirectory(dataDir, "kiwix");
+    makeDirectory(dataDir);
+    return dataDir;
+  }
+
+// Let's use the currentDirectory
+  return getCurrentDirectory();
+}
+
+static std::map<std::string, std::string> extMimeTypes = {
+ { "html", "text/html"},
+ { "htm", "text/html"},
+ { "png", "image/png"},
+ { "tiff", "image/tiff"},
+ { "tif", "image/tiff"},
+ { "jpeg", "image/jpeg"},
+ { "jpg", "image/jpeg"},
+ { "gif", "image/gif"},
+ { "svg", "image/svg+xml"},
+ { "txt", "text/plain"},
+ { "xml", "text/xml"},
+ { "pdf", "application/pdf"},
+ { "ogg", "application/ogg"},
+ { "js", "application/javascript"},
+ { "css", "text/css"},
+ { "otf", "application/vnd.ms-opentype"},
+ { "ttf", "application/font-ttf"},
+ { "woff", "application/font-woff"},
+ { "vtt", "text/vtt"}
+};
+
+/* Try to get the mimeType from the file extension */
+std::string getMimeTypeForFile(const std::string& filename)
+{
+  std::string mimeType = "text/plain";
+  auto pos = filename.find_last_of(".");
+
+  if (pos != std::string::npos) {
+    std::string extension = filename.substr(pos + 1);
+
+    auto it = extMimeTypes.find(extension);
+    if (it != extMimeTypes.end()) {
+      mimeType = it->second;
+    } else {
+      it = extMimeTypes.find(kiwix::lcAll(extension));
+      if (it != extMimeTypes.end()) {
+        mimeType = it->second;
+      }
+    }
+  }
+
+  return mimeType;
+}
+

src/tools/regexTools.cpp

@@ -17,41 +17,48 @@
  * MA 02110-1301, USA.
  */
 
-#include <common/regexTools.h>
+#include <tools/regexTools.h>
+#include <tools/lock.h>
 
-std::map<std::string, icu::RegexMatcher*> regexCache;
+#include <unicode/regex.h>
+#include <unicode/ucnv.h>
 
-icu::RegexMatcher* buildRegex(const std::string& regex)
+#include <memory>
+#include <map>
+#include <stdexcept>
+#include <pthread.h>
+
+std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;
+static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
+
+std::unique_ptr<icu::RegexMatcher> buildMatcher(const std::string& regex, icu::UnicodeString& content)
 {
-  icu::RegexMatcher* matcher;
-  auto itr = regexCache.find(regex);
-
+  std::shared_ptr<icu::RegexPattern> pattern;
   /* Regex is in cache */
-  if (itr != regexCache.end()) {
-    matcher = itr->second;
+  try {
+    pattern = regexCache.at(regex);
+  } catch (std::out_of_range&) {
+    // Redo the search with a lock to avoid race condition.
+    kiwix::Lock l(&regexLock);
+    try {
+      pattern = regexCache.at(regex);
+    } catch (std::out_of_range&) {
+      UErrorCode status = U_ZERO_ERROR;
+      UParseError pe;
+      icu::UnicodeString uregex(regex.c_str());
+      pattern.reset(icu::RegexPattern::compile(uregex, UREGEX_CASE_INSENSITIVE, pe, status));
+      regexCache[regex] = pattern;
+    }
   }
-
-  /* Regex needs to be parsed (and cached) */
-  else {
-    UErrorCode status = U_ZERO_ERROR;
-    icu::UnicodeString uregex(regex.c_str());
-    matcher = new icu::RegexMatcher(uregex, UREGEX_CASE_INSENSITIVE, status);
-    regexCache[regex] = matcher;
-  }
-
-  return matcher;
+  UErrorCode status = U_ZERO_ERROR;
+  return std::unique_ptr<icu::RegexMatcher>(pattern->matcher(content, status));
 }
 
-/* todo */
-void freeRegexCache()
-{
-}
 bool matchRegex(const std::string& content, const std::string& regex)
 {
   ucnv_setDefaultName("UTF-8");
   icu::UnicodeString ucontent(content.c_str());
-  auto matcher = buildRegex(regex);
-  matcher->reset(ucontent);
+  auto matcher = buildMatcher(regex, ucontent);
   return matcher->find();
 }
 
@@ -60,10 +67,9 @@ std::string replaceRegex(const std::string& content,
                          const std::string& regex)
 {
   ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
   icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildRegex(regex);
-  matcher->reset(ucontent);
+  icu::UnicodeString ucontent(content.c_str());
+  auto matcher = buildMatcher(regex, ucontent);
   UErrorCode status = U_ZERO_ERROR;
   auto uresult = matcher->replaceAll(ureplacement, status);
   std::string tmp;
@@ -72,15 +78,13 @@ std::string replaceRegex(const std::string& content,
 }
 
 std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string regex,
+                                   const std::string& regex,
                                    const std::string& replacement)
 {
   ucnv_setDefaultName("UTF-8");
   icu::UnicodeString ucontent(content.c_str());
   icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildRegex(regex);
-  matcher->reset(ucontent);
-
+  auto matcher = buildMatcher(regex, ucontent);
   if (matcher->find()) {
     UErrorCode status = U_ZERO_ERROR;
     ucontent.insert(matcher->end(status), ureplacement);

src/tools/stringTools.cpp

@@ -17,8 +17,9 @@
  * MA 02110-1301, USA.
  */
 
-#include <common/stringTools.h>
+#include <tools/stringTools.h>
 
+#include <tools/pathTools.h>
 #include <unicode/normlzr.h>
 #include <unicode/rep.h>
 #include <unicode/translit.h>
@@ -26,6 +27,10 @@
 #include <unicode/uniset.h>
 #include <unicode/ustring.h>
 
+
+#include <iostream>
+#include <iomanip>
+
 /* tell ICU where to find its dat file (tables) */
 void kiwix::loadICUExternalTables()
 {
@@ -36,7 +41,7 @@ void kiwix::loadICUExternalTables()
       = computeAbsolutePath(executableDirectory, "icudt58l.dat");
   try {
     u_setDataDirectory(datPath.c_str());
-  } catch (exception& e) {
+  } catch (std::exception& e) {
     std::cerr << e.what() << std::endl;
   }
 #endif
@@ -57,8 +62,6 @@ std::string kiwix::removeAccents(const std::string& text)
   return unaccentedText;
 }
 
-#ifndef __ANDROID__
-
 /* Prepare integer for display */
 std::string kiwix::beautifyInteger(uint64_t number)
 {
@@ -138,8 +141,6 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-#endif
-
 /* urlEncode() based on javascript encodeURI() &
    encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
 
@@ -266,36 +267,42 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
-                                      const std::string& delims = " *-")
+                                      const std::string& delims,
+                                      bool trimEmpty)
 {
-  std::string::size_type lastPos = str.find_first_not_of(delims, 0);
-  std::string::size_type pos = str.find_first_of(delims, lastPos);
+  std::string::size_type lastPos = 0;
+  std::string::size_type pos = 0;
   std::vector<std::string> tokens;
-
-  while (std::string::npos != pos || std::string::npos != lastPos) {
-    tokens.push_back(str.substr(lastPos, pos - lastPos));
-    lastPos = str.find_first_not_of(delims, pos);
-    pos = str.find_first_of(delims, lastPos);
+  while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
+  {
+    auto token = str.substr(lastPos, pos - lastPos);
+    if (!trimEmpty || !token.empty()) {
+      tokens.push_back(token);
+    }
+    lastPos = pos + 1;
   }
 
+  auto token = str.substr(lastPos);
+  if (!trimEmpty || !token.empty()) {
+    tokens.push_back(token);
+  }
   return tokens;
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
+std::string kiwix::join(const std::vector<std::string>& list, const std::string& sep)
 {
-  const std::string m1(lhs), m2(rhs);
-  return split(m1, m2);
+  std::stringstream ss;
+  bool first = true;
+  for (auto& s:list) {
+    if (!first) {
+      ss << sep;
+    }
+    first = false;
+    ss << s;
+  }
+  return ss.str();
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
-{
-  return split(lhs, rhs.c_str());
-}
-
-std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
-{
-  return split(lhs.c_str(), rhs);
-}
 
 std::string kiwix::ucFirst(const std::string& word)
 {
@@ -376,3 +383,11 @@ std::string kiwix::normalize(const std::string& word)
 {
   return kiwix::lcAll(word);
 }
+
+
+bool kiwix::startsWith(const std::string& base, const std::string& start)
+{
+   return start.length() <= base.length()
+        && std::equal(start.begin(), start.end(), base.begin());
+}
+

src/wrapper/java/book.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Book.h"
+
+#include "utils.h"
+#include "book.h"
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_allocate(
+  JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Book>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Book>(env, thisObj);
+}
+
+#define BOOK (getPtr<kiwix::Book>(env, thisObj))
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
+{
+  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
+}
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
+{
+  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
+}
+
+#define GETTER(retType, name) JNIEXPORT retType JNICALL \
+Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
+{ \
+  auto cRet = BOOK->name(); \
+  retType ret = c2jni(cRet, env); \
+  return ret; \
+}
+
+GETTER(jstring, getId)
+GETTER(jstring, getPath)
+GETTER(jboolean, isPathValid)
+GETTER(jstring, getTitle)
+GETTER(jstring, getDescription)
+GETTER(jstring, getLanguage)
+GETTER(jstring, getCreator)
+GETTER(jstring, getPublisher)
+GETTER(jstring, getDate)
+GETTER(jstring, getUrl)
+GETTER(jstring, getName)
+GETTER(jstring, getFlavour)
+GETTER(jstring, getTags)
+GETTER(jlong, getArticleCount)
+GETTER(jlong, getMediaCount)
+GETTER(jlong, getSize)
+GETTER(jstring, getFavicon)
+GETTER(jstring, getFaviconUrl)
+GETTER(jstring, getFaviconMimeType)
+
+METHOD(jstring, Book, getTagStr, jstring tagName) try {
+  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
+  return c2jni(cRet, env);
+} catch(...) {
+  return c2jni<std::string>("", env);
+}
+
+#undef GETTER

src/wrapper/java/filter.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Filter.h"
+
+#include "library.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+METHOD0(void, Filter, allocate) {
+  allocate<kiwix::Filter>(env, thisObj);
+}
+
+METHOD0(void, Filter, dispose) {
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
+
+#define FORWARD(name, args_type) \
+METHOD(jobject, Filter, name, args_type value) { \
+  FILTER->name(jni2c(value, env)); \
+  return thisObj; \
+}
+
+#define FORWARDA(name, args_type) \
+METHOD(jobject, Filter, name, jobjectArray value) { \
+  FILTER->name(jni2c<args_type>(value, env)); \
+  return thisObj; \
+}
+
+
+
+FORWARD(local, jboolean)
+FORWARD(remote, jboolean)
+FORWARD(valid, jboolean)
+FORWARDA(acceptTags, jstring)
+FORWARDA(rejectTags, jstring)
+FORWARD(lang, jstring)
+FORWARD(publisher, jstring)
+FORWARD(creator, jstring)
+FORWARD(maxSize, jlong)
+FORWARD(query, jstring)
+
+

src/wrapper/java/kiwixicu.cpp

@@ -19,7 +19,7 @@
  */
 
 #include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwix.h"
+#include "org_kiwix_kiwixlib_JNIICU.h"
 
 #include <iostream>
 #include <string>
@@ -28,10 +28,14 @@
 
 #include "utils.h"
 
+#if __ANDROID__
 pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER;
+#else
+pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+#endif
 
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
-    JNIEnv* env, jobject obj, jstring dirStr)
+JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
+    JNIEnv* env, jclass kclass, jstring dirStr)
 {
   std::string cPath = jni2c(dirStr, env);
 

src/wrapper/java/kiwixreader.cpp

@@ -21,10 +21,10 @@
 
 #include <jni.h>
 #include <zim/file.h>
-#include <android/log.h>
+#include <exception>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
-#include "common/base64.h"
+#include "tools/base64.h"
 #include "reader.h"
 #include "utils.h"
 
@@ -34,14 +34,14 @@ JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
 {
   std::string cPath = jni2c(filename, env);
 
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create reader with: %s", cPath.c_str());
+  LOG("Attempting to create reader with: %s", cPath.c_str());
   Lock l;
   try {
     kiwix::Reader* reader = new kiwix::Reader(cPath);
     return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error opening ZIM file");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Error opening ZIM file");
+    LOG(e.what());
     return 0;
   }
 }
@@ -64,8 +64,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
     std::string cUrl = READER->getMainPage().getPath();
     url = c2jni(cUrl, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM main page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM main page");
+    LOG(e.what());
     url = NULL;
   }
   return url;
@@ -80,8 +80,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
     std::string cId = READER->getId();
     id = c2jni(cId, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM id");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
     id = NULL;
   }
 
@@ -91,14 +91,14 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
 JNIEXPORT jint JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
 {
-  jint size;
+  jint size = 0;
 
   try {
     int cSize = READER->getFileSize();
-    size = c2jni(cSize);
+    size = c2jni(cSize, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM file size");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM file size");
+    LOG(e.what());
   }
 
   return size;
@@ -113,8 +113,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
     std::string cCreator = READER->getCreator();
     creator = c2jni(cCreator, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM creator");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM creator");
+    LOG(e.what());
     creator = NULL;
   }
 
@@ -130,8 +130,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
     std::string cPublisher = READER->getPublisher();
     publisher = c2jni(cPublisher, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM publish");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM publish");
+    LOG(e.what());
     publisher = NULL;
   }
   return publisher;
@@ -146,8 +146,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
     std::string cName = READER->getName();
     name = c2jni(cName, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM name");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM name");
+    LOG(e.what());
     name = NULL;
   }
   return name;
@@ -166,8 +166,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
         base64_encode(cContent),
         env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM favicon");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM favicon");
+    LOG(e.what());
     favicon = NULL;
   }
   return favicon;
@@ -182,8 +182,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
     std::string cDate = READER->getDate();
     date = c2jni(cDate, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM date");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM date");
+    LOG(e.what());
     date = NULL;
   }
   return date;
@@ -198,8 +198,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
     std::string cLanguage = READER->getLanguage();
     language = c2jni(cLanguage, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM language");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM language");
+    LOG(e.what());
     language = NULL;
   }
 
@@ -217,15 +217,30 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
     auto cMimeType = entry.getMimetype();
     mimeType = c2jni(cMimeType, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get mime-type for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
+    LOG(e.what());
     mimeType = NULL;
   }
   return mimeType;
 }
 
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
+     JNIEnv* env, jobject obj, jstring url)
+{
+  jstring finalUrl;
+  std::string cUrl = jni2c(url, env);
+  try {
+    auto entry = READER->getEntryFromEncodedPath(cUrl);
+    entry = entry.getFinalEntry();
+    finalUrl = c2jni(entry.getPath(), env);
+  } catch (std::exception& e) {
+    finalUrl = c2jni(std::string(), env);
+  }
+  return finalUrl;
+}
+
 JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
-    JNIEnv* env, jobject obj, jstring url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
+    JNIEnv* env, jobject obj, jobject url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
 {
   /* Default values */
   setStringObjValue("", titleObj, env);
@@ -234,24 +249,27 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
   jbyteArray data = env->NewByteArray(0);
 
   /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
+  std::string cUrl = getStringObjValue(url, env);
   unsigned int cSize = 0;
 
   try {
     auto entry = READER->getEntryFromEncodedPath(cUrl);
+    bool isRedirect = entry.isRedirect();
     entry = entry.getFinalEntry();
     cSize = entry.getSize();
     setIntObjValue(cSize, sizeObj, env);
-
-    data = env->NewByteArray(cSize);
-    env->SetByteArrayRegion(
-        data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
-
     setStringObjValue(entry.getMimetype(), mimeTypeObj, env);
     setStringObjValue(entry.getTitle(), titleObj, env);
+    if (isRedirect) {
+      setStringObjValue(entry.getPath(), url, env);
+    } else {
+      data = env->NewByteArray(cSize);
+      env->SetByteArrayRegion(
+          data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
+    }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get content for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get content for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
 
   return data;
@@ -266,8 +284,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
   /* Default values */
   /* Retrieve the content */
   std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset);
-  unsigned int cLen = jni2c(len);
+  unsigned int cOffset = jni2c(offset, env);
+  unsigned int cLen = jni2c(len, env);
   try {
     auto entry = READER->getEntryFromEncodedPath(cUrl);
     entry = entry.getFinalEntry();
@@ -282,12 +300,28 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
       setIntObjValue(cLen, sizeObj, env);
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
+    LOG(e.what());
   }
   return data;
 }
 
+JNIEXPORT jlong JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
+    JNIEnv* env, jobject obj, jstring url)
+{
+  std::string cUrl = jni2c(url, env);
+  try {
+    auto entry = READER->getEntryFromEncodedPath(cUrl);
+    entry = entry.getFinalEntry();
+    return c2jni(entry.getSize(), env);
+  } catch(std::exception& e) {
+    LOG("Unable to get size for url : %s", cUrl.c_str());
+    LOG(e.what());
+  }
+  return c2jni(0, env);
+}
+
 JNIEXPORT jobject JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     JNIEnv* env, jobject obj, jstring url)
@@ -304,8 +338,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     auto part_info = entry.getDirectAccessInfo();
     setPairObjValue(part_info.first, part_info.second, pair, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get direct access info for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
   return pair;
 }
@@ -318,15 +352,15 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 {
   jboolean retVal = JNI_FALSE;
   std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count);
+  unsigned int cCount = jni2c(count, env);
 
   try {
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get search results for pattern: %s", cPrefix.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
+    LOG(e.what());
   }
 
   return retVal;
@@ -335,19 +369,22 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
                                                          jobject obj,
-                                                         jobject titleObj)
+                                                         jobject titleObj,
+                                                         jobject urlObj)
 {
   jboolean retVal = JNI_FALSE;
   std::string cTitle;
+  std::string cUrl;
 
   try {
-    if (READER->getNextSuggestion(cTitle)) {
+    if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
+      setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get next suggestion");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get next suggestion");
+    LOG(e.what());
   }
 
   return retVal;
@@ -367,8 +404,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
     setStringObjValue(entry.getPath(), urlObj, env);
     return JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get url for title %s: ", cTitle.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get url for title %s: ", cTitle.c_str());
+    LOG(e.what());
   }
 
   return JNI_FALSE;
@@ -383,8 +420,8 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
     std::string cTitle = READER->getTitle();
     title = c2jni(cTitle, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim title");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim title");
+    LOG(e.what());
     title = NULL;
   }
   return title;
@@ -399,13 +436,42 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
     std::string cDescription = READER->getDescription();
     description = c2jni(cDescription, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim description");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim description");
+    LOG(e.what());
     description = NULL;
   }
   return description;
 }
 
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
+{
+  jint articleCount = 0;
+  try {
+    auto cArticleCount = READER->getArticleCount();
+    articleCount = c2jni(cArticleCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get article count.");
+    LOG(e.what());
+  }
+  return articleCount;
+}
+
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
+{
+  jint mediaCount = 0;
+  try {
+    auto cMediaCount = READER->getMediaCount();
+    mediaCount = c2jni(cMediaCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get media count.");
+    LOG(e.what());
+  }
+  return mediaCount;
+}
+
+
 JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     JNIEnv* env, jobject obj, jobject urlObj)
 {
@@ -417,8 +483,8 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     setStringObjValue(cUrl, urlObj, env);
     retVal = JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get random page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get random page");
+    LOG(e.what());
   }
   return retVal;
 }

src/wrapper/java/kiwixsearcher.cpp

@@ -52,14 +52,14 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
 {
   auto searcher = SEARCHER;
 
-  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)), "");
+  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)));
 }
 
 JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
     JNIEnv* env, jobject obj, jstring query, jint count)
 {
   std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count);
+  unsigned int ccount = jni2c(count, env);
 
   SEARCHER->search(cquery, 0, ccount);
 }

src/wrapper/java/kiwixserver.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include <zim/file.h>
+#include "org_kiwix_kiwixlib_JNIKiwixServer.h"
+
+#include "tools/base64.h"
+#include "server.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
+    JNIEnv* env, jobject obj, jobject jLibrary)
+{
+  LOG("Attempting to create server");
+  Lock l;
+  try {
+    auto library = getPtr<kiwix::Library>(env, jLibrary);
+    kiwix::Server* server = new kiwix::Server(library);
+    return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
+  } catch (std::exception& e) {
+    LOG("Error creating the server");
+    LOG(e.what());
+    return 0;
+  }
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_dispose(JNIEnv* env, jobject obj)
+{
+  Handle<kiwix::Server>::dispose(env, obj);
+}
+
+#define SERVER (Handle<kiwix::Server>::getHandle(env, obj))
+
+/* Kiwix library functions */
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setRoot(JNIEnv* env, jobject obj, jstring jRoot)
+{
+  std::string root = jni2c(jRoot, env);
+  SERVER->setRoot(root);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setAddress(JNIEnv* env, jobject obj, jstring jAddress)
+{
+  std::string address = jni2c(jAddress, env);
+  SERVER->setAddress(address);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setPort(JNIEnv* env, jobject obj, int port)
+{
+  SERVER->setPort(port);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setNbThreads(JNIEnv* env, jobject obj, int threads)
+{
+  SERVER->setNbThreads(threads);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboolean withTaskbar, jboolean withLibraryButton)
+{
+  SERVER->setTaskbar(withTaskbar, withLibraryButton);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
+{
+  SERVER->setBlockExternalLinks(blockExternalLinks);
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
+{
+  return SERVER->start();
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_stop(JNIEnv* env, jobject obj)
+{
+  SERVER->stop();
+}

src/wrapper/java/library.cpp

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Library.h"
+
+#include "library.h"
+#include "reader.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_allocate(
+    JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Library>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
+
+/* Kiwix library functions */
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Library_addBook(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    kiwix::Reader reader(cPath);
+    kiwix::Book book;
+    book.update(reader);
+    return LIBRARY->addBook(book);
+  } catch (std::exception& e) {
+    LOG("Unable to add the book");
+    LOG(e.what()); }
+  return false;
+}
+
+METHOD(jobject, Library, getBookById, jstring id) {
+  auto cId = jni2c(id, env);
+  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
+  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
+  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
+  jobject book = env->NewObject(cls, constructorId);
+  setPtr(env, book, cBook);
+  return book;
+}
+
+METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
+  return LIBRARY->getBookCount(localBooks, remoteBooks);
+}
+
+METHOD0(jobjectArray, Library, getBooksIds) {
+  return c2jni(LIBRARY->getBooksIds(), env);
+}
+
+METHOD(jobjectArray, Library, filter, jobject filterObj) {
+  auto filter = getPtr<kiwix::Filter>(env, filterObj);
+  return c2jni(LIBRARY->filter(*filter), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksLanguages) {
+  return c2jni(LIBRARY->getBooksLanguages(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksCreators) {
+  return c2jni(LIBRARY->getBooksCreators(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksPublisher) {
+  return c2jni(LIBRARY->getBooksPublishers(), env);
+}
+