New version 9.2

On the appveyor CI, the link of the `server` test fails with
`libmicrohttpd`.

Do not build the test on windows to not break the CI.

.codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+
+ignore:
+  - "test"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/

.github/move.yml

@@ -0,0 +1,27 @@
+# Configuration for Move Issues - https://github.com/dessant/move-issues
+
+# Delete the command comment when it contains no other content
+deleteCommand: true
+
+# Close the source issue after moving
+closeSourceIssue: true
+
+# Lock the source issue after moving
+lockSourceIssue: false
+
+# Mention issue and comment authors
+mentionAuthors: true
+
+# Preserve mentions in the issue content
+keepContentMentions: true
+
+# Move labels that also exist on the target repository
+moveLabels: true
+
+# Set custom aliases for targets
+# aliases:
+#   r: repo
+#   or: owner/repo
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

.github/workflows/ci.yml

@@ -0,0 +1,152 @@
+name: CI
+
+on: [push]
+
+jobs:
+  Macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v1
+      - name: Setup python 3.5
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.5'
+      - name: Install packages
+        uses: mstksg/get-package@v1
+        with:
+          brew: gcovr pkg-config ninja
+      - name: Install python modules
+        run: pip3 install meson==0.49.2 pytest
+      - name: Install deps
+        shell: bash
+        run: |
+          ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
+          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
+      - name: Compile
+        shell: bash
+        run: |
+          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
+          meson . build --default-library=shared -Db_coverage=true
+          cd build
+          ninja
+      - name: Test
+        shell: bash
+        run: |
+          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
+          cd build
+          meson test --verbose
+          ninja coverage
+        env:
+          SKIP_BIG_MEMORY_TEST: 1
+      - name: Publish coverage
+        shell: bash
+        run: |
+          curl https://codecov.io/bash -o codecov.sh
+          bash codecov.sh -n osx_native_dyn -Z
+          rm codecov.sh
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - native_static
+          - native_dyn
+          - android_arm
+          - android_arm64
+          - win32_static
+          - win32_dyn
+        include:
+          - target: native_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: native_dyn
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: android_arm
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: android_arm64
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: win32_static
+            image_variant: f31
+            lib_postfix: '64'
+          - target: win32_dyn
+            image_variant: f31
+            lib_postfix: '64'
+    env:
+      HOME: /home/runner
+    runs-on: ubuntu-latest
+    container:
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
+    steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+    - name: Checkout code
+      shell: python
+      run: |
+        from subprocess import check_call
+        from os import environ
+        command = [
+          'git', 'clone',
+          'https://github.com/${{github.repository}}',
+          '--depth=1',
+          '--branch', '${{steps.extract_branch.outputs.branch}}'
+        ]
+        check_call(command, cwd=environ['HOME'])
+    - name: Install deps
+      shell: bash
+      run: |
+        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
+        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
+    - name: Compile
+      shell: bash
+      run: |
+        meson --version
+        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
+          MESON_OPTION="--default-library=shared"
+        else
+          MESON_OPTION="--default-library=static"
+        fi
+        if [[ "${{matrix.target}}" =~ native_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
+        else
+          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
+        fi
+        if [[ "${{matrix.target}}" =~ android_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+        fi
+        cd $HOME/kiwix-lib
+        meson . build ${MESON_OPTION}
+        cd build
+        ninja
+      env:
+        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
+    - name: Test
+      if: startsWith(matrix.target, 'native_')
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib/build
+        meson test --verbose
+        ninja coverage
+      env:
+        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
+        SKIP_BIG_MEMORY_TEST: 1
+    - name: Publish coverage
+      shell: bash
+      run: |
+        cd $HOME/kiwix-lib
+        curl https://codecov.io/bash -o codecov.sh
+        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
+        rm codecov.sh
+      if: startsWith(matrix.target, 'native_')
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,2 +1,4 @@
 .idea/
 *.swp
+subprojects/googletest-release*
+*.class

.travis.yml

@@ -1,37 +0,0 @@
-language: cpp
-dist: xenial
-sudo: true
-cache: ccache
-before_install:
-- PATH=$PATH:$HOME/bin
-install: travis/install_deps.sh
-script: travis/compile.sh
-env:
-  matrix:
-    - PLATFORM="native_static"
-    - PLATFORM="native_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-addons:
-  apt:
-    packages:
-      - cmake
-      - python3.5
-      - python3-pip
-      - libbz2-dev
-      - ccache
-      - zlib1g-dev
-      - uuid-dev
-      - libctpp2-dev
-      - ctpp2-utils
-      - libmicrohttpd-dev
-      - g++-mingw-w64-i686
-      - gcc-mingw-w64-i686
-      - gcc-mingw-w64-base
-      - mingw-w64-tools
-matrix:
-  include:
-    - env: PLATFORM="native_dyn"
-      os: osx

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,112 @@
+kiwix-lib 9.2
+=============
+
+ * Add tests
+ * Refactoring server code.
+ * [SERVER] Add HEAD, Etag and If-None-Match support.
+ * [SERVER] Compress opds catalog answers.
+
+kiwix-lib 9.1.2
+===============
+
+ * Do not use the pathToSave if it is empty.
+
+kiwix-lib 9.1.1
+===============
+
+ * Fix the detection of the dataDirectory on windows.
+
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
+kiwix-lib 9.0.1
+===============
+
+ * [JAVA] Use a long to store the offset of an article in the zim file instead
+    of an int.
+
+kiwix-lib 9.0.0
+===============
+
+ * [OPDS] Correctly set the id of the OPDS stream.
+ * [OPDS] Do not try to filter the catalog if no filter field is given in the
+   request.
+ * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
+ * [LIBRARY] Remove the function the read file using a native path.
+   All path must be utf8, no need to pass a native path along the utf8 path.
+ * [TEST] Fix tests using the main function of gtest instead of custom one.
+ * [CI] Move to github CI instead of Travis.
+ * The `Book::update` method always update the book's fields. Even if they are
+   not empty.
+ * [JAVA] Add wrapping around the library manager (opds parsing)
+ * [ARIA2] Add api option to start download with option (destination folder)
+ * [OPDS] Fixes about opds parsing, generation (missing attributes)
+   and requesting (server)
+ * Add methods on `Book` to get specific tag values (was on `Reader` only)
+ * Add flavour attribute to `Book`
+ * Fix opensearch description.
+ * Trust the given library.xml (by default) instead of reading the value from
+   the zim files.
+ * [OPDS] Be able to filter the content by name or size.
+ * [WINDOWS] Fix launching subcommand when there is spaces in the path.
+
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
+kiwix-lib 8.1.0
+===============
+
+ * Fix pathTools manipulation.
+ * Add missing implementation of getArticleCount and getMediaCount on android.
+ * Correctly convert windows path to utf8.
+ * Add code coverage in the CI
+
+kiwix-lib 8.0.1
+===============
+
+ * Fix join function
+
+kiwix-lib 8.0.0
+===============
+
+ * Add new methods to get all (and new) metadata from the zim file.
+ * Add methods to get the value of a specific tag.
+ * [API Change] Convert tags value to the new convention.
+ * [API Change] Rename `getMatatag` method to `getMetadata`
+ * [ABI Change] Correctly detect executable path in appimage.
+
+kiwix-lib 7.0.0
+===============
+
+ * [API break] Add a argument to kiwix-serve to specify the library to use.
+
+kiwix-lib 6.0.4
+===============
+
+ * Fix HTML rendering of the search result if there is no result.
+ * Do not crash at html rendering if request ask for 0 results (start == end)
+ * Correctly find the executable path if we are using AppImage
+
 kiwix-lib 6.0.3
 ===============
 

README.md

@@ -3,11 +3,13 @@ Kiwix library
 
 The Kiwix library provides the [Kiwix](https://kiwix.org) software
 suite core. It contains the code shared by all Kiwix ports (Windows,
-Linux, OSX, Android, ...).
+GNU/Linux, macOS, Android, iOS, ...).
 
 [![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
-[![Build Status](https://travis-ci.org/kiwix/kiwix-lib.svg?branch=master)](https://travis-ci.org/kiwix/kiwix-lib)
+[![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
+[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
+[![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -39,7 +41,7 @@ libraries need to be available:
 * [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
-set CPPFLAGS with correct `-I` option)
+set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -83,6 +85,15 @@ Meson. If you want statically linked libraries, you can add
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+Testing
+-------
+
+To run the automated tests:
+```bash
+cd build
+meson test
+```
+
 Installation
 ------------
 

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '6.0.3' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '9.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,5 +1,5 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="kiwix.org.kiwixlib">
+    package="org.kiwix.kiwixlib">
 
     <application
         android:allowBackup="true"

include/book.h

@@ -60,6 +60,9 @@ class Book
   const std::string& getUrl() const { return m_url; }
   const std::string& getName() const { return m_name; }
   const std::string& getTags() const { return m_tags; }
+  std::string getTagStr(const std::string& tagName) const;
+  bool getTagBool(const std::string& tagName) const;
+  const std::string& getFlavour() const { return m_flavour; }
   const std::string& getOrigId() const { return m_origId; }
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
@@ -81,6 +84,7 @@ class Book
   void setDate(const std::string& date) { m_date = date; }
   void setUrl(const std::string& url) { m_url = url; }
   void setName(const std::string& name) { m_name = name; }
+  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
   void setTags(const std::string& tags) { m_tags = tags; }
   void setOrigId(const std::string& origId) { m_origId = origId; }
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
@@ -94,7 +98,7 @@ class Book
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
-  bool m_pathValid;
+  bool m_pathValid = false;
   std::string m_title;
   std::string m_description;
   std::string m_language;
@@ -103,12 +107,13 @@ class Book
   std::string m_date;
   std::string m_url;
   std::string m_name;
+  std::string m_flavour;
   std::string m_tags;
   std::string m_origId;
-  uint64_t m_articleCount;
-  uint64_t m_mediaCount;
-  bool m_readOnly;
-  uint64_t m_size;
+  uint64_t m_articleCount = 0;
+  uint64_t m_mediaCount = 0;
+  bool m_readOnly = false;
+  uint64_t m_size = 0;
   mutable std::string m_favicon;
   std::string m_faviconUrl;
   std::string m_faviconMimeType;

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <pthread.h>
 #include <memory>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -92,7 +93,7 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri);
+  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
   Download* getDownload(const std::string& did);
 
   size_t getNbDownload() { return m_knownDownloads.size(); }

include/entry.h

@@ -64,14 +64,14 @@ class Entry
      * @return the path of the entry.
      */
     std::string getPath() const;
-    
+
     /**
      * Get the title of the entry.
      *
      * @return the title of the entry.
      */
     std::string getTitle() const;
-        
+
     /**
      * Get the content of the entry.
      *
@@ -81,7 +81,7 @@ class Entry
      * @return the content of the entry.
      */
     std::string getContent() const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -91,7 +91,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset = 0) const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -102,7 +102,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset, size_type size) const;
-        
+
     /**
      * Get the info for direct access to the content of the entry.
      *
@@ -118,7 +118,7 @@ class Entry
      *         Return <"",0> if is not possible to read directly.
      */
     std::pair<std::string, offset_type> getDirectAccessInfo() const;
-        
+
     /**
      * Get the size of the entry.
      *
@@ -132,8 +132,8 @@ class Entry
      * @return the mime_type of the entry.
      */
     std::string getMimetype() const;
-    
-    
+
+
     /**
      * Get if the entry is a redirect entry.
      *

include/kiwixserve.h

@@ -2,6 +2,7 @@
 #define KIWIXLIB_KIWIX_SERVE_H_
 
 #include <memory>
+#include <string>
 
 class Subprocess;
 namespace kiwix {
@@ -9,17 +10,19 @@ namespace kiwix {
 class KiwixServe
 {
   public:
-    KiwixServe(int port = 8181);
+    KiwixServe(const std::string& libraryPath, int port = 8181);
     ~KiwixServe();
 
     void run();
     void shutDown();
     bool isRunning();
     int getPort() { return m_port; }
+    int setPort(int port);
 
   private:
   std::unique_ptr<Subprocess> mp_kiwixServe;
   int m_port;
+  std::string m_libraryPath;
 };
 
 }; //end namespace kiwix

include/library.h

@@ -57,6 +57,7 @@ class Filter {
     std::string _creator;
     size_t _maxSize;
     std::string _query;
+    std::string _name;
 
   public:
     Filter();
@@ -100,6 +101,7 @@ class Filter {
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
     Filter& query(std::string query);
+    Filter& name(std::string name);
 
     bool accept(const Book& book) const;
 };
@@ -147,6 +149,7 @@ class Library
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
   Book& getBookById(const std::string& id);
+  Book& getBookByPath(const std::string& path);
   std::shared_ptr<Reader> getReaderById(const std::string& id);
 
   /**
@@ -208,7 +211,7 @@ class Library
    *
    * @return A list of bookmarks
    */
-  const std::vector<kiwix::Bookmark>& getBookmarks() { return m_bookmarks; }
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
 
   /**
    * Get all book ids of the books in the library.

include/manager.h

@@ -69,26 +69,12 @@ class Manager
   /**
    * Read a `library.xml` and add book in the file to the library.
    *
-   * @param path The path to the `library.xml`.
+   * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const std::string& path, const bool readOnly = true);
-
-  /**
-   * Read a `library.xml` and add book in the file to the library.
-   *
-   * @param nativePath The path of the `library.xml`
-   * @param UTF8Path The utf8 version (?) of the path. Also the path where the
-   *                 library will be writen i readOnly is False.
-   * @param readOnly Set if the libray path could be overwritten latter with
-   *                 updated content.
-   * @return True if file has been properly parsed.
-   */
-  bool readFile(const std::string& nativePath,
-                const std::string& UTF8Path,
-                const bool readOnly = true);
+  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
   /**
    * Load a library content store in the string.
@@ -101,7 +87,8 @@ class Manager
    */
   bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const std::string& libraryPath = "");
+               const std::string& libraryPath = "",
+               bool trustLibrary = true);
 
   /**
    * Load a library content stored in a OPDS stream.
@@ -168,8 +155,9 @@ class Manager
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   const bool readOnly,
-                   const std::string& libraryPath);
+                   bool readOnly,
+                   const std::string& libraryPath,
+                   bool trustLibrary);
   bool parseOpdsDom(const pugi::xml_document& doc,
                     const std::string& urlHost);
 

include/reader.h

@@ -158,31 +158,7 @@ class Reader
    * @param[out] value The value will be set to the content of the metadata.
    * @return True if it was possible to get the content of the metadata.
    */
-  bool getMetatag(const string& name, string& value) const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
+  bool getMetadata(const string& name, string& value) const;
 
   /**
    * Get the name of the zim file.
@@ -192,18 +168,13 @@ class Reader
   string getName() const;
 
   /**
-   * Get the tags of the zim file.
+   * Get the title of the zim file.
    *
-   * @return The tags of the zim file as specified in the zim metadata.
+   * @return The title of zim file as specified in the zim metadata.
+   *         If no title has been set, return a title computed from the
+   *         file path.
    */
-  string getTags() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
+  string getTitle() const;
 
   /**
    * Get the creator of the zim file.
@@ -219,6 +190,100 @@ class Reader
    */
   string getPublisher() const;
 
+  /**
+   * Get the date of the zim file.
+   *
+   * @return The date of the zim file as specified in the zim metadata.
+   */
+  string getDate() const;
+
+  /**
+   * Get the description of the zim file.
+   *
+   * @return The description of the zim file as specified in the zim metadata.
+   *         If no description has been set, return the subtitle.
+   */
+  string getDescription() const;
+
+  /**
+   * Get the long description of the zim file.
+   *
+   * @return The long description of the zim file as specifed in the zim metadata.
+   */
+  string getLongDescription() const;
+
+  /**
+   * Get the language of the zim file.
+   *
+   * @return The language of the zim file as specified in the zim metadata.
+   */
+  string getLanguage() const;
+
+  /**
+   * Get the license of the zim file.
+   *
+   * @return The license of the zim file as specified in the zim metadata.
+   */
+  string getLicense() const;
+
+  /**
+   * Get the tags of the zim file.
+   *
+   * @param original If true, return the original tags as specified in the zim metadata.
+   *                 Else, try to convert it to the new 'normalized' format.
+   * @return The tags of the zim file.
+   */
+  string getTags(bool original=false) const;
+
+  /**
+   * Get the value (as a string) of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   */
+  string getTagStr(const std::string& tagName) const;
+
+  /**
+   * Get the boolean value of a specific tag.
+   *
+   * According to https://wiki.openzim.org/wiki/Tags
+   *
+   * @return The boolean value of the specified tag.
+   * @throw  std::out_of_range if the specified tag is not found.
+   *         std::domain_error if the value of the tag cannot be convert to bool.
+   */
+  bool getTagBool(const std::string& tagName) const;
+
+  /**
+   * Get the relations of the zim file.
+   *
+   * @return The relation of the zim file as specified in the zim metadata.
+   */
+  string getRelation() const;
+
+  /**
+   * Get the flavour of the zim file.
+   *
+   * @return The flavour of the zim file as specified in the zim metadata.
+   */
+  string getFlavour() const;
+
+  /**
+   * Get the source of the zim file.
+   *
+   * @return The source of the zim file as specified in the zim metadata.
+   */
+  string getSource() const;
+
+  /**
+   * Get the scraper of the zim file.
+   *
+   * @return The scraper of the zim file as specified in the zim metadata.
+   */
+  string getScraper() const;
+
   /**
    * Get the origId of the zim file.
    *

include/searcher.h

@@ -89,7 +89,7 @@ class Searcher
    * @param resultEnd the end offset of the search results (used for pagination).
    * @param verbose print some info on stdout if true.
    */
-  void search(std::string& search,
+  void search(const std::string& search,
               unsigned int resultStart,
               unsigned int resultEnd,
               const bool verbose = false);

include/server.h

@@ -56,7 +56,9 @@ namespace kiwix
        void setNbThreads(int threads) { m_nbThreads = threads; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
-         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
 
      protected:
        Library* mp_library;
@@ -68,6 +70,7 @@ namespace kiwix
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

include/tools/otherTools.h

@@ -21,6 +21,7 @@
 #define KIWIX_OTHERTOOLS_H
 
 #include <string>
+#include <vector>
 
 namespace pugi {
   class xml_node;
@@ -28,9 +29,18 @@ namespace pugi {
 
 namespace kiwix
 {
-void sleep(unsigned int milliseconds);
-std::string nodeToString(const pugi::xml_node& node);
-std::string converta2toa3(const std::string& a2code);
+  void sleep(unsigned int milliseconds);
+  std::string nodeToString(const pugi::xml_node& node);
+  std::string converta2toa3(const std::string& a2code);
+
+  /*
+   * Convert all format tag string to new format
+   */
+  std::vector<std::string> convertTags(const std::string& tags_str);
+  std::string getTagValueFromTagList(const std::vector<std::string>& tagList,
+                                     const std::string& tagName);
+  bool convertStrToBool(const std::string& value);
+
 }
 
 #endif

include/tools/pathTools.h

@@ -22,12 +22,14 @@
 
 #include <string>
 
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr);
+std::wstring Utf8ToWide(const std::string& str);
+#endif
 bool isRelativePath(const std::string& path);
 std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
-std::string removeLastPathElement(const std::string& path,
-                                  const bool removePreSeparator = false,
-                                  const bool removePostSeparator = false);
+std::string removeLastPathElement(const std::string& path);
 std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
 
 unsigned int getFileSize(const std::string& path);
@@ -38,7 +40,7 @@ bool makeDirectory(const std::string& path);
 std::string makeTmpDirectory();
 bool copyFile(const std::string& sourcePath, const std::string& destPath);
 std::string getLastPathElement(const std::string& path);
-std::string getExecutablePath();
+std::string getExecutablePath(bool realPathOnly = false);
 std::string getCurrentDirectory();
 std::string getDataDirectory();
 bool writeTextFile(const std::string& path, const std::string& content);

include/tools/stringTools.h

@@ -43,10 +43,8 @@ void loadICUExternalTables();
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string&, const std::string&);
-std::vector<std::string> split(const char*, const char*);
-std::vector<std::string> split(const std::string&, const char*);
-std::vector<std::string> split(const char*, const std::string&);
+std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
+std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);
 std::string lcAll(const std::string& word);

meson.build

@@ -1,20 +1,26 @@
 project('kiwix-lib', 'cpp',
-  version : '6.0.3', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '9.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
-static_deps = get_option('android') or get_option('default_library') == 'static'
-if get_option('android')
+wrapper = get_option('wrapper')
+
+static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
+if 'android' in wrapper
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
+if 'java' in wrapper
+  add_languages('java')
+endif
+
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)

meson_options.txt

@@ -1,2 +1,2 @@
-option('android', type : 'boolean', value : false,
-  description : 'Do we make a kiwix-lib for android')
+option('wrapper', type:'array', choices:['java', 'android'], value:[],
+  description: 'The wrapper to generate.')

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,11 +95,11 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
+
 
 
 master_c_template = """//This file is automaically generated. Do not modify it.
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

src/android/kiwixlibrary.cpp

@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <android/log.h>
-#include "org_kiwix_kiwixlib_JNIKiwixLibrary.h"
-
-#include "library.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixLibrary_getNativeLibrary(
-    JNIEnv* env, jobject obj)
-{
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create library");
-  Lock l;
-  try {
-    kiwix::Library* library = new kiwix::Library();
-    return reinterpret_cast<jlong>(new Handle<kiwix::Library>(library));
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating ZIM library");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixLibrary_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Library>::dispose(env, obj);
-}
-
-#define LIBRARY (Handle<kiwix::Library>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixLibrary_addBook(JNIEnv* env, jobject obj, jstring path)
-{
-  std::string cPath = jni2c(path, env);
-  bool ret;
-
-  try {
-    kiwix::Reader reader(cPath);
-    kiwix::Book book;
-    book.update(reader);
-    ret = LIBRARY->addBook(book);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to add the book");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    ret = false;
-  }
-  return ret;
-}

src/android/meson.build

@@ -1,32 +0,0 @@
-
-kiwix_jni = custom_target('jni',
-  input: ['org/kiwix/kiwixlib/JNIICU.java',
-          'org/kiwix/kiwixlib/JNIKiwixReader.java',
-          'org/kiwix/kiwixlib/JNIKiwixLibrary.java',
-          'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-          'org/kiwix/kiwixlib/JNIKiwixServer.java',
-          'org/kiwix/kiwixlib/JNIKiwixInt.java',
-          'org/kiwix/kiwixlib/JNIKiwixString.java',
-          'org/kiwix/kiwixlib/JNIKiwixBool.java',
-          'org/kiwix/kiwixlib/JNIKiwixException.java',
-          'org/kiwix/kiwixlib/Pair.java'],
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_JNIKiwixLibrary.h',
-           'org_kiwix_kiwixlib_JNIKiwixServer.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-kiwix_sources += [
-    'android/kiwixicu.cpp',
-    'android/kiwixreader.cpp',
-    'android/kiwixlibrary.cpp',
-    'android/kiwixsearcher.cpp',
-    'android/kiwixserver.cpp',
-    kiwix_jni]
-
-install_subdir('org', install_dir: 'kiwix-lib/java')
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/aria2.cpp

@@ -49,7 +49,7 @@ Aria2::Aria2():
   m_secret = "token:"+m_secret;
 
   std::string aria2cmd = appendToDirectory(
-    removeLastPathElement(getExecutablePath(), true, true),
+    removeLastPathElement(getExecutablePath(true)),
     ARIA2_CMD);
   if (fileExists(aria2cmd)) {
     // A local aria2c exe exists (packaged with kiwix-desktop), use it.
@@ -78,23 +78,38 @@ Aria2::Aria2():
   callCmd.push_back("--max-concurrent-downloads=42");
   callCmd.push_back("--rpc-max-request-size=6M");
   callCmd.push_back("--file-allocation=none");
+  std::string launchCmd;
+  for (auto &cmd : callCmd) {
+      launchCmd.append(cmd).append(" ");
+  }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
+  char errbuf[CURL_ERROR_SIZE];
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
+    errbuf[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
+    } else if (watchdog == 1) {
+      std::cerr <<" curl_easy_perform() failed." << std::endl;
+      fprintf(stderr, "\nlibcurl: (%d) ", res);
+      if (errbuf[0] != 0) {
+        std::cerr << errbuf << std::endl;
+      } else {
+        std::cerr << curl_easy_strerror(res) << std::endl;
+      }
     }
   }
   if (!watchdog) {
     curl_easy_cleanup(mp_curl);
-    throw std::runtime_error("Cannot connect to aria2c rpc");
+    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
@@ -145,13 +160,16 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
   throw std::runtime_error("Cannot perform request");
 }
 
-std::string Aria2::addUri(const std::vector<std::string>& uris)
+std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
 {
   MethodCall methodCall("aria2.addUri", m_secret);
   auto uriParams = methodCall.newParamValue().getArray();
   for (auto& uri : uris) {
     uriParams.addValue().set(uri);
   }
+  for (auto& option : options) {
+    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
+  }
   auto ret = doRequest(methodCall);
   MethodResponse response(ret);
   return response.getParamValue(0).getAsS();

src/aria2.h

@@ -34,7 +34,7 @@ class Aria2
     virtual ~Aria2();
     void close();
 
-    std::string addUri(const std::vector<std::string>& uri);
+    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
     std::vector<std::string> tellActive();
     std::vector<std::string> tellWaiting();

src/book.cpp

@@ -23,6 +23,7 @@
 #include "tools/base64.h"
 #include "tools/regexTools.h"
 #include "tools/networkTools.h"
+#include "tools/otherTools.h"
 
 #include <pugixml.hpp>
 
@@ -44,43 +45,48 @@ bool Book::update(const kiwix::Book& other)
   if (m_readOnly)
     return false;
 
+  if (m_id != other.m_id)
+    return false;
+
   m_readOnly = other.m_readOnly;
+  m_path = other.m_path;
+  m_pathValid = other.m_pathValid;
+  m_title = other.m_title;
+  m_description = other.m_description;
+  m_language = other.m_language;
+  m_creator = other.m_creator;
+  m_publisher = other.m_publisher;
+  m_date = other.m_date;
+  m_url = other.m_url;
+  m_name = other.m_name;
+  m_flavour = other.m_flavour;
+  m_tags = other.m_tags;
+  m_origId = other.m_origId;
+  m_articleCount = other.m_articleCount;
+  m_mediaCount = other.m_mediaCount;
+  m_size = other.m_size;
+  m_favicon = other.m_favicon;
+  m_faviconMimeType = other.m_faviconMimeType;
+  m_faviconUrl = other.m_faviconUrl;
 
-  if (m_path.empty()) {
-    m_path = other.m_path;
-    m_pathValid = other.m_pathValid;
-  }
+  m_downloadId = other.m_downloadId;
 
-  if (m_url.empty()) {
-    m_url = other.m_url;
-  }
-
-  if (m_tags.empty()) {
-    m_tags = other.m_tags;
-  }
-
-  if (m_name.empty()) {
-    m_name = other.m_name;
-  }
-
-  if (m_faviconMimeType.empty()) {
-    m_favicon = other.m_favicon;
-    m_faviconMimeType = other.m_faviconMimeType;
-  }
   return true;
 }
 
 void Book::update(const kiwix::Reader& reader)
 {
   m_path = reader.getZimFilePath();
+  m_pathValid = true;
   m_id = reader.getId();
+  m_title = reader.getTitle();
   m_description = reader.getDescription();
   m_language = reader.getLanguage();
-  m_date = reader.getDate();
   m_creator = reader.getCreator();
   m_publisher = reader.getPublisher();
-  m_title = reader.getTitle();
+  m_date = reader.getDate();
   m_name = reader.getName();
+  m_flavour = reader.getFlavour();
   m_tags = reader.getTags();
   m_origId = reader.getOrigId();
   m_articleCount = reader.getArticleCount();
@@ -100,21 +106,24 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
+  m_pathValid = fileExists(path);
   m_title = ATTR("title");
-  m_name = ATTR("name");
-  m_tags = ATTR("tags");
   m_description = ATTR("description");
   m_language = ATTR("language");
-  m_date = ATTR("date");
   m_creator = ATTR("creator");
   m_publisher = ATTR("publisher");
+  m_date = ATTR("date");
   m_url = ATTR("url");
+  m_name = ATTR("name");
+  m_flavour = ATTR("flavour");
+  m_tags = ATTR("tags");
   m_origId = ATTR("origId");
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
   m_favicon = base64_decode(ATTR("favicon"));
   m_faviconMimeType = ATTR("faviconMimeType");
+  m_faviconUrl = ATTR("faviconUrl");
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
@@ -137,12 +146,18 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   if (!m_id.compare(0, 9, "urn:uuid:")) {
     m_id.erase(0, 9);
   }
+  // No path on opds.
   m_title = VALUE("title");
-  m_description = VALUE("description");
+  m_description = VALUE("summary");
   m_language = VALUE("language");
-  m_date = fromOpdsDate(VALUE("updated"));
   m_creator = node.child("author").child("name").child_value();
+  m_publisher = node.child("publisher").child("name").child_value();
+  m_date = fromOpdsDate(VALUE("updated"));
+  m_name = VALUE("name");
+  m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();
@@ -197,4 +212,12 @@ const std::string& Book::getFavicon() const {
   return m_favicon;
 }
 
+std::string Book::getTagStr(const std::string& tagName) const {
+  return getTagValueFromTagList(convertTags(m_tags), tagName);
+}
+
+bool Book::getTagBool(const std::string& tagName) const {
+  return convertStrToBool(getTagStr(tagName));
+}
+
 }

src/downloader.cpp

@@ -127,17 +127,24 @@ void Download::cancelDownload()
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  for (auto gid : mp_aria->tellActive()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what();
   }
-  for (auto gid : mp_aria->tellWaiting()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellWaiting()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what();
   }
 }
 
-
 /* Destructor */
 Downloader::~Downloader()
 {
@@ -156,7 +163,7 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri)
+Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
@@ -165,7 +172,7 @@ Download* Downloader::startDownload(const std::string& uri)
       return d.get();
   }
   std::vector<std::string> uris = {uri};
-  auto gid = mp_aria->addUri(uris);
+  auto gid = mp_aria->addUri(uris, options);
   m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
   return m_knownDownloads[gid].get();
 }

src/kiwixserve.cpp

@@ -14,7 +14,9 @@
 
 namespace kiwix {
 
-KiwixServe::KiwixServe(int port) : m_port(port)
+KiwixServe::KiwixServe(const std::string& libraryPath, int port)
+  : m_port(port),
+    m_libraryPath(libraryPath)
 {
 }
 
@@ -34,7 +36,7 @@ void KiwixServe::run()
 
     std::vector<const char*> callCmd;
     std::string kiwixServeCmd = appendToDirectory(
-        removeLastPathElement(getExecutablePath(), true, true),
+        removeLastPathElement(getExecutablePath(true)),
         KIWIXSERVE_CMD);
     if (fileExists(kiwixServeCmd)) {
         // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
@@ -43,13 +45,12 @@ void KiwixServe::run()
         // Try to use a potential installed kiwix-serve.
         callCmd.push_back(KIWIXSERVE_CMD);
     }
-    std::string libraryPath = getDataDirectory() + "/library.xml";
     std::string attachProcessOpt = "-a" + to_string(pid);
     std::string portOpt = "-p" + to_string(m_port);
     callCmd.push_back(attachProcessOpt.c_str());
     callCmd.push_back(portOpt.c_str());
     callCmd.push_back("-l");
-    callCmd.push_back(libraryPath.c_str());
+    callCmd.push_back(m_libraryPath.c_str());
     mp_kiwixServe = Subprocess::run(callCmd);
 }
 
@@ -67,4 +68,14 @@ bool KiwixServe::isRunning()
     return false;
 }
 
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
 }

src/library.cpp

@@ -85,6 +85,18 @@ Book& Library::getBookById(const std::string& id)
   return m_books.at(id);
 }
 
+Book& Library::getBookByPath(const std::string& path)
+{
+  for(auto& it: m_books) {
+    auto& book = it.second;
+    if (book.getPath() == path)
+      return book;
+  }
+  std::ostringstream ss;
+  ss << "No book with path " << path << " in the library." << std::endl;
+  throw std::out_of_range(ss.str());
+}
+
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {
@@ -115,7 +127,7 @@ unsigned int Library::getBookCount(const bool localBooks,
 
 bool Library::writeToFile(const std::string& path)
 {
-  auto baseDir = removeLastPathElement(path, true, false);
+  auto baseDir = removeLastPathElement(path);
   LibXMLDumper dumper(this);
   dumper.setBaseDir(baseDir);
   return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
@@ -184,6 +196,21 @@ std::vector<std::string> Library::getBooksPublishers()
   return booksPublishers;
 }
 
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
+{
+  if (!onlyValidBookmarks) {
+    return m_bookmarks;
+  }
+  std::vector<kiwix::Bookmark> validBookmarks;
+  auto booksId = getBooksIds();
+  for(auto& bookmark:m_bookmarks) {
+    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
+      validBookmarks.push_back(bookmark);
+    }
+  }
+  return validBookmarks;
+}
+
 std::vector<std::string> Library::getBooksIds()
 {
   std::vector<std::string> bookIds;
@@ -363,6 +390,7 @@ enum filterTypes {
   _CREATOR = FLAG(10),
   MAXSIZE = FLAG(11),
   QUERY = FLAG(12),
+  NAME = FLAG(13),
 };
 
 Filter& Filter::local(bool accept)
@@ -450,24 +478,35 @@ Filter& Filter::query(std::string query)
   return *this;
 }
 
+Filter& Filter::name(std::string name)
+{
+  _name = name;
+  activeFilters |= NAME;
+  return *this;
+}
+
 #define ACTIVE(X) (activeFilters & (X))
+#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
 bool Filter::accept(const Book& book) const
 {
   auto local = !book.getPath().empty();
-  if (ACTIVE(_LOCAL) && !local)
-    return false;
-  if (ACTIVE(_NOLOCAL) && local)
-    return false;
+  FILTER(_LOCAL, local)
+  FILTER(_NOLOCAL, !local)
+
   auto valid = book.isPathValid();
-  if (ACTIVE(_VALID) && !valid)
-    return false;
-  if (ACTIVE(_NOVALID) && valid)
-    return false;
+  FILTER(_VALID, valid)
+  FILTER(_NOVALID, !valid)
+
   auto remote = !book.getUrl().empty();
-  if (ACTIVE(_REMOTE) && !remote)
-    return false;
-  if (ACTIVE(_NOREMOTE) && remote)
-    return false;
+  FILTER(_REMOTE, remote)
+  FILTER(_NOREMOTE, !remote)
+
+  FILTER(MAXSIZE, book.getSize() <= _maxSize)
+  FILTER(LANG, book.getLanguage() == _lang)
+  FILTER(_PUBLISHER, book.getPublisher() == _publisher)
+  FILTER(_CREATOR, book.getCreator() == _creator)
+  FILTER(NAME, book.getName() == _name)
+
   if (ACTIVE(ACCEPTTAGS)) {
     if (!_acceptTags.empty()) {
       auto vBookTags = split(book.getTags(), ";");
@@ -490,18 +529,6 @@ bool Filter::accept(const Book& book) const
       }
     }
   }
-  if (ACTIVE(MAXSIZE) && book.getSize() > _maxSize)
-    return false;
-
-  if (ACTIVE(LANG) && book.getLanguage() != _lang)
-    return false;
-
-  if (ACTIVE(_PUBLISHER) && book.getPublisher() != _publisher)
-    return false;
-
-  if (ACTIVE(_CREATOR) && book.getCreator() != _creator)
-    return false;
-
   if ( ACTIVE(QUERY)
     && !(matchRegex(book.getTitle(), "\\Q" + _query + "\\E")
         || matchRegex(book.getDescription(), "\\Q" + _query + "\\E")))

src/libxml_dumper.cpp

@@ -53,13 +53,15 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
 
   if (book.getOrigId().empty()) {
     ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
-    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
-    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
     ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
     ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
     ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
     ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
     ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
+    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
     if (!book.getFavicon().empty())
       ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
   } else {

src/manager.cpp

@@ -19,6 +19,8 @@
 
 #include "manager.h"
 
+#include "tools/pathTools.h"
+
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -46,8 +48,9 @@ Manager::~Manager()
   }
 }
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          const bool readOnly,
-                          const std::string& libraryPath)
+                          bool readOnly,
+                          const std::string& libraryPath,
+                          bool trustLibrary)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
@@ -59,14 +62,10 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 
     book.setReadOnly(readOnly);
     book.updateFromXml(bookNode,
-                       removeLastPathElement(libraryPath, true, false));
+                       removeLastPathElement(libraryPath));
 
-    /* Update the book properties with the new importer */
-    if (libraryVersion.empty()
-        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.getPath().empty()) {
-        this->readBookFromPath(book.getPath(), &book);
-      }
+    if (!trustLibrary && !book.getPath().empty()) {
+      this->readBookFromPath(book.getPath(), &book);
     }
     manipulator->addBookToLibrary(book);
   }
@@ -75,15 +74,16 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 }
 
 bool Manager::readXml(const std::string& xml,
-                      const bool readOnly,
-                      const std::string& libraryPath)
+                      bool readOnly,
+                      const std::string& libraryPath,
+                      bool trustLibrary)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
       = doc.load_buffer_inplace((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath);
+    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
   }
 
   return true;
@@ -134,21 +134,22 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
   return false;
 }
 
-bool Manager::readFile(const std::string& path, const bool readOnly)
-{
-  return this->readFile(path, path, readOnly);
-}
-
-bool Manager::readFile(const std::string& nativePath,
-                       const std::string& UTF8Path,
-                       const bool readOnly)
+bool Manager::readFile(
+  const std::string& path,
+  bool readOnly,
+  bool trustLibrary)
 {
   bool retVal = true;
   pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
+
+#ifdef _WIN32
+  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
+#else
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+#endif
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, UTF8Path);
+    this->parseXmlDom(doc, readOnly, path, trustLibrary);
   } else {
     retVal = false;
   }
@@ -157,7 +158,7 @@ bool Manager::readFile(const std::string& nativePath,
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = UTF8Path;
+    this->writableLibraryPath = path;
   }
 
   return retVal;
@@ -174,10 +175,10 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (pathToSave != pathToOpen) {
+    if (!pathToSave.empty() && pathToSave != pathToOpen) {
       book.setPath(isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
+                      removeLastPathElement(writableLibraryPath),
                       pathToSave)
                 : pathToSave);
     }

src/meson.build

@@ -21,6 +21,7 @@ kiwix_sources = [
   'tools/otherTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
+  'server/etag.cpp',
   'server/request_context.cpp',
   'server/response.cpp'
 ]
@@ -32,13 +33,16 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if get_option('android')
-  subdir('android')
+if 'android' in wrapper
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
+if 'android' in wrapper or 'java' in wrapper
+  subdir('wrapper/java')
+endif
+
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
                input : 'config.h.in')

src/opds_dumper.cpp

@@ -70,12 +70,17 @@ void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 
 pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
+  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
   ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
+  ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage());
+  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
+  ADD_TEXT_ENTRY(entry_node, "name", book.getName());
+  ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
   ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
+  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
+  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
+  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
 
   auto content_node = entry_node.append_child("link");
   content_node.append_attribute("type") = "text/html";
@@ -84,6 +89,9 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   auto author_node = entry_node.append_child("author");
   ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
 
+  auto publisher_node = entry_node.append_child("publisher");
+  ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher());
+
   if (! book.getUrl().empty()) {
     auto acquisition_link = entry_node.append_child("link");
     acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";

src/reader.cpp

@@ -22,6 +22,8 @@
 
 #include <zim/search.h>
 
+#include "tools/otherTools.h"
+
 inline char hi(char v)
 {
   char hex[] = "0123456789abcdef";
@@ -278,7 +280,7 @@ string Reader::getZimFilePath() const
   return this->zimFilePath;
 }
 /* Return a metatag value */
-bool Reader::getMetatag(const string& name, string& value) const
+bool Reader::getMetadata(const string& name, string& value) const
 {
   try {
     auto entry = getEntryFromPath("M/"+name);
@@ -289,10 +291,17 @@ bool Reader::getMetatag(const string& name, string& value) const
   }
 }
 
+#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
+
+string Reader::getName() const
+{
+  METADATA("Name")
+}
+
 string Reader::getTitle() const
 {
   string value;
-  this->getMetatag("Title", value);
+  this->getMetadata("Title", value);
   if (value.empty()) {
     value = getLastPathElement(zimFileHandler->getFilename());
     std::replace(value.begin(), value.end(), '_', ' ');
@@ -302,65 +311,98 @@ string Reader::getTitle() const
   return value;
 }
 
-string Reader::getName() const
+string Reader::getCreator() const
 {
-  string value;
-  this->getMetatag("Name", value);
-  return value;
+  METADATA("Creator")
 }
 
-string Reader::getTags() const
+string Reader::getPublisher() const
 {
-  string value;
-  this->getMetatag("Tags", value);
-  return value;
+  METADATA("Publisher")
+}
+
+string Reader::getDate() const
+{
+  METADATA("Date")
 }
 
 string Reader::getDescription() const
 {
   string value;
-  this->getMetatag("Description", value);
+  this->getMetadata("Description", value);
 
   /* Mediawiki Collection tends to use the "Subtitle" name */
   if (value.empty()) {
-    this->getMetatag("Subtitle", value);
+    this->getMetadata("Subtitle", value);
   }
 
   return value;
 }
 
+string Reader::getLongDescription() const
+{
+  METADATA("LongDescription")
+}
+
 string Reader::getLanguage() const
 {
-  string value;
-  this->getMetatag("Language", value);
-  return value;
+  METADATA("Language")
 }
 
-string Reader::getDate() const
+string Reader::getLicense() const
 {
-  string value;
-  this->getMetatag("Date", value);
-  return value;
+  METADATA("License")
 }
 
-string Reader::getCreator() const
+string Reader::getTags(bool original) const
 {
-  string value;
-  this->getMetatag("Creator", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  if (original) {
+    return tags_str;
+  }
+  auto tags = convertTags(tags_str);
+  return join(tags, ";");
 }
 
-string Reader::getPublisher() const
+
+string Reader::getTagStr(const std::string& tagName) const
 {
-  string value;
-  this->getMetatag("Publisher", value);
-  return value;
+  string tags_str;
+  getMetadata("Tags", tags_str);
+  return getTagValueFromTagList(convertTags(tags_str), tagName);
 }
 
+bool Reader::getTagBool(const std::string& tagName) const
+{
+  return convertStrToBool(getTagStr(tagName));
+}
+
+string Reader::getRelation() const
+{
+  METADATA("Relation")
+}
+
+string Reader::getFlavour() const
+{
+  METADATA("Flavour")
+}
+
+string Reader::getSource() const
+{
+  METADATA("Source")
+}
+
+string Reader::getScraper() const
+{
+  METADATA("Scraper")
+}
+#undef METADATA
+
 string Reader::getOrigId() const
 {
   string value;
-  this->getMetatag("startfileuid", value);
+  this->getMetadata("startfileuid", value);
   if (value.empty()) {
     return "";
   }
@@ -672,7 +714,6 @@ bool Reader::searchSuggestions(const string& prefix,
                                const bool reset)
 {
   bool retVal = false;
-  zim::File::const_iterator articleItr;
 
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
@@ -690,7 +731,7 @@ bool Reader::searchSuggestions(const string& prefix,
     return false;
   }
 
-  for (articleItr = zimFileHandler->findByTitle('A', prefix);
+  for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
        && this->suggestions.size() < suggestionsCount;

src/search_renderer.cpp

@@ -96,27 +96,29 @@ std::string SearchRenderer::getHtml()
   auto resultEnd = mp_searcher->getResultEnd();
   auto resultCountPerPage = resultEnd - resultStart;
   auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
-
-  unsigned int pageStart
-      = resultStart / resultCountPerPage >= 5
-            ? resultStart / resultCountPerPage - 4
-            : 0;
-  unsigned int pageCount
-      = estimatedResultCount / resultCountPerPage + 1 - pageStart;
-
-  if (pageCount > 10) {
-    pageCount = 10;
-  } else if (pageCount == 1) {
-    pageCount = 0;
+  auto currentPage = 0U;
+  auto pageStart = 0U;
+  auto pageEnd = 0U;
+  auto lastPageStart = 0U;
+  if (resultCountPerPage) {
+    currentPage = resultStart/resultCountPerPage;
+    pageStart = currentPage > 4 ? currentPage-4 : 0;
+    pageEnd = currentPage + 5;
+    if (pageEnd > estimatedResultCount / resultCountPerPage) {
+      pageEnd = estimatedResultCount / resultCountPerPage;
+    }
+    if (estimatedResultCount > resultCountPerPage) {
+      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
+    }
   }
 
-  for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
+  for (unsigned int i = pageStart; i < pageEnd; i++) {
     kainjow::mustache::data page;
     page.set("label", to_string(i + 1));
     page.set("start", to_string(i * resultCountPerPage));
     page.set("end", to_string((i + 1) * resultCountPerPage));
 
-    if (i * resultCountPerPage == resultStart) {
+    if (i == currentPage) {
       page.set("selected", true);
     }
     pages.push_back(page);
@@ -128,16 +130,15 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data allData;
   allData.set("results", results);
   allData.set("pages", pages);
-  allData.set("hasResult", estimatedResultCount != 0);
+  allData.set("hasResults", estimatedResultCount != 0);
+  allData.set("hasPages", pageStart != pageEnd);
   allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
   allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
   allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
   allData.set("resultStart", to_string(resultStart + 1));
   allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
   allData.set("resultRange", to_string(resultCountPerPage));
-  allData.set("resultLastPageStart", to_string(estimatedResultCount > resultCountPerPage
-             ? round(estimatedResultCount / resultCountPerPage) * resultCountPerPage
-             : 0));
+  allData.set("resultLastPageStart", to_string(lastPageStart));
   allData.set("lastResult", to_string(estimatedResultCount));
   allData.set("protocolPrefix", this->protocolPrefix);
   allData.set("searchProtocolPrefix", this->searchProtocolPrefix);

src/searcher.cpp

@@ -97,7 +97,7 @@ Reader* Searcher::get_reader(int readerIndex)
 }
 
 /* Search strings in the database */
-void Searcher::search(std::string& search,
+void Searcher::search(const std::string& search,
                       unsigned int resultStart,
                       unsigned int resultEnd,
                       const bool verbose)
@@ -108,12 +108,12 @@ void Searcher::search(std::string& search,
     cout << "Performing query `" << search << "'" << endl;
   }
 
+  this->searchPattern = search;
+  this->resultStart = resultStart;
+  this->resultEnd = resultEnd;
   /* Try to find results */
   if (resultStart != resultEnd) {
     /* Perform the search */
-    this->searchPattern = search;
-    this->resultStart = resultStart;
-    this->resultEnd = resultEnd;
     string unaccentedSearch = removeAccents(search);
     std::vector<const zim::File*> zims;
     for (auto current = this->readers.begin(); current != this->readers.end();
@@ -146,11 +146,6 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
     cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
   }
 
-  /* Try to find results */
-  if (resultStart == resultEnd) {
-    return;
-  }
-
   /* Perform the search */
   std::ostringstream oss;
   oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
@@ -158,6 +153,11 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
   this->resultStart = resultStart;
   this->resultEnd = resultEnd;
 
+  /* Try to find results */
+  if (resultStart == resultEnd) {
+    return;
+  }
+
   std::vector<const zim::File*> zims;
   for (auto current = this->readers.begin(); current != this->readers.end();
        current++) {

src/server.cpp

@@ -75,6 +75,8 @@ namespace kiwix {
 
 static IdNameMapper defaultNameMapper;
 
+typedef kainjow::mustache::data MustacheData;
+
 static int staticHandlerCallback(void* cls,
                                  struct MHD_Connection* connection,
                                  const char* url,
@@ -95,7 +97,8 @@ class InternalServer {
                    int nbThreads,
                    bool verbose,
                    bool withTaskbar,
-                   bool withLibraryButton);
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
     virtual ~InternalServer() = default;
 
     int handlerCallback(struct MHD_Connection* connection,
@@ -108,10 +111,12 @@ class InternalServer {
     bool start();
     void stop();
 
-  private:
+  private: // functions
     Response handle_request(const RequestContext& request);
     Response build_500(const std::string& msg);
     Response build_404(const RequestContext& request, const std::string& zimName);
+    Response build_304(const RequestContext& request, const ETag& etag) const;
+    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
     Response build_homepage(const RequestContext& request);
     Response handle_skin(const RequestContext& request);
     Response handle_catalog(const RequestContext& request);
@@ -119,11 +124,18 @@ class InternalServer {
     Response handle_search(const RequestContext& request);
     Response handle_suggest(const RequestContext& request);
     Response handle_random(const RequestContext& request);
+    Response handle_captured_external(const RequestContext& request);
     Response handle_content(const RequestContext& request);
 
-    kainjow::mustache::data get_default_data();
-    Response get_default_response();
+    MustacheData get_default_data() const;
+    MustacheData homepage_data() const;
+    Response get_default_response() const;
 
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
     std::string m_addr;
     int m_port;
     std::string m_root;
@@ -131,10 +143,13 @@ class InternalServer {
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
+    bool m_blockExternalLinks;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
     NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
 };
 
 
@@ -157,7 +172,8 @@ bool Server::start() {
     m_nbThreads,
     m_verbose,
     m_withTaskbar,
-    m_withLibraryButton));
+    m_withLibraryButton,
+    m_blockExternalLinks));
   return mp_server->start();
 }
 
@@ -186,7 +202,8 @@ InternalServer::InternalServer(Library* library,
                                int nbThreads,
                                bool verbose,
                                bool withTaskbar,
-                               bool withLibraryButton) :
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
   m_addr(addr),
   m_port(port),
   m_root(root),
@@ -194,6 +211,7 @@ InternalServer::InternalServer(Library* library,
   m_verbose(verbose),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
   mp_daemon(nullptr),
   mp_library(library),
   mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
@@ -239,6 +257,8 @@ bool InternalServer::start() {
               << std::endl;
     return false;
   }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
   return true;
 }
 
@@ -288,7 +308,8 @@ int InternalServer::handlerCallback(struct MHD_Connection* connection,
   }
   /* Unexpected method */
   if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST) {
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
     printf("Reject request because of unhandled request method.\n");
     printf("----------------------\n");
     return MHD_NO;
@@ -305,6 +326,9 @@ int InternalServer::handlerCallback(struct MHD_Connection* connection,
     }
   }
 
+  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response.set_server_id(m_server_id);
+
   auto ret = response.send(request, connection);
   auto end_time = std::chrono::steady_clock::now();
   auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
@@ -315,6 +339,14 @@ int InternalServer::handlerCallback(struct MHD_Connection* connection,
   return ret;
 }
 
+Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
+{
+  auto response = get_default_response();
+  response.set_code(MHD_HTTP_NOT_MODIFIED);
+  response.set_etag(etag);
+  response.set_content("");
+  return response;
+}
 
 Response InternalServer::handle_request(const RequestContext& request)
 {
@@ -322,6 +354,10 @@ Response InternalServer::handle_request(const RequestContext& request)
     if (! request.is_valid_url())
       return build_404(request, "");
 
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return build_304(request, etag);
+
     if (kiwix::startsWith(request.get_url(), "/skin/"))
       return handle_skin(request);
 
@@ -340,6 +376,9 @@ Response InternalServer::handle_request(const RequestContext& request)
     if (request.get_url() == "/random")
       return handle_random(request);
 
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
     return handle_content(request);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
@@ -350,23 +389,23 @@ Response InternalServer::handle_request(const RequestContext& request)
   }
 }
 
-kainjow::mustache::data InternalServer::get_default_data()
+MustacheData InternalServer::get_default_data() const
 {
-  kainjow::mustache::data data;
+  MustacheData data;
   data.set("root", m_root);
   return data;
 }
 
-Response InternalServer::get_default_response()
+Response InternalServer::get_default_response() const
 {
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton);
+  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
 }
 
 
 Response InternalServer::build_404(const RequestContext& request,
                                    const std::string& bookName)
 {
-  kainjow::mustache::data results;
+  MustacheData results;
   results.set("url", request.get_full_url());
 
   auto response = get_default_response();
@@ -381,24 +420,24 @@ Response InternalServer::build_404(const RequestContext& request,
 
 Response InternalServer::build_500(const std::string& msg)
 {
-  kainjow::mustache::data data;
+  MustacheData data;
   data.set("error", msg);
-  Response response(m_root, true, false, false);
+  Response response(m_root, true, false, false, false);
   response.set_template(RESOURCE::templates::_500_html, data);
   response.set_mimeType("text/html");
   response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
   return response;
 }
 
-Response InternalServer::build_homepage(const RequestContext& request)
+MustacheData InternalServer::homepage_data() const
 {
   auto data = get_default_data();
 
-  kainjow::mustache::data books{kainjow::mustache::data::type::list};
+  MustacheData books{MustacheData::type::list};
   for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
     auto& currentBook = mp_library->getBookById(bookId);
 
-    kainjow::mustache::data book;
+    MustacheData book;
     book.set("name", mp_nameMapper->getNameForId(bookId));
     book.set("title", currentBook.getTitle());
     book.set("description", currentBook.getDescription());
@@ -408,9 +447,34 @@ Response InternalServer::build_homepage(const RequestContext& request)
   }
 
   data.set("books", books);
+  return data;
+}
 
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+Response InternalServer::build_homepage(const RequestContext& request)
+{
   auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, data);
+  response.set_template(RESOURCE::templates::index_html, homepage_data());
   response.set_mimeType("text/html; charset=utf-8");
   response.set_compress(true);
   response.set_taskbar("", "");
@@ -465,7 +529,7 @@ Response InternalServer::handle_meta(const RequestContext& request)
   response.set_content(content);
   response.set_mimeType(mimeType);
   response.set_compress(false);
-  response.set_cache(true);
+  response.set_cacheable();
   return response;
 }
 
@@ -498,14 +562,14 @@ Response InternalServer::handle_suggest(const RequestContext& request)
     printf("Searching suggestions for: \"%s\"\n", term.c_str());
   }
 
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
+  MustacheData results{MustacheData::type::list};
 
   bool first = true;
   if (reader != nullptr) {
     /* Get the suggestions */
     reader->searchSuggestionsSmart(term, maxSuggestionCount);
     while (reader->getNextSuggestion(suggestion)) {
-      kainjow::mustache::data result;
+      MustacheData result;
       result.set("label", suggestion);
       result.set("value", suggestion);
       result.set("first", first);
@@ -517,7 +581,7 @@ Response InternalServer::handle_suggest(const RequestContext& request)
 
   /* Propose the fulltext search if possible */
   if (reader->hasFulltextIndex()) {
-    kainjow::mustache::data result;
+    MustacheData result;
     result.set("label", "containing '" + term + "'...");
     result.set("value", term + " ");
     result.set("first", first);
@@ -549,7 +613,7 @@ Response InternalServer::handle_skin(const RequestContext& request)
   }
   response.set_mimeType(getMimeTypeForFile(resourceName));
   response.set_compress(true);
-  response.set_cache(true);
+  response.set_cacheable();
   return response;
 }
 
@@ -702,15 +766,32 @@ Response InternalServer::handle_random(const RequestContext& request)
 
   try {
     auto entry = reader->getRandomPage();
-    entry = entry.getFinalEntry();
-    auto response = get_default_response();
-    response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
-    return response;
+    return build_redirect(bookName, entry.getFinalEntry());
   } catch(kiwix::NoEntry& e) {
     return build_404(request, bookName);
   }
 }
 
+Response InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return build_404(request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::captured_external_html, data);
+  response.set_mimeType("text/html; charset=utf-8");
+  response.set_compress(true);
+  response.set_taskbar("", "");
+  return response;
+}
+
 Response InternalServer::handle_catalog(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -742,26 +823,30 @@ Response InternalServer::handle_catalog(const RequestContext& request)
   kiwix::OPDSDumper opdsDumper;
   opdsDumper.setRootLocation(m_root);
   opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setId(kiwix::to_string(uuid));
   opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8");
+  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
   std::vector<std::string> bookIdsToDump;
   if (url == "root.xml") {
     opdsDumper.setTitle("All zims");
     uuid = zim::Uuid::generate(host);
     bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
   } else if (url == "search") {
-    std::string query;
-    std::string language;
-    std::vector<std::string> tags;
-    std::vector<std::string> noTags;
+    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
+    string query("<Empty query>");
     size_t count(10);
     size_t startIndex(0);
     try {
       query = request.get_argument("q");
+      filter.query(query);
     } catch (const std::out_of_range&) {}
     try {
-      language = request.get_argument("lang");
+      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
     } catch (const std::out_of_range&) {}
     try {
       count = extractFromString<unsigned long>(request.get_argument("count"));
@@ -770,20 +855,14 @@ Response InternalServer::handle_catalog(const RequestContext& request)
       startIndex = extractFromString<unsigned long>(request.get_argument("start"));
     } catch (...) {}
     try {
-      tags = kiwix::split(request.get_argument("notag"), ";");
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
     } catch (...) {}
     try {
-      noTags = kiwix::split(request.get_argument("notag"), ";");
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
     } catch (...) {}
     opdsDumper.setTitle("Search result for " + query);
     uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(
-      kiwix::Filter().valid(true).local(true).remote(true)
-        .query(query)
-        .lang(language)
-        .acceptTags(tags)
-        .rejectTags(noTags)
-      );
+    bookIdsToDump = mp_library->filter(filter);
     auto totalResults = bookIdsToDump.size();
     bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
     if (count>0 && bookIdsToDump.size() > count) {
@@ -792,40 +871,57 @@ Response InternalServer::handle_catalog(const RequestContext& request)
     opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
   }
 
+  opdsDumper.setId(kiwix::to_string(uuid));
   response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
   return response;
 }
 
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+Response
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto response = get_default_response();
+  response.set_redirection(m_root + "/" + bookName + "/" +
+                           kiwix::urlEncode(entry.getPath()));
+  return response;
+}
+
 Response InternalServer::handle_content(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_content\n");
   }
 
-  std::string baseUrl;
-  std::string content;
-  std::string mimeType;
-
-  kiwix::Entry entry;
-
-  std::string bookName;
-  try {
-    bookName = request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return build_homepage(request);
-  }
+  const std::string bookName = get_book_name(request);
   if (bookName.empty())
     return build_homepage(request);
 
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
   if (reader == nullptr) {
     return build_404(request, bookName);
   }
@@ -835,16 +931,14 @@ Response InternalServer::handle_content(const RequestContext& request)
     urlStr = urlStr.substr(1);
   }
 
+  kiwix::Entry entry;
+
   try {
     entry = reader->getEntryFromPath(urlStr);
     if (entry.isRedirect() || urlStr.empty()) {
       // If urlStr is empty, we want to mainPage.
       // We must do a redirection to the real page.
-      entry = entry.getFinalEntry();
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" +
-        kiwix::urlEncode(entry.getPath()));
-      return response;
+      return build_redirect(bookName, entry.getFinalEntry());
     }
   } catch(kiwix::NoEntry& e) {
     if (m_verbose.load())
@@ -853,59 +947,19 @@ Response InternalServer::handle_content(const RequestContext& request)
     return build_404(request, bookName);
   }
 
-  try {
-    mimeType = entry.getMimetype();
-  } catch (exception& e) {
-    mimeType = "application/octet-stream";
-  }
+  auto response = get_default_response();
+
+  response.set_entry(entry, request);
 
   if (m_verbose.load()) {
-    printf("Found %s\n", urlStr.c_str());
-    printf("mimeType: %s\n", mimeType.c_str());
+    printf("Found %s\n", entry.getPath().c_str());
+    printf("mimeType: %s\n", response.get_mimeType().c_str());
   }
 
-  if (mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/json") != string::npos) {
-    zim::Blob raw_content = entry.getBlob();
-    content = string(raw_content.data(), raw_content.size());
-    auto response = get_default_response();
-    response.set_mimeType(mimeType);
+  if (response.get_mimeType().find("text/html") != string::npos)
+    response.set_taskbar(bookName, reader->getTitle());
 
-    /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
-     * /A/Kiwix */
-    if (mimeType.find("text/html") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(href|src)(=[\"|\']{0,1})/([A-Z|\\-])/");
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(@import[ ]+)([\"|\']{0,1})/([A-Z|\\-])/");
-      response.set_taskbar(bookName, reader->getTitle());
-    } else if (mimeType.find("text/css") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(url|URL)(\\([\"|\']{0,1})/([A-Z|\\-])/");
-    }
-    response.set_content(content);
-    response.set_compress(true);
-    response.set_cache(true);
-    return response;
-  } else {
-    int range_len;
-    if (request.get_range().second == -1) {
-       range_len = entry.getSize() - request.get_range().first;
-    } else {
-       range_len = request.get_range().second - request.get_range().first;
-    }
-    auto response = get_default_response();
-    response.set_entry(entry);
-    response.set_mimeType(mimeType);
-    response.set_range_first(request.get_range().first);
-    response.set_range_len(range_len);
-    response.set_cache(true);
-    return response;
-  }
+  return response;
 }
 
 }

src/server/etag.cpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "etag.h"
+
+#include "tools/stringTools.h"
+
+#include <algorithm>
+#include <sstream>
+
+namespace kiwix {
+
+namespace {
+
+// Characters in the options part of the ETag could in principle be picked up
+// from the latin alphabet in natural order (the character corresponding to
+// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
+// this file). However it is better to have some mnemonics in the option names,
+// hence below variable: all_options[opt] corresponds to the character going
+// into the ETag for ETag::Option opt.
+// IMPORTANT: The characters in all_options must come in sorted order (so that
+// IMPORTANT: isValidOptionsString() works correctly).
+const char all_options[] = "cz";
+
+static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
+
+bool isValidServerId(const std::string& s)
+{
+  return !s.empty() && s.find_first_of("\"/") == std::string::npos;
+}
+
+bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
+{
+  std::string::size_type i = 0;
+  for ( const char c : s )
+  {
+    const std::string::size_type j = sortedString.find(c, i);
+    if ( j == std::string::npos )
+      return false;
+    i = j+1;
+  }
+  return true;
+}
+
+bool isValidOptionsString(const std::string& s)
+{
+  return isSubsequenceOf(s, all_options);
+}
+
+} // namespace
+
+
+void ETag::set_option(Option opt)
+{
+  if ( ! get_option(opt) )
+  {
+    m_options.push_back(all_options[opt]);
+    std::sort(m_options.begin(), m_options.end());
+  }
+}
+
+bool ETag::get_option(Option opt) const
+{
+  return m_options.find(all_options[opt]) != std::string::npos;
+}
+
+std::string ETag::get_etag() const
+{
+  if ( m_serverId.empty() )
+    return std::string();
+
+  return "\"" + m_serverId + "/" + m_options + "\"";
+}
+
+ETag::ETag(const std::string& serverId, const std::string& options)
+{
+  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  {
+    m_serverId = serverId;
+    m_options = options;
+  }
+}
+
+ETag ETag::parse(std::string s)
+{
+  if ( kiwix::startsWith("W/", s) )
+    s = s.substr(2);
+
+  if ( s.front() != '"' || s.back() != '"' )
+    return ETag();
+
+  s = s.substr(1, s.size()-2);
+
+  const std::string::size_type i = s.find('/');
+  if ( i == std::string::npos )
+    return ETag();
+
+  return ETag(s.substr(0, i), s.substr(i+1));
+}
+
+ETag ETag::match(const std::string& etags, const std::string& server_id)
+{
+  std::istringstream ss(etags);
+  std::string etag_str;
+  while ( ss >> etag_str )
+  {
+    if ( etag_str.back() == ',' )
+      etag_str.pop_back();
+
+    const ETag etag = parse(etag_str);
+    if ( etag && etag.m_serverId == server_id )
+      return etag;
+  }
+
+  return ETag();
+}
+
+} // namespace kiwix

src/server/etag.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_ETAG_H
+#define KIWIXLIB_SERVER_ETAG_H
+
+#include <string>
+
+namespace kiwix {
+
+// The ETag string used by Kiwix server (more precisely, its value inside the
+// double quotes) consists of two parts:
+//
+// 1. ServerId - The string obtained on server start up
+//
+// 2. Options -  Zero or more characters encoding the values of some of the
+//               headers of the response
+//
+// The two parts are separated with a slash (/) symbol (which is always present,
+// even when the the options part is empty). Neither portion of a Kiwix ETag
+// may contain the slash symbol.
+// Examples of valid Kiwix server ETags (including the double quotes):
+//
+//   "abcdefghijklmn/"
+//   "1234567890/z"
+//   "1234567890/cz"
+//
+// The options part of the Kiwix ETag allows to correctly set the required
+// headers when responding to a conditional If-None-Match request with a 304
+// (Not Modified) response without following the full code path that would
+// discover the necessary options.
+
+class ETag
+{
+  public: // types
+    enum Option {
+      CACHEABLE_ENTITY,
+      COMPRESSED_CONTENT,
+      OPTION_COUNT
+    };
+
+  public: // functions
+    ETag() {}
+
+    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_option(Option opt);
+
+    explicit operator bool() const { return !m_serverId.empty(); }
+
+    bool get_option(Option opt) const;
+    std::string get_etag() const;
+
+
+    static ETag match(const std::string& etags, const std::string& server_id);
+
+  private: // functions
+    ETag(const std::string& serverId, const std::string& options);
+
+    static ETag parse(std::string s);
+
+  private: // data
+    std::string m_serverId;
+    std::string m_options;
+};
+
+} // namespace kiwix
+
+#endif // KIWIXLIB_SERVER_ETAG_H

src/server/request_context.cpp

@@ -30,60 +30,56 @@ namespace kiwix {
 
 static std::atomic_ullong s_requestIndex(0);
 
+namespace {
+
+RequestMethod str2RequestMethod(const std::string& method) {
+  if      (method == "GET")     return RequestMethod::GET;
+  else if (method == "HEAD")    return RequestMethod::HEAD;
+  else if (method == "POST")    return RequestMethod::POST;
+  else if (method == "PUT")     return RequestMethod::PUT;
+  else if (method == "DELETE")  return RequestMethod::DELETE_;
+  else if (method == "CONNECT") return RequestMethod::CONNECT;
+  else if (method == "OPTIONS") return RequestMethod::OPTIONS;
+  else if (method == "TRACE")   return RequestMethod::TRACE;
+  else if (method == "PATCH")   return RequestMethod::PATCH;
+  else                          return RequestMethod::OTHER;
+}
+
+std::string
+fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
+{
+  if (rootLocation.empty()) {
+    // nothing special to handle.
+    return full_url;
+  } else if (full_url == rootLocation) {
+    return "/";
+  } else if (full_url.size() > rootLocation.size() &&
+             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
+    return full_url.substr(rootLocation.size());
+  } else {
+    return "";
+  }
+}
+
+} // unnamed namespace
+
 RequestContext::RequestContext(struct MHD_Connection* connection,
                                std::string rootLocation,
                                const std::string& _url,
-                               const std::string& method,
+                               const std::string& _method,
                                const std::string& version) :
   full_url(_url),
-  url(_url),
-  valid_url(true),
+  url(fullURL2LocalURL(_url, rootLocation)),
+  method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingDeflate(false),
   accept_range(false),
   range_pair(0, -1)
 {
-  if (method == "GET") {
-    this->method = RequestMethod::GET;
-  } else if (method == "HEAD") {
-    this->method = RequestMethod::HEAD;
-  } else if (method == "POST") {
-    this->method = RequestMethod::POST;
-  } else if (method == "PUT") {
-    this->method = RequestMethod::PUT;
-  } else if (method == "DELETE") {
-    this->method = RequestMethod::DELETE_;
-  } else if (method == "CONNECT") {
-    this->method = RequestMethod::CONNECT;
-  } else if (method == "OPTIONS") {
-    this->method = RequestMethod::OPTIONS;
-  } else if (method == "TRACE") {
-    this->method = RequestMethod::TRACE;
-  } else if (method == "PATCH") {
-    this->method = RequestMethod::PATCH;
-  } else {
-    this->method = RequestMethod::OTHER;
-  }
-
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
 
-  valid_url = true;
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    url = full_url;
-  } else {
-    if (full_url == rootLocation) {
-      url = "/";
-    } else if (full_url.size() > rootLocation.size() &&
-               full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-      url = full_url.substr(rootLocation.size());
-    } else {
-      valid_url = false;
-    }
-  }
-
   try {
     acceptEncodingDeflate =
         (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
@@ -147,10 +143,11 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("Parsed : \n");
+  printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
   printf("has_range : %d\n", accept_range);
-  printf("is_valid_url : %d\n", valid_url);
+  printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
 }
 
@@ -188,7 +185,7 @@ std::string RequestContext::get_full_url() const {
 }
 
 bool RequestContext::is_valid_url() const {
-  return valid_url;
+  return !url.empty();
 }
 
 bool RequestContext::has_range() const {

src/server/request_context.h

@@ -51,7 +51,10 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
-  public:
+  public: // types
+  typedef std::pair<int, int> ByteRange;
+
+  public: // functions
     RequestContext(struct MHD_Connection* connection,
                    std::string rootLocation,
                    const std::string& url,
@@ -79,14 +82,13 @@ class RequestContext {
     std::string get_full_url() const;
 
     bool has_range() const;
-    std::pair<int, int> get_range() const;
+    ByteRange get_range() const;
 
     bool can_compress() const { return acceptEncodingDeflate; }
 
-  private:
+  private: // data
     std::string full_url;
     std::string url;
-    bool valid_url;
     RequestMethod method;
     std::string version;
     unsigned long long requestIndex;
@@ -94,10 +96,11 @@ class RequestContext {
     bool acceptEncodingDeflate;
 
     bool accept_range;
-    std::pair<int, int> range_pair;
+    ByteRange range_pair;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::string> arguments;
 
+  private: // functions
     static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
     static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };

src/server/response.cpp

@@ -17,7 +17,38 @@
 
 namespace kiwix {
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton)
+namespace
+{
+// some utilities
+
+std::string get_mime_type(const kiwix::Entry& entry)
+{
+  try {
+    return entry.getMimetype();
+  } catch (exception& e) {
+    return "application/octet-stream";
+  }
+}
+
+bool is_compressible_mime_type(const std::string& mimeType)
+{
+  return mimeType.find("text/") != string::npos
+      || mimeType.find("application/javascript") != string::npos
+      || mimeType.find("application/atom") != string::npos
+      || mimeType.find("application/opensearchdescription") != string::npos
+      || mimeType.find("application/json") != string::npos;
+}
+
+int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
+{
+  return range.second == -1
+       ? entry.getSize() - range.first
+       : range.second - range.first;
+}
+
+} // unnamed namespace
+
+Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
   : m_verbose(verbose),
     m_root(root),
     m_content(""),
@@ -25,7 +56,7 @@ Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool
     m_returnCode(MHD_HTTP_OK),
     m_withTaskbar(withTaskbar),
     m_withLibraryButton(withLibraryButton),
-    m_useCache(false),
+    m_blockExternalLinks(blockExternalLinks),
     m_addTaskbar(false),
     m_bookName(""),
     m_startRange(0),
@@ -127,86 +158,134 @@ void Response::introduce_taskbar()
 }
 
 
-
-int Response::send(const RequestContext& request, MHD_Connection* connection)
+void Response::inject_externallinks_blocker()
 {
-  MHD_Response* response = nullptr;
-  switch (m_mode) {
-    case ResponseMode::RAW_CONTENT : {
-      if (m_addTaskbar) {
-        introduce_taskbar();
-      }
+  kainjow::mustache::data data;
+  data.set("root", m_root);
+  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<head>",
+    script_tag);
+}
 
-      bool shouldCompress = m_compress && request.can_compress();
-      shouldCompress &= m_mimeType.find("text/") != string::npos
-                     || m_mimeType.find("application/javascript") != string::npos
-                     || m_mimeType.find("application/json") != string::npos; 
+bool
+Response::can_compress(const RequestContext& request) const
+{
+  return request.can_compress()
+      && is_compressible_mime_type(m_mimeType)
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+}
 
-      shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+MHD_Response*
+Response::create_raw_content_mhd_response(const RequestContext& request)
+{
+  if (m_addTaskbar) {
+    introduce_taskbar();
+  }
+  if ( m_blockExternalLinks ) {
+    inject_externallinks_blocker();
+  }
 
-      if (shouldCompress) {
-        std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-        uLongf comprLen = compr_buffer.capacity();
-        int err = compress(&compr_buffer[0],
-                           &comprLen,
-                           (const Bytef*)(m_content.data()),
-                           m_content.size());
-        if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-          /* /!\ Internet Explorer has a bug with deflate compression.
-             It can not handle the first two bytes (compression headers)
-             We need to chunk them off (move the content 2bytes)
-             It has no incidence on other browsers
-             See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-          m_content = string((char*)&compr_buffer[2], comprLen - 2);
-        } else {
-          shouldCompress = false;
-        }
-      }
-
-      response = MHD_create_response_from_buffer(
-        m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
-
-      if (shouldCompress) {
-        MHD_add_response_header(
-            response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-        MHD_add_response_header(
-            response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
-      }
-      MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-      break;
-    }
-
-    case ResponseMode::REDIRECTION : {
-      response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-      MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-      break;
-    }
-
-    case ResponseMode::ENTRY : {
-      response = MHD_create_response_from_callback(m_entry.getSize(),
-                                                   16384,
-                                                   callback_reader_from_entry,
-                                                   new RunningResponse(m_entry, m_startRange),
-                                                   callback_free_response);
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-      std::ostringstream oss;
-      oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-          << "/" << m_entry.getSize();
-
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-      MHD_add_response_header(response,
-        MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
-      break;
+  bool shouldCompress = m_compress && can_compress(request);
+  if (shouldCompress) {
+    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
+    uLongf comprLen = compr_buffer.capacity();
+    int err = compress(&compr_buffer[0],
+                       &comprLen,
+                       (const Bytef*)(m_content.data()),
+                       m_content.size());
+    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
+      /* /!\ Internet Explorer has a bug with deflate compression.
+         It can not handle the first two bytes (compression headers)
+         We need to chunk them off (move the content 2bytes)
+         It has no incidence on other browsers
+         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
+      m_content = string((char*)&compr_buffer[2], comprLen - 2);
+      m_etag.set_option(ETag::COMPRESSED_CONTENT);
+    } else {
+      shouldCompress = false;
     }
   }
 
+  MHD_Response* response = MHD_create_response_from_buffer(
+    m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
+
+  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
+  // shouldCompress can have different values. This can happen for a 304 (Not
+  // Modified) response generated while handling a conditional If-None-Match
+  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
+  // option) is obtained from the ETag list of the If-None-Match header and the
+  // response has no body (which shouldn't be compressed).
+  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  if (shouldCompress) {
+    MHD_add_response_header(
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+  }
+  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_redirection_mhd_response() const
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
+  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_entry_mhd_response() const
+{
+  MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
+                                               16384,
+                                               callback_reader_from_entry,
+                                               new RunningResponse(m_entry, m_startRange),
+                                               callback_free_response);
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
+  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
+  std::ostringstream oss;
+  oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
+      << "/" << m_entry.getSize();
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
+  return response;
+}
+
+MHD_Response*
+Response::create_mhd_response(const RequestContext& request)
+{
+  switch (m_mode) {
+    case ResponseMode::RAW_CONTENT :
+      return create_raw_content_mhd_response(request);
+
+    case ResponseMode::REDIRECTION :
+      return create_redirection_mhd_response();
+
+    case ResponseMode::ENTRY :
+      return create_entry_mhd_response();
+  }
+  return nullptr;
+}
+
+int Response::send(const RequestContext& request, MHD_Connection* connection)
+{
+  MHD_Response* response = create_mhd_response(request);
+
   MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_useCache ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+  const std::string etag = m_etag.get_etag();
+  if ( ! etag.empty() )
+      MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
 
   if (m_returnCode == MHD_HTTP_OK && request.has_range())
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
@@ -234,9 +313,25 @@ void Response::set_redirection(const std::string& url) {
   m_returnCode = MHD_HTTP_FOUND;
 }
 
-void Response::set_entry(const Entry& entry) {
+void Response::set_entry(const Entry& entry, const RequestContext& request) {
   m_entry = entry;
   m_mode = ResponseMode::ENTRY;
+
+  const std::string mimeType = get_mime_type(entry);
+  set_mimeType(mimeType);
+  set_cacheable();
+
+  if ( is_compressible_mime_type(mimeType) ) {
+    zim::Blob raw_content = entry.getBlob();
+    const std::string content = string(raw_content.data(), raw_content.size());
+
+    set_content(content);
+    set_compress(true);
+  } else {
+    const int range_len = get_range_len(entry, request.get_range());
+    set_range_first(request.get_range().first);
+    set_range_len(range_len);
+  }
 }
 
 void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)

src/server/response.h

@@ -25,6 +25,7 @@
 
 #include <mustache.hpp>
 #include "entry.h"
+#include "etag.h"
 
 extern "C" {
 #include <microhttpd.h>
@@ -42,7 +43,7 @@ class RequestContext;
 
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton);
+    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
     ~Response() = default;
 
     int send(const RequestContext& request, MHD_Connection* connection);
@@ -50,22 +51,34 @@ class Response {
     void set_template(const std::string& template_str, kainjow::mustache::data data);
     void set_content(const std::string& content);
     void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry);
+    void set_entry(const Entry& entry, const RequestContext& request);
 
 
     void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
     void set_code(int code) { m_returnCode = code; }
-    void set_cache(bool cache) { m_useCache = cache; }
+    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
+    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_etag(const ETag& etag) { m_etag = etag; }
     void set_compress(bool compress) { m_compress = compress; }
     void set_taskbar(const std::string& bookName, const std::string& bookTitle);
     void set_range_first(uint64_t start) { m_startRange = start; }
     void set_range_len(uint64_t len) { m_lenRange = len; }
 
-    int getReturnCode() { return m_returnCode; }
+    int getReturnCode() const { return m_returnCode; }
+    std::string get_mimeType() const { return m_mimeType; }
 
     void introduce_taskbar();
+    void inject_externallinks_blocker();
 
-  private:
+    bool can_compress(const RequestContext& request) const;
+
+  private: // functions
+    MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
+    MHD_Response* create_redirection_mhd_response() const;
+    MHD_Response* create_entry_mhd_response() const;
+
+  private: // data
     bool m_verbose;
     ResponseMode m_mode;
     std::string m_root;
@@ -75,13 +88,14 @@ class Response {
     int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
-    bool m_useCache;
+    bool m_blockExternalLinks;
     bool m_compress;
     bool m_addTaskbar;
     std::string m_bookName;
     std::string m_bookTitle;
     uint64_t m_startRange;
     uint64_t m_lenRange;
+    ETag m_etag;
 };
 
 }

src/subprocess_windows.cpp

@@ -4,6 +4,7 @@
 
 #include <windows.h>
 #include <winbase.h>
+#include <shlwapi.h>
 #include <iostream>
 #include <sstream>
 
@@ -34,11 +35,11 @@ DWORD WINAPI WinImpl::waitForPID(void* _self)
   return 0;
 }
 
-std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
+std::unique_ptr<wchar_t[]> toWideChar(const std::string& value, size_t min_size = 0)
 {
-  auto size = MultiByteToWideChar(CP_UTF8, 0,
+  size_t size = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, nullptr, 0);
-  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size]);
+  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size>min_size?size:min_size]);
   auto ret = MultiByteToWideChar(CP_UTF8, 0,
                 value.c_str(), -1, wdata.get(), size);
   if (0 == ret) {
@@ -46,6 +47,9 @@ std::unique_ptr<wchar_t[]> toWideChar(const std::string& value)
     oss << "Cannot convert to wchar : " << GetLastError();
     throw std::runtime_error(oss.str());
   }
+  if (size < min_size) {
+    memset(wdata.get() + size, 0, min_size-size);
+  }
   return wdata;
 }
 
@@ -55,14 +59,16 @@ void WinImpl::run(commandLine_t& commandLine)
   STARTUPINFOW startInfo = {0};
   PROCESS_INFORMATION procInfo;
   startInfo.cb = sizeof(startInfo);
-  std::ostringstream oss;
+  std::wostringstream oss;
   for(auto& item: commandLine) {
-    oss << item << " ";
+    auto witem = toWideChar(item, MAX_PATH);
+    PathQuoteSpacesW(witem.get());
+    oss << witem.get() << " ";
   }
-  auto wCommandLine = toWideChar(oss.str());
+  auto wCommandLine = oss.str();
   if (CreateProcessW(
     NULL,
-    wCommandLine.get(),
+    const_cast<wchar_t*>(wCommandLine.c_str()),
     NULL,
     NULL,
     false,
@@ -70,7 +76,8 @@ void WinImpl::run(commandLine_t& commandLine)
     NULL,
     NULL,
     &startInfo,
-    &procInfo)) {
+    &procInfo))
+  {
     m_pid = procInfo.dwProcessId;
     m_handle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);

src/tools/base64.cpp

@@ -1,4 +1,4 @@
-/* 
+/*
    base64.cpp and base64.h
 
    Copyright (C) 2004-2008 René Nyffenegger
@@ -27,7 +27,7 @@
 #include <tools/base64.h>
 #include <iostream>
 
-static const std::string base64_chars = 
+static const std::string base64_chars =
              "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
              "abcdefghijklmnopqrstuvwxyz"
   "0123456789+/";

src/tools/otherTools.cpp

@@ -25,7 +25,10 @@
 #include <unistd.h>
 #endif
 
+#include "tools/stringTools.h"
+
 #include <map>
+#include <sstream>
 #include <pugixml.hpp>
 
 
@@ -204,3 +207,76 @@ std::string kiwix::nodeToString(const pugi::xml_node& node)
 std::string kiwix::converta2toa3(const std::string& a2code){
   return codeisomapping.at(a2code);
 }
+
+std::vector<std::string> kiwix::convertTags(const std::string& tags_str)
+{
+  auto tags = kiwix::split(tags_str, ";");
+  std::vector<std::string> tagsList;
+  bool picSeen(false), vidSeen(false), detSeen(false), indexSeen(false);
+  for (auto tag: tags) {
+    picSeen |= (tag == "nopic" || startsWith(tag, "_pictures:"));
+    vidSeen |= (tag == "novid" || startsWith(tag, "_videos:"));
+    detSeen |= (tag == "nodet" || startsWith(tag, "_details:"));
+    indexSeen |= kiwix::startsWith(tag, "_ftindex");
+    if (tag == "nopic") {
+      tagsList.push_back("_pictures:no");
+    } else if (tag == "novid") {
+      tagsList.push_back("_videos:no");
+    } else if (tag == "nodet") {
+      tagsList.push_back("_details:no");
+    } else if (tag == "_ftindex") {
+      tagsList.push_back("_ftindex:yes");
+    } else {
+      tagsList.push_back(tag);
+    }
+  }
+  if (!indexSeen) {
+    tagsList.push_back("_ftindex:no");
+  }
+  if (!picSeen) {
+    tagsList.push_back("_pictures:yes");
+  }
+  if (!vidSeen) {
+    tagsList.push_back("_videos:yes");
+  }
+  if (!detSeen) {
+    tagsList.push_back("_details:yes");
+  }
+  return tagsList;
+}
+
+std::string kiwix::getTagValueFromTagList(
+  const std::vector<std::string>& tagList, const std::string& tagName)
+{
+  for (auto tag: tagList) {
+    if (tag[0] == '_') {
+      auto delimPos = tag.find(':');
+      if (delimPos == std::string::npos) {
+        // No delimiter... what to do ?
+        continue;
+      }
+      auto cTagName = tag.substr(1, delimPos-1);
+      auto cTagValue = tag.substr(delimPos+1);
+      if (cTagName == tagName) {
+        return cTagValue;
+      }
+    }
+  }
+  std::stringstream ss;
+  ss << tagName << " cannot be found";
+  throw std::out_of_range(ss.str());
+}
+
+bool kiwix::convertStrToBool(const std::string& value)
+{
+  if (value == "yes") {
+    return true;
+  } else if (value == "no") {
+    return false;
+  }
+
+  std::stringstream ss;
+  ss << "Tag value '" << value << "' cannot be converted to bool.";
+  throw std::domain_error(ss.str());
+}
+

src/tools/pathTools.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "tools/pathTools.h"
+#include <stdexcept>
 
 #ifdef __APPLE__
 #include <limits.h>
@@ -39,20 +40,44 @@
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <algorithm>
 
 #ifdef _WIN32
-const std::string SEPARATOR("\\");
+# define SEPARATOR "\\"
+# include <io.h>
 #else
-const std::string SEPARATOR("/");
-#include <unistd.h>
+# define SEPARATOR "/"
+# include <unistd.h>
+# include <sys/stat.h>
 #endif
 
+#include <fcntl.h>
+
 #include <stdlib.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX 1024
 #endif
 
+
+#ifdef _WIN32
+std::string WideToUtf8(const std::wstring& wstr)
+{
+  auto needed_size = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), NULL, 0, NULL, NULL);
+  std::string ret(needed_size, 0);
+  WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr.size(), &ret[0], needed_size, NULL, NULL);
+  return ret;
+}
+
+std::wstring Utf8ToWide(const std::string& str)
+{
+  auto needed_size = MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), NULL, 0);
+  std::wstring ret(needed_size, 0);
+  MultiByteToWideChar(CP_UTF8, 0, str.data(), str.size(), &ret[0], needed_size);
+  return ret;
+}
+#endif
+
 bool isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
@@ -62,11 +87,65 @@ bool isRelativePath(const std::string& path)
 #endif
 }
 
+std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
+{
+  std::vector<std::string> ret;
+#ifdef _WIN32
+  //Special case if we have a drive directory not at first.
+  //Starts from there.
+  auto it = find_if(parts.rbegin(), parts.rend(),
+                    [](const std::string& p) ->bool
+                      { return p.length() == 2 && p[1] == ':'; });
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+  }
+#endif
+
+  size_t index = 0;
+  for (auto& part: parts) {
+    index++;
+    if (part == "..") {
+      if (absolute) {
+        // We try to remove as far as possible.
+        if (ret.size() > 1) {
+          ret.pop_back();
+        }
+      } else {
+        // We remove only if we can remove it.
+        // Else we add it.
+        if (!ret.empty() && ret.back() != "..") {
+          ret.pop_back();
+        } else {
+          ret.push_back("..");
+        }
+      }
+      continue;
+    }
+    if (part == "") {
+#ifndef _WIN32
+      if (ret.empty() && (absolute || index<parts.size())) {
+        ret.push_back("");
+      }
+#endif
+      continue;
+    }
+    if (part == ".") {
+      continue;
+    }
+    ret.push_back(part);
+  }
+#ifndef _WIN32
+  if (absolute && ret.size() == 1 && ret.back() == "") {
+    ret.push_back("");
+  }
+#endif
+  return ret;
+}
+
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
-  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
-  std::vector<std::string> absolutePathParts
-      = kiwix::split(absolutePath, SEPARATOR);
+  auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
 
   unsigned int commonCount = 0;
   while (commonCount < pathParts.size()
@@ -75,108 +154,60 @@ std::string computeRelativePath(const std::string& path, const std::string& abso
       commonCount++;
   }
 
-  std::string relativePath;
-#ifdef _WIN32
-  /* On Windows you have a token more because the root is represented
-     by a letter */
-  if (commonCount == 0) {
-    relativePath = ".." + SEPARATOR;
-  }
-#endif
-
+  std::vector<std::string> relativeParts;
   for (unsigned int i = commonCount; i < pathParts.size(); i++) {
-    relativePath += ".." + SEPARATOR;
+    relativeParts.push_back("..");
   }
   for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
-    relativePath += absolutePathParts[i];
-    relativePath += i + 1 < absolutePathParts.size() ? SEPARATOR : "";
+    relativeParts.push_back(absolutePathParts[i]);
   }
-  return relativePath;
+  auto ret = kiwix::join(normalizeParts(relativeParts, false), SEPARATOR);
+  return ret;
 }
 
-#ifdef _WIN32
-# define STRTOK strtok_s
-#else
-# define STRTOK strtok_r
-#endif
-
-/* Warning: the relative path must be with slashes */
 std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
 {
-  std::string absolutePath;
-
+  std::string absolutePath = path;
   if (path.empty()) {
-    char* path = NULL;
-    size_t size = 0;
-
-#ifdef _WIN32
-    path = _getcwd(path, size);
-#else
-    path = getcwd(path, size);
-#endif
-
-    absolutePath = std::string(path) + SEPARATOR;
-  } else {
-    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
-                       ? path
-                       : path + SEPARATOR;
+    absolutePath = getCurrentDirectory();
   }
 
-#if _WIN32
-  char* cRelativePath = _strdup(relativePath.c_str());
-#else
-  char* cRelativePath = strdup(relativePath.c_str());
-#endif
-  char* saveptr = nullptr;
-  char* token = STRTOK(cRelativePath, "/", &saveptr);
+  auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
+  auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
 
-  while (token != NULL) {
-    if (std::string(token) == "..") {
-      absolutePath = removeLastPathElement(absolutePath, true, false);
-      token = STRTOK(NULL, "/", &saveptr);
-    } else if (strcmp(token, ".") && strcmp(token, "")) {
-      absolutePath += std::string(token);
-      token = STRTOK(NULL, "/", &saveptr);
-      if (token != NULL) {
-        absolutePath += SEPARATOR;
-      }
-    } else {
-      token = STRTOK(NULL, "/", &saveptr);
-    }
-  }
-  free(cRelativePath);
-
-  return absolutePath;
+  absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
+  auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
+  return ret;
 }
 
-std::string removeLastPathElement(const std::string& path,
-                                  const bool removePreSeparator,
-                                  const bool removePostSeparator)
+std::string removeLastPathElement(const std::string& path)
 {
-  std::string newPath = path;
-  size_t offset = newPath.find_last_of(SEPARATOR);
-  if (removePreSeparator &&
-#ifndef _WIN32
-      offset != newPath.find_first_of(SEPARATOR) &&
-#endif
-      offset == newPath.length() - 1) {
-    newPath = newPath.substr(0, offset);
-    offset = newPath.find_last_of(SEPARATOR);
+  auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  if (!parts.empty()) {
+    parts.pop_back();
   }
-  newPath = removePostSeparator ? newPath.substr(0, offset)
-                                : newPath.substr(0, offset + 1);
-  return newPath;
+  auto ret = kiwix::join(parts, SEPARATOR);
+  return ret;
 }
 
 std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
 {
-  std::string newPath = directoryPath + SEPARATOR + filename;
+  std::string newPath = directoryPath;
+  if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
+    newPath += SEPARATOR;
+  }
+  newPath += filename;
   return newPath;
 }
 
 std::string getLastPathElement(const std::string& path)
 {
-  return path.substr(path.find_last_of(SEPARATOR) + 1);
+  auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
+  if (parts.empty()) {
+    return "";
+  }
+  auto ret = parts.back();
+  return ret;
 }
 
 unsigned int getFileSize(const std::string& path)
@@ -201,14 +232,33 @@ std::string getFileSizeAsString(const std::string& path)
 
 std::string getFileContent(const std::string& path)
 {
-  std::ifstream f(path, std::ios::in|std::ios::ate);
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_RDONLY | _O_BINARY);
+#else
+  auto fd = open(path.c_str(), O_RDONLY);
+#endif
   std::string content;
-  if (f.is_open()) {
-    auto size = f.tellg();
-    content.reserve(size);
-    f.seekg(0, std::ios::beg);
-    content.assign((std::istreambuf_iterator<char>(f)),
-                    std::istreambuf_iterator<char>());
+  if (fd != -1) {
+#ifdef _WIN32
+    auto size = _lseeki64(fd, 0, SEEK_END);
+#else
+    auto size = lseek(fd, 0, SEEK_END);
+#endif
+    content.resize(size);
+#ifdef _WIN32
+    _lseeki64(fd, 0, SEEK_SET);
+#else
+    lseek(fd, 0, SEEK_SET);
+#endif
+    auto p = (char*)content.data();
+    while (size) {
+      auto readsize = size > 2048 ? 2048 : size;
+      readsize = ::read(fd, p, readsize);
+      p += readsize;
+      size -= readsize;
+    }
+    close(fd);
   }
   return content;
 }
@@ -216,7 +266,7 @@ std::string getFileContent(const std::string& path)
 bool fileExists(const std::string& path)
 {
 #ifdef _WIN32
-  return PathFileExists(path.c_str());
+  return PathFileExistsW(Utf8ToWide(path).c_str());
 #else
   bool flag = false;
   std::fstream fin;
@@ -232,7 +282,7 @@ bool fileExists(const std::string& path)
 bool makeDirectory(const std::string& path)
 {
 #ifdef _WIN32
-  int status = _mkdir(path.c_str());
+  int status = _wmkdir(Utf8ToWide(path).c_str());
 #else
   int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 #endif
@@ -242,15 +292,15 @@ bool makeDirectory(const std::string& path)
 std::string makeTmpDirectory()
 {
 #ifdef _WIN32
-  char cbase[MAX_PATH];
-  char ctmp[MAX_PATH];
-  GetTempPath(MAX_PATH-14, cbase);
+  wchar_t cbase[MAX_PATH];
+  wchar_t ctmp[MAX_PATH];
+  GetTempPathW(MAX_PATH-14, cbase);
   // This create a file for us, ensure it is unique.
   // So we need to delete it and create the directory using the same name.
-  GetTempFileName(cbase, "kiwix", 0, ctmp);
-  DeleteFile(ctmp);
-  _mkdir(ctmp);
-  return std::string(ctmp);
+  GetTempFileNameW(cbase, L"kiwix", 0, ctmp);
+  DeleteFileW(ctmp);
+  _wmkdir(ctmp);
+  return WideToUtf8(ctmp);
 #else
   char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
   std::string dir = mkdtemp(_template_array);
@@ -261,86 +311,136 @@ std::string makeTmpDirectory()
 /* Try to create a link and if does not work then make a copy */
 bool copyFile(const std::string& sourcePath, const std::string& destPath)
 {
+#ifdef _WIN32
+  return CopyFileW(Utf8ToWide(sourcePath).c_str(), Utf8ToWide(destPath).c_str(), 1);
+#else
   try {
-#ifndef _WIN32
     if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
-#endif
       std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
       std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
       outfile << infile.rdbuf();
-#ifndef _WIN32
     }
-#endif
   } catch (std::exception& e) {
     std::cerr << e.what() << std::endl;
     return false;
   }
 
   return true;
+#endif
 }
 
-std::string getExecutablePath()
+std::string getExecutablePath(bool realPathOnly)
 {
-  char binRootPath[PATH_MAX];
+  if (!realPathOnly) {
+    char* cAppImage = ::getenv("APPIMAGE");
+    if (cAppImage) {
+      char* cArgv0 = ::getenv("ARGV0");
+      char* cOwd = ::getenv("OWD");
+      if (cArgv0 && cOwd) {
+        auto ret = appendToDirectory(cOwd, cArgv0);
+        return ret;
+      }
+    }
+  }
 
 #ifdef _WIN32
-  GetModuleFileName(NULL, binRootPath, PATH_MAX);
-  return std::string(binRootPath);
+  std::wstring binRootPath(PATH_MAX, 0);
+  GetModuleFileNameW(NULL, &binRootPath[0], PATH_MAX);
+  std::string ret = WideToUtf8(binRootPath);
+  return ret;
 #elif __APPLE__
+  char binRootPath[PATH_MAX];
   uint32_t max = (uint32_t)PATH_MAX;
   _NSGetExecutablePath(binRootPath, &max);
   return std::string(binRootPath);
 #else
+  char binRootPath[PATH_MAX];
   ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
   if (size != -1) {
     return std::string(binRootPath, size);
   }
-#endif
-
   return "";
+#endif
 }
 
 bool writeTextFile(const std::string& path, const std::string& content)
 {
-  std::ofstream file;
-  file.open(path.c_str());
-  file << content;
-  file.close();
+#ifdef _WIN32
+  auto wpath = Utf8ToWide(path);
+  auto fd = _wopen(wpath.c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC, S_IWRITE);
+#else
+  auto fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC,
+                               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+#endif
+  if (fd == -1)
+    return false;
+
+  if (write(fd, content.c_str(), content.size()) != (long)content.size()) {
+    close(fd);
+    return false;
+  }
+  close(fd);
   return true;
 }
 
 std::string getCurrentDirectory()
 {
-  char* a_cwd = getcwd(NULL, 0);
-  std::string s_cwd(a_cwd);
+#ifdef _WIN32
+  wchar_t* a_cwd = _wgetcwd(NULL, 0);
+  std::string ret = WideToUtf8(a_cwd);
   free(a_cwd);
-  return s_cwd;
+#else
+  char* a_cwd = getcwd(NULL, 0);
+  std::string ret(a_cwd);
+  free(a_cwd);
+#endif
+  return ret;
 }
 
 std::string getDataDirectory()
 {
+// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
 #ifdef _WIN32
-  char* cDataDir = ::getenv("APPDATA");
+  wchar_t* cDataDir = ::_wgetenv(L"KIWIX_DATA_DIR");
+  if (cDataDir != nullptr) {
+    return WideToUtf8(cDataDir);
+  }
 #else
   char* cDataDir = ::getenv("KIWIX_DATA_DIR");
-#endif
-  std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (!dataDir.empty())
-    return dataDir;
-#ifdef _WIN32
-  cDataDir = ::getenv("USERPROFILE");
-  dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-#else
-  cDataDir = ::getenv("XDG_DATA_HOME");
-  dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (dataDir.empty()) {
-    cDataDir = ::getenv("HOME");
-    dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-    dataDir = appendToDirectory(dataDir, ".local");
-    dataDir = appendToDirectory(dataDir, "share");
+  if (cDataDir != nullptr) {
+    return cDataDir;
   }
 #endif
-  return appendToDirectory(dataDir, "kiwix");
+
+// Compute the dataDir from the user directory.
+  std::string dataDir;
+#ifdef _WIN32
+  cDataDir = ::_wgetenv(L"APPDATA");
+  if (cDataDir == nullptr)
+    cDataDir = ::_wgetenv(L"USERPROFILE");
+  if (cDataDir != nullptr)
+    dataDir = WideToUtf8(cDataDir);
+#else
+  cDataDir = ::getenv("XDG_DATA_HOME");
+  if (cDataDir != nullptr) {
+    dataDir = cDataDir;
+  } else {
+    cDataDir = ::getenv("HOME");
+    if (cDataDir != nullptr) {
+      dataDir = cDataDir;
+      dataDir = appendToDirectory(dataDir, ".local");
+      dataDir = appendToDirectory(dataDir, "share");
+    }
+  }
+#endif
+  if (!dataDir.empty()) {
+    dataDir = appendToDirectory(dataDir, "kiwix");
+    makeDirectory(dataDir);
+    return dataDir;
+  }
+
+// Let's use the currentDirectory
+  return getCurrentDirectory();
 }
 
 static std::map<std::string, std::string> extMimeTypes = {

src/tools/regexTools.cpp

@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <map>
+#include <stdexcept>
 #include <pthread.h>
 
 std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;

src/tools/stringTools.cpp

@@ -267,36 +267,42 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
-                                      const std::string& delims = " *-")
+                                      const std::string& delims,
+                                      bool trimEmpty)
 {
-  std::string::size_type lastPos = str.find_first_not_of(delims, 0);
-  std::string::size_type pos = str.find_first_of(delims, lastPos);
+  std::string::size_type lastPos = 0;
+  std::string::size_type pos = 0;
   std::vector<std::string> tokens;
-
-  while (std::string::npos != pos || std::string::npos != lastPos) {
-    tokens.push_back(str.substr(lastPos, pos - lastPos));
-    lastPos = str.find_first_not_of(delims, pos);
-    pos = str.find_first_of(delims, lastPos);
+  while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
+  {
+    auto token = str.substr(lastPos, pos - lastPos);
+    if (!trimEmpty || !token.empty()) {
+      tokens.push_back(token);
+    }
+    lastPos = pos + 1;
   }
 
+  auto token = str.substr(lastPos);
+  if (!trimEmpty || !token.empty()) {
+    tokens.push_back(token);
+  }
   return tokens;
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
+std::string kiwix::join(const std::vector<std::string>& list, const std::string& sep)
 {
-  const std::string m1(lhs), m2(rhs);
-  return split(m1, m2);
+  std::stringstream ss;
+  bool first = true;
+  for (auto& s:list) {
+    if (!first) {
+      ss << sep;
+    }
+    first = false;
+    ss << s;
+  }
+  return ss.str();
 }
 
-std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
-{
-  return split(lhs, rhs.c_str());
-}
-
-std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
-{
-  return split(lhs.c_str(), rhs);
-}
 
 std::string kiwix::ucFirst(const std::string& word)
 {

src/wrapper/java/book.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Book.h"
+
+#include "utils.h"
+#include "book.h"
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_allocate(
+  JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Book>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Book>(env, thisObj);
+}
+
+#define BOOK (getPtr<kiwix::Book>(env, thisObj))
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
+{
+  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
+}
+
+METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
+{
+  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
+}
+
+#define GETTER(retType, name) JNIEXPORT retType JNICALL \
+Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
+{ \
+  auto cRet = BOOK->name(); \
+  retType ret = c2jni(cRet, env); \
+  return ret; \
+}
+
+GETTER(jstring, getId)
+GETTER(jstring, getPath)
+GETTER(jboolean, isPathValid)
+GETTER(jstring, getTitle)
+GETTER(jstring, getDescription)
+GETTER(jstring, getLanguage)
+GETTER(jstring, getCreator)
+GETTER(jstring, getPublisher)
+GETTER(jstring, getDate)
+GETTER(jstring, getUrl)
+GETTER(jstring, getName)
+GETTER(jstring, getFlavour)
+GETTER(jstring, getTags)
+GETTER(jlong, getArticleCount)
+GETTER(jlong, getMediaCount)
+GETTER(jlong, getSize)
+GETTER(jstring, getFavicon)
+GETTER(jstring, getFaviconUrl)
+GETTER(jstring, getFaviconMimeType)
+
+METHOD(jstring, Book, getTagStr, jstring tagName) try {
+  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
+  return c2jni(cRet, env);
+} catch(...) {
+  return c2jni<std::string>("", env);
+}
+
+#undef GETTER

src/wrapper/java/filter.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Filter.h"
+
+#include "library.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+METHOD0(void, Filter, allocate) {
+  allocate<kiwix::Filter>(env, thisObj);
+}
+
+METHOD0(void, Filter, dispose) {
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
+
+#define FORWARD(name, args_type) \
+METHOD(jobject, Filter, name, args_type value) { \
+  FILTER->name(jni2c(value, env)); \
+  return thisObj; \
+}
+
+#define FORWARDA(name, args_type) \
+METHOD(jobject, Filter, name, jobjectArray value) { \
+  FILTER->name(jni2c<args_type>(value, env)); \
+  return thisObj; \
+}
+
+
+
+FORWARD(local, jboolean)
+FORWARD(remote, jboolean)
+FORWARD(valid, jboolean)
+FORWARDA(acceptTags, jstring)
+FORWARDA(rejectTags, jstring)
+FORWARD(lang, jstring)
+FORWARD(publisher, jstring)
+FORWARD(creator, jstring)
+FORWARD(maxSize, jlong)
+FORWARD(query, jstring)
+
+

src/wrapper/java/kiwixicu.cpp

@@ -28,7 +28,11 @@
 
 #include "utils.h"
 
+#if __ANDROID__
 pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER;
+#else
+pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+#endif
 
 JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
     JNIEnv* env, jclass kclass, jstring dirStr)

src/wrapper/java/kiwixreader.cpp

@@ -21,7 +21,7 @@
 
 #include <jni.h>
 #include <zim/file.h>
-#include <android/log.h>
+#include <exception>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
 #include "tools/base64.h"
@@ -34,14 +34,14 @@ JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
 {
   std::string cPath = jni2c(filename, env);
 
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create reader with: %s", cPath.c_str());
+  LOG("Attempting to create reader with: %s", cPath.c_str());
   Lock l;
   try {
     kiwix::Reader* reader = new kiwix::Reader(cPath);
     return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error opening ZIM file");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Error opening ZIM file");
+    LOG(e.what());
     return 0;
   }
 }
@@ -64,8 +64,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
     std::string cUrl = READER->getMainPage().getPath();
     url = c2jni(cUrl, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM main page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM main page");
+    LOG(e.what());
     url = NULL;
   }
   return url;
@@ -80,8 +80,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
     std::string cId = READER->getId();
     id = c2jni(cId, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM id");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
     id = NULL;
   }
 
@@ -95,10 +95,10 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
 
   try {
     int cSize = READER->getFileSize();
-    size = c2jni(cSize);
+    size = c2jni(cSize, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM file size");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM file size");
+    LOG(e.what());
   }
 
   return size;
@@ -113,8 +113,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
     std::string cCreator = READER->getCreator();
     creator = c2jni(cCreator, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM creator");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM creator");
+    LOG(e.what());
     creator = NULL;
   }
 
@@ -130,8 +130,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
     std::string cPublisher = READER->getPublisher();
     publisher = c2jni(cPublisher, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM publish");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM publish");
+    LOG(e.what());
     publisher = NULL;
   }
   return publisher;
@@ -146,8 +146,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
     std::string cName = READER->getName();
     name = c2jni(cName, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM name");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM name");
+    LOG(e.what());
     name = NULL;
   }
   return name;
@@ -166,8 +166,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
         base64_encode(cContent),
         env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM favicon");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM favicon");
+    LOG(e.what());
     favicon = NULL;
   }
   return favicon;
@@ -182,8 +182,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
     std::string cDate = READER->getDate();
     date = c2jni(cDate, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM date");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM date");
+    LOG(e.what());
     date = NULL;
   }
   return date;
@@ -198,8 +198,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
     std::string cLanguage = READER->getLanguage();
     language = c2jni(cLanguage, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM language");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get ZIM language");
+    LOG(e.what());
     language = NULL;
   }
 
@@ -217,8 +217,8 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
     auto cMimeType = entry.getMimetype();
     mimeType = c2jni(cMimeType, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get mime-type for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
+    LOG(e.what());
     mimeType = NULL;
   }
   return mimeType;
@@ -234,7 +234,7 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
     entry = entry.getFinalEntry();
     finalUrl = c2jni(entry.getPath(), env);
   } catch (std::exception& e) {
-    finalUrl = c2jni("", env);
+    finalUrl = c2jni(std::string(), env);
   }
   return finalUrl;
 }
@@ -268,8 +268,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
           data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get content for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get content for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
 
   return data;
@@ -284,8 +284,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
   /* Default values */
   /* Retrieve the content */
   std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset);
-  unsigned int cLen = jni2c(len);
+  unsigned int cOffset = jni2c(offset, env);
+  unsigned int cLen = jni2c(len, env);
   try {
     auto entry = READER->getEntryFromEncodedPath(cUrl);
     entry = entry.getFinalEntry();
@@ -300,12 +300,28 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
       setIntObjValue(cLen, sizeObj, env);
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
+    LOG(e.what());
   }
   return data;
 }
 
+JNIEXPORT jlong JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
+    JNIEnv* env, jobject obj, jstring url)
+{
+  std::string cUrl = jni2c(url, env);
+  try {
+    auto entry = READER->getEntryFromEncodedPath(cUrl);
+    entry = entry.getFinalEntry();
+    return c2jni(entry.getSize(), env);
+  } catch(std::exception& e) {
+    LOG("Unable to get size for url : %s", cUrl.c_str());
+    LOG(e.what());
+  }
+  return c2jni(0, env);
+}
+
 JNIEXPORT jobject JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     JNIEnv* env, jobject obj, jstring url)
@@ -322,8 +338,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     auto part_info = entry.getDirectAccessInfo();
     setPairObjValue(part_info.first, part_info.second, pair, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get direct access info for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
+    LOG(e.what());
   }
   return pair;
 }
@@ -336,15 +352,15 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 {
   jboolean retVal = JNI_FALSE;
   std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count);
+  unsigned int cCount = jni2c(count, env);
 
   try {
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get search results for pattern: %s", cPrefix.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
+    LOG(e.what());
   }
 
   return retVal;
@@ -353,19 +369,22 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
                                                          jobject obj,
-                                                         jobject titleObj)
+                                                         jobject titleObj,
+                                                         jobject urlObj)
 {
   jboolean retVal = JNI_FALSE;
   std::string cTitle;
+  std::string cUrl;
 
   try {
-    if (READER->getNextSuggestion(cTitle)) {
+    if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
+      setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get next suggestion");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get next suggestion");
+    LOG(e.what());
   }
 
   return retVal;
@@ -385,8 +404,8 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
     setStringObjValue(entry.getPath(), urlObj, env);
     return JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get url for title %s: ", cTitle.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Unable to get url for title %s: ", cTitle.c_str());
+    LOG(e.what());
   }
 
   return JNI_FALSE;
@@ -401,8 +420,8 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
     std::string cTitle = READER->getTitle();
     title = c2jni(cTitle, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim title");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim title");
+    LOG(e.what());
     title = NULL;
   }
   return title;
@@ -417,13 +436,42 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
     std::string cDescription = READER->getDescription();
     description = c2jni(cDescription, env);
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim description");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get zim description");
+    LOG(e.what());
     description = NULL;
   }
   return description;
 }
 
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
+{
+  jint articleCount = 0;
+  try {
+    auto cArticleCount = READER->getArticleCount();
+    articleCount = c2jni(cArticleCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get article count.");
+    LOG(e.what());
+  }
+  return articleCount;
+}
+
+JNIEXPORT jint JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
+{
+  jint mediaCount = 0;
+  try {
+    auto cMediaCount = READER->getMediaCount();
+    mediaCount = c2jni(cMediaCount, env);
+  } catch (std::exception& e) {
+    LOG("Unable to get media count.");
+    LOG(e.what());
+  }
+  return mediaCount;
+}
+
+
 JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     JNIEnv* env, jobject obj, jobject urlObj)
 {
@@ -435,8 +483,8 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
     setStringObjValue(cUrl, urlObj, env);
     retVal = JNI_TRUE;
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get random page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
+    LOG("Unable to get random page");
+    LOG(e.what());
   }
   return retVal;
 }

src/wrapper/java/kiwixsearcher.cpp

@@ -59,7 +59,7 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
     JNIEnv* env, jobject obj, jstring query, jint count)
 {
   std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count);
+  unsigned int ccount = jni2c(count, env);
 
   SEARCHER->search(cquery, 0, ccount);
 }

src/wrapper/java/kiwixserver.cpp

@@ -21,7 +21,6 @@
 
 #include <jni.h>
 #include <zim/file.h>
-#include <android/log.h>
 #include "org_kiwix_kiwixlib_JNIKiwixServer.h"
 
 #include "tools/base64.h"
@@ -32,15 +31,15 @@
 JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
     JNIEnv* env, jobject obj, jobject jLibrary)
 {
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create server");
+  LOG("Attempting to create server");
   Lock l;
   try {
-    auto library = Handle<kiwix::Library>::getHandle(env, jLibrary);
-    kiwix::Server* server = new kiwix::Server(*library);
+    auto library = getPtr<kiwix::Library>(env, jLibrary);
+    kiwix::Server* server = new kiwix::Server(library);
     return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
   } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating the server");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
+    LOG("Error creating the server");
+    LOG(e.what());
     return 0;
   }
 }
@@ -86,6 +85,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboo
   SERVER->setTaskbar(withTaskbar, withLibraryButton);
 }
 
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
+{
+  SERVER->setBlockExternalLinks(blockExternalLinks);
+}
+
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
 {

src/wrapper/java/library.cpp

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_Library.h"
+
+#include "library.h"
+#include "reader.h"
+#include "utils.h"
+
+/* Kiwix Reader JNI functions */
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_allocate(
+    JNIEnv* env, jobject thisObj)
+{
+  allocate<kiwix::Library>(env, thisObj);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Library>(env, thisObj);
+}
+
+#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
+
+/* Kiwix library functions */
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Library_addBook(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    kiwix::Reader reader(cPath);
+    kiwix::Book book;
+    book.update(reader);
+    return LIBRARY->addBook(book);
+  } catch (std::exception& e) {
+    LOG("Unable to add the book");
+    LOG(e.what()); }
+  return false;
+}
+
+METHOD(jobject, Library, getBookById, jstring id) {
+  auto cId = jni2c(id, env);
+  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
+  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
+  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
+  jobject book = env->NewObject(cls, constructorId);
+  setPtr(env, book, cBook);
+  return book;
+}
+
+METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
+  return LIBRARY->getBookCount(localBooks, remoteBooks);
+}
+
+METHOD0(jobjectArray, Library, getBooksIds) {
+  return c2jni(LIBRARY->getBooksIds(), env);
+}
+
+METHOD(jobjectArray, Library, filter, jobject filterObj) {
+  auto filter = getPtr<kiwix::Filter>(env, filterObj);
+  return c2jni(LIBRARY->filter(*filter), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksLanguages) {
+  return c2jni(LIBRARY->getBooksLanguages(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksCreators) {
+  return c2jni(LIBRARY->getBooksCreators(), env);
+}
+
+METHOD0(jobjectArray, Library, getBooksPublisher) {
+  return c2jni(LIBRARY->getBooksPublishers(), env);
+}
+

src/wrapper/java/manager.cpp

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <jni.h>
+#include <zim/file.h>
+#include "org_kiwix_kiwixlib_Manager.h"
+
+#include "manager.h"
+#include "utils.h"
+
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Manager_allocate(
+  JNIEnv* env, jobject thisObj, jobject libraryObj)
+{
+  auto lib = getPtr<kiwix::Library>(env, libraryObj);
+  allocate<kiwix::Manager>(env, thisObj, lib);
+}
+
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_Manager_dispose(JNIEnv* env, jobject thisObj)
+{
+  dispose<kiwix::Manager>(env, thisObj);
+}
+
+#define MANAGER (getPtr<kiwix::Manager>(env, thisObj))
+
+/* Kiwix manager functions */
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readFile(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    return MANAGER->readFile(cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get readFile");
+    LOG(e.what());
+  }
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readXml(
+  JNIEnv* env, jobject thisObj, jstring content, jstring libraryPath)
+{
+  auto cContent = jni2c(content, env);
+  auto cPath = jni2c(libraryPath, env);
+
+  try {
+    return MANAGER->readXml(cContent, false, cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readOpds(
+  JNIEnv* env, jobject thisObj, jstring content, jstring urlHost)
+{
+  auto cContent = jni2c(content, env);
+  auto cUrl = jni2c(urlHost, env);
+
+  try {
+    return MANAGER->readOpds(cContent, cUrl);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_Manager_readBookmarkFile(
+  JNIEnv* env, jobject thisObj, jstring path)
+{
+  auto cPath = jni2c(path, env);
+
+  try {
+    return MANAGER->readBookmarkFile(cPath);
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM id");
+    LOG(e.what());
+  }
+
+  return false;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_Manager_addBookFromPath(
+  JNIEnv* env, jobject thisObj,
+  jstring pathToOpen, jstring pathToSave, jstring url, jboolean checkMetaData)
+{
+  auto cPathToOpen = jni2c(pathToOpen, env);
+  auto cPathToSave = jni2c(pathToSave, env);
+  auto cUrl = jni2c(url, env);
+  jstring id = NULL;
+
+  try {
+    auto cId = MANAGER->addBookFromPathAndGetId(cPathToOpen, cPathToSave, cUrl, checkMetaData);
+    if ( !cId.empty() ) {
+      id = c2jni(cId, env);
+    }
+  } catch (std::exception& e) {
+    LOG("Unable to get ZIM file size");
+    LOG(e.what());
+  }
+
+  return id;
+}

src/wrapper/java/meson.build

@@ -0,0 +1,55 @@
+
+java_sources = files([
+  'org/kiwix/kiwixlib/JNIICU.java',
+  'org/kiwix/kiwixlib/Book.java',
+  'org/kiwix/kiwixlib/JNIKiwixReader.java',
+  'org/kiwix/kiwixlib/Library.java',
+  'org/kiwix/kiwixlib/Manager.java',
+  'org/kiwix/kiwixlib/Filter.java',
+  'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
+  'org/kiwix/kiwixlib/JNIKiwixServer.java',
+  'org/kiwix/kiwixlib/JNIKiwixInt.java',
+  'org/kiwix/kiwixlib/JNIKiwixString.java',
+  'org/kiwix/kiwixlib/JNIKiwixBool.java',
+  'org/kiwix/kiwixlib/JNIKiwixException.java',
+  'org/kiwix/kiwixlib/Pair.java'
+])
+
+kiwix_jni = custom_target('jni',
+  input: java_sources,
+  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
+           'org_kiwix_kiwixlib_Book.h',
+           'org_kiwix_kiwixlib_JNIKiwixReader.h',
+           'org_kiwix_kiwixlib_Library.h',
+           'org_kiwix_kiwixlib_Manager.h',
+           'org_kiwix_kiwixlib_Filter.h',
+           'org_kiwix_kiwixlib_JNIKiwixServer.h',
+           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
+           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
+  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
+)
+
+jni_sources = files([
+     'kiwixicu.cpp',
+     'book.cpp',
+     'kiwixreader.cpp',
+     'library.cpp',
+     'manager.cpp',
+     'filter.cpp',
+     'kiwixsearcher.cpp',
+     'kiwixserver.cpp',
+])
+
+kiwix_sources += jni_sources + [kiwix_jni]
+
+if 'java' in wrapper
+  kiwix_jar = jar('kiwixlib', java_sources)
+  #junit_jar = files('org/kiwix/testing/junit-4.13.jar')
+  #test_jar = jar('testing', 'org/kiwix/testing/test.java',
+  #               link_with: [kiwix_jar, junit_jar])
+  #test('javatest', test_jar)
+endif
+
+install_subdir('org', install_dir: 'kiwix-lib/java', exclude_directories: ['kiwix/testing'])
+install_subdir('res', install_dir: 'kiwix-lib')
+install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -0,0 +1,47 @@
+
+package org.kiwix.kiwixlib;
+
+public class Book
+{
+  public Book() { allocate(); }
+
+
+  public native void update(Book book);
+  public native void update(JNIKiwixReader reader);
+
+  @Override
+  protected void finalize() { dispose();  }
+
+  public native String getId();
+  public native String getPath();
+  public native boolean isPathValid();
+  public native String getTitle();
+  public native String getDescription();
+  public native String getLanguage();
+  public native String getCreator();
+  public native String getPublisher();
+  public native String getDate();
+  public native String getUrl();
+  public native String getName();
+  public native String getFlavour();
+  public native String getTags();
+  /**
+   * Return the value associated to the tag tagName
+   *
+   * @param tagName the tag name to search for.
+   * @return The value of the tag. If the tag is not found, return empty string.
+   */
+  public native String getTagStr(String tagName);
+
+  public native long getArticleCount();
+  public native long getMediaCount();
+  public native long getSize();
+
+  public native String getFavicon();
+  public native String getFaviconUrl();
+  public native String getFaviconMimeType();
+
+  private native void allocate();
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/Filter.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+public class Filter
+{
+
+  public native Filter local(boolean accept);
+  public native Filter remote(boolean accept);
+  public native Filter valid(boolean accept);
+  public native Filter acceptTags(String[] tags);
+  public native Filter rejectTags(String[] tags);
+  public native Filter lang(String lang);
+  public native Filter publisher(String publisher);
+  public native Filter creator(String creator);
+  public native Filter maxSize(long size);
+  public native Filter query(String query);
+
+
+  public Filter() { allocate(); }
+
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate();
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -82,6 +82,16 @@ public class JNIKiwixReader
                                       int len,
                                       JNIKiwixInt size);
 
+  /**
+   *
+   * Get the size of an article.
+   *
+   * @param   url The url of the article.
+   * @return  The size of the final (redirections are resolved) article (in byte).
+   *          Return 0 if the article is not found.
+   */
+  public native long getArticleSize(String url);
+
   /**
    * getDirectAccessInformation.
    *
@@ -102,7 +112,7 @@ public class JNIKiwixReader
 
   public native boolean searchSuggestions(String prefix, int count);
 
-  public native boolean getNextSuggestion(JNIKiwixString title);
+  public native boolean getNextSuggestion(JNIKiwixString title, JNIKiwixString url);
 
   public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
 

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixServer.java

@@ -20,7 +20,7 @@
 package org.kiwix.kiwixlib;
 
 import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.JNIKiwixLibrary;
+import org.kiwix.kiwixlib.Library;
 
 public class JNIKiwixServer
 {
@@ -34,15 +34,17 @@ public class JNIKiwixServer
 
   public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
 
+  public native void setBlockExternalLinks(boolean blockExternalLinks);
+
   public native boolean start();
 
   public native void stop();
 
-  public JNIKiwixServer(JNIKiwixLibrary library)
+  public JNIKiwixServer(Library library)
   {
     nativeHandle = getNativeServer(library);
   }
 
-  private native long getNativeServer(JNIKiwixLibrary library);
+  private native long getNativeServer(Library library);
   private long nativeHandle;
 }

src/wrapper/java/org/kiwix/kiwixlib/Library.java

@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -20,19 +19,31 @@
 
 package org.kiwix.kiwixlib;
 
+import org.kiwix.kiwixlib.Book;
 import org.kiwix.kiwixlib.JNIKiwixException;
 
-public class JNIKiwixLibrary
+public class Library
 {
   public native boolean addBook(String path) throws JNIKiwixException;
 
-  public JNIKiwixLibrary()
+  public native Book getBookById(String id);
+  public native int  getBookCount(boolean localBooks, boolean remoteBooks);
+
+  public native String[] getBooksIds();
+  public native String[] filter(Filter filter);
+
+  public native String[] getBooksLanguages();
+  public native String[] getBooksCreators();
+  public native String[] getBooksPublishers();
+
+  public Library()
   {
-    nativeHandle = getNativeLibrary();
+    allocate();
   }
 
-  public native void dispose();
-
-  private native long getNativeLibrary();
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate();
+  private native void dispose();
   private long nativeHandle;
 }

src/wrapper/java/org/kiwix/kiwixlib/Manager.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+import org.kiwix.kiwixlib.Library;
+
+public class Manager
+{
+  /**
+   * Read a `library.xml` file and add books in the library.
+   *
+   * @param path The (utf8) path to the `library.xml` file.
+   * @return True if the file has been properly parsed.
+   */
+  public native boolean readFile(String path);
+
+  /**
+   * Load a library content stored in a string (at `library.xml` format).
+   *
+   * @param content The content corresponding of the library xml.
+   * @param libraryPath The library path (used to resolve relative paths)
+   * @return True if the content has been properly parsed.
+   */
+  public native boolean readXml(String content, String libraryPath);
+
+  /**
+   * Load a library content stored in a string (at OPDS stream format)
+   *
+   * @param content the content of the OPDS stream.
+   * @param urlHost the url of the stream (used to resolve relative url)
+   * @return True if the content has been properly parsed.
+   */
+  public native boolean readOpds(String content, String urlHost);
+
+  /**
+   * Load a bookmark file
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed
+   */
+  public native boolean readBookmarkFile(String path);
+
+  /**
+   * Add a book to the library.
+   *
+   * @param pathToOpen    The path of the zim file to add.
+   * @param pathToSave    The path to store in the library in place of
+   *                      pathToOpen.
+   * @param url           The url of the book to store in the library
+   *                      (useful for kiiwix-serve catalog)
+   * @param checkMetaData Tell if we check metadata before adding a book to the
+   *                      library.
+   * @return The id of te book if the book has been added to the library.
+   *         Empty string if not.
+   */
+  public native String addBookFromPath(String pathToOpen,
+                                       String pathToSave,
+                                       String url,
+                                       boolean checkMetaData);
+
+  public Manager(Library library) {
+    allocate(library);
+    _library = library;
+  }
+
+  private Library _library;
+
+  @Override
+  protected void finalize() { dispose(); }
+  private native void allocate(Library library);
+  private native void dispose();
+  private long nativeHandle;
+}

src/wrapper/java/org/kiwix/kiwixlib/Pair.java

@@ -22,5 +22,5 @@ package org.kiwix.kiwixlib;
 public class Pair
 {
   public String filename;
-  public int offset;
+  public long offset;
 }

src/wrapper/java/org/kiwix/testing/compile_test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/bash
+
+# This script compile the unit test to test the java wrapper.
+# This is not integrated in meson because ... this is not so easy.
+
+
+KIWIX_LIB_JAR=$1
+if [ -z $KIWIX_LIB_JAR ]
+then
+  echo "You must give the path to the kiwixlib.jar as first argument"
+  exit 1
+fi
+
+KIWIX_LIB_DIR=$2
+if [ -z $KIWIX_LIB_DIR ]
+then
+  echo "You must give the path to directory containing libkiwix.so as second argument"
+  exit 1
+fi
+TEST_SOURCE_DIR=$(dirname $(readlink -f $0))
+
+
+javac -g -d . -s . -cp $TEST_SOURCE_DIR/junit-4.13.jar:$KIWIX_LIB_JAR $TEST_SOURCE_DIR/test.java
+
+java -Djava.library.path=$KIWIX_LIB_DIR -cp $TEST_SOURCE_DIR/junit-4.13.jar:$TEST_SOURCE_DIR/hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:. org.junit.runner.JUnitCore test
+

src/wrapper/java/org/kiwix/testing/test.java

@@ -0,0 +1,53 @@
+
+import java.io.*;
+import java.util.*;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.kiwix.kiwixlib.*;
+
+public class test {
+static {
+  System.loadLibrary("kiwix");
+}
+
+private static String getCatalogContent()
+throws IOException
+{
+  BufferedReader reader = new BufferedReader(new FileReader("catalog.xml"));
+  String line;
+  StringBuilder sb = new StringBuilder();
+  while ((line = reader.readLine()) != null)
+  {
+    sb.append(line + "\n");
+  }
+  reader.close();
+  return sb.toString();
+}
+
+@Test
+public void testSome()
+throws IOException
+{
+  Library lib = new Library();
+  Manager manager = new Manager(lib);
+  String content = getCatalogContent();
+  manager.readOpds(content, "https://library.kiwix.org");
+  assertEquals(lib.getBookCount(true, true), 10);
+  String[] bookIds = lib.getBooksIds();
+  assertEquals(bookIds.length, 10);
+  Book book = lib.getBookById(bookIds[0]);
+  assertEquals(book.getTitle(), "Wikisource");
+  assertEquals(book.getTags(), "wikisource;_category:wikisource;_pictures:no;_videos:no;_details:yes;_ftindex:yes");
+  assertEquals(book.getFaviconUrl(), "https://library.kiwix.org/meta?name=favicon&content=wikisource_fr_all_nopic_2020-01");
+  assertEquals(book.getUrl(), "http://download.kiwix.org/zim/wikisource/wikisource_fr_all_nopic_2020-01.zim.meta4");
+}
+
+static
+public void main(String[] args) {
+  Library lib = new Library();
+  lib.getBookCount(true, true);
+}
+
+
+
+}

src/wrapper/java/utils.h

@@ -26,9 +26,55 @@
 
 #include <pthread.h>
 #include <string>
+#include <vector>
+#include <iostream>
+
+#if __ANDROID__
+ #include <android/log.h>
+ #define LOG(...) __android_log_print(ANDROID_LOG_ERROR, "kiwix", __VA_ARGS__)
+#else
+ #define LOG(...)
+#endif
 
 extern pthread_mutex_t globalLock;
 
+template<typename T>
+void setPtr(JNIEnv* env, jobject thisObj, T* ptr)
+{
+  jclass thisClass = env->GetObjectClass(thisObj);
+  jfieldID fieldId = env->GetFieldID(thisClass, "nativeHandle", "J");
+  env->SetLongField(thisObj, fieldId, reinterpret_cast<jlong>(ptr));
+}
+
+template<typename T, typename ...Args>
+void allocate(JNIEnv* env, jobject thisObj, Args && ...args)
+{
+  T* ptr = new T(std::forward<Args>(args)...);
+  setPtr(env, thisObj, ptr);
+}
+
+template<typename T>
+T* getPtr(JNIEnv* env, jobject thisObj)
+{
+  jclass thisClass = env->GetObjectClass(thisObj);
+  jfieldID fidNumber = env->GetFieldID(thisClass, "nativeHandle", "J");
+  return reinterpret_cast<T*>(env->GetLongField(thisObj, fidNumber));
+}
+
+template<typename T>
+void dispose(JNIEnv* env, jobject thisObj)
+{
+  delete getPtr<T>(env, thisObj);
+}
+
+#define METHOD0(retType, class, name) \
+JNIEXPORT retType JNICALL Java_org_kiwix_kiwixlib_##class##_##name( \
+ JNIEnv* env, jobject thisObj)
+
+#define METHOD(retType, class, name, ...) \
+JNIEXPORT retType JNICALL Java_org_kiwix_kiwixlib_##class##_##name( \
+  JNIEnv* env, jobject thisObj, __VA_ARGS__)
+
 inline jfieldID getHandleField(JNIEnv* env, jobject obj)
 {
   jclass c = env->GetObjectClass(obj);
@@ -36,6 +82,12 @@ inline jfieldID getHandleField(JNIEnv* env, jobject obj)
   return env->GetFieldID(c, "nativeHandle", "J");
 }
 
+inline jobjectArray createArray(JNIEnv* env, size_t length, const std::string& type_sig)
+{
+  jclass c = env->FindClass(type_sig.c_str());
+  return env->NewObjectArray(length, c, NULL);
+}
+
 class Lock
 {
  protected:
@@ -91,19 +143,67 @@ struct LockedHandle : public Lock {
   T* operator->() { return h->h; }
   T* operator*() { return h->h; }
   operator bool() const { return (h->h != nullptr); }
+  operator T*() const { return h->h; }
 };
 
-/* c2jni type conversion functions */
-inline jboolean c2jni(const bool& val) { return val ? JNI_TRUE : JNI_FALSE; }
+template<typename T>
+struct JType { };
+
+template<> struct JType<bool>{ typedef jboolean type_t; };
+template<> struct JType<int>{ typedef jint type_t; };
+template<> struct JType<long>{ typedef jlong type_t; };
+template<> struct JType<uint64_t> { typedef jlong type_t; };
+template<> struct JType<uint32_t> { typedef jlong type_t; };
+template<> struct JType<std::string>{ typedef jstring type_t; };
+template<> struct JType<std::vector<std::string>>{ typedef jobjectArray type_t; };
+
+template<typename T>
+inline typename JType<T>::type_t c2jni(const T& val, JNIEnv* env) {
+  return static_cast<typename JType<T>::type_t>(val);
+}
+
+template<>
+inline jboolean c2jni(const bool& val, JNIEnv* env) { return val ? JNI_TRUE : JNI_FALSE; }
+
+template<>
 inline jstring c2jni(const std::string& val, JNIEnv* env)
 {
   return env->NewStringUTF(val.c_str());
 }
 
-inline jint c2jni(const int val) { return (jint)val; }
-inline jint c2jni(const unsigned val) { return (unsigned)val; }
+template<>
+inline jobjectArray c2jni(const std::vector<std::string>& val, JNIEnv* env)
+{
+  auto array = createArray(env, val.size(), "java/lang/String");
+  size_t index = 0;
+  for (auto& elem: val) {
+    auto jElem = c2jni(elem, env);
+    env->SetObjectArrayElement(array, index++, jElem);
+  }
+  return array;
+}
+
+template<typename T, typename U=T>
+struct CType { };
+
+template<> struct CType<jboolean>{ typedef bool type_t; };
+template<> struct CType<jint>{ typedef int type_t; };
+template<> struct CType<jlong>{ typedef long type_t; };
+template<> struct CType<jstring>{ typedef std::string type_t; };
+
+template<typename U>
+struct CType<jobjectArray, U>{ typedef std::vector<typename CType<U>::type_t> type_t; };
+
 /* jni2c type conversion functions */
-inline bool jni2c(const jboolean& val) { return val == JNI_TRUE; }
+template<typename T, typename U=T>
+inline typename CType<T>::type_t jni2c(const T& val, JNIEnv* env) {
+  return static_cast<typename CType<T>::type_t>(val);
+}
+
+template<>
+inline bool jni2c(const jboolean& val, JNIEnv* env) { return val == JNI_TRUE; }
+
+template<>
 inline std::string jni2c(const jstring& val, JNIEnv* env)
 {
   const char* chars = env->GetStringUTFChars(val, 0);
@@ -112,7 +212,21 @@ inline std::string jni2c(const jstring& val, JNIEnv* env)
   return ret;
 }
 
-inline int jni2c(const jint val) { return (int)val; }
+template<typename U>
+inline typename CType<jobjectArray, U>::type_t jni2c(const jobjectArray& val, JNIEnv* env)
+{
+  jsize length = env->GetArrayLength(val);
+  typename CType<jobjectArray, U>::type_t v(length);
+
+  int i;
+  for (i = 0; i < length; i++) {
+    U obj = (U) env->GetObjectArrayElement(val, i);
+    auto cobj = jni2c<U>(obj, env);
+    v.push_back(cobj);
+  }
+  return v;
+}
+
 /* Method to deal with variable passed by reference */
 inline std::string getStringObjValue(const jobject obj, JNIEnv* env)
 {
@@ -141,17 +255,17 @@ inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
 {
   jclass objClass = env->GetObjectClass(obj);
   jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
-  env->SetIntField(obj, objFid, c2jni(value));
+  env->SetIntField(obj, objFid, c2jni(value, env));
 }
 
-inline void setPairObjValue(const std::string& filename, const int offset,
+inline void setPairObjValue(const std::string& filename, const long offset,
                             const jobject obj, JNIEnv* env)
 {
   jclass objClass = env->GetObjectClass(obj);
   jfieldID filenameFid = env->GetFieldID(objClass, "filename", "Ljava/lang/String;");
   env->SetObjectField(obj, filenameFid, c2jni(filename, env));
-  jfieldID offsetFid = env->GetFieldID(objClass, "offset", "I");
-  env->SetIntField(obj, offsetFid, offset);
+  jfieldID offsetFid = env->GetFieldID(objClass, "offset", "J");
+  env->SetLongField(obj, offsetFid, offset);
 }
 
 #endif // _ANDROID_JNI_UTILS_H

static/meson.build

@@ -1,3 +1,8 @@
+resource_files = run_command(find_program('python3'),
+                    '-c',
+                    'import sys; f=open(sys.argv[1]); print(f.read())',
+                    files('resources_list.txt')
+                 ).stdout().strip().split('\n')
 
 lib_resources = custom_target('resources',
     input: 'resources_list.txt',
@@ -7,5 +12,5 @@ lib_resources = custom_target('resources',
              '--hfile', '@OUTPUT1@',
              '--source_dir', '@OUTDIR@',
              '@INPUT@'],
-    build_always_stale: true
+    depend_files: resource_files
 )

static/opensearchdescription.xml

@@ -2,8 +2,9 @@
 <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
   <ShortName>Zim catalog search</ShortName>
   <Description>Search zim files in the catalog.</Description>
-  <Url type="application/atom+xml;profile=opds-catalog" 
+  <Url type="application/atom+xml;profile=opds-catalog"
        xmlns:atom="http://www.w3.org/2005/Atom"
+       xmlns:k="http://kiwix.org/opensearchextension/1.0"
        indexOffset="0"
-       template="/{{root}}/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>
+       template="/{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}&notag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/>
 </OpenSearchDescription>

static/resources_list.txt

@@ -20,6 +20,7 @@ skin/jquery-ui/jquery-ui.min.css
 skin/caret.png
 skin/taskbar.js
 skin/taskbar.css
+skin/block_external.js
 templates/search_result.html
 templates/no_search_result.html
 templates/404.html
@@ -28,4 +29,6 @@ templates/index.html
 templates/suggestion.json
 templates/head_part.html
 templates/taskbar_part.html
+templates/external_blocker_part.html
+templates/captured_external.html
 opensearchdescription.xml

static/skin/block_external.js

@@ -0,0 +1,72 @@
+var block_path = "/catch/external";
+// called only on external links
+function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
+
+// called on all link clicks. filters external and call capture_event
+function on_click_event(e) {
+  var target = findParent("a", e.target);
+  if (target !== null && "href" in target) {
+    var href = target.href;
+    if (window.location.pathname.indexOf(block_path) == 0) // already in catch page
+      return;
+    if (href.indexOf(window.location.origin) == 0)
+      return;
+    if (href.substr(0, 2) == "//")
+      return capture_event(e, target);
+    if (href.substr(0, 5) == "http:")
+      return capture_event(e, target);
+    if (href.substr(0, 6) == "https:")
+      return capture_event(e, target);
+    return;
+  }
+}
+
+// script entrypoint (called on document ready)
+function run() { live('a', 'click', on_click_event); }
+
+// find first parent with tagname
+function findParent(tagname, el) {
+  while (el) {
+    if ((el.nodeName || el.tagName).toLowerCase() === tagname.toLowerCase()) {
+      return el;
+    }
+    el = el.parentNode;
+  }
+  return null;
+}
+
+// matches polyfill
+this.Element && function(ElementPrototype) {
+    ElementPrototype.matches = ElementPrototype.matches ||
+    ElementPrototype.matchesSelector ||
+    ElementPrototype.webkitMatchesSelector ||
+    ElementPrototype.msMatchesSelector ||
+    function(selector) {
+        var node = this, nodes = (node.parentNode || node.document).querySelectorAll(selector), i = -1;
+        while (nodes[++i] && nodes[i] != node);
+        return !!nodes[i];
+    }
+}(Element.prototype);
+
+// helper for enabling IE 8 event bindings
+function addEvent(el, type, handler) {
+    if (el.attachEvent) el.attachEvent('on'+type, handler); else el.addEventListener(type, handler);
+}
+
+// live binding helper using matchesSelector
+function live(selector, event, callback, context) {
+    addEvent(context || document, event, function(e) {
+        var found, el = e.target || e.srcElement;
+        while (el && el.matches && el !== context && !(found = el.matches(selector))) el = el.parentElement;
+        if (found) callback.call(el, e);
+    });
+}
+
+// in case the document is already rendered
+if (document.readyState!='loading') run();
+// modern browsers
+else if (document.addEventListener) document.addEventListener('DOMContentLoaded', run);
+// IE <= 8
+else document.attachEvent('onreadystatechange', function(){
+    if (document.readyState=='complete') run();
+});

static/skin/taskbar.css

@@ -124,7 +124,7 @@ label[for=kiwixsearchbox] {
 }
 
 body {
-    padding-top: 40px !important;
+    padding-top: 3em !important;
 }
 
 /* Try to fix buggy stuff in jquery-ui autocomplete */

static/templates/captured_external.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <meta charset="UTF-8" />
+  <title>External link blocked</title>
+</head>
+<body class="kiwix">
+<h1>External link blocked</h1>
+<p>This instance of Kiwix protects you from accidentaly going to external (out-of ZIM) links.</p>
+<p>If you intend to go to such locations, please click the link below.</p>
+<p><a href="{{ source }}">Go to {{ source }}</a></p>
+<div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>
+</body>
+</html>

static/templates/external_blocker_part.html

@@ -0,0 +1 @@
+<script type="text/javascript" src="{{root}}/skin/block_external.js"></script>

static/templates/search_result.html

@@ -95,7 +95,7 @@
   </head>
   <body bgcolor="white">
     <div class="header">
-      {{#hasResult}}
+      {{#hasResults}}
         Results
         <b>
           {{resultStart}}-{{resultEnd}}
@@ -104,10 +104,10 @@
         </b> for <b>
           {{searchPattern}}
         </b>
-      {{/hasResult}}
-      {{^hasResult}}
+      {{/hasResults}}
+      {{^hasResults}}
         No results were found for <b>{{searchPattern}}</b>
-      {{/hasResult}}
+      {{/hasResults}}
     </div>
 
     <div class="results">
@@ -129,30 +129,32 @@
     </div>
 
     <div class="footer">
-      <ul>
-        {{#resultLastPageStart}}
-          <li>
-            <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
-              ◀
-            </a>
-          </li>
-        {{/resultLastPageStart}}
-        {{#pages}}
-          <li>
-            <a {{#selected}}class="selected"{{/selected}}
-               href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
-              {{label}}
-            </a>
-          </li>
-        {{/pages}}
-        {{#resultLastPageStart}}
-          <li>
-            <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
-              ▶
-            </a>
-          </li>
-        {{/resultLastPageStart}}
-      </ul>
+      {{#hasPages}}
+        <ul>
+          {{#resultLastPageStart}}
+            <li>
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
+                ◀
+              </a>
+            </li>
+          {{/resultLastPageStart}}
+          {{#pages}}
+            <li>
+              <a {{#selected}}class="selected"{{/selected}}
+                 href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
+                {{label}}
+              </a>
+            </li>
+          {{/pages}}
+          {{#resultLastPageStart}}
+            <li>
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
+                ▶
+              </a>
+            </li>
+          {{/resultLastPageStart}}
+        </ul>
+      {{/hasPages}}
     </div>
   </body>
 </html>

test/book.cpp

@@ -0,0 +1,33 @@
+#include "gtest/gtest.h"
+#include "../include/book.h"
+
+TEST(BookTest, updateTest)
+{
+    kiwix::Book book;
+
+    book.setReadOnly(false);
+    book.setPath("/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim");
+    book.setPathValid(true);
+    book.setUrl("book-url");
+    book.setTags("youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes");
+    book.setName("skin-of-color-society_en_all");
+    book.setFavicon("book-favicon");
+    book.setFaviconMimeType("book-favicon-mimetype");
+
+    kiwix::Book newBook;
+
+    newBook.setReadOnly(true);
+    EXPECT_FALSE(newBook.update(book));
+
+    newBook.setReadOnly(false);
+    EXPECT_TRUE(newBook.update(book));
+
+    EXPECT_EQ(newBook.readOnly(), book.readOnly());
+    EXPECT_EQ(newBook.getPath(), book.getPath());
+    EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
+    EXPECT_EQ(newBook.getUrl(), book.getUrl());
+    EXPECT_EQ(newBook.getTags(), book.getTags());
+    EXPECT_EQ(newBook.getName(), book.getName());
+    EXPECT_EQ(newBook.getFavicon(), book.getFavicon());
+    EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType());
+}

test/kiwixserve.cpp

@@ -0,0 +1,12 @@
+#include "gtest/gtest.h"
+#include "../include/kiwixserve.h"
+
+TEST(KiwixServeTest, PortTest)
+{
+    kiwix::KiwixServe kiwixServe("libraryPath", 8181);
+    EXPECT_EQ(kiwixServe.getPort(), 8181);
+    kiwixServe.setPort(8484);
+    EXPECT_EQ(kiwixServe.getPort(), 8484);
+    EXPECT_EQ(kiwixServe.setPort(0), -1);
+    EXPECT_EQ(kiwixServe.setPort(3456789), -1);
+}