Update name of the project in meson.build.

The right name is `kiwix-lib`, not `kiwixlib`.
Also update the version to be able to do a small fix release.

.codecov.yml

@@ -1,12 +0,0 @@
-codecov:
-  notify:
-    require_ci_to_pass: yes
-
-coverage:
-  status:
-    project:
-      default:
-        threshold: 1%
-
-ignore:
-  - "test"

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: kiwix
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # https://kiwix.org/support-us/

.github/move.yml

@@ -1,27 +0,0 @@
-# Configuration for Move Issues - https://github.com/dessant/move-issues
-
-# Delete the command comment when it contains no other content
-deleteCommand: true
-
-# Close the source issue after moving
-closeSourceIssue: true
-
-# Lock the source issue after moving
-lockSourceIssue: false
-
-# Mention issue and comment authors
-mentionAuthors: true
-
-# Preserve mentions in the issue content
-keepContentMentions: true
-
-# Move labels that also exist on the target repository
-moveLabels: true
-
-# Set custom aliases for targets
-# aliases:
-#   r: repo
-#   or: owner/repo
-
-# Repository to extend settings from
-# _extends: repo

.github/stale.yml

@@ -1,15 +0,0 @@
-daysUntilClose: false
-staleLabel: stale
-
-issues:
-  daysUntilStale: 60
-  markComment: >
-    This issue has been automatically marked as stale because it has not had
-    recent activity. It will be now be reviewed manually. Thank you
-    for your contributions.
-pulls:
-   daysUntilStale: 7
-   markComment: >
-     This pull request has been automatically marked as stale because it has not had
-     recent activity. It will be now be reviewed manually. Thank you
-     for your contributions.

.github/workflows/ci.yml

@@ -1,152 +0,0 @@
-name: CI
-
-on: [push]
-
-jobs:
-  Macos:
-    runs-on: macos-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v1
-      - name: Setup python 3.5
-        uses: actions/setup-python@v1
-        with:
-          python-version: '3.5'
-      - name: Install packages
-        uses: mstksg/get-package@v1
-        with:
-          brew: gcovr pkg-config ninja
-      - name: Install python modules
-        run: pip3 install meson==0.49.2 pytest
-      - name: Install deps
-        shell: bash
-        run: |
-          ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
-          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
-      - name: Compile
-        shell: bash
-        run: |
-          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
-          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
-          meson . build --default-library=shared -Db_coverage=true
-          cd build
-          ninja
-      - name: Test
-        shell: bash
-        run: |
-          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
-          cd build
-          meson test --verbose
-          ninja coverage
-        env:
-          SKIP_BIG_MEMORY_TEST: 1
-      - name: Publish coverage
-        shell: bash
-        run: |
-          curl https://codecov.io/bash -o codecov.sh
-          bash codecov.sh -n osx_native_dyn -Z
-          rm codecov.sh
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  Linux:
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-          - native_static
-          - native_dyn
-          - android_arm
-          - android_arm64
-          - win32_static
-          - win32_dyn
-        include:
-          - target: native_static
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: native_dyn
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm64
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: win32_static
-            image_variant: f31
-            lib_postfix: '64'
-          - target: win32_dyn
-            image_variant: f31
-            lib_postfix: '64'
-    env:
-      HOME: /home/runner
-    runs-on: ubuntu-latest
-    container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
-    steps:
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
-    - name: Checkout code
-      shell: python
-      run: |
-        from subprocess import check_call
-        from os import environ
-        command = [
-          'git', 'clone',
-          'https://github.com/${{github.repository}}',
-          '--depth=1',
-          '--branch', '${{steps.extract_branch.outputs.branch}}'
-        ]
-        check_call(command, cwd=environ['HOME'])
-    - name: Install deps
-      shell: bash
-      run: |
-        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
-        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
-    - name: Compile
-      shell: bash
-      run: |
-        meson --version
-        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
-          MESON_OPTION="--default-library=shared"
-        else
-          MESON_OPTION="--default-library=static"
-        fi
-        if [[ "${{matrix.target}}" =~ native_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
-        else
-          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
-        fi
-        if [[ "${{matrix.target}}" =~ android_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Dandroid=true"
-        fi
-        cd $HOME/kiwix-lib
-        meson . build ${MESON_OPTION}
-        cd build
-        ninja
-      env:
-        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
-        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
-    - name: Test
-      if: startsWith(matrix.target, 'native_')
-      shell: bash
-      run: |
-        cd $HOME/kiwix-lib/build
-        meson test --verbose
-        ninja coverage
-      env:
-        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
-        SKIP_BIG_MEMORY_TEST: 1
-    - name: Publish coverage
-      shell: bash
-      run: |
-        cd $HOME/kiwix-lib
-        curl https://codecov.io/bash -o codecov.sh
-        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
-        rm codecov.sh
-      if: startsWith(matrix.target, 'native_')
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,4 +0,0 @@
-.idea/
-*.swp
-subprojects/googletest-release*
-*.class

.travis.yml

@@ -0,0 +1,13 @@
+language: cpp
+dist: trusty
+sudo: required
+cache: ccache
+install: travis/install_deps.sh
+script: travis/compile.sh
+env:
+ - PLATFORM="native_static"
+ - PLATFORM="native_dyn"
+ - PLATFORM="win32_static"
+ - PLATFORM="win32_dyn"
+ - PLATFORM="android_arm"
+ - PLATFORM="android_arm64"

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
-
+ 
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-
+  
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,335 +1,3 @@
-kiwix-lib 9.2
-=============
-
- * Add tests
- * Refactoring server code.
- * [SERVER] Add HEAD, Etag and If-None-Match support.
- * [SERVER] Compress opds catalog answers.
-
-kiwix-lib 9.1.2
-===============
-
- * Do not use the pathToSave if it is empty.
-
-kiwix-lib 9.1.1
-===============
-
- * Fix the detection of the dataDirectory on windows.
-
-kiwix-lib 9.1.0
-===============
-
- * [JAVA] Add a method to get the size of an article.
- * Add a new method to the libray to get the book by path.
- * Add a option to make the server blocks external link.
-   Links are intercepted by js an redirected to a "portail" page.
- * [ODPS] Correctly handle book's articleCount and mediaCount.
-
-kiwix-lib 9.0.1
-===============
-
- * [JAVA] Use a long to store the offset of an article in the zim file instead
-    of an int.
-
-kiwix-lib 9.0.0
-===============
-
- * [OPDS] Correctly set the id of the OPDS stream.
- * [OPDS] Do not try to filter the catalog if no filter field is given in the
-   request.
- * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
- * [LIBRARY] Remove the function the read file using a native path.
-   All path must be utf8, no need to pass a native path along the utf8 path.
- * [TEST] Fix tests using the main function of gtest instead of custom one.
- * [CI] Move to github CI instead of Travis.
- * The `Book::update` method always update the book's fields. Even if they are
-   not empty.
- * [JAVA] Add wrapping around the library manager (opds parsing)
- * [ARIA2] Add api option to start download with option (destination folder)
- * [OPDS] Fixes about opds parsing, generation (missing attributes)
-   and requesting (server)
- * Add methods on `Book` to get specific tag values (was on `Reader` only)
- * Add flavour attribute to `Book`
- * Fix opensearch description.
- * Trust the given library.xml (by default) instead of reading the value from
-   the zim files.
- * [OPDS] Be able to filter the content by name or size.
- * [WINDOWS] Fix launching subcommand when there is spaces in the path.
-
-kiwix-lib 8.2.2
-===============
-
- * Improve a few compilation scripts
-
-kiwix-lib 8.2.1
-===============
-
- * Reintroduce kiwix-serve taskbar
-
-kiwix-lib 8.2.0
-===============
-
- * More debug information if aria2c command fails
- * Allow to set kiwix-serve port
- * Better (dead) bookmarks mgmt
-
-kiwix-lib 8.1.0
-===============
-
- * Fix pathTools manipulation.
- * Add missing implementation of getArticleCount and getMediaCount on android.
- * Correctly convert windows path to utf8.
- * Add code coverage in the CI
-
-kiwix-lib 8.0.1
-===============
-
- * Fix join function
-
-kiwix-lib 8.0.0
-===============
-
- * Add new methods to get all (and new) metadata from the zim file.
- * Add methods to get the value of a specific tag.
- * [API Change] Convert tags value to the new convention.
- * [API Change] Rename `getMatatag` method to `getMetadata`
- * [ABI Change] Correctly detect executable path in appimage.
-
-kiwix-lib 7.0.0
-===============
-
- * [API break] Add a argument to kiwix-serve to specify the library to use.
-
-kiwix-lib 6.0.4
-===============
-
- * Fix HTML rendering of the search result if there is no result.
- * Do not crash at html rendering if request ask for 0 results (start == end)
- * Correctly find the executable path if we are using AppImage
-
-kiwix-lib 6.0.3
-===============
-
- * force one column suggestion in kiwix-serve suggestions
- * fix fulltext search link in suggestions
- * UI fixes in kiwix-serve rendering
-
-kiwix-lib 6.0.2
-===============
-
- * Correctly set the groupId in the pom file.
-
-kiwix-lib 6.0.1
-===============
-
- * Generate the pom file for android/maven
-
-kiwix-lib 6.0.0
-===============
-
- * Move the server code in kiwix-lib (from kiwix-serve).
- * Add unit test on regex functions.
- * Fix computerAbsolutePath (thread safe, memory leak).
- * Correctly set the book's path as valid if we construct the book from a
-   reader.
- * [JNI] Add a method to know if a article is a redirection.
- * Do not embed the gtest dependency.
- * [JNI] Add a constructor to JNIKiwixString.
- * Change order of search of the favicon urls.
- * Clean a lot of unecessary includes in headers. (potential "API break")
-
-kiwix-lib 5.2.0
-===============
-
- * kiwix-serve integration (as a seperated process).
- * Fix crash in the suggestion search.
- * Better API to filter the library books.
- * New kiwix-lib application for android. (.aar)
- * Use ReLinker to link with libkiwix.so in android.
- * Correctly set the verbosity of zim search.
-
-kiwix-lib 5.1.0
-===============
-
- * Add function to pause, resume and stop downloads.
- * Add zim's tags in the opds stream.
- * Addapt to new libzim 5.0.0 API.
-
-kiwix-lib 5.0.0
-===============
-
- * Remove error message when trying to open an wrong zim file.
- * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
- * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
- * Remove rpath
- * Detect infinite (and too long) redirect loops.
-
-kiwix-lib 4.1.0
-===============
-
- * Allow the library to be filtered by tags.
- * Fix language mapping.
- * Update README about mustache dependency.
-
-kiwix-lib 4.0.1
-===============
-
- * Fix "maybe uninitialize variable" issue.
- * Ensure path are stored correctly (absolute path) in the library.
- * [CI] Use the new deps archive xz
-
-kiwix-lib 4.0.0
-===============
-
- * [API break] Remove support for external index.
- * Move to the mustache templating system instead of ctpp2.
- * Make meson.build works for meson>=0.43.0
- * [API break] Move the basic tools from the `common` directory to `tools`.
-
-kiwix-lib 3.1.1
-===============
-
- * The OPDS feed book's date must be the date of the book, not the date of the
- feed generation.
- * Convert the standard opds date to our format (YYYY-MM-DD)
- * Remove duplicate language attribute in the libxml dumper.
- * Create the datadirectory to not fail to write a file in a non-existent
-   directory
-
-kiwix-lib 3.1.0
-===============
-
- * Add a method to get the favicon url of book (if available).
- * Move dump code of library.xml in a specific class.
- * Add a first support to bookmarks
-
-kiwix-lib 3.0.3
-===============
-
- * Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
- ('eng').
- * Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
- * Correctly fill the book size for the zim file size.
- * Fix launch of aria2c.
-
-kiwix-lib 3.0.2
-===============
-
- * Use the correct path separator when computing relativePath on Windows.
-
-kiwix-lib 3.0.1
-===============
-
- * Small fix about parsing the opdsStream.
-
-kiwix-lib 3.0.0
-===============
-
- * Change the downloader to use aria2 using a separated process (with rpc)
-   instead of using the libaria2. This simplify a lot the link process to
-   libaria2 on Windows.
-    - kiwix-lib doesn't depend on libaria2 anymore.
-    - kiwix-lib now depends on libcurl.
- * [API break] Library class API has been updated :
-    - Books are referenced by id, not index. A lot of methods have been
-      updated this way.
-    - Books "list" is now private.
-    - There is no more "current" book.
-    - listBooksIds's filters have been updated.
- * [API break] Book class API has been updated :
-    - Move the definition of Book in `book.h`.
-    - Use getter/setter methods instead public members.
-    - Size (getSize/setSize) is now returned in bytes, not kB.
-    - Dependending of how the book has been initialized (opdsfeed), the
-      faviconUrl may be stored in the book, the favicon being downloaded when
-      using `getFavicon`.
-    - The path (and indexPath) are always absolute path.
-    - Book has now a downloadId, corresponding to the aria2 download id (if
-      exists)
-  * [API break] Manager class API has been updated :
-    - The manager is mainly use to fill a Libray from a "library.xml" file or
-      opds feed. Other operations (has removeBookById, setBookPath, filter, ...)
-      have been removed.
-    - The manager use a intermediate class (LibraryManipulator) to add book to
-      the library. This dependency injection allow caller code to hook the add
-      of a book to the library.
-    - The manager work on a existing Library. It doesn't how a internal
-      Library.
-  * [API break] OpdsDumper class API has been updated :
-    - dumpOPDSFeed method now take the list of bookIds to dump instead of
-      dumping all books in the library.
-    - OpdsDumper can now dump openSearch result information (total result
-      count, start index, ...).
-  * [API break] Common tools API has been updated :
-    - `base64_encode` and `base64_decode` take std::string as arguments.
-    - New `download` function in networkTools.h using libcurl.
-    - New `getDataDirectory` function in pathTools.
-    - Better `beautifyInteger` and `beautifyFileSize` functions.
-    - New `nodeToString` function serializing a pugi::xml_node to a string.
-    - New `converta2toa3` function to convert alpha2 language code to aplha3
-      language code.
-
-
-kiwix-lib 2.0.2
-===============
-
- * [Android] Forward c++ errors message de Java world.
- * Follow redirection of favicon.
- * Make aria2 dependency optional.
- * Inculde unistd.h only on unix platform.
-
-kiwix-lib 2.0.1
-===============
-
- * Fix parsing of url.
- * Remove unused static resources.
- * Correctly decode reserved characters in URLs.
- * Explicitly use icu namespace to allow use of packaged icu lib.
-
-kiwix-lib 2.0.0
-===============
-
- * Introduce a new API to retrive content from a reader.
-   * Introduce the `Entry` class.
-   * Reader's methods return an `Entry`.
-   * Content and other information can be retrieved from the `Entry`.
-   * Older Reader's methods are depreciated.
- * Add an `OPDSDumper` class to dump a whole `Library` as an OPDS feed.
- * Add a tool function to get the content of a file.
- * Add a tool function to create a tempory directory.
- * Add a `Downloader` class to download a file.
- * Allow the manager to populate a `Library` from an OPDS feed.
- * Try to locate libctpp2 in default system libdir and then fallback in 'lib'
-   directory.
- * Build kiwix-lib setting RPATH.
- * Build kiwix-lib without warning (werror=true)
- * Build kiwix-lib on macos.
-
-kiwix-lib 1.1.1
-===============
-
- * Correct the name of kiwix-lib (from `kiwixlib`) in meson.build to generate
-   dist archive with the correct name.
- * Libzim version need to be at least 3.2.0
-
-kiwix-lib 1.1.0
-===============
-
- * Allow for more than 70 search result per page in html results rendering
-   (kiwix/kiwix-tools#92)
- * Add a small api to do geo queries.
- * Add multi-search support in the JNI (#67)
- * Add an API to get only one part of an article.
- * Add an API to get direct location of an article content in the zim file.
- * Improve urlencoding
- * Fix pagination in html results rendering.
- * Compile using gcc-5 on Travis.
- * Allow JNI to access search snippets.
- * JNI throw an exception instead of returning an invalid object if something
-   goes wrong.
- * Add doctext documentation. (#116)
- * Various bug fixes.
-
 kiwix-lib 1.0.0
 ===============
 

README.md

@@ -1,16 +1,8 @@
 Kiwix library
 =============
 
-The Kiwix library provides the [Kiwix](https://kiwix.org) software
-suite core. It contains the code shared by all Kiwix ports (Windows,
-GNU/Linux, macOS, Android, iOS, ...).
-
-[![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
-[![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
-[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
-[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
-[![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
-[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+The Kiwix library provides the Kiwix software core. It contains the
+code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
 
 Disclaimer
 ----------
@@ -23,9 +15,9 @@ to [kiwix-build](https://github.com/kiwix/kiwix-build).
 Preamble
 --------
 
-Although the Kiwix library can be (cross-)compiled on/for many
+Although the Kiwix library can be compiled/cross-compiled on/for many
 sytems, the following documentation explains how to do it on POSIX
-ones. It is primarly thought for GNU/Linux systems and has been tested
+ones. It is primarly though for GNU/Linux systems and has been tested
 on recent releases of Ubuntu and Fedora.
 
 Dependencies
@@ -35,13 +27,16 @@ The Kiwix library relies on many third parts software libraries. They
 are prerequisites to the Kiwix library compilation. Following
 libraries need to be available:
 
-* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
-* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
-* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
-* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
-* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
-header `mustache.hpp` somewhere it can be found by the compiler and/or
-set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
+* ICU ................................... http://site.icu-project.org/
+(package libicu-dev on Ubuntu)
+* ZIM ........................................ http://www.openzim.org/
+(package libzim-dev on Ubuntu)
+* Pugixml ........................................ http://pugixml.org/
+(package libpugixml-dev on Ubuntu)
+* ctpp2 ........................................ http://ctpp.havoc.ru/
+(package libctpp2-dev on Ubuntu)
+* Xapian ......................................... https://xapian.org/
+(package libxapian-dev on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -50,104 +45,76 @@ In the worse case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
-`kiwix-lib` directory as install prefix.
+kiwix-lib directory as install prefix.
 
-Environment
+If you compile ctpp2 from source and want to compile the Kiwix library
+statically then you will probably need to rename ctpp2 static library
+from ctpp2-st.a to ctpp2.a.
+
+Environnement
 -------------
 
-The Kiwix library builds using [Meson](https://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+The Kiwix library builds using [Meson](http://mesonbuild.com/) version
+0.34 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
-* [Meson](https://mesonbuild.com/)
-* [Ninja](https://ninja-build.org/)
-* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
+* Automake
+* Libtool
+* Virtualenv
+* Pkg-config
 
-These tools should be packaged if you use a cutting edge operating
-system. If not, have a look to the [Troubleshooting](#Troubleshooting)
-section.
-
-Compilation
------------
-
-Once all dependencies are installed, you can compile the Kiwix library
-with:
-```bash
-meson . build
-ninja -C build
+Then install Meson itself:
 ```
-
-By default, it will compile dynamic linked libraries. All binary files
-will be created in the "build" directory created automatically by
-Meson. If you want statically linked libraries, you can add
-`--default-library=static` option to the Meson command.
-
-Depending of you system, `ninja` may be called `ninja-build`.
-
-Testing
--------
-
-To run the automated tests:
-```bash
-cd build
-meson test
-```
-
-Installation
-------------
-
-If you want to install the Kiwix library and the headers you just have
-compiled on your system, here we go:
-```bash
-ninja -C build install
-```
-
-You might need to run the command as root (or using `sudo`), depending
-where you want to install the libraries. After the installation
-succeeded, you may need to run `ldconfig` (as `root`).
-
-Uninstallation
-------------
-
-If you want to uninstall the Kiwix library:
-```bash
-ninja -C build uninstall
-```
-
-Like for the installation, you might need to run the command as `root`
-(or using `sudo`).
-
-Troubleshooting
----------------
-
-If you need to install Meson "manually":
-```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
-pip3 install meson       # Install Meson
+pip install meson        # Install Meson
 hash -r                  # Refresh bash paths
 ```
 
-If you need to install Ninja "manually":
-```bash
+Finally download and build Ninja locally:
+```
 git clone git://github.com/ninja-build/ninja.git
 cd ninja
 git checkout release
 ./configure.py --bootstrap
-mkdir ../bin
+mkdir ../bin                      
 cp ninja ../bin
 cd ..
 ```
 
-If the compilation still fails, you might need to get a more recent
-version of a dependency than the one packaged by your Linux
-distribution. Try then with a source tarball distributed by the
-problematic upstream project or even directly from the source code
-repository.
+Compilation
+-----------
+
+Once all dependencies are installed, you can compile kiwix-lib with:
+```
+mkdir build
+meson . build
+cd build
+ninja
+```
+
+By default, it will compile dynamic linked libraries. If you want
+statically linked libraries, you can add `--default-library=static`
+option to the Meson command.
+
+Depending of you system, `ninja` may be called `ninja-build`.
+
+Installation
+------------
+
+If you want to install the libraries you just have compiled on your
+system, here we go:
+
+```
+ninja install
+cd ..
+```
+
+You might need to run the command as root, depending where you want to
+install the libraries.
 
 License
 -------
 
-[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
-[COPYING](COPYING) for more details.
+GPLv3 or later, see COPYING for more details.

android-kiwix-lib-publisher/.gitignore

@@ -1,13 +0,0 @@
-*.iml
-.gradle
-/local.properties
-/.idea/caches
-/.idea/libraries
-/.idea/modules.xml
-/.idea/workspace.xml
-/.idea/navEditor.xml
-/.idea/assetWizardSettings.xml
-.DS_Store
-/build
-/captures
-.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -1,25 +0,0 @@
-// Top-level build file where you can add configuration options common to all sub-projects/modules.
-
-buildscript {
-    repositories {
-        google()
-        jcenter()
-
-    }
-    dependencies {
-        classpath 'com.android.tools.build:gradle:3.4.1'
-        // NOTE: Do not place your application dependencies here; they belong
-        // in the individual module build.gradle files
-    }
-}
-
-allprojects {
-    repositories {
-        google()
-        jcenter()
-    }
-}
-
-task clean(type: Delete) {
-    delete rootProject.buildDir
-}

android-kiwix-lib-publisher/gradle.properties

@@ -1,15 +0,0 @@
-# Project-wide Gradle settings.
-# IDE (e.g. Android Studio) users:
-# Gradle settings configured through the IDE *will override*
-# any settings specified in this file.
-# For more details on how to configure your build environment visit
-# http://www.gradle.org/docs/current/userguide/build_environment.html
-# Specifies the JVM arguments used for the daemon process.
-# The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx1536m
-# When configured, Gradle will run in incubating parallel mode.
-# This option should only be used with decoupled projects. More details, visit
-# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
-# org.gradle.parallel=true
-# Kotlin code style for this project: "official" or "obsolete":
-kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +0,0 @@
-#Wed Jun 19 15:28:39 BST 2019
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -1,172 +0,0 @@
-#!/usr/bin/env sh
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
-exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -1,84 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -1 +0,0 @@
-/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -1,64 +0,0 @@
-apply plugin: 'com.android.library'
-apply plugin: 'maven'
-
-android {
-    compileSdkVersion 28
-    defaultConfig {
-        minSdkVersion 15
-        targetSdkVersion 28
-        versionCode 1
-        versionName "1.0"
-    }
-    buildTypes {
-        release {
-            minifyEnabled false
-            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
-        }
-    }
-}
-
-dependencies {
-    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
-}
-
-task writePom {
-  pom {
-    project {
-        groupId 'org.kiwix.kiwixlib'
-        artifactId 'kiwixlib'
-        version '9.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
-        packaging 'aar'
-        name 'kiwixlib'
-        url 'https://github.com/kiwix/kiwix-lib'
-        licenses {
-            license {
-                name 'GPLv3'
-                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
-            }
-        }
-      developers {
-          developer {
-              id 'kiwix'
-              name 'kiwix'
-              email 'contact@kiwix.org'
-          }
-      }
-      scm {
-          connection 'https://github.com/kiwix/kiwix-lib.git'
-          developerConnection 'https://github.com/kiwix/kiwix-lib.git'
-          url 'https://github.com/kiwix/kiwix-lib'
-      }
-    }
-  }.withXml {
-    def dependenciesNode = asNode().appendNode('dependencies')
-
-    //Iterate over the implementation dependencies, adding a <dependency> node for each
-    configurations.implementation.allDependencies.each {
-      def dependencyNode = dependenciesNode.appendNode('dependency')
-      dependencyNode.appendNode('groupId', it.group)
-      dependencyNode.appendNode('artifactId', it.name)
-      dependencyNode.appendNode('version', it.version)
-    }
-  }.writeTo("$buildDir/pom.xml")
-}
-

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -1,21 +0,0 @@
-# Add project specific ProGuard rules here.
-# You can control the set of applied configuration files using the
-# proguardFiles setting in build.gradle.
-#
-# For more details, see
-#   http://developer.android.com/guide/developing/tools/proguard.html
-
-# If your project uses WebView with JS, uncomment the following
-# and specify the fully qualified class name to the JavaScript interface
-# class:
-#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
-#   public *;
-#}
-
-# Uncomment this to preserve the line number information for
-# debugging stack traces.
-#-keepattributes SourceFile,LineNumberTable
-
-# If you keep the line number information, uncomment this to
-# hide the original source file name.
-#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,10 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="org.kiwix.kiwixlib">
-
-    <application
-        android:allowBackup="true"
-        android:supportsRtl="true">
-
-    </application>
-
-</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -1 +0,0 @@
-include ':kiwixLibAndroid'

include/book.h

@@ -1,124 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_BOOK_H
-#define KIWIX_BOOK_H
-
-#include <string>
-
-namespace pugi {
-class xml_node;
-}
-
-namespace kiwix
-{
-
-class OPDSDumper;
-class Reader;
-
-/**
- * A class to store information about a book (a zim file)
- */
-class Book
-{
- public:
-  Book();
-  ~Book();
-
-  bool update(const Book& other);
-  void update(const Reader& reader);
-  void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
-  void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
-  std::string getHumanReadableIdFromPath() const;
-
-  bool readOnly() const { return m_readOnly; }
-  const std::string& getId() const { return m_id; }
-  const std::string& getPath() const { return m_path; }
-  bool isPathValid() const { return m_pathValid; }
-  const std::string& getTitle() const { return m_title; }
-  const std::string& getDescription() const { return m_description; }
-  const std::string& getLanguage() const { return m_language; }
-  const std::string& getCreator() const { return m_creator; }
-  const std::string& getPublisher() const { return m_publisher; }
-  const std::string& getDate() const { return m_date; }
-  const std::string& getUrl() const { return m_url; }
-  const std::string& getName() const { return m_name; }
-  const std::string& getTags() const { return m_tags; }
-  std::string getTagStr(const std::string& tagName) const;
-  bool getTagBool(const std::string& tagName) const;
-  const std::string& getFlavour() const { return m_flavour; }
-  const std::string& getOrigId() const { return m_origId; }
-  const uint64_t& getArticleCount() const { return m_articleCount; }
-  const uint64_t& getMediaCount() const { return m_mediaCount; }
-  const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const;
-  const std::string& getFaviconUrl() const { return m_faviconUrl; }
-  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
-  const std::string& getDownloadId() const { return m_downloadId; }
-
-  void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
-  void setId(const std::string& id) { m_id = id; }
-  void setPath(const std::string& path);
-  void setPathValid(bool valid) { m_pathValid = valid; }
-  void setTitle(const std::string& title) { m_title = title; }
-  void setDescription(const std::string& description) { m_description = description; }
-  void setLanguage(const std::string& language) { m_language = language; }
-  void setCreator(const std::string& creator) { m_creator = creator; }
-  void setPublisher(const std::string& publisher) { m_publisher = publisher; }
-  void setDate(const std::string& date) { m_date = date; }
-  void setUrl(const std::string& url) { m_url = url; }
-  void setName(const std::string& name) { m_name = name; }
-  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
-  void setTags(const std::string& tags) { m_tags = tags; }
-  void setOrigId(const std::string& origId) { m_origId = origId; }
-  void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
-  void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
-  void setSize(uint64_t size) { m_size = size; }
-  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
-  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
-  void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
-
- protected:
-  std::string m_id;
-  std::string m_downloadId;
-  std::string m_path;
-  bool m_pathValid = false;
-  std::string m_title;
-  std::string m_description;
-  std::string m_language;
-  std::string m_creator;
-  std::string m_publisher;
-  std::string m_date;
-  std::string m_url;
-  std::string m_name;
-  std::string m_flavour;
-  std::string m_tags;
-  std::string m_origId;
-  uint64_t m_articleCount = 0;
-  uint64_t m_mediaCount = 0;
-  bool m_readOnly = false;
-  uint64_t m_size = 0;
-  mutable std::string m_favicon;
-  std::string m_faviconUrl;
-  std::string m_faviconMimeType;
-};
-
-}
-
-#endif

include/bookmark.h

@@ -1,68 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_BOOKMARK_H
-#define KIWIX_BOOKMARK_H
-
-#include <string>
-
-namespace pugi {
-class xml_node;
-}
-
-namespace kiwix
-{
-
-/**
- * A class to store information about a bookmark (an article in a book)
- */
-class Bookmark
-{
- public:
-  Bookmark();
-  ~Bookmark();
-
-  void updateFromXml(const pugi::xml_node& node);
-
-  const std::string& getBookId() const { return m_bookId; }
-  const std::string& getBookTitle()  const { return m_bookTitle; }
-  const std::string& getUrl() const { return m_url; }
-  const std::string& getTitle() const { return m_title; }
-  const std::string& getLanguage() const { return m_language; }
-  const std::string& getDate() const { return m_date; }
-
-  void setBookId(const std::string& bookId) { m_bookId = bookId; }
-  void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
-  void setUrl(const std::string& url) { m_url = url; }
-  void setTitle(const std::string& title) { m_title = title; }
-  void setLanguage(const std::string& language) { m_language = language; }
-  void setDate(const std::string& date) { m_date = date; }
-
- protected:
-  std::string m_bookId;
-  std::string m_bookTitle;
-  std::string m_url;
-  std::string m_title;
-  std::string m_language;
-  std::string m_date;
-};
-
-}
-
-#endif

include/common.h

@@ -1,24 +0,0 @@
-
-#ifndef _KIWIX_COMMON_H_
-#define _KIWIX_COMMON_H_
-
-#include <zim/zim.h>
-
-#ifdef __GNUC__
-#define DEPRECATED __attribute__((deprecated))
-#elif defined(_MSC_VER)
-#define DEPRECATED __declspec(deprecated)
-#else
-#praga message("WARNING: You need to implement DEPRECATED for this compiler")
-#define DEPRECATED
-#endif
-
-
-namespace kiwix {
-
-typedef zim::size_type size_type;
-typedef zim::offset_type offset_type;
-
-}
-
-#endif //_KIWIX_COMMON_H_

include/common/base64.h

@@ -0,0 +1,4 @@
+#include <string>
+
+std::string base64_encode(unsigned char const* , unsigned int len);
+std::string base64_decode(std::string const& s);

include/common/networkTools.h

@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -18,19 +17,33 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_NETWORKTOOLS_H
+#define KIWIX_NETWORKTOOLS_H
 
-import android.content.Context;
-import com.getkeepsafe.relinker.ReLinker;
-import org.kiwix.kiwixlib.JNIICU;
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <net/if.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
 
-public class JNIKiwix
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace kiwix
 {
-  public JNIKiwix(final Context context){
-    ReLinker.loadLibrary(context, "kiwix");
-  }
-
-  public void setDataDirectory(String icuDataDir) {
-    JNIICU.setDataDirectory(icuDataDir);
-  }
+std::map<std::string, std::string> getNetworkInterfaces();
+std::string getBestPublicIp();
 }
+
+#endif

include/common/otherTools.h

@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -18,9 +17,18 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_OTHERTOOLS_H
+#define KIWIX_OTHERTOOLS_H
 
-public class JNIICU
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace kiwix
 {
-  static public native void setDataDirectory(String icuDataDir);
+void sleep(unsigned int milliseconds);
 }
+
+#endif

include/common/pathTools.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_PATHTOOLS_H
+#define KIWIX_PATHTOOLS_H
+
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fstream>
+#include <ios>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#ifdef _WIN32
+#include <direct.h>
+#endif
+
+#include "stringTools.h"
+
+using namespace std;
+
+bool isRelativePath(const string& path);
+string computeAbsolutePath(const string path, const string relativePath);
+string computeRelativePath(const string path, const string absolutePath);
+string removeLastPathElement(const string path,
+                             const bool removePreSeparator = false,
+                             const bool removePostSeparator = false);
+string appendToDirectory(const string& directoryPath, const string& filename);
+
+unsigned int getFileSize(const string& path);
+string getFileSizeAsString(const string& path);
+bool fileExists(const string& path);
+bool makeDirectory(const string& path);
+bool copyFile(const string& sourcePath, const string& destPath);
+string getLastPathElement(const string& path);
+string getExecutablePath();
+string getCurrentDirectory();
+bool writeTextFile(const string& path, const string& content);
+#endif

include/common/regexTools.h

@@ -20,6 +20,9 @@
 #ifndef KIWIX_REGEXTOOLS_H
 #define KIWIX_REGEXTOOLS_H
 
+#include <unicode/regex.h>
+#include <unicode/ucnv.h>
+#include <map>
 #include <string>
 
 bool matchRegex(const std::string& content, const std::string& regex);
@@ -27,7 +30,7 @@ std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
 std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
+                                   const std::string regex,
                                    const std::string& replacement);
 
 #endif

include/common/stringTools.h

@@ -22,29 +22,38 @@
 
 #include <unicode/unistr.h>
 
+#include <fstream>
+#include <iostream>
+#include <sstream>
 #include <string>
 #include <vector>
-#include <sstream>
+
+#include "pathTools.h"
 
 namespace kiwix
 {
-std::string beautifyInteger(uint64_t number);
-std::string beautifyFileSize(uint64_t number);
+#ifndef __ANDROID__
+
+std::string beautifyInteger(const unsigned int number);
+std::string beautifyFileSize(const unsigned int number);
+std::string urlEncode(const std::string& c);
 void printStringInHexadecimal(const char* s);
-void printStringInHexadecimal(icu::UnicodeString s);
+void printStringInHexadecimal(UnicodeString s);
 void stringReplacement(std::string& str,
                        const std::string& oldStr,
                        const std::string& newStr);
 std::string encodeDiples(const std::string& str);
 
+#endif
+
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
+std::string urlDecode(const std::string& c);
 
-std::string urlEncode(const std::string& value, bool encodeReserved = false);
-std::string urlDecode(const std::string& value, bool component = false);
-
-std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
-std::string join(const std::vector<std::string>& list, const std::string& sep);
+std::vector<std::string> split(const std::string&, const std::string&);
+std::vector<std::string> split(const char*, const char*);
+std::vector<std::string> split(const std::string&, const char*);
+std::vector<std::string> split(const char*, const std::string&);
 
 std::string ucAll(const std::string& word);
 std::string lcAll(const std::string& word);
@@ -53,22 +62,6 @@ std::string lcFirst(const std::string& word);
 std::string toTitle(const std::string& word);
 
 std::string normalize(const std::string& word);
-template<typename T>
-std::string to_string(T value)
-{
-    std::ostringstream oss;
-    oss << value;
-    return oss.str();
 }
 
-template<typename T>
-T extractFromString(const std::string& str) {
-    std::istringstream iss(str);
-    T ret;
-    iss >> ret;
-    return ret;
-}
-
-bool startsWith(const std::string& base, const std::string& start);
-} //namespace kiwix
 #endif

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef _CTPP2_VM_STRING_LOADER_HPP__
+#define _CTPP2_VM_STRING_LOADER_HPP__ 1
+
+#include <ctpp2/CTPP2VMLoader.hpp>
+#include <ctpp2/CTPP2Util.hpp>
+#include <ctpp2/CTPP2Exception.hpp>
+#include <ctpp2/CTPP2VMExecutable.hpp>
+#include <ctpp2/CTPP2VMInstruction.hpp>
+#include <ctpp2/CTPP2VMMemoryCore.hpp>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <iostream>
+#include <string>
+
+/**
+  @file VMStringLoader.hpp
+  @brief Load program core from file
+*/
+
+namespace CTPP // C++ Template Engine
+{
+// FWD
+struct VMExecutable;
+
+/**
+  @class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
+  @brief Load program core from file
+*/
+class CTPP2DECL VMStringLoader:
+  public VMLoader
+{
+public:
+	/**
+	*/
+	VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
+	/**
+	  @brief Get ready-to-run program
+	*/
+	const VMMemoryCore * GetCore() const;
+
+	/**
+	  @brief A destructor
+	*/
+	~VMStringLoader() throw();
+private:
+	/** Program core             */
+	VMExecutable  * oCore;
+	/** Ready-to-run program     */
+	VMMemoryCore  * pVMMemoryCore;
+};
+
+} // namespace CTPP
+#endif // _CTPP2_VM_STRING_LOADER_HPP__
+// End.

include/downloader.h

@@ -1,108 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_DOWNLOADER_H
-#define KIWIX_DOWNLOADER_H
-
-#include <string>
-#include <vector>
-#include <map>
-#include <pthread.h>
-#include <memory>
-#include <stdexcept>
-
-namespace kiwix
-{
-
-class Aria2;
-struct DownloadedFile {
-  DownloadedFile()
-   : success(false) {}
-  bool success;
-  std::string path;
-};
-
-class AriaError : public std::runtime_error {
- public:
-  AriaError(const std::string& message) : std::runtime_error(message) {}
-};
-
-
-class Download {
- public:
-  typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
-
-  Download() :
-    m_status(K_UNKNOWN) {}
-  Download(std::shared_ptr<Aria2> p_aria, std::string did)
-    : mp_aria(p_aria),
-      m_status(K_UNKNOWN),
-      m_did(did) {};
-  void updateStatus(bool follow=false);
-  void pauseDownload();
-  void resumeDownload();
-  void cancelDownload();
-  StatusResult getStatus()          { return m_status; }
-  std::string  getDid()             { return m_did; }
-  std::string  getFollowedBy()      { return m_followedBy; }
-  uint64_t     getTotalLength()     { return m_totalLength; }
-  uint64_t     getCompletedLength() { return m_completedLength; }
-  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
-  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
-  std::string  getPath()            { return m_path; }
-  std::vector<std::string>&  getUris()             { return m_uris; }
-
- protected:
-  std::shared_ptr<Aria2> mp_aria;
-  StatusResult m_status;
-  std::string m_did = "";
-  std::string m_followedBy = "";
-  uint64_t m_totalLength;
-  uint64_t m_completedLength;
-  uint64_t m_downloadSpeed;
-  uint64_t m_verifiedLength;
-  std::vector<std::string> m_uris;
-  std::string m_path;
-};
-
-/**
- * A tool to download things.
- *
- */
-class Downloader
-{
- public:
-  Downloader();
-  virtual ~Downloader();
-
-  void close();
-
-  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
-  Download* getDownload(const std::string& did);
-
-  size_t getNbDownload() { return m_knownDownloads.size(); }
-  std::vector<std::string> getDownloadIds();
-
- private:
-  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
-  std::shared_ptr<Aria2> mp_aria;
-};
-}
-
-#endif

include/entry.h

@@ -1,192 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_ENTRY_H
-#define KIWIX_ENTRY_H
-
-#include <stdio.h>
-#include <zim/article.h>
-#include <exception>
-#include <string>
-
-#include "common.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-
-class NoEntry : public std::exception {};
-
-/**
- * A entry represent an.. entry in a zim file.
- */
-class Entry
-{
-  public:
-    /**
-     * Default constructor.
-     *
-     * Construct an invalid entry.
-     */
-    Entry() = default;
-
-    /**
-     * Construct an entry making reference to an zim article.
-     *
-     * @param article a zim::Article object
-     */
-    Entry(zim::Article article);
-    virtual ~Entry() = default;
-
-    /**
-     * Get the path of the entry.
-     *
-     * The path is the "key" of an entry.
-     *
-     * @return the path of the entry.
-     */
-    std::string getPath() const;
-
-    /**
-     * Get the title of the entry.
-     *
-     * @return the title of the entry.
-     */
-    std::string getTitle() const;
-
-    /**
-     * Get the content of the entry.
-     *
-     * The string is a copy of the content.
-     * If you don't want to do a copy, use get_blob.
-     *
-     * @return the content of the entry.
-     */
-    std::string getContent() const;
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset = 0) const;
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @param size The size of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset, size_type size) const;
-
-    /**
-     * Get the info for direct access to the content of the entry.
-     *
-     * Some entry (ie binary ones) have their content plain stored
-     * in the zim file. Knowing the offset where the content is stored
-     * an user can directly read the content in the zim file bypassing the
-     * kiwix-lib/libzim.
-     *
-     * @return A pair specifying where to read the content.
-     *         The string is the real file to read (may be different that .zim
-     *         file if zim is cut).
-     *         The offset is the offset to read in the file.
-     *         Return <"",0> if is not possible to read directly.
-     */
-    std::pair<std::string, offset_type> getDirectAccessInfo() const;
-
-    /**
-     * Get the size of the entry.
-     *
-     * @return the size of the entry.
-     */
-    size_type   getSize() const;
-
-    /**
-     * Get the mime_type of the entry.
-     *
-     * @return the mime_type of the entry.
-     */
-    std::string getMimetype() const;
-
-
-    /**
-     * Get if the entry is a redirect entry.
-     *
-     * @return True if the entry is a redirect.
-     */
-    bool isRedirect() const;
-
-    /**
-     * Get if the entry is a link target entry.
-     *
-     * @return True if the entry is a link target.
-     */
-    bool isLinkTarget() const;
-
-    /**
-     * Get if the entry is a deleted entry.
-     *
-     * @return True if the entry is a deleted entry.
-     */
-    bool isDeleted() const;
-
-    /**
-     * Get the entry pointed by this entry.
-     *
-     * @return the entry pointed.
-     * @throw NoEntry if the entry is not a redirected entry.
-     */
-    Entry getRedirectEntry() const;
-
-    /**
-     * Get the final entry pointed by this entry.
-     *
-     * Follow the redirection until a "not redirecting" entry is found.
-     * If the entry is not a redirected entry, return the entry itself.
-     *
-     * @return the final entry.
-     */
-    Entry getFinalEntry() const;
-
-    /**
-     * Convert the entry to a boolean value.
-     *
-     * @return True if the entry is valid.
-     */
-    explicit operator bool() const { return good(); }
-
-  private:
-    zim::Article article;
-    mutable zim::Article final_article;
-
-    bool good() const { return article.good(); }
-};
-
-}
-
-#endif // KIWIX_ENTRY_H

include/kiwixserve.h

@@ -1,30 +0,0 @@
-#ifndef KIWIXLIB_KIWIX_SERVE_H_
-#define KIWIXLIB_KIWIX_SERVE_H_
-
-#include <memory>
-#include <string>
-
-class Subprocess;
-namespace kiwix {
-
-class KiwixServe
-{
-  public:
-    KiwixServe(const std::string& libraryPath, int port = 8181);
-    ~KiwixServe();
-
-    void run();
-    void shutDown();
-    bool isRunning();
-    int getPort() { return m_port; }
-    int setPort(int port);
-
-  private:
-  std::unique_ptr<Subprocess> mp_kiwixServe;
-  int m_port;
-  std::string m_libraryPath;
-};
-
-}; //end namespace kiwix
-
-#endif // KIWIXLIB_KIWIX_SERVE_H_

include/library.h

@@ -20,269 +20,86 @@
 #ifndef KIWIX_LIBRARY_H
 #define KIWIX_LIBRARY_H
 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stack>
 #include <string>
 #include <vector>
-#include <map>
-#include <memory>
 
-#include "book.h"
-#include "bookmark.h"
-#include "common.h"
+#include "common/regexTools.h"
+#include "common/stringTools.h"
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
+using namespace std;
+
 namespace kiwix
 {
+enum supportedIndexType { UNKNOWN, XAPIAN };
 
-class OPDSDumper;
+class Book
+{
+ public:
+  Book();
+  ~Book();
 
-enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
-enum supportedListMode {
-  ALL = 0,
-  LOCAL = 1,
-  REMOTE = 1 << 1,
-  NOLOCAL = 1 << 2,
-  NOREMOTE = 1 << 3,
-  VALID = 1 << 4,
-  NOVALID = 1 << 5
+  static bool sortByLastOpen(const Book& a, const Book& b);
+  static bool sortByTitle(const Book& a, const Book& b);
+  static bool sortBySize(const Book& a, const Book& b);
+  static bool sortByDate(const Book& a, const Book& b);
+  static bool sortByCreator(const Book& a, const Book& b);
+  static bool sortByPublisher(const Book& a, const Book& b);
+  static bool sortByLanguage(const Book& a, const Book& b);
+  string getHumanReadableIdFromPath();
+
+  string id;
+  string path;
+  string pathAbsolute;
+  string last;
+  string indexPath;
+  string indexPathAbsolute;
+  supportedIndexType indexType;
+  string title;
+  string description;
+  string language;
+  string creator;
+  string publisher;
+  string date;
+  string url;
+  string name;
+  string tags;
+  string origId;
+  string articleCount;
+  string mediaCount;
+  bool readOnly;
+  string size;
+  string favicon;
+  string faviconMimeType;
 };
 
-class Filter {
-  private:
-    uint64_t activeFilters;
-    std::vector<std::string> _acceptTags;
-    std::vector<std::string> _rejectTags;
-    std::string _lang;
-    std::string _publisher;
-    std::string _creator;
-    size_t _maxSize;
-    std::string _query;
-    std::string _name;
-
-  public:
-    Filter();
-    ~Filter() = default;
-
-    /**
-     *  Set the filter to check local.
-     *
-     *  A local book is a book with a path.
-     *  If accept is true, only local book are accepted.
-     *  If accept is false, only non local book are accepted.
-     */
-    Filter& local(bool accept);
-
-    /**
-     *  Set the filter to check remote.
-     *
-     *  A remote book is a book with a url.
-     *  If accept is true, only remote book are accepted.
-     *  If accept is false, only non remote book are accepted.
-     */
-    Filter& remote(bool accept);
-
-    /**
-     *  Set the filter to check validity.
-     *
-     *  A valid book is a book with a path pointing to a existing zim file.
-     *  If accept is true, only valid book are accepted.
-     *  If accept is false, only non valid book are accepted.
-     */
-    Filter& valid(bool accept);
-
-    /**
-     * Set the filter to only accept book with corresponding tag.
-     */
-    Filter& acceptTags(std::vector<std::string> tags);
-    Filter& rejectTags(std::vector<std::string> tags);
-
-    Filter& lang(std::string lang);
-    Filter& publisher(std::string publisher);
-    Filter& creator(std::string creator);
-    Filter& maxSize(size_t size);
-    Filter& query(std::string query);
-    Filter& name(std::string name);
-
-    bool accept(const Book& book) const;
-};
-
-
-/**
- * A Library store several books.
- */
 class Library
 {
-  std::map<std::string, kiwix::Book> m_books;
-  std::map<std::string, std::shared_ptr<Reader>> m_readers;
-  std::vector<kiwix::Bookmark> m_bookmarks;
-
  public:
   Library();
   ~Library();
 
-  /**
-   * Add a book to the library.
-   *
-   * If a book already exist in the library with the same id, update
-   * the existing book instead of adding a new one.
-   *
-   * @param book The book to add.
-   * @return True if the book has been added.
-   *         False if a book has been updated.
-   */
+  string version;
   bool addBook(const Book& book);
+  bool removeBookByIndex(const unsigned int bookIndex);
+  vector<kiwix::Book> books;
 
-  /**
-   * Add a bookmark to the library.
-   *
-   * @param bookmark the book to add.
+  /*
+   * 'current' is the variable storing the current content/book id
+   * in the library. This is used to be able to load per default a
+   * content. As Kiwix may work with many library XML files, you may
+   * have "current" defined many time with different values. The
+   * last XML file read has the priority, Although we do not have an
+   * library object for each file, we want to be able to fallback to
+   * an 'old' current book if the one which should be load
+   * failed. That is the reason why we need a stack here
    */
-  void addBookmark(const Bookmark& bookmark);
-
-  /**
-   * Remove a bookmarkk
-   *
-   * @param zimId The zimId of the bookmark.
-   * @param url The url of the bookmark.
-   * @return True if the bookmark has been removed.
-   */
-  bool removeBookmark(const std::string& zimId, const std::string& url);
-
-  Book& getBookById(const std::string& id);
-  Book& getBookByPath(const std::string& path);
-  std::shared_ptr<Reader> getReaderById(const std::string& id);
-
-  /**
-   * Remove a book from the library.
-   *
-   * @param id the id of the book to remove.
-   * @return True if the book were in the lirbrary and has been removed.
-   */
-  bool removeBookById(const std::string& id);
-
-  /**
-   * Write the library to a file.
-   *
-   * @param path the path of the file to write to.
-   * @return True if the library has been correctly saved.
-   */
-  bool writeToFile(const std::string& path);
-
-  /**
-   * Write the library bookmarks to a file.
-   *
-   * @param path the path of the file to write to.
-   * @return True if the library has been correctly saved.
-   */
-  bool writeBookmarksToFile(const std::string& path);
-
-  /**
-   * Get the number of book in the library.
-   *
-   * @param localBooks If we must count local books (books with a path).
-   * @param remoteBooks If we must count remote books (books with an url)
-   * @return The number of books.
-   */
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
-
-  /**
-   * Get all langagues of the books in the library.
-   *
-   * @return A list of languages.
-   */
-  std::vector<std::string> getBooksLanguages();
-
-  /**
-   * Get all book creators of the books in the library.
-   *
-   * @return A list of book creators.
-   */
-  std::vector<std::string> getBooksCreators();
-
-  /**
-   * Get all book publishers of the books in the library.
-   *
-   * @return A list of book publishers.
-   */
-  std::vector<std::string> getBooksPublishers();
-
-  /**
-   * Get all bookmarks.
-   *
-   * @return A list of bookmarks
-   */
-  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
-
-  /**
-   * Get all book ids of the books in the library.
-   *
-   * @return A list of book ids.
-   */
-  std::vector<std::string> getBooksIds();
-
-  /**
-   * Filter the library and generate a new one with the keep elements.
-   *
-   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
-   *
-   * @param search List only books with search in the title or description.
-   * @return The list of bookIds corresponding to the query.
-   */
-  DEPRECATED std::vector<std::string> filter(const std::string& search);
-
-
-  /**
-   * Filter the library and return the id of the keep elements.
-   *
-   * @param filter The filter to use.
-   * @return The list of bookIds corresponding to the filter.
-   */
-  std::vector<std::string> filter(const Filter& filter);
-
-
-  /**
-   * Sort (in place) bookIds using the given comparator.
-   *
-   * @param bookIds the list of book Ids to sort
-   * @param comparator how to sort the books
-   * @return The sorted list of books
-   */
-  void sort(std::vector<std::string>& bookIds, supportedListSortBy sortBy, bool ascending);
-
-  /**
-   * List books in the library.
-   *
-   * @param mode The mode of listing :
-   *             - LOCAL    : list only local books (with a path).
-   *             - REMOTE   : list only remote books (with an url).
-   *             - VALID    : list only valid books (without a path or with a
-   *                          path pointing to a valid zim file).
-   *             - NOLOCAL  : list only books without valid path.
-   *             - NOREMOTE : list only books without url.
-   *             - NOVALID  : list only books not valid.
-   *             - ALL : Do not do any filter (LOCAL or REMOTE)
-   *             - Flags can be combined.
-   * @param sortBy Attribute to sort by the book list.
-   * @param search List only books with search in the title, description.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param maxSize Do not list book bigger than maxSize.
-   *                Set to 0 to cancel this filter.
-   * @return The list of bookIds corresponding to the query.
-   */
-  DEPRECATED std::vector<std::string> listBooksIds(
-    int supportedListMode = ALL,
-    supportedListSortBy sortBy = UNSORTED,
-    const std::string& search = "",
-    const std::string& language = "",
-    const std::string& creator = "",
-    const std::string& publisher = "",
-    const std::vector<std::string>& tags = {},
-    size_t maxSize = 0);
-
-  friend class OPDSDumper;
-  friend class libXMLDumper;
+  stack<string> current;
 };
 }
 

include/libxml_dumper.h

@@ -1,83 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_LIBXML_DUMPER_H
-#define KIWIX_LIBXML_DUMPER_H
-
-#include <string>
-#include <vector>
-
-#include <pugixml.hpp>
-
-#include "library.h"
-
-namespace kiwix
-{
-
-/**
- * A tool to dump a `Library` into a basic library.xml
- *
- */
-class LibXMLDumper
-{
- public:
-  LibXMLDumper() = default;
-  LibXMLDumper(Library* library);
-  ~LibXMLDumper();
-
-  /**
-   * Dump the library.xml
-   *
-   * @param id The id of the library.
-   * @return The library.xml content.
-   */
-  std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
-
-
-  /**
-   * Dump the bookmark of the library.
-   *
-   * @return The bookmark.xml content.
-   */
-  std::string dumpLibXMLBookmark();
-
-  /**
-   * Set the base directory used.
-   *
-   * @param baseDir the base directory to use.
-   */
-  void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
-
-  /**
-   * Set the library to dump.
-   *
-   * @param library The library to dump.
-   */
-  void setLibrary(Library* library) { this->library = library; }
-
- protected:
-  kiwix::Library* library;
-  std::string baseDir;
- private:
-  void handleBook(Book book, pugi::xml_node root_node);
-  void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
-};
-}
-
-#endif // KIWIX_OPDS_DUMPER_H

include/manager.h

@@ -20,147 +20,88 @@
 #ifndef KIWIX_MANAGER_H
 #define KIWIX_MANAGER_H
 
-#include "book.h"
+#include <time.h>
+#include <sstream>
+#include <string>
+
+#include <pugixml.hpp>
+
+#include "common/base64.h"
+#include "common/pathTools.h"
+#include "common/regexTools.h"
 #include "library.h"
 #include "reader.h"
 
-#include <string>
-#include <vector>
-
-namespace pugi {
-class xml_document;
-}
+using namespace std;
 
 namespace kiwix
 {
+enum supportedListMode { LASTOPEN, REMOTE, LOCAL };
+enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
 
-class LibraryManipulator {
- public:
-  virtual ~LibraryManipulator() {}
-  virtual bool addBookToLibrary(Book book) = 0;
-  virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
-};
-
-class DefaultLibraryManipulator : public LibraryManipulator {
- public:
-  DefaultLibraryManipulator(Library* library) :
-    library(library) {}
-  virtual ~DefaultLibraryManipulator() {}
-  bool addBookToLibrary(Book book) {
-    return library->addBook(book);
-  }
-  void addBookmarkToLibrary(Bookmark bookmark) {
-    library->addBookmark(bookmark);
-  }
- private:
-   kiwix::Library* library;
-};
-
-/**
- * A tool to manage a `Library`.
- */
 class Manager
 {
  public:
-  Manager(LibraryManipulator* manipulator);
-  Manager(Library* library);
+  Manager();
   ~Manager();
 
-  /**
-   * Read a `library.xml` and add book in the file to the library.
-   *
-   * @param path The (utf8) path to the `library.xml`.
-   * @param readOnly Set if the libray path could be overwritten latter with
-   *                 updated content.
-   * @return True if file has been properly parsed.
-   */
-  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
-
-  /**
-   * Load a library content store in the string.
-   *
-   * @param xml The content corresponding of the library xml
-   * @param readOnly Set if the libray path could be overwritten latter with
-   *                 updated content.
-   * @param libraryPath The library path (used to resolve relative path)
-   * @return True if the content has been properly parsed.
-   */
-  bool readXml(const std::string& xml,
+  bool readFile(const string path, const bool readOnly = true);
+  bool readFile(const string nativePath,
+                const string UTF8Path,
+                const bool readOnly = true);
+  bool readXml(const string xml,
                const bool readOnly = true,
-               const std::string& libraryPath = "",
-               bool trustLibrary = true);
-
-  /**
-   * Load a library content stored in a OPDS stream.
-   *
-   * @param content The content of the OPDS stream.
-   * @param readOnly Set if the library path could be overwritten later with
-   *                 updated content.
-   * @param libraryPath The library path (used to resolve relative path)
-   * @return True if the content has been properly parsed.
-   */
-  bool readOpds(const std::string& content, const std::string& urlHost);
-
-
-  /**
-   * Load a bookmark file.
-   *
-   * @param path The path of the file to read.
-   * @return True if the content has been properly parsed.
-   */
-  bool readBookmarkFile(const std::string& path);
-
-  /**
-   * Add a book to the library.
-   *
-   * @param pathToOpen The path to the zim file to add.
-   * @param pathToSave The path to store in the library in place of pathToOpen.
-   * @param url        The url of the book to store in the library.
-   * @param checMetaData Tell if we check metadata before adding book to the
-   *                     library.
-   * @return The id of the book if the book has been added to the library.
-   *         Else, an empty string.
-   */
-  std::string addBookFromPathAndGetId(const std::string& pathToOpen,
-                                 const std::string& pathToSave = "",
-                                 const std::string& url = "",
+               const string libraryPath = "");
+  bool writeFile(const string path);
+  bool removeBookByIndex(const unsigned int bookIndex);
+  bool removeBookById(const string id);
+  bool setCurrentBookId(const string id);
+  string getCurrentBookId();
+  bool setBookIndex(const string id,
+                    const string path,
+                    const supportedIndexType type);
+  bool setBookIndex(const string id, const string path);
+  bool setBookPath(const string id, const string path);
+  string addBookFromPathAndGetId(const string pathToOpen,
+                                 const string pathToSave = "",
+                                 const string url = "",
                                  const bool checkMetaData = false);
-
-  /**
-   * Add a book to the library.
-   *
-   * @param pathToOpen The path to the zim file to add.
-   * @param pathToSave The path to store in the library in place of pathToOpen.
-   * @param url        The url of the book to store in the library.
-   * @param checMetaData Tell if we check metadata before adding book to the
-   *                     library.
-   * @return True if the book has been added to the library.
-   */
-
-  bool addBookFromPath(const std::string& pathToOpen,
-                       const std::string& pathToSave = "",
-                       const std::string& url = "",
+  bool addBookFromPath(const string pathToOpen,
+                       const string pathToSave = "",
+                       const string url = "",
                        const bool checkMetaData = false);
+  Library cloneLibrary();
+  bool getBookById(const string id, Book& book);
+  bool getCurrentBook(Book& book);
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
+  bool updateBookLastOpenDateById(const string id);
+  void removeBookPaths();
+  bool listBooks(const supportedListMode mode,
+                 const supportedListSortBy sortBy,
+                 const unsigned int maxSize,
+                 const string language,
+                 const string creator,
+                 const string publisher,
+                 const string search);
+  vector<string> getBooksLanguages();
+  vector<string> getBooksCreators();
+  vector<string> getBooksPublishers();
+  vector<string> getBooksIds();
 
-  std::string writableLibraryPath;
+  string writableLibraryPath;
 
-  bool m_hasSearchResult = false;
-  uint64_t m_totalBooks = 0;
-  uint64_t m_startIndex = 0;
-  uint64_t m_itemsPerPage = 0;
+  vector<std::string> bookIdList;
 
  protected:
-  kiwix::LibraryManipulator* manipulator;
-  bool mustDeleteManipulator;
+  kiwix::Library library;
 
-  bool readBookFromPath(const std::string& path, Book* book);
+  bool readBookFromPath(const string path, Book* book = NULL);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   bool readOnly,
-                   const std::string& libraryPath,
-                   bool trustLibrary);
-  bool parseOpdsDom(const pugi::xml_document& doc,
-                    const std::string& urlHost);
+                   const bool readOnly,
+                   const string libraryPath);
 
+ private:
+  void checkAndCleanBookPaths(Book& book, const string& libraryPath);
 };
 }
 

include/meson.build

@@ -1,31 +1,30 @@
 headers = [
-  'book.h',
-  'bookmark.h',
-  'common.h',
   'library.h',
   'manager.h',
-  'libxml_dumper.h',
-  'opds_dumper.h',
-  'downloader.h',
   'reader.h',
-  'entry.h',
-  'searcher.h',
-  'search_renderer.h',
-  'server.h',
-  'kiwixserve.h',
-  'name_mapper.h'
+  'searcher.h'
 ]
 
+if xapian_dep.found()
+  headers += ['xapianSearcher.h']
+endif
+
 install_headers(headers, subdir:'kiwix')
 
 install_headers(
-  'tools/base64.h',
-  'tools/networkTools.h',
-  'tools/otherTools.h',
-  'tools/pathTools.h',
-  'tools/regexTools.h',
-  'tools/stringTools.h',
-  'tools/lock.h',
-  subdir:'kiwix/tools'
+  'common/base64.h',
+  'common/networkTools.h',
+  'common/otherTools.h',
+  'common/pathTools.h',
+  'common/regexTools.h',
+  'common/stringTools.h',
+  subdir:'kiwix/common'
 )
 
+if has_ctpp2_dep
+  install_headers(
+    'ctpp2/CTPP2VMStringLoader.hpp',
+    subdir:'kiwix/ctpp2'
+  )
+endif
+

include/name_mapper.h

@@ -1,61 +0,0 @@
-/*
- * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_NAMEMAPPER_H
-#define KIWIX_NAMEMAPPER_H
-
-#include <string>
-#include <map>
-
-namespace kiwix
-{
-
-class Library;
-
-class NameMapper {
-  public:
-    virtual ~NameMapper() = default;
-    virtual std::string getNameForId(const std::string& id) = 0;
-    virtual std::string getIdForName(const std::string& name) = 0;
-};
-
-
-class IdNameMapper : public NameMapper {
-  public:
-    virtual std::string getNameForId(const std::string& id) { return id; };
-    virtual std::string getIdForName(const std::string& name) { return name; };
-};
-
-class HumanReadableNameMapper : public NameMapper {
-  private:
-    std::map<std::string, std::string> m_idToName;
-    std::map<std::string, std::string> m_nameToId;
-
-  public:
-    HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
-    virtual ~HumanReadableNameMapper() = default;
-    virtual std::string getNameForId(const std::string& id);
-    virtual std::string getIdForName(const std::string& name);
-};
-
-
-
-}
-
-#endif

include/opds_dumper.h

@@ -1,120 +0,0 @@
-/*
- * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_OPDS_DUMPER_H
-#define KIWIX_OPDS_DUMPER_H
-
-#include <time.h>
-#include <sstream>
-#include <string>
-
-#include <pugixml.hpp>
-
-#include "tools/base64.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "library.h"
-#include "reader.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-/**
- * A tool to dump a `Library` into a opds stream.
- *
- */
-class OPDSDumper
-{
- public:
-  OPDSDumper() = default;
-  OPDSDumper(Library* library);
-  ~OPDSDumper();
-
-  /**
-   * Dump the OPDS feed.
-   *
-   * @param id The id of the library.
-   * @return The OPDS feed.
-   */
-  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds);
-
-  /**
-   * Set the id of the opds stream.
-   *
-   * @param id the id to use.
-   */
-  void setId(const std::string& id) { this->id = id;}
-
-  /**
-   * Set the title oft the opds stream.
-   *
-   * @param title the title to use.
-   */
-  void setTitle(const std::string& title) { this->title = title; }
-
-  /**
-   * Set the root location used when generating url.
-   *
-   * @param rootLocation the root location to use.
-   */
-  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
-
-  /**
-   * Set the search url.
-   *
-   * @param searchUrl the search url to use.
-   */
-  void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; }
-
-  /**
-   * Set some informations about the search results.
-   *
-   * @param totalResult the total number of results of the search.
-   * @param startIndex the start index of the result.
-   * @param count the number of result of the current set (or page).
-   */
-  void setOpenSearchInfo(int totalResult, int startIndex, int count);
-
-  /**
-   * Set the library to dump.
-   *
-   * @param library The library to dump.
-   */
-  void setLibrary(Library* library) { this->library = library; }
-
- protected:
-  kiwix::Library* library;
-  std::string id;
-  std::string title;
-  std::string date;
-  std::string rootLocation;
-  std::string searchDescriptionUrl;
-  int m_totalResults;
-  int m_startIndex;
-  int m_count;
-  bool m_isSearchResult = false;
-
- private:
-  pugi::xml_node handleBook(Book book, pugi::xml_node root_node);
-};
-}
-
-#endif // KIWIX_OPDS_DUMPER_H

include/reader.h

@@ -29,532 +29,91 @@
 #include <map>
 #include <sstream>
 #include <string>
-#include "common.h"
-#include "entry.h"
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
+#include "common/pathTools.h"
+#include "common/stringTools.h"
 
 using namespace std;
 
 namespace kiwix
 {
-
-/**
- * The Reader class is the class who allow to get an entry content from a zim
- * file.
- */
 class Reader
 {
  public:
-  /**
-   * Create a Reader to read a zim file specified by zimFilePath.
-   *
-   * @param zimFilePath The path to the zim file to read.
-   *                    The zim file can be splitted (.zimaa, .zimab, ...).
-   *                    In this case, the file path must still point to the
-   *                    unsplitted path as if the file were not splitted
-   *                    (.zim extesion).
-   */
   Reader(const string zimFilePath);
   ~Reader();
 
-  /**
-   * Get the number of "displayable" entries in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'text/html' MIMEtype specified in the metadata.
-   *         Else return the number of entries in the 'A' namespace.
-   */
+  void reset();
   unsigned int getArticleCount() const;
-
-  /**
-   * Get the number of media in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'image/jpeg', 'image/gif' and 'image/png' in
-   *         the metadata.
-   *         Else return the number of entries in the 'I' namespace.
-   */
   unsigned int getMediaCount() const;
-
-  /**
-   * Get the number of all entries in the zim file.
-   *
-   * @return Return the number of all the entries, whatever their MIMEtype or
-   *         their namespace.
-   */
   unsigned int getGlobalCount() const;
-
-  /**
-   * Get the path of the zim file.
-   *
-   * @return the path of the zim file as given in the constructor.
-   */
   string getZimFilePath() const;
-
-  /**
-   * Get the Id of the zim file.
-   *
-   * @return The uuid stored in the zim file.
-   */
   string getId() const;
-
-  /**
-   * Get the url of a random page.
-   *
-   * Deprecated : Use `getRandomPage` instead.
-   *
-   * @return Url of a random page. The page is picked from all entries in
-   *         the 'A' namespace.
-   *         The main page is excluded from the potential results.
-   */
-  DEPRECATED string getRandomPageUrl() const;
-
-  /**
-   * Get a random page.
-   *
-   * @return A random Entry. The entry is picked from all entries in
-   *         the 'A' namespace.
-   *         The main entry is excluded from the potential results.
-   */
-  Entry getRandomPage() const;
-
-  /**
-   * Get the url of the first page.
-   *
-   * Deprecated : Use `getFirstPage` instead.
-   *
-   * @return Url of the first entry in the 'A' namespace.
-   */
-  DEPRECATED string getFirstPageUrl() const;
-
-  /**
-   * Get the entry of the first page.
-   *
-   * @return The first entry in the 'A' namespace.
-   */
-  Entry getFirstPage() const;
-
-  /**
-   * Get the url of the main page.
-   *
-   * Deprecated : Use `getMainPage` instead.
-   *
-   * @return Url of the main page as specified in the zim file.
-   */
-  DEPRECATED string getMainPageUrl() const;
-
-  /**
-   * Get the entry of the main page.
-   *
-   * @return Entry of the main page as specified in the zim file.
-   */
-  Entry getMainPage() const;
-
-  /**
-   * Get the content of a metadata.
-   *
-   * @param[in]  name The name of the metadata.
-   * @param[out] value The value will be set to the content of the metadata.
-   * @return True if it was possible to get the content of the metadata.
-   */
-  bool getMetadata(const string& name, string& value) const;
-
-  /**
-   * Get the name of the zim file.
-   *
-   * @return The name of the zim file as specified in the zim metadata.
-   */
-  string getName() const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
+  string getRandomPageUrl() const;
+  string getFirstPageUrl() const;
+  string getMainPageUrl() const;
+  bool getMetatag(const string& url, string& content) const;
   string getTitle() const;
-
-  /**
-   * Get the creator of the zim file.
-   *
-   * @return The creator of the zim file as specified in the zim metadata.
-   */
-  string getCreator() const;
-
-  /**
-   * Get the publisher of the zim file.
-   *
-   * @return The publisher of the zim file as specified in the zim metadata.
-   */
-  string getPublisher() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
   string getDescription() const;
-
-  /**
-   * Get the long description of the zim file.
-   *
-   * @return The long description of the zim file as specifed in the zim metadata.
-   */
-  string getLongDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
   string getLanguage() const;
-
-  /**
-   * Get the license of the zim file.
-   *
-   * @return The license of the zim file as specified in the zim metadata.
-   */
-  string getLicense() const;
-
-  /**
-   * Get the tags of the zim file.
-   *
-   * @param original If true, return the original tags as specified in the zim metadata.
-   *                 Else, try to convert it to the new 'normalized' format.
-   * @return The tags of the zim file.
-   */
-  string getTags(bool original=false) const;
-
-  /**
-   * Get the value (as a string) of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   */
-  string getTagStr(const std::string& tagName) const;
-
-  /**
-   * Get the boolean value of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The boolean value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   *         std::domain_error if the value of the tag cannot be convert to bool.
-   */
-  bool getTagBool(const std::string& tagName) const;
-
-  /**
-   * Get the relations of the zim file.
-   *
-   * @return The relation of the zim file as specified in the zim metadata.
-   */
-  string getRelation() const;
-
-  /**
-   * Get the flavour of the zim file.
-   *
-   * @return The flavour of the zim file as specified in the zim metadata.
-   */
-  string getFlavour() const;
-
-  /**
-   * Get the source of the zim file.
-   *
-   * @return The source of the zim file as specified in the zim metadata.
-   */
-  string getSource() const;
-
-  /**
-   * Get the scraper of the zim file.
-   *
-   * @return The scraper of the zim file as specified in the zim metadata.
-   */
-  string getScraper() const;
-
-  /**
-   * Get the origId of the zim file.
-   *
-   * The origId is only used in the case of patch zim file and is the Id
-   * of the original zim file.
-   *
-   * @return The origId of the zim file as specified in the zim metadata.
-   */
+  string getName() const;
+  string getTags() const;
+  string getDate() const;
+  string getCreator() const;
+  string getPublisher() const;
   string getOrigId() const;
-
-  /**
-   * Get the favicon of the zim file.
-   *
-   * @param[out] content The content of the favicon.
-   * @param[out] mimeType The mimeType of the favicon.
-   * @return True if a favicon has been found.
-   */
   bool getFavicon(string& content, string& mimeType) const;
-
-  /**
-   * Get an entry associated to an path.
-   *
-   * @param path The path of the entry.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromPath(const std::string& path) const;
-
-  /**
-   * Get an entry associated to an url encoded path.
-   *
-   * Equivalent to `getEntryFromPath(urlDecode(path));`
-   *
-   * @param path The url encoded path.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromEncodedPath(const std::string& path) const;
-
-  /**
-   * Get un entry associated to a title.
-   *
-   * @param title The title.
-   * @return The entry
-   * throw NoEntry If no entry correspond to the url.
-   */
-  Entry getEntryFromTitle(const std::string& title) const;
-
-  /**
-   * Get the url of a page specified by a title.
-   *
-   * @param[in] title the title of the page.
-   * @param[out] url the url of the page.
-   * @return True if the page can be found.
-   */
-  DEPRECATED bool getPageUrlFromTitle(const string& title, string& url) const;
-
-  /**
-   * Get the mimetype of a entry specified by a url.
-   *
-   * @param[in] url the url of the entry.
-   * @param[out] mimeType the mimeType of the entry.
-   * @return True if the mimeType has been found.
-   */
-  DEPRECATED bool getMimeTypeByUrl(const string& url, string& mimeType) const;
-
-  /**
-   * Get the content of an entry specifed by a url.
-   *
-   * Alias to `getContentByEncodedUrl`
-   */
-  DEPRECATED bool getContentByUrl(const string& url,
+  bool getPageUrlFromTitle(const string& title, string& url) const;
+  bool getMimeTypeByUrl(const string& url, string& mimeType) const;
+  bool getContentByUrl(const string& url,
                        string& content,
                        string& title,
                        unsigned int& contentLength,
                        string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url encoded url.
-   *
-   * Equivalent to getContentByDecodedUrl(urlDecode(url), ...).
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
+  bool getContentByEncodedUrl(const string& url,
                               string& content,
                               string& title,
                               unsigned int& contentLength,
                               string& contentType,
                               string& baseUrl) const;
-
-  /**
-   * Get the content of an entry specified by an url encoded url.
-   *
-   * Equivalent to getContentByEncodedUrl but without baseUrl.
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
+  bool getContentByEncodedUrl(const string& url,
                               string& content,
                               string& title,
                               unsigned int& contentLength,
                               string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * @param[in] url The url of the entry.
-   * @param[out] content The content of the entry.
-   * @param[out] title the title of the entry.
-   * @param[out] contentLength The size of the entry (size of content).
-   * @param[out] contentType The mimeType of the entry.
-   * @param[out] baseUrl Return the true url of the entry.
-   *                     If the specified entry is a redirection, contains
-   *                     the url of the targeted entry.
-   * @return True if the entry has been found.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
+  bool getContentByDecodedUrl(const string& url,
                               string& content,
                               string& title,
                               unsigned int& contentLength,
                               string& contentType,
                               string& baseUrl) const;
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * Equivalent to getContentByDecodedUrl but withou the baseUrl.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
+  bool getContentByDecodedUrl(const string& url,
                               string& content,
                               string& title,
                               unsigned int& contentLength,
                               string& contentType) const;
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are stored in an internal vector and can be retrieved using
-   * `getNextSuggestion` method.
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param reset If true, remove previous suggestions in the internal vector.
-   *              If false, add suggestions to the internal vector
-   *              (until internal vector size is suggestionCount (or no more
-   *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
-   */
   bool searchSuggestions(const string& prefix,
                          unsigned int suggestionsCount,
                          const bool reset = true);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   */
   bool searchSuggestionsSmart(const string& prefix,
                               unsigned int suggestionsCount);
-
-  /**
-   * Check if the url exists in the zim file.
-   *
-   * Deprecated : Use `pathExists` instead.
-   *
-   * @param url the url to check.
-   * @return True if the url exits in the zim file.
-   */
-  DEPRECATED bool urlExists(const string& url) const;
-
-  /**
-   * Check if the path exists in the zim file.
-   *
-   * @param path the path to check.
-   * @return True if the path exists in the zim file.
-   */
-  bool pathExists(const string& path) const;
-
-  /**
-   * Check if the zim file has a embedded fulltext index.
-   *
-   * @return True if the zim file has a embedded fulltext index
-   *         and is not split (else the fulltext is not accessible).
-   */
+  bool urlExists(const string& url) const;
   bool hasFulltextIndex() const;
-
-  /**
-   * Get potential case title variations for a title.
-   *
-   * @param title a title.
-   * @return the list of variantions.
-   */
   std::vector<std::string> getTitleVariants(const std::string& title) const;
-
-  /**
-   * Get the next suggestion title.
-   *
-   * @param[out] title the title of the suggestion.
-   * @return True if title has been set.
-   */
   bool getNextSuggestion(string& title);
-
-  /**
-   * Get the next suggestion title and url.
-   *
-   * @param[out] title the title of the suggestion.
-   * @param[out] url the url of the suggestion.
-   * @return True if title and url have been set.
-   */
   bool getNextSuggestion(string& title, string& url);
-
-  /**
-   * Get if we can check zim file integrity (has a checksum).
-   *
-   * @return True if zim file have a checksum.
-   */
   bool canCheckIntegrity() const;
-
-  /**
-   * Check is zim file is corrupted.
-   *
-   * @return True if zim file is corrupted.
-   */
   bool isCorrupted() const;
-
-  /**
-   * Parse a full url into a namespace and url.
-   *
-   * @param[in] url The full url ("/N/url").
-   * @param[out] ns The namespace (N).
-   * @param[out] title The url (url).
-   * @return True
-   */
-  DEPRECATED bool parseUrl(const string& url, char* ns, string& title) const;
-
-  /**
-   * Return the total size of the zim file.
-   *
-   * If zim file is split, return the sum of all parts' size.
-   *
-   * @return Size of the size file is KiB.
-   */
+  bool parseUrl(const string& url, char* ns, string& title) const;
   unsigned int getFileSize() const;
-
-  /**
-   * Get the zim file handler.
-   *
-   * @return The libzim file handler.
-   */
   zim::File* getZimFileHandler() const;
-
-  /**
-   * Get the zim article object associated to a url.
-   *
-   * @param[in] url The url of the article.
-   * @param[out] article The libzim article object.
-   * @return True if the url is good (article.good()).
-   */
-  DEPRECATED bool getArticleObjectByDecodedUrl(const string& url,
+  bool getArticleObjectByDecodedUrl(const string& url,
                                     zim::Article& article) const;
 
  protected:
   zim::File* zimFileHandler;
   zim::size_type firstArticleOffset;
   zim::size_type lastArticleOffset;
+  zim::size_type currentArticleOffset;
   zim::size_type nsACount;
   zim::size_type nsICount;
   std::string zimFilePath;

include/search_renderer.h

@@ -1,85 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCH_RENDERER_H
-#define KIWIX_SEARCH_RENDERER_H
-
-#include <string>
-
-namespace kiwix
-{
-
-class Searcher;
-class NameMapper;
-/**
- * The SearcherRenderer class is used to render a search result to a html page.
- */
-class SearchRenderer
-{
- public:
-  /**
-   * The default constructor.
-   *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
-   */
-  SearchRenderer(Searcher* searcher, NameMapper* mapper);
-
-  ~SearchRenderer();
-
-  void setSearchPattern(const std::string& pattern);
-
-  /**
-   * Set the search content id.
-   */
-  void setSearchContent(const std::string& name);
-
-  /**
-   * Set protocol prefix.
-   */
-  void setProtocolPrefix(const std::string& prefix);
-
-  /**
-   * Set search protocol prefix.
-   */
-  void setSearchProtocolPrefix(const std::string& prefix);
-
-  /**
-   * Generate the html page with the resutls of the search.
-   */
-  std::string getHtml();
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  Searcher* mp_searcher;
-  NameMapper* mp_nameMapper;
-  std::string searchContent;
-  std::string searchPattern;
-  std::string protocolPrefix;
-  std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int resultEnd;
-};
-
-
-}
-
-#endif

include/searcher.h

@@ -29,8 +29,8 @@
 #include <string>
 #include <vector>
 #include <vector>
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
+#include "common/pathTools.h"
+#include "common/stringTools.h"
 #include "kiwix_config.h"
 
 using namespace std;
@@ -53,98 +53,31 @@ class Result
 };
 
 struct SearcherInternal;
-/**
- * The Searcher class is reponsible to do different kind of search using the
- * fulltext index.
- */
 class Searcher
 {
  public:
-  /**
-   * The default constructor.
-   */
   Searcher();
-
+  Searcher(const string& xapianDirectoryPath,
+           Reader* reader,
+           const string& humanReadableName);
   ~Searcher();
 
-  /**
-   * Add a reader (containing embedded fulltext index) to the search.
-   *
-   * @param reader The Reader for the zim containing the fulltext index.
-   * @return true if the reader has been added.
-   *         false if the reader cannot be added (no embedded fulltext index present)
-   */
-  bool add_reader(Reader* reader);
-
-
-  Reader* get_reader(int index);
-
-  /**
-   * Start a search on the zim associated to the Searcher.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param search The search query.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void search(const std::string& search,
+  void add_reader(Reader* reader, const std::string& humanReaderName);
+  void search(std::string& search,
               unsigned int resultStart,
               unsigned int resultEnd,
               const bool verbose = false);
-
-  /**
-   * Start a geographique search.
-   * The search return result for entry in a disc of center latitude/longitude
-   * and radius distance.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param latitude The latitude of the center point.
-   * @param longitude The longitude of the center point.
-   * @param distance The radius of the disc.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void geo_search(float latitude, float longitude, float distance,
-                  unsigned int resultStart,
-                  unsigned int resultEnd,
-                  const bool verbose = false);
-
-  /**
-   * Start a suggestion search.
-   * The search made depend of the "version" of the embedded index.
-   *  - If the index is newer enough and have a title namespace, the search is
-   *    made in the titles only.
-   *  - Else the search is made on the whole article content.
-   * In any case, the search is made "partial" (as adding '*' at the end of the query)
-   *
-   * @param search The search query.
-   * @param verbose print some info on stdout if true.
-   */
   void suggestions(std::string& search, const bool verbose = false);
-
-  /**
-   * Get the next result of a started search.
-   * This is the method to use to loop hover the search results.
-   */
   Result* getNextResult();
-
-  /**
-   * Restart the previous search.
-   * Next call to getNextResult will return the first result.
-   */
   void restart_search();
-
-  /**
-   * Get a estimation of the result count.
-   */
   unsigned int getEstimatedResultCount();
+  bool setProtocolPrefix(const std::string prefix);
+  bool setSearchProtocolPrefix(const std::string prefix);
+  void reset();
 
-  unsigned int getResultStart() { return resultStart; }
-  unsigned int getResultEnd() { return resultEnd; }
+#ifdef ENABLE_CTPP2
+  string getHtml();
+#endif
 
  protected:
   std::string beautifyInteger(const unsigned int number);
@@ -155,18 +88,17 @@ class Searcher
                      const bool verbose = false);
 
   std::vector<Reader*> readers;
+  std::vector<std::string> humanReaderNames;
   SearcherInternal* internal;
   std::string searchPattern;
+  std::string protocolPrefix;
+  std::string searchProtocolPrefix;
+  unsigned int resultCountPerPage;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
   unsigned int resultEnd;
-
- private:
-  void reset();
-
+  std::string contentHumanReadableId;
 };
-
-
 }
 
 #endif

include/server.h

@@ -1,78 +0,0 @@
-/*
- * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SERVER_H
-#define KIWIX_SERVER_H
-
-#include <string>
-#include <memory>
-
-namespace kiwix
-{
-  class Library;
-  class NameMapper;
-  class InternalServer;
-
-  class Server {
-     public:
-       /**
-        * The default constructor.
-        *
-        * @param library The library to serve.
-        */
-       Server(Library* library, NameMapper* nameMapper=nullptr);
-
-       virtual ~Server();
-
-       /**
-        * Serve the content.
-        */
-       bool start();
-
-       /**
-        * Stop the daemon.
-        */
-       void stop();
-
-       void setRoot(const std::string& root);
-       void setAddress(const std::string& addr) { m_addr = addr; }
-       void setPort(int port) { m_port = port; }
-       void setNbThreads(int threads) { m_nbThreads = threads; }
-       void setVerbose(bool verbose) { m_verbose = verbose; }
-       void setTaskbar(bool withTaskbar, bool withLibraryButton)
-        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
-       void setBlockExternalLinks(bool blockExternalLinks)
-        { m_blockExternalLinks = blockExternalLinks; }
-
-     protected:
-       Library* mp_library;
-       NameMapper* mp_nameMapper;
-       std::string m_root = "";
-       std::string m_addr = "";
-       int m_port = 80;
-       int m_nbThreads = 1;
-       bool m_verbose = false;
-       bool m_withTaskbar = true;
-       bool m_withLibraryButton = true;
-       bool m_blockExternalLinks = false;
-       std::unique_ptr<InternalServer> mp_server;
-  };
-}
-
-#endif

include/tools/base64.h

@@ -1,4 +0,0 @@
-#include <string>
-
-std::string base64_encode(const std::string& inString);
-std::string base64_decode(const std::string& s);

include/tools/lock.h

@@ -1,46 +0,0 @@
-
-
-#ifndef KIWIXLIB_TOOL_LOCK_H
-#define KIWIXLIB_TOOL_LOCK_H
-
-#include <pthread.h>
-
-namespace kiwix {
-
-class Lock
-{
-  public:
-    explicit Lock(pthread_mutex_t* mutex) :
-      mp_mutex(mutex)
-    {
-      pthread_mutex_lock(mp_mutex);
-    }
-    ~Lock() {
-      if (mp_mutex != nullptr) {
-        pthread_mutex_unlock(mp_mutex);
-      }
-    }
-    Lock(Lock && other) :
-      mp_mutex(other.mp_mutex)
-    {
-      other.mp_mutex = nullptr;
-    }
-    Lock & operator=(Lock && other)
-    {
-      mp_mutex = other.mp_mutex;
-      other.mp_mutex = nullptr;
-      return *this;
-    }
-
-
-  private:
-    pthread_mutex_t* mp_mutex;
-
-    Lock(Lock const &) = delete;
-    Lock & operator=(Lock const &) = delete;
-};
-
-
-}
-
-#endif //KIWIXLIB_TOOL_LOCK_H

include/tools/otherTools.h

@@ -1,46 +0,0 @@
-/*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_OTHERTOOLS_H
-#define KIWIX_OTHERTOOLS_H
-
-#include <string>
-#include <vector>
-
-namespace pugi {
-  class xml_node;
-}
-
-namespace kiwix
-{
-  void sleep(unsigned int milliseconds);
-  std::string nodeToString(const pugi::xml_node& node);
-  std::string converta2toa3(const std::string& a2code);
-
-  /*
-   * Convert all format tag string to new format
-   */
-  std::vector<std::string> convertTags(const std::string& tags_str);
-  std::string getTagValueFromTagList(const std::vector<std::string>& tagList,
-                                     const std::string& tagName);
-  bool convertStrToBool(const std::string& value);
-
-}
-
-#endif

include/tools/pathTools.h

@@ -1,48 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_PATHTOOLS_H
-#define KIWIX_PATHTOOLS_H
-
-#include <string>
-
-#ifdef _WIN32
-std::string WideToUtf8(const std::wstring& wstr);
-std::wstring Utf8ToWide(const std::string& str);
-#endif
-bool isRelativePath(const std::string& path);
-std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
-std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
-std::string removeLastPathElement(const std::string& path);
-std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
-
-unsigned int getFileSize(const std::string& path);
-std::string getFileSizeAsString(const std::string& path);
-std::string getFileContent(const std::string& path);
-bool fileExists(const std::string& path);
-bool makeDirectory(const std::string& path);
-std::string makeTmpDirectory();
-bool copyFile(const std::string& sourcePath, const std::string& destPath);
-std::string getLastPathElement(const std::string& path);
-std::string getExecutablePath(bool realPathOnly = false);
-std::string getCurrentDirectory();
-std::string getDataDirectory();
-bool writeTextFile(const std::string& path, const std::string& content);
-std::string getMimeTypeForFile(const std::string& filename);
-#endif

include/xapianSearcher.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_XAPIAN_SEARCHER_H
+#define KIWIX_XAPIAN_SEARCHER_H
+
+#include <xapian.h>
+#include "reader.h"
+#include "searcher.h"
+
+#include <map>
+#include <string>
+
+using namespace std;
+
+namespace kiwix
+{
+class XapianSearcher;
+
+class XapianResult : public Result
+{
+ public:
+  XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
+  virtual ~XapianResult(){};
+
+  virtual std::string get_url();
+  virtual std::string get_title();
+  virtual int get_score();
+  virtual std::string get_snippet();
+  virtual std::string get_content();
+  virtual int get_wordCount();
+  virtual int get_size();
+  virtual int get_readerIndex() { return 0; };
+
+ private:
+  XapianSearcher* searcher;
+  Xapian::MSetIterator iterator;
+  Xapian::Document document;
+};
+
+class NoXapianIndexInZim : public exception
+{
+  virtual const char* what() const throw()
+  {
+    return "There is no fulltext index in the zim file";
+  }
+};
+
+class XapianSearcher
+{
+  friend class XapianResult;
+
+ public:
+  XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
+  virtual ~XapianSearcher(){};
+  void searchInIndex(string& search,
+                     const unsigned int resultStart,
+                     const unsigned int resultEnd,
+                     const bool verbose = false);
+  virtual Result* getNextResult();
+  void restart_search();
+
+  Xapian::MSet results;
+
+ protected:
+  void closeIndex();
+  void openIndex(const string& xapianDirectoryPath);
+  void setup_queryParser();
+
+  Reader* reader;
+  Xapian::Database readableDatabase;
+  std::string language;
+  std::string stopwords;
+  Xapian::QueryParser queryParser;
+  Xapian::Stem stemmer;
+  Xapian::SimpleStopper stopper;
+  Xapian::MSetIterator current_result;
+  std::map<std::string, int> valuesmap;
+};
+}
+
+#endif

meson.build

@@ -1,60 +1,100 @@
 project('kiwix-lib', 'cpp',
-  version : '9.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '1.0.2',
   license : 'GPL',
-  default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
+  default_options : ['c_std=c11', 'cpp_std=c++11'])
 
 compiler = meson.get_compiler('cpp')
+find_library_in_compiler = meson.version().version_compare('>=0.31.0')
 
-wrapper = get_option('wrapper')
-
-static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
-if 'android' in wrapper
-  extra_libs = ['-llog']
-else
-  extra_libs = []
-endif
-
-if 'java' in wrapper
-  add_languages('java')
-endif
+static_deps = get_option('android') or get_option('default_library') == 'static'
 
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=3.0.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
-libcurl_dep = dependency('libcurl', static:static_deps)
-microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
 
-if not compiler.has_header('mustache.hpp')
-  error('Cannot found header mustache.hpp')
+ctpp2_include_path = ''
+has_ctpp2_dep = false
+ctpp2_prefix_install = get_option('ctpp2-install-prefix')
+ctpp2_link_args = []
+if ctpp2_prefix_install == ''
+  if compiler.has_header('ctpp2/CTPP2Logger.hpp')
+    if find_library_in_compiler
+      ctpp2_lib = compiler.find_library('ctpp2')
+    else
+      ctpp2_lib = find_library('ctpp2')
+    endif
+    ctpp2_link_args = ['-lctpp2']
+    if meson.is_cross_build() and host_machine.system() == 'windows'
+      if find_library_in_compiler
+        iconv_lib = compiler.find_library('iconv', required:false)
+      else
+        iconv_lib = find_library('iconv', required:false)
+      endif
+      if iconv_lib.found()
+        ctpp2_link_args += ['-liconv']
+      endif
+    endif
+    has_ctpp2_dep = true
+    ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
+  else
+    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
+  endif
+else
+  if not find_library_in_compiler
+    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
+  endif
+  ctpp2_include_path = ctpp2_prefix_install + '/include'
+  ctpp2_include_args = ['-I'+ctpp2_include_path]
+  if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
+    ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
+    ctpp2_lib_path = ctpp2_prefix_install+'/lib'
+    ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
+    ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
+    if meson.is_cross_build() and host_machine.system() == 'windows'
+      iconv_lib = compiler.find_library('iconv', required:false)
+      if iconv_lib.found()
+        ctpp2_link_args += ['-liconv']
+      endif
+    endif
+    has_ctpp2_dep = true
+    ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
+  else
+    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
+  endif
 endif
 
-extra_cflags = ''
-if target_machine.system() == 'windows' and static_deps
-  add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
-  extra_cflags += '-DCURL_STATICLIB'
-endif
+xapian_dep = dependency('xapian-core', required:false, static:static_deps)
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
+all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep]
+if has_ctpp2_dep
+  all_deps += [ctpp2_dep]
+endif
 
 inc = include_directories('include')
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
-
-if build_machine.system() == 'windows'
-     extra_link_args = ['-lshlwapi', '-lwinmm']
-else
-     extra_link_args = []
-endif
+conf.set('ENABLE_CTPP2', has_ctpp2_dep)
 
 subdir('include')
 subdir('scripts')
 subdir('static')
 subdir('src')
-subdir('test')
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml']
+if xapian_dep.found()
+    pkg_requires += ['xapian-core']
+endif
+
+extra_libs = []
+extra_cflags = ''
+if has_ctpp2_dep
+  extra_libs += ctpp2_link_args
+  if ctpp2_include_path != ''
+    extra_cflags = '-I'+ctpp2_include_path
+  endif
+endif
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,2 +1,4 @@
-option('wrapper', type:'array', choices:['java', 'android'], value:[],
-  description: 'The wrapper to generate.')
+option('ctpp2-install-prefix', type : 'string', value : '',
+  description : 'Prefix where ctpp libs has been installed')
+option('android', type : 'boolean', value : false,
+  description : 'Do we make a kiwix-lib for android')

scripts/ctpp2c.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+
+ctpp2c=$1
+SOURCE=$(pwd)/$2
+DEST=$3
+
+$ctpp2c $SOURCE $DEST

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-
+    
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,11 +95,11 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-
+    
 
 
 master_c_template = """//This file is automaically generated. Do not modify it.
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-
+    
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
-
+ 
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

scripts/meson.build

@@ -1,4 +1,5 @@
 
 res_compiler = find_program('kiwix-compile-resources')
+intermediate_ctpp2c = find_program('ctpp2c.sh')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))

src/android/gen_kiwix.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_PATH=$(pwd)
+
+javac -d $BUILD_PATH/src/android $1 $2 $3 $4
+
+cd $BUILD_PATH/src/android
+javah -jni org.kiwix.kiwixlib.JNIKiwix
+cd $BUILD_PATH

src/android/kiwix.cpp

@@ -0,0 +1,559 @@
+/*
+ * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <jni.h>
+#include "org_kiwix_kiwixlib_JNIKiwix.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include <iostream>
+#include <string>
+
+#include "common/base64.h"
+#include "reader.h"
+#include "searcher.h"
+#include "unicode/putil.h"
+
+#include <android/log.h>
+#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, "kiwix", __VA_ARGS__)
+
+#include <xapian.h>
+#include <zim/article.h>
+#include <zim/error.h>
+#include <zim/file.h>
+#include <zim/zim.h>
+
+/* global variables */
+kiwix::Reader* reader = NULL;
+kiwix::Searcher* searcher = NULL;
+
+static pthread_mutex_t readerLock = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t searcherLock = PTHREAD_MUTEX_INITIALIZER;
+
+/* c2jni type conversion functions */
+jboolean c2jni(const bool& val)
+{
+  return val ? JNI_TRUE : JNI_FALSE;
+}
+jstring c2jni(const std::string& val, JNIEnv* env)
+{
+  return env->NewStringUTF(val.c_str());
+}
+
+jint c2jni(const int val)
+{
+  return (jint)val;
+}
+jint c2jni(const unsigned val)
+{
+  return (unsigned)val;
+}
+/* jni2c type conversion functions */
+bool jni2c(const jboolean& val)
+{
+  return val == JNI_TRUE;
+}
+std::string jni2c(const jstring& val, JNIEnv* env)
+{
+  return std::string(env->GetStringUTFChars(val, 0));
+}
+
+int jni2c(const jint val)
+{
+  return (int)val;
+}
+/* Method to deal with variable passed by reference */
+void setStringObjValue(const std::string& value, const jobject obj, JNIEnv* env)
+{
+  jclass objClass = env->GetObjectClass(obj);
+  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
+  env->SetObjectField(obj, objFid, c2jni(value, env));
+}
+
+void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
+{
+  jclass objClass = env->GetObjectClass(obj);
+  jfieldID objFid = env->GetFieldID(objClass, "value", "I");
+  env->SetIntField(obj, objFid, value);
+}
+
+void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
+{
+  jclass objClass = env->GetObjectClass(obj);
+  jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
+  env->SetIntField(obj, objFid, c2jni(value));
+}
+
+/* Kiwix library functions */
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv* env, jobject obj)
+{
+  jstring url;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cUrl = reader->getMainPageUrl();
+      url = c2jni(cUrl, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM main page" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return url;
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv* env,
+                                                                 jobject obj)
+{
+  jstring id;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cId = reader->getId();
+      id = c2jni(cId, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM id" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return id;
+}
+
+JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv* env,
+                                                                    jobject obj)
+{
+  jint size;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      int cSize = reader->getFileSize();
+      size = c2jni(cSize);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM file size" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return size;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv* env, jobject obj)
+{
+  jstring creator;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cCreator = reader->getCreator();
+      creator = c2jni(cCreator, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM creator" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return creator;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv* env, jobject obj)
+{
+  jstring publisher;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cPublisher = reader->getPublisher();
+      publisher = c2jni(cPublisher, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM creator" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return publisher;
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv* env,
+                                                                   jobject obj)
+{
+  jstring name;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cName = reader->getName();
+      name = c2jni(cName, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM name" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return name;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv* env, jobject obj)
+{
+  jstring favicon;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cContent;
+      std::string cMime;
+      reader->getFavicon(cContent, cMime);
+      favicon
+          = c2jni(base64_encode(
+                      reinterpret_cast<const unsigned char*>(cContent.c_str()),
+                      cContent.length()),
+                  env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM favicon" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return favicon;
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv* env,
+                                                                   jobject obj)
+{
+  jstring date;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cDate = reader->getDate();
+      date = c2jni(cDate, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM date" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return date;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv* env, jobject obj)
+{
+  jstring language;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cLanguage = reader->getLanguage();
+      language = c2jni(cLanguage, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM language" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return language;
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(
+    JNIEnv* env, jobject obj, jstring url)
+{
+  jstring mimeType;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    std::string cUrl = jni2c(url, env);
+    try {
+      std::string cMimeType;
+      reader->getMimeTypeByUrl(cUrl, cMimeType);
+      mimeType = c2jni(cMimeType, env);
+    } catch (...) {
+      std::cerr << "Unable to get mime-type for url " << cUrl << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return mimeType;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv* env, jobject obj, jstring path)
+{
+  jboolean retVal = JNI_TRUE;
+  std::string cPath = jni2c(path, env);
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      delete reader;
+    }
+    reader = new kiwix::Reader(cPath);
+  } catch (...) {
+    std::cerr << "Unable to load ZIM " << cPath << std::endl;
+    reader = NULL;
+    retVal = JNI_FALSE;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(
+    JNIEnv* env, jobject obj, jstring url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
+{
+  /* Default values */
+  setStringObjValue("", titleObj, env);
+  setStringObjValue("", mimeTypeObj, env);
+  setIntObjValue(0, sizeObj, env);
+  jbyteArray data = env->NewByteArray(0);
+
+  /* Retrieve the content */
+  if (reader != NULL) {
+    std::string cUrl = jni2c(url, env);
+    std::string cData;
+    std::string cTitle;
+    std::string cMimeType;
+    unsigned int cSize = 0;
+
+    pthread_mutex_lock(&readerLock);
+    try {
+      if (reader->getContentByUrl(cUrl, cData, cTitle, cSize, cMimeType)) {
+        data = env->NewByteArray(cSize);
+        env->SetByteArrayRegion(
+            data, 0, cSize, reinterpret_cast<const jbyte*>(cData.c_str()));
+        setStringObjValue(cMimeType, mimeTypeObj, env);
+        setStringObjValue(cTitle, titleObj, env);
+        setIntObjValue(cSize, sizeObj, env);
+      }
+    } catch (...) {
+      std::cerr << "Unable to get content for url " << cUrl << std::endl;
+    }
+    pthread_mutex_unlock(&readerLock);
+  }
+
+  return data;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions(
+    JNIEnv* env, jobject obj, jstring prefix, jint count)
+{
+  jboolean retVal = JNI_FALSE;
+  std::string cPrefix = jni2c(prefix, env);
+  unsigned int cCount = jni2c(count);
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      if (reader->searchSuggestionsSmart(cPrefix, cCount)) {
+        retVal = JNI_TRUE;
+      }
+    }
+  } catch (...) {
+    std::cerr << "Unable to search suggestions for pattern " << cPrefix
+              << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion(
+    JNIEnv* env, jobject obj, jobject titleObj)
+{
+  jboolean retVal = JNI_FALSE;
+  std::string cTitle;
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      if (reader->getNextSuggestion(cTitle)) {
+        setStringObjValue(cTitle, titleObj, env);
+        retVal = JNI_TRUE;
+      }
+    }
+  } catch (...) {
+    std::cerr << "Unable to get next suggestion" << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle(
+    JNIEnv* env, jobject obj, jstring title, jobject urlObj)
+{
+  jboolean retVal = JNI_FALSE;
+  std::string cTitle = jni2c(title, env);
+  std::string cUrl;
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      if (reader->getPageUrlFromTitle(cTitle, cUrl)) {
+        setStringObjValue(cUrl, urlObj, env);
+        retVal = JNI_TRUE;
+      }
+    }
+  } catch (...) {
+    std::cerr << "Unable to get URL for title " << cTitle << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle(
+    JNIEnv* env, jobject obj, jobject titleObj)
+{
+  jboolean retVal = JNI_FALSE;
+  std::string cTitle;
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      std::string cTitle = reader->getTitle();
+      setStringObjValue(cTitle, titleObj, env);
+      retVal = JNI_TRUE;
+    }
+  } catch (...) {
+    std::cerr << "Unable to get ZIM title" << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT jstring JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv* env, jobject obj)
+{
+  jstring description;
+
+  pthread_mutex_lock(&readerLock);
+  if (reader != NULL) {
+    try {
+      std::string cDescription = reader->getDescription();
+      description = c2jni(cDescription, env);
+    } catch (...) {
+      std::cerr << "Unable to get ZIM description" << std::endl;
+    }
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return description;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage(
+    JNIEnv* env, jobject obj, jobject urlObj)
+{
+  jboolean retVal = JNI_FALSE;
+  std::string cUrl;
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    if (reader != NULL) {
+      std::string cUrl = reader->getRandomPageUrl();
+      setStringObjValue(cUrl, urlObj, env);
+      retVal = JNI_TRUE;
+    }
+  } catch (...) {
+    std::cerr << "Unable to get random page" << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+
+  return retVal;
+}
+
+JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
+    JNIEnv* env, jobject obj, jstring dirStr)
+{
+  std::string cPath = jni2c(dirStr, env);
+
+  pthread_mutex_lock(&readerLock);
+  try {
+    u_setDataDirectory(cPath.c_str());
+  } catch (...) {
+    std::cerr << "Unable to set data directory " << cPath << std::endl;
+  }
+  pthread_mutex_unlock(&readerLock);
+}
+
+JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(
+    JNIEnv* env, jobject obj, jstring path)
+{
+  jboolean retVal = JNI_TRUE;
+  std::string cPath = jni2c(path, env);
+
+  pthread_mutex_lock(&searcherLock);
+  try {
+    if (searcher != NULL) {
+      delete searcher;
+    }
+    if (!reader || !reader->hasFulltextIndex()) {
+      // Use old API (no embedded full text index).
+      searcher = new kiwix::Searcher(cPath, reader, "");
+    } else {
+      // Use the new API. We don't care about the human readable name as
+      // we don't use it (in android).
+      searcher = new kiwix::Searcher();
+      searcher->add_reader(reader, "");
+    }
+  } catch (...) {
+    searcher = NULL;
+    retVal = JNI_FALSE;
+    std::cerr << "Unable to load full text index " << cPath << std::endl;
+  }
+  pthread_mutex_unlock(&searcherLock);
+
+  return retVal;
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery(
+    JNIEnv* env, jclass obj, jstring query, jint count)
+{
+  std::string cQuery = jni2c(query, env);
+  unsigned int cCount = jni2c(count);
+  kiwix::Result* p_result;
+  std::string result;
+
+  pthread_mutex_lock(&searcherLock);
+  try {
+    if (searcher != NULL) {
+      searcher->search(cQuery, 0, count);
+      while ((p_result = searcher->getNextResult())
+             && !(p_result->get_title().empty())
+             && !(p_result->get_url().empty())) {
+        result += p_result->get_title() + "\n";
+        delete p_result;
+      }
+    }
+  } catch (...) {
+    std::cerr << "Unable to make indexed query " << cQuery << std::endl;
+  }
+  pthread_mutex_unlock(&searcherLock);
+
+  return env->NewStringUTF(result.c_str());
+}

src/android/meson.build

@@ -0,0 +1,17 @@
+
+jni_generator = find_program('gen_kiwix.sh')
+
+kiwix_jni = custom_target('jni',
+  input: ['org/kiwix/kiwixlib/JNIKiwix.java',
+          'org/kiwix/kiwixlib/JNIKiwixInt.java',
+          'org/kiwix/kiwixlib/JNIKiwixString.java',
+          'org/kiwix/kiwixlib/JNIKiwixBool.java'],
+  output: ['org_kiwix_kiwixlib_JNIKiwix.h'],
+  command:[jni_generator, '@INPUT@']
+)
+
+kiwix_sources += ['android/kiwix.cpp', kiwix_jni]
+
+install_subdir('org', install_dir: 'kiwix-lib/java')
+install_subdir('res', install_dir: 'kiwix-lib')
+install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/android/org/kiwix/kiwixlib/JNIKiwix.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+package org.kiwix.kiwixlib;
+
+import org.kiwix.kiwixlib.JNIKiwixString;
+import org.kiwix.kiwixlib.JNIKiwixBool;
+import org.kiwix.kiwixlib.JNIKiwixInt;
+
+public class JNIKiwix
+{
+  static { System.loadLibrary("kiwix"); }
+  public native String getMainPage();
+
+  public native String getId();
+
+  public native String getLanguage();
+
+  public native String getMimeType(String url);
+
+  public native boolean loadZIM(String path);
+
+  public native boolean loadFulltextIndex(String path);
+
+  public native byte[] getContent(String url, JNIKiwixString title, JNIKiwixString mimeType, JNIKiwixInt size);
+
+  public native boolean searchSuggestions(String prefix, int count);
+
+  public native boolean getNextSuggestion(JNIKiwixString title);
+
+  public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
+
+  public native boolean getTitle(JNIKiwixString title);
+
+  public native String getDescription();
+
+  public native String getDate();
+
+  public native String getFavicon();
+
+  public native String getCreator();
+
+  public native String getPublisher();
+
+  public native String getName();
+
+  public native int getFileSize();
+
+  public native int getArticleCount();
+
+  public native int getMediaCount();
+
+  public native boolean getRandomPage(JNIKiwixString url);
+
+  public native void setDataDirectory(String icuDataDir);
+
+  public static native String indexedQuery(String db, int count);
+}

src/android/org/kiwix/kiwixlib/JNIKiwixString.java

@@ -22,16 +22,4 @@ package org.kiwix.kiwixlib;
 public class JNIKiwixString
 {
   public String value;
-
-  public JNIKiwixString(String value) {
-    this.value = value;
-  }
-
-  public JNIKiwixString() {
-    this("");
-  }
-
-  public String getValue() {
-    return value;
-  }
 }

src/aria2.cpp

@@ -1,265 +0,0 @@
-
-
-#include "aria2.h"
-#include "xmlrpc.h"
-#include <iostream>
-#include <sstream>
-#include <thread>
-#include <chrono>
-#include <tools/otherTools.h>
-#include <tools/pathTools.h>
-#include <tools/stringTools.h>
-#include <downloader.h> // For AriaError
-
-#ifdef _WIN32
-# define ARIA2_CMD "aria2c.exe"
-#else
-# define ARIA2_CMD "aria2c"
-# include <unistd.h>
-#endif
-
-
-
-namespace kiwix {
-
-Aria2::Aria2():
-  mp_aria(nullptr),
-  m_port(42042),
-  m_secret("kiwixariarpc"),
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
-{
-  m_downloadDir = getDataDirectory();
-  makeDirectory(m_downloadDir);
-  std::vector<const char*> callCmd;
-
-  std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
-  std::string download_dir = "--dir=" + getDataDirectory();
-  std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
-  std::string session = "--save-session=" + session_file;
-  std::string inputFile = "--input-file=" + session_file;
-//  std::string log_dir = "--log=\"" + logDir + "\"";
-#ifdef _WIN32
-  int pid = GetCurrentProcessId();
-#else
-  pid_t pid = getpid();
-#endif
-  std::string stop_with_pid = "--stop-with-process=" + to_string(pid);
-  std::string rpc_secret = "--rpc-secret=" + m_secret;
-  m_secret = "token:"+m_secret;
-
-  std::string aria2cmd = appendToDirectory(
-    removeLastPathElement(getExecutablePath(true)),
-    ARIA2_CMD);
-  if (fileExists(aria2cmd)) {
-    // A local aria2c exe exists (packaged with kiwix-desktop), use it.
-    callCmd.push_back(aria2cmd.c_str());
-  } else {
-    // Try to use a potential installed aria2c.
-    callCmd.push_back(ARIA2_CMD);
-  }
-  callCmd.push_back("--enable-rpc");
-  callCmd.push_back(rpc_secret.c_str());
-  callCmd.push_back(rpc_port.c_str());
-  callCmd.push_back(download_dir.c_str());
-  if (fileExists(session_file)) {
-    callCmd.push_back(inputFile.c_str());
-  }
-  callCmd.push_back(session.c_str());
-//  callCmd.push_back(log_dir.c_str());
-  callCmd.push_back("--auto-save-interval=10");
-  callCmd.push_back(stop_with_pid.c_str());
-  callCmd.push_back("--allow-overwrite=true");
-  callCmd.push_back("--dht-entry-point=router.bittorrent.com:6881");
-  callCmd.push_back("--dht-entry-point6=router.bittorrent.com:6881");
-  callCmd.push_back("--quiet=true");
-  callCmd.push_back("--bt-enable-lpd=true");
-  callCmd.push_back("--always-resume=true");
-  callCmd.push_back("--max-concurrent-downloads=42");
-  callCmd.push_back("--rpc-max-request-size=6M");
-  callCmd.push_back("--file-allocation=none");
-  std::string launchCmd;
-  for (auto &cmd : callCmd) {
-      launchCmd.append(cmd).append(" ");
-  }
-  mp_aria = Subprocess::run(callCmd);
-  mp_curl = curl_easy_init();
-  char errbuf[CURL_ERROR_SIZE];
-  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
-  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
-  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
-
-  int watchdog = 50;
-  while(--watchdog) {
-    sleep(10);
-    errbuf[0] = 0;
-    auto res = curl_easy_perform(mp_curl);
-    if (res == CURLE_OK) {
-      break;
-    } else if (watchdog == 1) {
-      std::cerr <<" curl_easy_perform() failed." << std::endl;
-      fprintf(stderr, "\nlibcurl: (%d) ", res);
-      if (errbuf[0] != 0) {
-        std::cerr << errbuf << std::endl;
-      } else {
-        std::cerr << curl_easy_strerror(res) << std::endl;
-      }
-    }
-  }
-  if (!watchdog) {
-    curl_easy_cleanup(mp_curl);
-    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
-  }
-}
-
-Aria2::~Aria2()
-{
-  curl_easy_cleanup(mp_curl);
-}
-
-void Aria2::close()
-{
-  saveSession();
-  shutdown();
-}
-
-size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
-{
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
-  return nmemb;
-}
-
-std::string Aria2::doRequest(const MethodCall& methodCall)
-{
-  pthread_mutex_lock(&m_lock);
-  auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
-  CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
-  res = curl_easy_perform(mp_curl);
-  if (res == CURLE_OK) {
-    long response_code;
-    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
-    pthread_mutex_unlock(&m_lock);
-    if (response_code != 200) {
-      throw std::runtime_error("Invalid return code from aria");
-    }
-    auto responseContent = stringstream.str();
-    MethodResponse response(responseContent);
-    if (response.isFault()) {
-      throw AriaError(response.getFault().getFaultString());
-    }
-    return responseContent;
-  }
-  pthread_mutex_unlock(&m_lock);
-  throw std::runtime_error("Cannot perform request");
-}
-
-std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
-{
-  MethodCall methodCall("aria2.addUri", m_secret);
-  auto uriParams = methodCall.newParamValue().getArray();
-  for (auto& uri : uris) {
-    uriParams.addValue().set(uri);
-  }
-  for (auto& option : options) {
-    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
-  }
-  auto ret = doRequest(methodCall);
-  MethodResponse response(ret);
-  return response.getParamValue(0).getAsS();
-}
-
-std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::string>& statusKey)
-{
-  MethodCall methodCall("aria2.tellStatus", m_secret);
-  methodCall.newParamValue().set(gid);
-  if (!statusKey.empty()) {
-    auto statusArray = methodCall.newParamValue().getArray();
-    for (auto& key : statusKey) {
-      statusArray.addValue().set(key);
-    }
-  }
-  return doRequest(methodCall);
-}
-
-std::vector<std::string> Aria2::tellActive()
-{
-  MethodCall methodCall("aria2.tellActive", m_secret);
-  auto statusArray = methodCall.newParamValue().getArray();
-  statusArray.addValue().set(std::string("gid"));
-  auto responseContent = doRequest(methodCall);
-  MethodResponse response(responseContent);
-  std::vector<std::string> activeGID;
-  int index = 0;
-  while(true) {
-    try {
-      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
-      auto gidNode = structNode.getMember("gid");
-      activeGID.push_back(gidNode.getValue().getAsS());
-    } catch (InvalidRPCNode& e) { break; }
-  }
-  return activeGID;
-}
-
-std::vector<std::string> Aria2::tellWaiting()
-{
-  MethodCall methodCall("aria2.tellWaiting", m_secret);
-  methodCall.newParamValue().set(0);
-  methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
-  auto statusArray = methodCall.newParamValue().getArray();
-  statusArray.addValue().set(std::string("gid"));
-  auto responseContent = doRequest(methodCall);
-  MethodResponse response(responseContent);
-  std::vector<std::string> waitingGID;
-  int index = 0;
-  while(true) {
-    try {
-      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
-      auto gidNode = structNode.getMember("gid");
-      waitingGID.push_back(gidNode.getValue().getAsS());
-    } catch (InvalidRPCNode& e) { break; }
-  }
-  return waitingGID;
-}
-
-void Aria2::saveSession()
-{
-  MethodCall methodCall("aria2.saveSession", m_secret);
-  doRequest(methodCall);
-  std::cout << "session saved" << std::endl;
-}
-
-void Aria2::shutdown()
-{
-  MethodCall methodCall("aria2.shutdown", m_secret);
-  doRequest(methodCall);
-}
-
-void Aria2::pause(const std::string& gid)
-{
-    MethodCall methodCall("aria2.pause", m_secret);
-    methodCall.newParamValue().set(gid);
-    doRequest(methodCall);
-}
-
-void Aria2::unpause(const std::string& gid)
-{
-    MethodCall methodCall("aria2.unpause", m_secret);
-    methodCall.newParamValue().set(gid);
-    doRequest(methodCall);
-}
-
-void Aria2::remove(const std::string& gid)
-{
-    MethodCall methodCall("aria2.remove", m_secret);
-    methodCall.newParamValue().set(gid);
-    doRequest(methodCall);
-}
-
-} // end namespace kiwix

src/aria2.h

@@ -1,50 +0,0 @@
-
-
-#ifndef KIWIXLIB_ARIA2_H_
-#define KIWIXLIB_ARIA2_H_
-
-#ifdef _WIN32
-// winsock2.h need to be included before windows.h (included by curl.h)
-# include <winsock2.h>
-#endif
-
-#include "subprocess.h"
-#include "xmlrpc.h"
-
-#include <memory>
-#include <curl/curl.h>
-#include <pthread.h>
-
-namespace kiwix {
-
-class Aria2
-{
-  private:
-    std::unique_ptr<Subprocess> mp_aria;
-    int m_port;
-    std::string m_secret;
-    std::string m_downloadDir;
-    CURL* mp_curl;
-    pthread_mutex_t m_lock;
-
-    std::string doRequest(const MethodCall& methodCall);
-
-  public:
-    Aria2();
-    virtual ~Aria2();
-    void close();
-
-    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
-    std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
-    std::vector<std::string> tellActive();
-    std::vector<std::string> tellWaiting();
-    void saveSession();
-    void shutdown();
-    void pause(const std::string& gid);
-    void unpause(const std::string& gid);
-    void remove(const std::string& gid);
-};
-
-}; //end namespace kiwix
-
-#endif // KIWIXLIB_ARIA2_H_

src/book.cpp

@@ -1,223 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "book.h"
-#include "reader.h"
-
-#include "tools/base64.h"
-#include "tools/regexTools.h"
-#include "tools/networkTools.h"
-#include "tools/otherTools.h"
-
-#include <pugixml.hpp>
-
-namespace kiwix
-{
-/* Constructor */
-Book::Book() :
-  m_pathValid(false),
-  m_readOnly(false)
-{
-}
-/* Destructor */
-Book::~Book()
-{
-}
-
-bool Book::update(const kiwix::Book& other)
-{
-  if (m_readOnly)
-    return false;
-
-  if (m_id != other.m_id)
-    return false;
-
-  m_readOnly = other.m_readOnly;
-  m_path = other.m_path;
-  m_pathValid = other.m_pathValid;
-  m_title = other.m_title;
-  m_description = other.m_description;
-  m_language = other.m_language;
-  m_creator = other.m_creator;
-  m_publisher = other.m_publisher;
-  m_date = other.m_date;
-  m_url = other.m_url;
-  m_name = other.m_name;
-  m_flavour = other.m_flavour;
-  m_tags = other.m_tags;
-  m_origId = other.m_origId;
-  m_articleCount = other.m_articleCount;
-  m_mediaCount = other.m_mediaCount;
-  m_size = other.m_size;
-  m_favicon = other.m_favicon;
-  m_faviconMimeType = other.m_faviconMimeType;
-  m_faviconUrl = other.m_faviconUrl;
-
-  m_downloadId = other.m_downloadId;
-
-  return true;
-}
-
-void Book::update(const kiwix::Reader& reader)
-{
-  m_path = reader.getZimFilePath();
-  m_pathValid = true;
-  m_id = reader.getId();
-  m_title = reader.getTitle();
-  m_description = reader.getDescription();
-  m_language = reader.getLanguage();
-  m_creator = reader.getCreator();
-  m_publisher = reader.getPublisher();
-  m_date = reader.getDate();
-  m_name = reader.getName();
-  m_flavour = reader.getFlavour();
-  m_tags = reader.getTags();
-  m_origId = reader.getOrigId();
-  m_articleCount = reader.getArticleCount();
-  m_mediaCount = reader.getMediaCount();
-  m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
-  m_pathValid = true;
-
-  reader.getFavicon(m_favicon, m_faviconMimeType);
-}
-
-#define ATTR(name) node.attribute(name).value()
-void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
-{
-  m_id = ATTR("id");
-  std::string path = ATTR("path");
-  if (isRelativePath(path)) {
-    path = computeAbsolutePath(baseDir, path);
-  }
-  m_path = path;
-  m_pathValid = fileExists(path);
-  m_title = ATTR("title");
-  m_description = ATTR("description");
-  m_language = ATTR("language");
-  m_creator = ATTR("creator");
-  m_publisher = ATTR("publisher");
-  m_date = ATTR("date");
-  m_url = ATTR("url");
-  m_name = ATTR("name");
-  m_flavour = ATTR("flavour");
-  m_tags = ATTR("tags");
-  m_origId = ATTR("origId");
-  m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
-  m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
-  m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  m_favicon = base64_decode(ATTR("favicon"));
-  m_faviconMimeType = ATTR("faviconMimeType");
-  m_faviconUrl = ATTR("faviconUrl");
-  try {
-    m_downloadId = ATTR("downloadId");
-  } catch(...) {}
-}
-#undef ATTR
-
-
-static std::string fromOpdsDate(const std::string& date)
-{
-  //The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
-  //and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
-  return date.substr(0, 10);
-}
-
-
-#define VALUE(name) node.child(name).child_value()
-void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
-{
-  m_id = VALUE("id");
-  if (!m_id.compare(0, 9, "urn:uuid:")) {
-    m_id.erase(0, 9);
-  }
-  // No path on opds.
-  m_title = VALUE("title");
-  m_description = VALUE("summary");
-  m_language = VALUE("language");
-  m_creator = node.child("author").child("name").child_value();
-  m_publisher = node.child("publisher").child("name").child_value();
-  m_date = fromOpdsDate(VALUE("updated"));
-  m_name = VALUE("name");
-  m_flavour = VALUE("flavour");
-  m_tags = VALUE("tags");
-  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
-  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
-  for(auto linkNode = node.child("link"); linkNode;
-           linkNode = linkNode.next_sibling("link")) {
-    std::string rel = linkNode.attribute("rel").value();
-
-    if (rel == "http://opds-spec.org/acquisition/open-access") {
-      m_url = linkNode.attribute("href").value();
-      m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
-    }
-    if (rel == "http://opds-spec.org/image/thumbnail") {
-      m_faviconUrl = urlHost + linkNode.attribute("href").value();
-      m_faviconMimeType = linkNode.attribute("type").value();
-    }
- }
-
-}
-#undef VALUE
-
-std::string Book::getHumanReadableIdFromPath() const
-{
-  std::string id = m_path;
-  if (!id.empty()) {
-    kiwix::removeAccents(id);
-
-#ifdef _WIN32
-    id = replaceRegex(id, "", "^.*\\\\");
-#else
-    id = replaceRegex(id, "", "^.*/");
-#endif
-
-    id = replaceRegex(id, "", "\\.zim[a-z]*$");
-    id = replaceRegex(id, "_", " ");
-    id = replaceRegex(id, "plus", "\\+");
-  }
-  return id;
-}
-
-void Book::setPath(const std::string& path)
-{
- m_path = isRelativePath(path)
-   ? computeAbsolutePath(getCurrentDirectory(), path)
-   : path;
-}
-
-const std::string& Book::getFavicon() const {
-  if (m_favicon.empty() && !m_faviconUrl.empty()) {
-    try {
-      m_favicon = download(m_faviconUrl);
-    } catch(...) {
-      std::cerr << "Cannot download favicon from " << m_faviconUrl;
-    }
-  }
-  return m_favicon;
-}
-
-std::string Book::getTagStr(const std::string& tagName) const {
-  return getTagValueFromTagList(convertTags(m_tags), tagName);
-}
-
-bool Book::getTagBool(const std::string& tagName) const {
-  return convertStrToBool(getTagStr(tagName));
-}
-
-}

src/bookmark.cpp

@@ -1,47 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "bookmark.h"
-
-#include <pugixml.hpp>
-
-namespace kiwix
-{
-/* Constructor */
-Bookmark::Bookmark()
-{
-}
-
-/* Destructor */
-Bookmark::~Bookmark()
-{
-}
-
-void Bookmark::updateFromXml(const pugi::xml_node& node)
-{
-  auto bookNode = node.child("book");
-  m_bookId = bookNode.child("id").child_value();
-  m_bookTitle = bookNode.child("title").child_value();
-  m_language = bookNode.child("language").child_value();
-  m_date = bookNode.child("date").child_value();
-  m_title = node.child("title").child_value();
-  m_url = node.child("url").child_value();
-}
-
-}

src/common/base64.cpp

@@ -1,4 +1,4 @@
-/*
+/* 
    base64.cpp and base64.h
 
    Copyright (C) 2004-2008 René Nyffenegger
@@ -24,10 +24,10 @@
    René Nyffenegger rene.nyffenegger@adp-gmbh.ch
 */
 
-#include <tools/base64.h>
+#include <common/base64.h>
 #include <iostream>
 
-static const std::string base64_chars =
+static const std::string base64_chars = 
              "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
              "abcdefghijklmnopqrstuvwxyz"
   "0123456789+/";
@@ -37,10 +37,8 @@ static inline bool is_base64(unsigned char c) {
   return (isalnum(c) || (c == '+') || (c == '/'));
 }
 
-std::string base64_encode(const std::string& inString) {
+std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) {
   std::string ret;
-  auto in_len = inString.size();
-  const unsigned char* bytes_to_encode = reinterpret_cast<const unsigned char*>(inString.data());
   int i = 0;
   int j = 0;
   unsigned char char_array_3[3];

src/common/networkTools.cpp

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <common/networkTools.h>
+
+std::map<std::string, std::string> kiwix::getNetworkInterfaces()
+{
+  std::map<std::string, std::string> interfaces;
+
+#ifdef _WIN32
+  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
+  if (sd == SOCKET_ERROR) {
+    std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
+              << std::endl;
+    return interfaces;
+  }
+
+  INTERFACE_INFO InterfaceList[20];
+  unsigned long nBytesReturned;
+  if (WSAIoctl(sd,
+               SIO_GET_INTERFACE_LIST,
+               0,
+               0,
+               &InterfaceList,
+               sizeof(InterfaceList),
+               &nBytesReturned,
+               0,
+               0)
+      == SOCKET_ERROR) {
+    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
+              << std::endl;
+    return interfaces;
+  }
+
+  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
+  for (int i = 0; i < nNumInterfaces; ++i) {
+    sockaddr_in* pAddress;
+    pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
+
+    /* Add to the map */
+    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
+    std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
+    interfaces.insert(
+        std::pair<std::string, std::string>(interfaceName, interfaceIp));
+  }
+#else
+  /* Get Network interfaces information */
+  char buf[16384];
+  struct ifconf ifconf;
+  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
+  ifconf.ifc_len = sizeof buf;
+  ifconf.ifc_buf = buf;
+  if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
+    perror("ioctl(SIOCGIFCONF)");
+    exit(EXIT_FAILURE);
+  }
+
+  /* Go through each interface */
+  int i;
+  size_t len;
+  struct ifreq* ifreq;
+  ifreq = ifconf.ifc_req;
+  for (i = 0; i < ifconf.ifc_len;) {
+    if (ifreq->ifr_addr.sa_family == AF_INET) {
+      /* Get the network interface ip */
+      char host[128] = {0};
+      const int error = getnameinfo(&(ifreq->ifr_addr),
+                                    sizeof ifreq->ifr_addr,
+                                    host,
+                                    sizeof host,
+                                    0,
+                                    0,
+                                    NI_NUMERICHOST);
+      if (!error) {
+        std::string interfaceName = std::string(ifreq->ifr_name);
+        std::string interfaceIp = std::string(host);
+        /* Add to the map */
+        interfaces.insert(
+            std::pair<std::string, std::string>(interfaceName, interfaceIp));
+      } else {
+        perror("getnameinfo()");
+      }
+    }
+
+/* some systems have ifr_addr.sa_len and adjust the length that
+ * way, but not mine. weird */
+#ifndef __linux__
+    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
+#else
+    len = sizeof *ifreq;
+#endif
+    ifreq = (struct ifreq*)((char*)ifreq + len);
+    i += len;
+  }
+#endif
+  return interfaces;
+}
+
+std::string kiwix::getBestPublicIp()
+{
+  std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
+
+#ifndef _WIN32
+  const char* const prioritizedNames[]
+      = {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
+  const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
+  for (int i = 0; i < count; ++i) {
+    std::map<std::string, std::string>::const_iterator it
+        = interfaces.find(prioritizedNames[i]);
+    if (it != interfaces.end()) {
+      return it->second;
+    }
+  }
+#endif
+
+  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
+       iter != interfaces.end();
+       ++iter) {
+    std::string interfaceIp = iter->second;
+    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
+      return interfaceIp;
+    }
+  }
+
+  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
+       iter != interfaces.end();
+       ++iter) {
+    std::string interfaceIp = iter->second;
+    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
+      return interfaceIp;
+    }
+  }
+
+  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
+       iter != interfaces.end();
+       ++iter) {
+    std::string interfaceIp = iter->second;
+    if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
+      return interfaceIp;
+    }
+  }
+
+  return "127.0.0.1";
+}

src/common/otherTools.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,14 +17,13 @@
  * MA 02110-1301, USA.
  */
 
-#ifndef KIWIX_NETWORKTOOLS_H
-#define KIWIX_NETWORKTOOLS_H
+#include <common/otherTools.h>
 
-#include <string>
-
-namespace kiwix
+void kiwix::sleep(unsigned int milliseconds)
 {
-std::string download(const std::string& url);
-}
-
+#ifdef _WIN32
+  Sleep(milliseconds);
+#else
+  usleep(1000 * milliseconds);
 #endif
+}

src/common/pathTools.cpp

@@ -0,0 +1,274 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <common/pathTools.h>
+
+#ifdef __APPLE__
+#include <limits.h>
+#include <mach-o/dyld.h>
+#elif _WIN32
+#include <direct.h>
+#include <windows.h>
+#include "shlwapi.h"
+#define getcwd _getcwd  // stupid MSFT "deprecation" warning
+#endif
+
+#ifdef _WIN32
+#else
+#include <unistd.h>
+#endif
+
+#ifdef _WIN32
+#define SEPARATOR "\\"
+#else
+#define SEPARATOR "/"
+#include <unistd.h>
+#endif
+
+#include <stdlib.h>
+
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
+bool isRelativePath(const string& path)
+{
+#ifdef _WIN32
+  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+#else
+  return path.empty() || path.substr(0, 1) == "/" ? false : true;
+#endif
+}
+
+string computeRelativePath(const string path, const string absolutePath)
+{
+  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
+  std::vector<std::string> absolutePathParts
+      = kiwix::split(absolutePath, SEPARATOR);
+
+  unsigned int commonCount = 0;
+  while (commonCount < pathParts.size()
+         && commonCount < absolutePathParts.size()
+         && pathParts[commonCount] == absolutePathParts[commonCount]) {
+    if (!pathParts[commonCount].empty()) {
+      commonCount++;
+    }
+  }
+
+  string relativePath;
+#ifdef _WIN32
+  /* On Windows you have a token more because the root is represented
+     by a letter */
+  if (commonCount == 0) {
+    relativePath = "../";
+  }
+#endif
+
+  for (unsigned int i = commonCount; i < pathParts.size(); i++) {
+    relativePath += "../";
+  }
+  for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
+    relativePath += absolutePathParts[i];
+    relativePath += i + 1 < absolutePathParts.size() ? "/" : "";
+  }
+
+  return relativePath;
+}
+
+/* Warning: the relative path must be with slashes */
+string computeAbsolutePath(const string path, const string relativePath)
+{
+  string absolutePath;
+
+  if (path.empty()) {
+    char* path = NULL;
+    size_t size = 0;
+
+#ifdef _WIN32
+    path = _getcwd(path, size);
+#else
+    path = getcwd(path, size);
+#endif
+
+    absolutePath = string(path) + SEPARATOR;
+  } else {
+    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
+                       ? path
+                       : path + SEPARATOR;
+  }
+
+#if _WIN32
+  char* cRelativePath = _strdup(relativePath.c_str());
+#else
+  char* cRelativePath = strdup(relativePath.c_str());
+#endif
+  char* token = strtok(cRelativePath, "/");
+
+  while (token != NULL) {
+    if (string(token) == "..") {
+      absolutePath = removeLastPathElement(absolutePath, true, false);
+      token = strtok(NULL, "/");
+    } else if (strcmp(token, ".") && strcmp(token, "")) {
+      absolutePath += string(token);
+      token = strtok(NULL, "/");
+      if (token != NULL) {
+        absolutePath += SEPARATOR;
+      }
+    } else {
+      token = strtok(NULL, "/");
+    }
+  }
+
+  return absolutePath;
+}
+
+string removeLastPathElement(const string path,
+                             const bool removePreSeparator,
+                             const bool removePostSeparator)
+{
+  string newPath = path;
+  size_t offset = newPath.find_last_of(SEPARATOR);
+  if (removePreSeparator &&
+#ifndef _WIN32
+      offset != newPath.find_first_of(SEPARATOR) &&
+#endif
+      offset == newPath.length() - 1) {
+    newPath = newPath.substr(0, offset);
+    offset = newPath.find_last_of(SEPARATOR);
+  }
+  newPath = removePostSeparator ? newPath.substr(0, offset)
+                                : newPath.substr(0, offset + 1);
+  return newPath;
+}
+
+string appendToDirectory(const string& directoryPath, const string& filename)
+{
+  string newPath = directoryPath + SEPARATOR + filename;
+  return newPath;
+}
+
+string getLastPathElement(const string& path)
+{
+  return path.substr(path.find_last_of(SEPARATOR) + 1);
+}
+
+unsigned int getFileSize(const string& path)
+{
+#ifdef _WIN32
+  struct _stat filestatus;
+  _stat(path.c_str(), &filestatus);
+#else
+  struct stat filestatus;
+  stat(path.c_str(), &filestatus);
+#endif
+
+  return filestatus.st_size / 1024;
+}
+
+string getFileSizeAsString(const string& path)
+{
+  ostringstream convert;
+  convert << getFileSize(path);
+  return convert.str();
+}
+
+bool fileExists(const string& path)
+{
+#ifdef _WIN32
+  return PathFileExists(path.c_str());
+#else
+  bool flag = false;
+  fstream fin;
+  fin.open(path.c_str(), ios::in);
+  if (fin.is_open()) {
+    flag = true;
+  }
+  fin.close();
+  return flag;
+#endif
+}
+
+bool makeDirectory(const string& path)
+{
+#ifdef _WIN32
+  int status = _mkdir(path.c_str());
+#else
+  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+#endif
+  return status == 0;
+}
+
+/* Try to create a link and if does not work then make a copy */
+bool copyFile(const string& sourcePath, const string& destPath)
+{
+  try {
+#ifndef _WIN32
+    if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
+#endif
+      std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
+      std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
+      outfile << infile.rdbuf();
+#ifndef _WIN32
+    }
+#endif
+  } catch (exception& e) {
+    cerr << e.what() << endl;
+    return false;
+  }
+
+  return true;
+}
+
+string getExecutablePath()
+{
+  char binRootPath[PATH_MAX];
+
+#ifdef _WIN32
+  GetModuleFileName(NULL, binRootPath, PATH_MAX);
+  return std::string(binRootPath);
+#elif __APPLE__
+  uint32_t max = (uint32_t)PATH_MAX;
+  _NSGetExecutablePath(binRootPath, &max);
+  return std::string(binRootPath);
+#else
+  ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
+  if (size != -1) {
+    return std::string(binRootPath, size);
+  }
+#endif
+
+  return "";
+}
+
+bool writeTextFile(const string& path, const string& content)
+{
+  std::ofstream file;
+  file.open(path.c_str());
+  file << content;
+  file.close();
+  return true;
+}
+
+string getCurrentDirectory()
+{
+  char* a_cwd = getcwd(NULL, 0);
+  string s_cwd(a_cwd);
+  free(a_cwd);
+  return s_cwd;
+}

src/common/regexTools.cpp

@@ -17,48 +17,41 @@
  * MA 02110-1301, USA.
  */
 
-#include <tools/regexTools.h>
-#include <tools/lock.h>
+#include <common/regexTools.h>
 
-#include <unicode/regex.h>
-#include <unicode/ucnv.h>
+std::map<std::string, RegexMatcher*> regexCache;
 
-#include <memory>
-#include <map>
-#include <stdexcept>
-#include <pthread.h>
-
-std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;
-static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
-
-std::unique_ptr<icu::RegexMatcher> buildMatcher(const std::string& regex, icu::UnicodeString& content)
+RegexMatcher* buildRegex(const std::string& regex)
 {
-  std::shared_ptr<icu::RegexPattern> pattern;
+  RegexMatcher* matcher;
+  std::map<std::string, RegexMatcher*>::iterator itr = regexCache.find(regex);
+
   /* Regex is in cache */
-  try {
-    pattern = regexCache.at(regex);
-  } catch (std::out_of_range&) {
-    // Redo the search with a lock to avoid race condition.
-    kiwix::Lock l(&regexLock);
-    try {
-      pattern = regexCache.at(regex);
-    } catch (std::out_of_range&) {
-      UErrorCode status = U_ZERO_ERROR;
-      UParseError pe;
-      icu::UnicodeString uregex(regex.c_str());
-      pattern.reset(icu::RegexPattern::compile(uregex, UREGEX_CASE_INSENSITIVE, pe, status));
-      regexCache[regex] = pattern;
-    }
+  if (itr != regexCache.end()) {
+    matcher = itr->second;
   }
-  UErrorCode status = U_ZERO_ERROR;
-  return std::unique_ptr<icu::RegexMatcher>(pattern->matcher(content, status));
+
+  /* Regex needs to be parsed (and cached) */
+  else {
+    UErrorCode status = U_ZERO_ERROR;
+    UnicodeString uregex = UnicodeString(regex.c_str());
+    matcher = new RegexMatcher(uregex, UREGEX_CASE_INSENSITIVE, status);
+    regexCache[regex] = matcher;
+  }
+
+  return matcher;
 }
 
+/* todo */
+void freeRegexCache()
+{
+}
 bool matchRegex(const std::string& content, const std::string& regex)
 {
   ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
+  UnicodeString ucontent = UnicodeString(content.c_str());
+  RegexMatcher* matcher = buildRegex(regex);
+  matcher->reset(ucontent);
   return matcher->find();
 }
 
@@ -67,24 +60,27 @@ std::string replaceRegex(const std::string& content,
                          const std::string& regex)
 {
   ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ureplacement(replacement.c_str());
-  icu::UnicodeString ucontent(content.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
+  UnicodeString ucontent = UnicodeString(content.c_str());
+  UnicodeString ureplacement = UnicodeString(replacement.c_str());
+  RegexMatcher* matcher = buildRegex(regex);
+  matcher->reset(ucontent);
   UErrorCode status = U_ZERO_ERROR;
-  auto uresult = matcher->replaceAll(ureplacement, status);
+  UnicodeString uresult = matcher->replaceAll(ureplacement, status);
   std::string tmp;
   uresult.toUTF8String(tmp);
   return tmp;
 }
 
 std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
+                                   const std::string regex,
                                    const std::string& replacement)
 {
   ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
+  UnicodeString ucontent = UnicodeString(content.c_str());
+  UnicodeString ureplacement = UnicodeString(replacement.c_str());
+  RegexMatcher* matcher = buildRegex(regex);
+  matcher->reset(ucontent);
+
   if (matcher->find()) {
     UErrorCode status = U_ZERO_ERROR;
     ucontent.insert(matcher->end(status), ureplacement);

src/common/stringTools.cpp

@@ -17,9 +17,8 @@
  * MA 02110-1301, USA.
  */
 
-#include <tools/stringTools.h>
+#include <common/stringTools.h>
 
-#include <tools/pathTools.h>
 #include <unicode/normlzr.h>
 #include <unicode/rep.h>
 #include <unicode/translit.h>
@@ -27,10 +26,6 @@
 #include <unicode/uniset.h>
 #include <unicode/ustring.h>
 
-
-#include <iostream>
-#include <iomanip>
-
 /* tell ICU where to find its dat file (tables) */
 void kiwix::loadICUExternalTables()
 {
@@ -38,10 +33,10 @@ void kiwix::loadICUExternalTables()
   std::string executablePath = getExecutablePath();
   std::string executableDirectory = removeLastPathElement(executablePath);
   std::string datPath
-      = computeAbsolutePath(executableDirectory, "icudt58l.dat");
+      = computeAbsolutePath(executableDirectory, "icudt49l.dat");
   try {
     u_setDataDirectory(datPath.c_str());
-  } catch (std::exception& e) {
+  } catch (exception& e) {
     std::cerr << e.what() << std::endl;
   }
 #endif
@@ -52,9 +47,9 @@ std::string kiwix::removeAccents(const std::string& text)
   loadICUExternalTables();
   ucnv_setDefaultName("UTF-8");
   UErrorCode status = U_ZERO_ERROR;
-  auto removeAccentsTrans = icu::Transliterator::createInstance(
+  Transliterator* removeAccentsTrans = Transliterator::createInstance(
       "Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
-  icu::UnicodeString ustring(text.c_str());
+  UnicodeString ustring = UnicodeString(text.c_str());
   removeAccentsTrans->transliterate(ustring);
   delete removeAccentsTrans;
   std::string unaccentedText;
@@ -62,8 +57,10 @@ std::string kiwix::removeAccents(const std::string& text)
   return unaccentedText;
 }
 
+#ifndef __ANDROID__
+
 /* Prepare integer for display */
-std::string kiwix::beautifyInteger(uint64_t number)
+std::string kiwix::beautifyInteger(const unsigned int number)
 {
   std::stringstream numberStream;
   numberStream << number;
@@ -78,22 +75,17 @@ std::string kiwix::beautifyInteger(uint64_t number)
   return numberString;
 }
 
-std::string kiwix::beautifyFileSize(uint64_t number)
+std::string kiwix::beautifyFileSize(const unsigned int number)
 {
-  std::stringstream ss;
-  ss << std::fixed << std::setprecision(2);
-  if (number>>30)
-    ss << (number/(1024.0*1024*1024)) << " GB";
-  else if (number>>20)
-    ss << (number/(1024.0*1024)) << " MB";
-  else if (number>>10)
-    ss << (number/1024.0) << " KB";
-  else
-    ss << number << " B";
-  return ss.str();
+  if (number > 1024 * 1024) {
+    return kiwix::beautifyInteger(number / (1024 * 1024)) + " GB";
+  } else {
+    return kiwix::beautifyInteger(number / 1024 != 0 ? number / 1024 : 1)
+           + " MB";
+  }
 }
 
-void kiwix::printStringInHexadecimal(icu::UnicodeString s)
+void kiwix::printStringInHexadecimal(UnicodeString s)
 {
   std::cout << std::showbase << std::hex;
   for (int i = 0; i < s.length(); i++) {
@@ -141,168 +133,107 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-/* urlEncode() based on javascript encodeURI() &
-   encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
+// Urlencode
+// based on javascript encodeURIComponent()
 
-bool isReservedUrlChar(char c)
+std::string char2hex(char dec)
 {
-  switch (c) {
-  case ';':
-  case ',':
-  case '/':
-  case '?':
-  case ':':
-  case '@':
-  case '&':
-  case '=':
-  case '+':
-  case '$':
-    return true;
-  default:
-    return false;
+  char dig1 = (dec & 0xF0) >> 4;
+  char dig2 = (dec & 0x0F);
+  if (0 <= dig1 && dig1 <= 9) {
+    dig1 += 48;  // 0,48inascii
   }
+  if (10 <= dig1 && dig1 <= 15) {
+    dig1 += 97 - 10;  // a,97inascii
+  }
+  if (0 <= dig2 && dig2 <= 9) {
+    dig2 += 48;
+  }
+  if (10 <= dig2 && dig2 <= 15) {
+    dig2 += 97 - 10;
+  }
+
+  std::string r;
+  r.append(&dig1, 1);
+  r.append(&dig2, 1);
+  return r;
 }
 
-bool needsEscape(char c, bool encodeReserved)
+std::string kiwix::urlEncode(const std::string& c)
 {
-  if (c >= 'a' && c <= 'z')
-    return false;
-  if (c >= 'A' && c <= 'Z')
-    return false;
-  if (c >= '0' && c <= '9')
-    return false;
-  if (isReservedUrlChar(c))
-    return encodeReserved;
-  switch (c) {
-  case '-':
-  case '_':
-  case '.':
-  case '!':
-  case '~':
-  case '*':
-  case '\'':
-  case '(':
-  case ')':
-    return false;
-  }
-  return true;
-}
-
-int hexToInt(char c) {
-  switch (c) {
-  case '0': return 0;
-  case '1': return 1;
-  case '2': return 2;
-  case '3': return 3;
-  case '4': return 4;
-  case '5': return 5;
-  case '6': return 6;
-  case '7': return 7;
-  case '8': return 8;
-  case '9': return 9;
-  case 'A': case 'a': return 10;
-  case 'B': case 'b': return 11;
-  case 'C': case 'c': return 12;
-  case 'D': case 'd': return 13;
-  case 'E': case 'e': return 14;
-  case 'F': case 'f': return 15;
-  default: return -1;
-  }
-}
-
-std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
-{
-  std::ostringstream os;
-  os << std::hex << std::uppercase;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
-
-    if (!needsEscape(*it, encodeReserved)) {
-      os << *it;
+  std::string escaped = "";
+  int max = c.length();
+  for (int i = 0; i < max; i++) {
+    if ((48 <= c[i] && c[i] <= 57) ||  // 0-9
+        (65 <= c[i] && c[i] <= 90)
+        ||  // abc...xyz
+        (97 <= c[i] && c[i] <= 122)
+        ||  // ABC...XYZ
+        (c[i] == '~' || c[i] == '!' || c[i] == '*' || c[i] == '(' || c[i] == ')'
+         || c[i] == '\'')) {
+      escaped.append(&c[i], 1);
     } else {
-      os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
+      escaped.append("%");
+      escaped.append(char2hex(c[i]));  // converts char 255 to string "ff"
     }
   }
-  return os.str();
+  return escaped;
 }
 
-std::string kiwix::urlDecode(const std::string& value, bool component)
+#endif
+
+static char charFromHex(std::string a)
 {
-  std::ostringstream os;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
+  std::istringstream Blat(a);
+  int Z;
+  Blat >> std::hex >> Z;
+  return char(Z);
+}
 
-    // If there aren't enough characters left for this to be a
-    // valid escape code, just use the character and move on
-    if (it > value.end() - 3) {
-      os << *it;
-      continue;
-    }
-
-    if (*it == '%') {
-      char hi = *(++it);
-      char lo = *(++it);
-      int iHi = hexToInt(hi);
-      int iLo = hexToInt(lo);
-      if (iHi < 0 || iLo < 0) {
-	// Invalid escape sequence
-	os << '%' << hi << lo;
-	continue;
-      }
-      char c = (char)(iHi << 4 | iLo);
-      if (!component && isReservedUrlChar(c)) {
-	os << '%' << hi << lo;
-      } else {
-	os << c;
-      }
-    } else {
-      os << *it;
-    }
+std::string kiwix::urlDecode(const std::string& originalUrl)
+{
+  std::string url = originalUrl;
+  std::string::size_type pos = 0;
+  while ((pos = url.find('%', pos)) != std::string::npos
+         && pos + 2 < url.length()) {
+    url.replace(pos, 3, 1, charFromHex(url.substr(pos + 1, 2)));
+    ++pos;
   }
-
-  return os.str();
+  return url;
 }
 
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
-                                      const std::string& delims,
-                                      bool trimEmpty)
+                                      const std::string& delims = " *-")
 {
-  std::string::size_type lastPos = 0;
-  std::string::size_type pos = 0;
+  std::string::size_type lastPos = str.find_first_not_of(delims, 0);
+  std::string::size_type pos = str.find_first_of(delims, lastPos);
   std::vector<std::string> tokens;
-  while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
-  {
-    auto token = str.substr(lastPos, pos - lastPos);
-    if (!trimEmpty || !token.empty()) {
-      tokens.push_back(token);
-    }
-    lastPos = pos + 1;
+
+  while (std::string::npos != pos || std::string::npos != lastPos) {
+    tokens.push_back(str.substr(lastPos, pos - lastPos));
+    lastPos = str.find_first_not_of(delims, pos);
+    pos = str.find_first_of(delims, lastPos);
   }
 
-  auto token = str.substr(lastPos);
-  if (!trimEmpty || !token.empty()) {
-    tokens.push_back(token);
-  }
   return tokens;
 }
 
-std::string kiwix::join(const std::vector<std::string>& list, const std::string& sep)
+std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
 {
-  std::stringstream ss;
-  bool first = true;
-  for (auto& s:list) {
-    if (!first) {
-      ss << sep;
-    }
-    first = false;
-    ss << s;
-  }
-  return ss.str();
+  const std::string m1(lhs), m2(rhs);
+  return split(m1, m2);
 }
 
+std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
+{
+  return split(lhs, rhs.c_str());
+}
+
+std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
+{
+  return split(lhs.c_str(), rhs);
+}
 
 std::string kiwix::ucFirst(const std::string& word)
 {
@@ -312,8 +243,8 @@ std::string kiwix::ucFirst(const std::string& word)
 
   std::string result;
 
-  icu::UnicodeString unicodeWord(word.c_str());
-  auto unicodeFirstLetter = icu::UnicodeString(unicodeWord, 0, 1).toUpper();
+  UnicodeString unicodeWord(word.c_str());
+  UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toUpper();
   unicodeWord.replace(0, 1, unicodeFirstLetter);
   unicodeWord.toUTF8String(result);
 
@@ -328,7 +259,7 @@ std::string kiwix::ucAll(const std::string& word)
 
   std::string result;
 
-  icu::UnicodeString unicodeWord(word.c_str());
+  UnicodeString unicodeWord(word.c_str());
   unicodeWord.toUpper().toUTF8String(result);
 
   return result;
@@ -342,8 +273,8 @@ std::string kiwix::lcFirst(const std::string& word)
 
   std::string result;
 
-  icu::UnicodeString unicodeWord(word.c_str());
-  auto unicodeFirstLetter = icu::UnicodeString(unicodeWord, 0, 1).toLower();
+  UnicodeString unicodeWord(word.c_str());
+  UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toLower();
   unicodeWord.replace(0, 1, unicodeFirstLetter);
   unicodeWord.toUTF8String(result);
 
@@ -358,7 +289,7 @@ std::string kiwix::lcAll(const std::string& word)
 
   std::string result;
 
-  icu::UnicodeString unicodeWord(word.c_str());
+  UnicodeString unicodeWord(word.c_str());
   unicodeWord.toLower().toUTF8String(result);
 
   return result;
@@ -372,7 +303,7 @@ std::string kiwix::toTitle(const std::string& word)
 
   std::string result;
 
-  icu::UnicodeString unicodeWord(word.c_str());
+  UnicodeString unicodeWord(word.c_str());
   unicodeWord = unicodeWord.toTitle(0);
   unicodeWord.toUTF8String(result);
 
@@ -383,11 +314,3 @@ std::string kiwix::normalize(const std::string& word)
 {
   return kiwix::lcAll(word);
 }
-
-
-bool kiwix::startsWith(const std::string& base, const std::string& start)
-{
-   return start.length() <= base.length()
-        && std::equal(start.begin(), start.end(), base.begin());
-}
-

src/config.h.in

@@ -1,3 +1,4 @@
 
 #mesondefine VERSION
 
+#mesondefine ENABLE_CTPP2

src/ctpp2/CTPP2VMStringLoader.cpp

@@ -0,0 +1,210 @@
+/*
+ * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <ctpp2/CTPP2VMStringLoader.hpp>
+
+namespace CTPP // C++ Template Engine
+{
+
+//
+// Convert byte order
+//
+static void ConvertExecutable(VMExecutable * oCore)
+{
+	// Code entry point
+	oCore -> entry_point = Swap32(oCore -> entry_point);
+	// Offset of code segment
+	oCore -> code_offset = Swap32(oCore -> code_offset);
+
+	// Code segment size
+	oCore -> code_size   = Swap32(oCore -> code_size);
+
+	// Offset of static text segment
+	oCore -> syscalls_offset    = Swap32(oCore -> syscalls_offset);
+	// Static text segment size
+	oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
+
+	// Offset of static text index segment
+	oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
+	// Static text index segment size
+	oCore -> syscalls_index_size   = Swap32(oCore -> syscalls_index_size);
+
+	// Offset of static data segment
+	oCore -> static_data_offset    = Swap32(oCore -> static_data_offset);
+
+	// Static data segment size
+	oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
+
+	// Offset of static text segment
+	oCore -> static_text_offset    = Swap32(oCore -> static_text_offset);
+	// Static text segment size
+	oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
+
+	// Offset of static text index segment
+	oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
+	// Static text index segment size
+	oCore -> static_text_index_size   = Swap32(oCore -> static_text_index_size);
+
+	// Version 2.2+
+	// Offset of static data bit index
+	oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
+	/// Offset of static data bit index
+	oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
+
+	// Platform
+	oCore -> platform      = Swap64(oCore -> platform);
+
+	// Ugly-jolly hack!
+	// ... dereferencing type-punned pointer will break strict-aliasing rules ...
+	UINT_64 iTMP;
+	memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
+	iTMP = Swap64(iTMP);
+	memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
+
+	// Cyclic Redundancy Check
+	oCore -> crc = 0;
+
+	// Convert data structures
+
+	// Convert code segment
+	VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
+	UINT_32 iI = 0;
+	UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
+	for(iI = 0; iI < iSteps; ++iI)
+	{
+		pInstructions -> instruction = Swap32(pInstructions -> instruction);
+		pInstructions -> argument    = Swap32(pInstructions -> argument);
+		pInstructions -> reserved    = Swap64(pInstructions -> reserved);
+		++pInstructions;
+	}
+
+	// Convert syscalls index
+	TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
+	iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
+	for(iI = 0; iI < iSteps; ++iI)
+	{
+		pTextIndex -> offset = Swap32(pTextIndex -> offset);
+		pTextIndex -> length = Swap32(pTextIndex -> length);
+		++pTextIndex;
+	}
+
+	// Convert static text index
+	pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
+	iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
+	for(iI = 0; iI < iSteps; ++iI)
+	{
+		pTextIndex -> offset = Swap32(pTextIndex -> offset);
+		pTextIndex -> length = Swap32(pTextIndex -> length);
+		++pTextIndex;
+	}
+
+	// Convert static data
+	StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
+	iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
+	for(iI = 0; iI < iSteps; ++iI)
+	{
+		(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
+		++pStaticDataVar;
+	}
+}
+
+//
+// Constructor
+//
+VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
+{
+    oCore = (VMExecutable *)malloc(rawContentSize + 1);
+    memcpy(oCore, rawContent, rawContentSize);
+
+	if (oCore -> magic[0] == 'C' &&
+	    oCore -> magic[1] == 'T' &&
+	    oCore -> magic[2] == 'P' &&
+	    oCore -> magic[3] == 'P')
+	{
+		// Check version
+		if (oCore -> version[0] >= 1)
+		{
+			// Platform-dependent data (byte order)
+			if (oCore -> platform == 0x4142434445464748ull)
+			{
+#ifdef _DEBUG
+				fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
+#endif
+
+				// Nothing to do, only check crc
+				UINT_32 iCRC = oCore -> crc;
+				oCore -> crc = 0;
+
+				// Calculate CRC of file
+                // KELSON: next line used to refer to oStat.st_size
+                // changed it to rawContentSize
+				if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
+				{
+					free(oCore);
+					throw CTPPLogicError("CRC checksum invalid");
+				}
+			}
+			// Platform-dependent data (byte order)
+			else if (oCore -> platform == 0x4847464544434241ull)
+			{
+				// Need to reconvert data
+#ifdef _DEBUG
+				fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
+#endif
+				ConvertExecutable(oCore);
+			}
+			else
+			{
+				free(oCore);
+				throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
+			}
+
+			// Check IEEE 754 format
+			if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
+			{
+				free(oCore);
+				throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
+			}
+		}
+
+		pVMMemoryCore = new VMMemoryCore(oCore);
+	}
+	else
+	{
+		free(oCore);
+		throw CTPPLogicError("Not an CTPP bytecode file.");
+	}
+}
+
+//
+// Get ready-to-run program
+//
+const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
+
+//
+// A destructor
+//
+VMStringLoader::~VMStringLoader() throw()
+{
+	delete pVMMemoryCore;
+	free(oCore);
+}
+
+} // namespace CTPP
+// End.

src/downloader.cpp

@@ -1,204 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "downloader.h"
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
-
-#include <algorithm>
-#include <thread>
-#include <chrono>
-
-#include <iostream>
-
-#include "aria2.h"
-#include "xmlrpc.h"
-#include "tools/otherTools.h"
-#include <pugixml.hpp>
-
-namespace kiwix
-{
-
-void Download::updateStatus(bool follow)
-{
-  if (m_status == Download::K_REMOVED)
-    return;
-  static std::vector<std::string> statusKey = {"status", "files", "totalLength",
-                                               "completedLength", "followedBy",
-                                               "downloadSpeed", "verifiedLength"};
-  std::string strStatus;
-  if(follow && !m_followedBy.empty()) {
-    strStatus = mp_aria->tellStatus(m_followedBy, statusKey);
-  } else {
-    strStatus = mp_aria->tellStatus(m_did, statusKey);
-  }
-//  std::cout << strStatus << std::endl;
-  MethodResponse response(strStatus);
-  if (response.isFault()) {
-    m_status = Download::K_UNKNOWN;
-    return;
-  }
-  auto structNode = response.getParams().getParam(0).getValue().getStruct();
-  auto _status = structNode.getMember("status").getValue().getAsS();
-  auto status = _status == "active" ? Download::K_ACTIVE
-              : _status == "waiting" ? Download::K_WAITING
-              : _status == "paused" ? Download::K_PAUSED
-              : _status == "error" ? Download::K_ERROR
-              : _status == "complete" ? Download::K_COMPLETE
-              : _status == "removed" ? Download::K_REMOVED
-              : Download::K_UNKNOWN;
-  if (status == K_COMPLETE) {
-    try {
-      auto followedByMember = structNode.getMember("followedBy");
-      m_followedBy = followedByMember.getValue().getArray().getValue(0).getAsS();
-      if (follow) {
-        status = K_ACTIVE;
-        updateStatus(true);
-        return;
-      }
-    } catch (InvalidRPCNode& e) { }
-  }
-  m_status = status;
-  m_totalLength = extractFromString<uint64_t>(structNode.getMember("totalLength").getValue().getAsS());
-  m_completedLength = extractFromString<uint64_t>(structNode.getMember("completedLength").getValue().getAsS());
-  m_downloadSpeed = extractFromString<uint64_t>(structNode.getMember("downloadSpeed").getValue().getAsS());
-  try {
-    auto verifiedLengthValue = structNode.getMember("verifiedLength").getValue();
-    m_verifiedLength = extractFromString<uint64_t>(verifiedLengthValue.getAsS());
-  } catch (InvalidRPCNode& e) { m_verifiedLength = 0; }
-  auto filesMember = structNode.getMember("files");
-  auto fileStruct = filesMember.getValue().getArray().getValue(0).getStruct();
-  m_path = fileStruct.getMember("path").getValue().getAsS();
-  auto urisArray = fileStruct.getMember("uris").getValue().getArray();
-  int index = 0;
-  m_uris.clear();
-  while(true) {
-    try {
-      auto uriNode = urisArray.getValue(index++).getStruct().getMember("uri");
-      m_uris.push_back(uriNode.getValue().getAsS());
-    } catch(InvalidRPCNode& e) { break; }
-  }
-}
-
-void Download::resumeDownload()
-{
-    if (!m_followedBy.empty())
-        mp_aria->unpause(m_followedBy);
-    else
-        mp_aria->unpause(m_did);
-    updateStatus(true);
-}
-
-void Download::pauseDownload()
-{
-    if (!m_followedBy.empty())
-        mp_aria->pause(m_followedBy);
-    else
-        mp_aria->pause(m_did);
-    updateStatus(true);
-}
-
-void Download::cancelDownload()
-{
-    if (!m_followedBy.empty())
-        mp_aria->remove(m_followedBy);
-    else
-        mp_aria->remove(m_did);
-    m_status = Download::K_REMOVED;
-}
-
-/* Constructor */
-Downloader::Downloader() :
-  mp_aria(new Aria2())
-{
-  try {
-    for (auto gid : mp_aria->tellActive()) {
-      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
-    }
-  } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what();
-  }
-  try {
-    for (auto gid : mp_aria->tellWaiting()) {
-      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
-    }
-  } catch (std::exception& e) {
-    std::cerr << "aria2 tellWaiting failed : " << e.what();
-  }
-}
-
-/* Destructor */
-Downloader::~Downloader()
-{
-}
-
-void Downloader::close()
-{
-  mp_aria->close();
-}
-
-std::vector<std::string> Downloader::getDownloadIds() {
-  std::vector<std::string> ret;
-  for(auto& p:m_knownDownloads) {
-    ret.push_back(p.first);
-  }
-  return ret;
-}
-
-Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
-{
-  for (auto& p: m_knownDownloads) {
-    auto& d = p.second;
-    auto& uris = d->getUris();
-    if (std::find(uris.begin(), uris.end(), uri) != uris.end())
-      return d.get();
-  }
-  std::vector<std::string> uris = {uri};
-  auto gid = mp_aria->addUri(uris, options);
-  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-  return m_knownDownloads[gid].get();
-}
-
-Download* Downloader::getDownload(const std::string& did)
-{
-  try {
-    m_knownDownloads.at(did).get()->updateStatus(true);
-    return m_knownDownloads.at(did).get();
-  } catch(std::exception& e) {
-    for (auto gid : mp_aria->tellActive()) {
-      if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
-      }
-    }
-    for (auto gid : mp_aria->tellWaiting()) {
-      if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
-      }
-  }
-    throw e;
-  }
-}
-
-}

src/entry.cpp

@@ -1,140 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-
-namespace kiwix
-{
-
-Entry::Entry(zim::Article article)
-  : article(article)
-{
-}
-
-#define RETURN_IF_INVALID(WHAT) if(!good()) { return (WHAT); }
-
-std::string Entry::getPath() const
-{
-  RETURN_IF_INVALID("");
-  return article.getLongUrl();
-}
-
-std::string Entry::getTitle() const
-{
-  RETURN_IF_INVALID("");
-  return article.getTitle();
-}
-
-std::string Entry::getContent() const
-{
-  RETURN_IF_INVALID("");
-  return article.getData();
-}
-
-zim::Blob Entry::getBlob(offset_type offset) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset);
-}
-
-zim::Blob Entry::getBlob(offset_type offset, size_type size) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset, size);
-}
-
-std::pair<std::string, offset_type> Entry::getDirectAccessInfo() const
-{
-  RETURN_IF_INVALID(std::make_pair("", 0));
-  return article.getDirectAccessInformation();
-}
-
-size_type Entry::getSize() const
-{
-  RETURN_IF_INVALID(0);
-  return article.getArticleSize();
-}
-
-std::string Entry::getMimetype() const
-{
-  RETURN_IF_INVALID("");
-  try {
-    return article.getMimeType();
-  } catch (exception& e) {
-    return "application/octet-stream";
-  }
-}
-
-bool Entry::isRedirect() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isRedirect();
-}
-
-bool Entry::isLinkTarget() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isLinktarget();
-}
-
-bool Entry::isDeleted() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isDeleted();
-}
-
-Entry Entry::getRedirectEntry() const
-{
-  RETURN_IF_INVALID(Entry());
-  if ( !article.isRedirect() ) {
-    throw NoEntry();
-  }
-
-  auto targeted_article = article.getRedirectArticle();
-  if ( !targeted_article.good()) {
-    throw NoEntry();
-  }
-  return targeted_article;
-}
-
-Entry Entry::getFinalEntry() const
-{
-  RETURN_IF_INVALID(Entry());
-  if (final_article.good()) {
-    return final_article;
-  }
-  int loopCounter = 42;
-  final_article = article;
-  while (final_article.isRedirect() && loopCounter--) {
-    final_article = final_article.getRedirectArticle();
-    if ( !final_article.good()) {
-      throw NoEntry();
-    }
-  }
-  // Prevent infinite loops.
-  if (final_article.isRedirect()) {
-    throw NoEntry();
-  }
-  return final_article;
-}
-
-}

src/kiwixserve.cpp

@@ -1,81 +0,0 @@
-#include "kiwixserve.h"
-#include "subprocess.h"
-
-#ifdef _WIN32
-    # define KIWIXSERVE_CMD "kiwix-serve.exe"
-    # include <windows.h>
-#else
-    # define KIWIXSERVE_CMD "kiwix-serve"
-    # include <unistd.h>
-#endif
-
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
-
-namespace kiwix {
-
-KiwixServe::KiwixServe(const std::string& libraryPath, int port)
-  : m_port(port),
-    m_libraryPath(libraryPath)
-{
-}
-
-KiwixServe::~KiwixServe()
-{
-    shutDown();
-}
-
-void KiwixServe::run()
-{
-    #ifdef _WIN32
-        int pid = GetCurrentProcessId();
-    #else
-        pid_t pid = getpid();
-
-    #endif
-
-    std::vector<const char*> callCmd;
-    std::string kiwixServeCmd = appendToDirectory(
-        removeLastPathElement(getExecutablePath(true)),
-        KIWIXSERVE_CMD);
-    if (fileExists(kiwixServeCmd)) {
-        // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
-        callCmd.push_back(kiwixServeCmd.c_str());
-    } else {
-        // Try to use a potential installed kiwix-serve.
-        callCmd.push_back(KIWIXSERVE_CMD);
-    }
-    std::string attachProcessOpt = "-a" + to_string(pid);
-    std::string portOpt = "-p" + to_string(m_port);
-    callCmd.push_back(attachProcessOpt.c_str());
-    callCmd.push_back(portOpt.c_str());
-    callCmd.push_back("-l");
-    callCmd.push_back(m_libraryPath.c_str());
-    mp_kiwixServe = Subprocess::run(callCmd);
-}
-
-void KiwixServe::shutDown()
-{
-    if (mp_kiwixServe)
-        mp_kiwixServe->kill();
-}
-
-bool KiwixServe::isRunning()
-{
-    if (mp_kiwixServe) {
-        return (mp_kiwixServe->isRunning());
-    }
-    return false;
-}
-
-int KiwixServe::setPort(int port)
-{
-    if (port >= 1 && port <= 65535) {
-        m_port = port;
-    } else {
-        return -1;
-    }
-    return m_port;
-}
-
-}

src/library.cpp

@@ -18,524 +18,137 @@
  */
 
 #include "library.h"
-#include "book.h"
-#include "reader.h"
-#include "libxml_dumper.h"
-
-#include "tools/base64.h"
-#include "tools/regexTools.h"
-#include "tools/pathTools.h"
-#include "tools/stringTools.h"
-
-#include <pugixml.hpp>
-#include <algorithm>
-#include <set>
 
 namespace kiwix
 {
+/* Constructor */
+Book::Book() : readOnly(false)
+{
+}
+/* Destructor */
+Book::~Book()
+{
+}
+/* Sort functions */
+bool Book::sortByLastOpen(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return atoi(a.last.c_str()) > atoi(b.last.c_str());
+}
+
+bool Book::sortByTitle(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return strcmp(a.title.c_str(), b.title.c_str()) < 0;
+}
+
+bool Book::sortByDate(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return strcmp(a.date.c_str(), b.date.c_str()) > 0;
+}
+
+bool Book::sortBySize(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return atoi(a.size.c_str()) < atoi(b.size.c_str());
+}
+
+bool Book::sortByPublisher(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return strcmp(a.publisher.c_str(), b.publisher.c_str()) < 0;
+}
+
+bool Book::sortByCreator(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return strcmp(a.creator.c_str(), b.creator.c_str()) < 0;
+}
+
+bool Book::sortByLanguage(const kiwix::Book& a, const kiwix::Book& b)
+{
+  return strcmp(a.language.c_str(), b.language.c_str()) < 0;
+}
+
+std::string Book::getHumanReadableIdFromPath()
+{
+  std::string id = pathAbsolute;
+  if (!id.empty()) {
+    kiwix::removeAccents(id);
+
+#ifdef _WIN32
+    id = replaceRegex(id, "", "^.*\\\\");
+#else
+    id = replaceRegex(id, "", "^.*/");
+#endif
+
+    id = replaceRegex(id, "", "\\.zim[a-z]*$");
+    id = replaceRegex(id, "_", " ");
+    id = replaceRegex(id, "plus", "\\+");
+  }
+  return id;
+}
 
 /* Constructor */
-Library::Library()
+Library::Library() : version(KIWIX_LIBRARY_VERSION)
 {
 }
 /* Destructor */
 Library::~Library()
 {
 }
-
-
 bool Library::addBook(const Book& book)
 {
   /* Try to find it */
-  try {
-    auto& oldbook = m_books.at(book.getId());
-    oldbook.update(book);
-    return false;
-  } catch (std::out_of_range&) {
-    m_books[book.getId()] = book;
-    return true;
-  }
-}
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = this->books.begin(); itr != this->books.end(); ++itr) {
+    if (itr->id == book.id) {
+      if (!itr->readOnly) {
+        itr->readOnly = book.readOnly;
 
-void Library::addBookmark(const Bookmark& bookmark)
-{
-  m_bookmarks.push_back(bookmark);
-}
+        if (itr->path.empty()) {
+          itr->path = book.path;
+        }
 
-bool Library::removeBookmark(const std::string& zimId, const std::string& url)
-{
-  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
-    if (it->getBookId() == zimId && it->getUrl() == url) {
-      m_bookmarks.erase(it);
-      return true;
-    }
-  }
-  return false;
-}
+        if (itr->pathAbsolute.empty()) {
+          itr->pathAbsolute = book.pathAbsolute;
+        }
 
+        if (itr->url.empty()) {
+          itr->url = book.url;
+        }
 
+        if (itr->tags.empty()) {
+          itr->tags = book.tags;
+        }
 
-bool Library::removeBookById(const std::string& id)
-{
-  return m_books.erase(id) == 1;
-}
+        if (itr->name.empty()) {
+          itr->name = book.name;
+        }
 
-Book& Library::getBookById(const std::string& id)
-{
-  return m_books.at(id);
-}
+        if (itr->indexPath.empty()) {
+          itr->indexPath = book.indexPath;
+          itr->indexType = book.indexType;
+        }
 
-Book& Library::getBookByPath(const std::string& path)
-{
-  for(auto& it: m_books) {
-    auto& book = it.second;
-    if (book.getPath() == path)
-      return book;
-  }
-  std::ostringstream ss;
-  ss << "No book with path " << path << " in the library." << std::endl;
-  throw std::out_of_range(ss.str());
-}
+        if (itr->indexPathAbsolute.empty()) {
+          itr->indexPathAbsolute = book.indexPathAbsolute;
+          itr->indexType = book.indexType;
+        }
 
-std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
-{
-  try {
-    return m_readers.at(id);
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
-    return nullptr;
-  auto sptr = make_shared<Reader>(book.getPath());
-  m_readers[id] = sptr;
-  return sptr;
-}
-
-unsigned int Library::getBookCount(const bool localBooks,
-                                   const bool remoteBooks)
-{
-  unsigned int result = 0;
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    if ((!book.getPath().empty() && localBooks)
-        || (book.getPath().empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
-}
-
-bool Library::writeToFile(const std::string& path)
-{
-  auto baseDir = removeLastPathElement(path);
-  LibXMLDumper dumper(this);
-  dumper.setBaseDir(baseDir);
-  return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
-}
-
-bool Library::writeBookmarksToFile(const std::string& path)
-{
-  LibXMLDumper dumper(this);
-  return writeTextFile(path, dumper.dumpLibXMLBookmark());
-}
-
-std::vector<std::string> Library::getBooksLanguages()
-{
-  std::vector<std::string> booksLanguages;
-  std::map<std::string, bool> booksLanguagesMap;
-
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    auto& language = book.getLanguage();
-    if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksLanguagesMap[language] = true;
-        booksLanguages.push_back(language);
-      }
-    }
-  }
-
-  return booksLanguages;
-}
-
-std::vector<std::string> Library::getBooksCreators()
-{
-  std::vector<std::string> booksCreators;
-  std::map<std::string, bool> booksCreatorsMap;
-
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    auto& creator = book.getCreator();
-    if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksCreatorsMap[creator] = true;
-        booksCreators.push_back(creator);
-      }
-    }
-  }
-
-  return booksCreators;
-}
-
-std::vector<std::string> Library::getBooksPublishers()
-{
-  std::vector<std::string> booksPublishers;
-  std::map<std::string, bool> booksPublishersMap;
-
-  for (auto& pair:m_books) {
-    auto& book = pair.second;
-    auto& publisher = book.getPublisher();
-    if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
-      if (book.getOrigId().empty()) {
-        booksPublishersMap[publisher] = true;
-        booksPublishers.push_back(publisher);
-      }
-    }
-  }
-
-  return booksPublishers;
-}
-
-const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
-{
-  if (!onlyValidBookmarks) {
-    return m_bookmarks;
-  }
-  std::vector<kiwix::Bookmark> validBookmarks;
-  auto booksId = getBooksIds();
-  for(auto& bookmark:m_bookmarks) {
-    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
-      validBookmarks.push_back(bookmark);
-    }
-  }
-  return validBookmarks;
-}
-
-std::vector<std::string> Library::getBooksIds()
-{
-  std::vector<std::string> bookIds;
-
-  for (auto& pair: m_books) {
-    bookIds.push_back(pair.first);
-  }
-
-  return bookIds;
-}
-
-std::vector<std::string> Library::filter(const std::string& search)
-{
-  if (search.empty()) {
-    return getBooksIds();
-  }
-
-  return filter(Filter().query(search));
-}
-
-
-std::vector<std::string> Library::filter(const Filter& filter)
-{
-  std::vector<std::string> bookIds;
-  for(auto& pair:m_books) {
-    auto book = pair.second;
-    if(filter.accept(book)) {
-      bookIds.push_back(pair.first);
-    }
-  }
-  return bookIds;
-}
-
-template<supportedListSortBy SORT>
-struct KEY_TYPE {
-  typedef std::string TYPE;
-};
-
-template<>
-struct KEY_TYPE<SIZE> {
-  typedef size_t TYPE;
-};
-
-template<supportedListSortBy sort>
-class Comparator {
-  private:
-    Library* lib;
-    bool     ascending;
-
-    inline typename KEY_TYPE<sort>::TYPE get_key(const std::string& id);
-
-  public:
-    Comparator(Library* lib, bool ascending) : lib(lib), ascending(ascending) {}
-    inline bool operator() (const std::string& id1, const std::string& id2) {
-      if (ascending) {
-        return get_key(id1) < get_key(id2);
-      } else {
-        return get_key(id2) < get_key(id1);
-      }
-    }
-};
-
-template<>
-std::string Comparator<TITLE>::get_key(const std::string& id)
-{
-  return lib->getBookById(id).getTitle();
-}
-
-template<>
-size_t Comparator<SIZE>::get_key(const std::string& id)
-{
-  return lib->getBookById(id).getSize();
-}
-
-template<>
-std::string Comparator<DATE>::get_key(const std::string& id)
-{
-  return lib->getBookById(id).getDate();
-}
-
-template<>
-std::string Comparator<CREATOR>::get_key(const std::string& id)
-{
-  return lib->getBookById(id).getCreator();
-}
-
-template<>
-std::string Comparator<PUBLISHER>::get_key(const std::string& id)
-{
-  return lib->getBookById(id).getPublisher();
-}
-
-void Library::sort(std::vector<std::string>& bookIds, supportedListSortBy sort, bool ascending)
-{
-  switch(sort) {
-    case TITLE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
-      break;
-    case SIZE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<SIZE>(this, ascending));
-      break;
-    case DATE:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<DATE>(this, ascending));
-      break;
-    case CREATOR:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<CREATOR>(this, ascending));
-      break;
-    case PUBLISHER:
-      std::sort(bookIds.begin(), bookIds.end(), Comparator<PUBLISHER>(this, ascending));
-      break;
-    default:
-      break;
-  }
-}
-
-
-std::vector<std::string> Library::listBooksIds(
-    int mode,
-    supportedListSortBy sortBy,
-    const std::string& search,
-    const std::string& language,
-    const std::string& creator,
-    const std::string& publisher,
-    const std::vector<std::string>& tags,
-    size_t maxSize) {
-
-  Filter _filter;
-  if (mode & LOCAL)
-    _filter.local(true);
-  if (mode & NOLOCAL)
-    _filter.local(false);
-  if (mode & VALID)
-    _filter.valid(true);
-  if (mode & NOVALID)
-    _filter.valid(false);
-  if (mode & REMOTE)
-    _filter.remote(true);
-  if (mode & NOREMOTE)
-    _filter.remote(false);
-  if (!tags.empty())
-    _filter.acceptTags(tags);
-  if (maxSize != 0)
-    _filter.maxSize(maxSize);
-  if (!language.empty())
-    _filter.lang(language);
-  if (!publisher.empty())
-    _filter.publisher(publisher);
-  if (!creator.empty())
-    _filter.creator(creator);
-  if (!search.empty())
-    _filter.query(search);
-
-  auto bookIds = filter(_filter);
-
-  sort(bookIds, sortBy, true);
-  return bookIds;
-}
-
-Filter::Filter()
-  : activeFilters(0),
-    _maxSize(0)
-{};
-
-#define FLAG(x) (1 << x)
-enum filterTypes {
-  NONE = 0,
-  _LOCAL = FLAG(0),
-  _REMOTE = FLAG(1),
-  _NOLOCAL = FLAG(2),
-  _NOREMOTE = FLAG(3),
-  _VALID = FLAG(4),
-  _NOVALID = FLAG(5),
-  ACCEPTTAGS = FLAG(6),
-  REJECTTAGS = FLAG(7),
-  LANG = FLAG(8),
-  _PUBLISHER = FLAG(9),
-  _CREATOR = FLAG(10),
-  MAXSIZE = FLAG(11),
-  QUERY = FLAG(12),
-  NAME = FLAG(13),
-};
-
-Filter& Filter::local(bool accept)
-{
-  if (accept) {
-    activeFilters |= _LOCAL;
-    activeFilters &= ~_NOLOCAL;
-  } else {
-    activeFilters |= _NOLOCAL;
-    activeFilters &= ~_LOCAL;
-  }
-  return *this;
-}
-
-Filter& Filter::remote(bool accept)
-{
-  if (accept) {
-    activeFilters |= _REMOTE;
-    activeFilters &= ~_NOREMOTE;
-  } else {
-    activeFilters |= _NOREMOTE;
-    activeFilters &= ~_REMOTE;
-  }
-  return *this;
-}
-
-Filter& Filter::valid(bool accept)
-{
-  if (accept) {
-    activeFilters |= _VALID;
-    activeFilters &= ~_NOVALID;
-  } else {
-    activeFilters |= _NOVALID;
-    activeFilters &= ~_VALID;
-  }
-  return *this;
-}
-
-Filter& Filter::acceptTags(std::vector<std::string> tags)
-{
-  _acceptTags = tags;
-  activeFilters |= ACCEPTTAGS;
-  return *this;
-}
-
-Filter& Filter::rejectTags(std::vector<std::string> tags)
-{
-  _rejectTags = tags;
-  activeFilters |= REJECTTAGS;
-  return *this;
-}
-
-Filter& Filter::lang(std::string lang)
-{
-  _lang = lang;
-  activeFilters |= LANG;
-  return *this;
-}
-
-Filter& Filter::publisher(std::string publisher)
-{
-  _publisher = publisher;
-  activeFilters |= _PUBLISHER;
-  return *this;
-}
-
-Filter& Filter::creator(std::string creator)
-{
-  _creator = creator;
-  activeFilters |= _CREATOR;
-  return *this;
-}
-
-Filter& Filter::maxSize(size_t maxSize)
-{
-  _maxSize = maxSize;
-  activeFilters |= MAXSIZE;
-  return *this;
-}
-
-Filter& Filter::query(std::string query)
-{
-  _query = query;
-  activeFilters |= QUERY;
-  return *this;
-}
-
-Filter& Filter::name(std::string name)
-{
-  _name = name;
-  activeFilters |= NAME;
-  return *this;
-}
-
-#define ACTIVE(X) (activeFilters & (X))
-#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
-bool Filter::accept(const Book& book) const
-{
-  auto local = !book.getPath().empty();
-  FILTER(_LOCAL, local)
-  FILTER(_NOLOCAL, !local)
-
-  auto valid = book.isPathValid();
-  FILTER(_VALID, valid)
-  FILTER(_NOVALID, !valid)
-
-  auto remote = !book.getUrl().empty();
-  FILTER(_REMOTE, remote)
-  FILTER(_NOREMOTE, !remote)
-
-  FILTER(MAXSIZE, book.getSize() <= _maxSize)
-  FILTER(LANG, book.getLanguage() == _lang)
-  FILTER(_PUBLISHER, book.getPublisher() == _publisher)
-  FILTER(_CREATOR, book.getCreator() == _creator)
-  FILTER(NAME, book.getName() == _name)
-
-  if (ACTIVE(ACCEPTTAGS)) {
-    if (!_acceptTags.empty()) {
-      auto vBookTags = split(book.getTags(), ";");
-      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
-      for (auto& t: _acceptTags) {
-        if (sBookTags.find(t) == sBookTags.end()) {
-          return false;
+        if (itr->faviconMimeType.empty()) {
+          itr->favicon = book.favicon;
+          itr->faviconMimeType = book.faviconMimeType;
         }
       }
-    }
-  }
-  if (ACTIVE(REJECTTAGS)) {
-    if (!_rejectTags.empty()) {
-      auto vBookTags = split(book.getTags(), ";");
-      std::set<std::string> sBookTags(vBookTags.begin(), vBookTags.end());
-      for (auto& t: _rejectTags) {
-        if (sBookTags.find(t) != sBookTags.end()) {
-          return false;
-        }
-      }
-    }
-  }
-  if ( ACTIVE(QUERY)
-    && !(matchRegex(book.getTitle(), "\\Q" + _query + "\\E")
-        || matchRegex(book.getDescription(), "\\Q" + _query + "\\E")))
-    return false;
 
+      return false;
+    }
+  }
+
+  /* otherwise */
+  this->books.push_back(book);
   return true;
-
 }
 
+bool Library::removeBookByIndex(const unsigned int bookIndex)
+{
+  books.erase(books.begin() + bookIndex);
+  return true;
+}
 }

src/libxml_dumper.cpp

@@ -1,142 +0,0 @@
-/*
- * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "libxml_dumper.h"
-#include "book.h"
-
-#include "tools/base64.h"
-#include "tools/stringTools.h"
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-
-namespace kiwix
-{
-/* Constructor */
-LibXMLDumper::LibXMLDumper(Library* library)
-  : library(library)
-{
-}
-/* Destructor */
-LibXMLDumper::~LibXMLDumper()
-{
-}
-
-#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
-#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
-
-void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
-  if (book.readOnly())
-    return;
-
-  auto entry_node = root_node.append_child("book");
-  ADD_ATTRIBUTE(entry_node, "id", book.getId());
-
-  if (!book.getPath().empty()) {
-    ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
-  }
-
-  if (book.getOrigId().empty()) {
-    ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
-    ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
-    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
-    ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
-    ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
-    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
-    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
-    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
-    ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
-    if (!book.getFavicon().empty())
-      ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
-  } else {
-    ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
-  }
-
-  ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
-  ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
-
-  if (book.getArticleCount())
-    ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
-
-  if (book.getMediaCount())
-    ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
-
-  if (book.getSize())
-    ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
-
-  ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
-}
-
-#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
-
-void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
-
-  auto entry_node = root_node.append_child("bookmark");
-  auto book_node = entry_node.append_child("book");
-
-  try {
-    auto book = library->getBookById(bookmark.getBookId());
-    ADD_TEXT_ENTRY(book_node, "id", book.getId());
-    ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
-    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
-    ADD_TEXT_ENTRY(book_node, "date", book.getDate());
-  } catch (...) {
-    ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
-    ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
-    ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
-    ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
-  }
-  ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
-  ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
-}
-
-
-std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
-{
-  pugi::xml_document doc;
-
-  /* Add the library node */
-  pugi::xml_node libraryNode = doc.append_child("library");
-
-  libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
-
-  if (library) {
-    for (auto& bookId: bookIds) {
-      handleBook(library->getBookById(bookId), libraryNode);
-    }
-  }
-  return nodeToString(libraryNode);
-}
-
-std::string LibXMLDumper::dumpLibXMLBookmark()
-{
-  pugi::xml_document doc;
-
-  /* Add the library node */
-  pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
-
-  if (library) {
-    for (auto& bookmark: library->getBookmarks()) {
-      handleBookmark(bookmark, bookmarksNode);
-    }
-  }
-  return nodeToString(bookmarksNode);
-}
-
-}

src/manager.cpp

@@ -19,137 +19,105 @@
 
 #include "manager.h"
 
-#include "tools/pathTools.h"
-
-#include <pugixml.hpp>
-
 namespace kiwix
 {
 /* Constructor */
-Manager::Manager(LibraryManipulator* manipulator):
-  writableLibraryPath(""),
-  manipulator(manipulator),
-  mustDeleteManipulator(false)
+Manager::Manager() : writableLibraryPath("")
 {
 }
-
-Manager::Manager(Library* library) :
-  writableLibraryPath(""),
-  manipulator(new DefaultLibraryManipulator(library)),
-  mustDeleteManipulator(true)
-{
-}
-
 /* Destructor */
 Manager::~Manager()
 {
-  if (mustDeleteManipulator) {
-    delete manipulator;
-  }
 }
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          bool readOnly,
-                          const std::string& libraryPath,
-                          bool trustLibrary)
+                          const bool readOnly,
+                          const string libraryPath)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
-  std::string libraryVersion = libraryNode.attribute("version").value();
+  if (strlen(libraryNode.attribute("current").value()))
+    this->setCurrentBookId(libraryNode.attribute("current").value());
+
+  string libraryVersion = libraryNode.attribute("version").value();
 
   for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode;
        bookNode = bookNode.next_sibling("book")) {
+    bool ok = true;
     kiwix::Book book;
 
-    book.setReadOnly(readOnly);
-    book.updateFromXml(bookNode,
-                       removeLastPathElement(libraryPath));
+    book.readOnly = readOnly;
+    book.id = bookNode.attribute("id").value();
+    book.path = bookNode.attribute("path").value();
+    book.last = (std::string(bookNode.attribute("last").value()) != "undefined"
+                     ? bookNode.attribute("last").value()
+                     : "");
+    book.indexPath = bookNode.attribute("indexPath").value();
+    book.indexType = XAPIAN;
+    book.title = bookNode.attribute("title").value();
+    book.name = bookNode.attribute("name").value();
+    book.tags = bookNode.attribute("tags").value();
+    book.description = bookNode.attribute("description").value();
+    book.language = bookNode.attribute("language").value();
+    book.date = bookNode.attribute("date").value();
+    book.creator = bookNode.attribute("creator").value();
+    book.publisher = bookNode.attribute("publisher").value();
+    book.url = bookNode.attribute("url").value();
+    book.origId = bookNode.attribute("origId").value();
+    book.articleCount = bookNode.attribute("articleCount").value();
+    book.mediaCount = bookNode.attribute("mediaCount").value();
+    book.size = bookNode.attribute("size").value();
+    book.favicon = bookNode.attribute("favicon").value();
+    book.faviconMimeType = bookNode.attribute("faviconMimeType").value();
 
-    if (!trustLibrary && !book.getPath().empty()) {
-      this->readBookFromPath(book.getPath(), &book);
+    /* Check absolute and relative paths */
+    this->checkAndCleanBookPaths(book, libraryPath);
+
+    /* Update the book properties with the new importer */
+    if (libraryVersion.empty()
+        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
+      if (!book.path.empty()) {
+        ok = this->readBookFromPath(book.pathAbsolute);
+      }
+    }
+
+    if (ok) {
+      library.addBook(book);
     }
-    manipulator->addBookToLibrary(book);
   }
 
   return true;
 }
 
-bool Manager::readXml(const std::string& xml,
-                      bool readOnly,
-                      const std::string& libraryPath,
-                      bool trustLibrary)
+bool Manager::readXml(const string xml,
+                      const bool readOnly,
+                      const string libraryPath)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
       = doc.load_buffer_inplace((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
+    this->parseXmlDom(doc, readOnly, libraryPath);
   }
 
   return true;
 }
 
-
-
-bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& urlHost)
+bool Manager::readFile(const string path, const bool readOnly)
 {
-  pugi::xml_node libraryNode = doc.child("feed");
-
-  try {
-    m_totalBooks = strtoull(libraryNode.child("totalResults").child_value(), 0, 0);
-    m_startIndex = strtoull(libraryNode.child("startIndex").child_value(), 0, 0);
-    m_itemsPerPage = strtoull(libraryNode.child("itemsPerPage").child_value(), 0, 0);
-    m_hasSearchResult = true;
-  } catch(...) {
-    m_hasSearchResult = false;
-  }
-
-  for (pugi::xml_node entryNode = libraryNode.child("entry"); entryNode;
-       entryNode = entryNode.next_sibling("entry")) {
-    kiwix::Book book;
-
-    book.setReadOnly(false);
-    book.updateFromOpds(entryNode, urlHost);
-
-    /* Update the book properties with the new importer */
-    manipulator->addBookToLibrary(book);
-  }
-
-  return true;
+  return this->readFile(path, path, readOnly);
 }
 
-
-
-bool Manager::readOpds(const std::string& content, const std::string& urlHost)
-{
-  pugi::xml_document doc;
-  pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)content.data(), content.size());
-
-  if (result) {
-    this->parseOpdsDom(doc, urlHost);
-    return true;
-  }
-
-  return false;
-}
-
-bool Manager::readFile(
-  const std::string& path,
-  bool readOnly,
-  bool trustLibrary)
+bool Manager::readFile(const string nativePath,
+                       const string UTF8Path,
+                       const bool readOnly)
 {
   bool retVal = true;
   pugi::xml_document doc;
-
-#ifdef _WIN32
-  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
-#else
-  pugi::xml_parse_result result = doc.load_file(path.c_str());
-#endif
+  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, path, trustLibrary);
+    this->parseXmlDom(doc, readOnly, UTF8Path);
   } else {
     retVal = false;
   }
@@ -158,37 +126,154 @@ bool Manager::readFile(
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = path;
+    this->writableLibraryPath = UTF8Path;
   }
 
   return retVal;
 }
 
+bool Manager::writeFile(const string path)
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node libraryNode = doc.append_child("library");
+
+  if (!getCurrentBookId().empty()) {
+    libraryNode.append_attribute("current") = getCurrentBookId().c_str();
+  }
+
+  if (!library.version.empty())
+    libraryNode.append_attribute("version") = library.version.c_str();
+
+  /* Add each book */
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (!itr->readOnly) {
+      this->checkAndCleanBookPaths(*itr, path);
+
+      pugi::xml_node bookNode = libraryNode.append_child("book");
+      bookNode.append_attribute("id") = itr->id.c_str();
+
+      if (!itr->path.empty()) {
+        bookNode.append_attribute("path") = itr->path.c_str();
+      }
+
+      if (!itr->last.empty() && itr->last != "undefined") {
+        bookNode.append_attribute("last") = itr->last.c_str();
+      }
+
+      if (!itr->indexPath.empty())
+        bookNode.append_attribute("indexPath") = itr->indexPath.c_str();
+
+      if (!itr->indexPath.empty() || !itr->indexPathAbsolute.empty()) {
+        if (itr->indexType == XAPIAN) {
+          bookNode.append_attribute("indexType") = "xapian";
+        }
+      }
+
+      if (itr->origId.empty()) {
+        if (!itr->title.empty())
+          bookNode.append_attribute("title") = itr->title.c_str();
+
+        if (!itr->name.empty())
+          bookNode.append_attribute("name") = itr->name.c_str();
+
+        if (!itr->tags.empty())
+          bookNode.append_attribute("tags") = itr->tags.c_str();
+
+        if (!itr->description.empty())
+          bookNode.append_attribute("description") = itr->description.c_str();
+
+        if (!itr->language.empty())
+          bookNode.append_attribute("language") = itr->language.c_str();
+
+        if (!itr->creator.empty())
+          bookNode.append_attribute("creator") = itr->creator.c_str();
+
+        if (!itr->publisher.empty())
+          bookNode.append_attribute("publisher") = itr->publisher.c_str();
+
+        if (!itr->favicon.empty())
+          bookNode.append_attribute("favicon") = itr->favicon.c_str();
+
+        if (!itr->faviconMimeType.empty())
+          bookNode.append_attribute("faviconMimeType")
+              = itr->faviconMimeType.c_str();
+      }
+
+      if (!itr->date.empty()) {
+        bookNode.append_attribute("date") = itr->date.c_str();
+      }
+
+      if (!itr->url.empty()) {
+        bookNode.append_attribute("url") = itr->url.c_str();
+      }
+
+      if (!itr->origId.empty())
+        bookNode.append_attribute("origId") = itr->origId.c_str();
+
+      if (!itr->articleCount.empty())
+        bookNode.append_attribute("articleCount") = itr->articleCount.c_str();
+
+      if (!itr->mediaCount.empty())
+        bookNode.append_attribute("mediaCount") = itr->mediaCount.c_str();
+
+      if (!itr->size.empty()) {
+        bookNode.append_attribute("size") = itr->size.c_str();
+      }
+    }
+  }
+
+  /* saving file */
+  doc.save_file(path.c_str());
+
+  return true;
+}
+
+bool Manager::setCurrentBookId(const string id)
+{
+  if (library.current.empty() || library.current.top() != id) {
+    if (id.empty() && !library.current.empty()) {
+      library.current.pop();
+    } else {
+      library.current.push(id);
+    }
+  }
+  return true;
+}
+
+string Manager::getCurrentBookId()
+{
+  return library.current.empty() ? "" : library.current.top();
+}
 
 /* Add a book to the library. Return empty string if failed, book id otherwise
  */
-std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
-                                             const std::string& pathToSave,
-                                             const std::string& url,
-                                             const bool checkMetaData)
+string Manager::addBookFromPathAndGetId(const string pathToOpen,
+                                        const string pathToSave,
+                                        const string url,
+                                        const bool checkMetaData)
 {
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (!pathToSave.empty() && pathToSave != pathToOpen) {
-      book.setPath(isRelativePath(pathToSave)
+    if (pathToSave != pathToOpen) {
+      book.path = pathToSave;
+      book.pathAbsolute
+          = isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath),
+                      removeLastPathElement(writableLibraryPath, true, false),
                       pathToSave)
-                : pathToSave);
+                : pathToSave;
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
-            && !book.getDate().empty())) {
-      book.setUrl(url);
-      manipulator->addBookToLibrary(book);
-      return book.getId();
+        || (checkMetaData && !book.title.empty() && !book.language.empty()
+            && !book.date.empty())) {
+      book.url = url;
+      library.addBook(book);
+      return book.id;
     }
   }
 
@@ -197,9 +282,9 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
 
 /* Wrapper over Manager::addBookFromPath which return a bool instead of a string
  */
-bool Manager::addBookFromPath(const std::string& pathToOpen,
-                              const std::string& pathToSave,
-                              const std::string& url,
+bool Manager::addBookFromPath(const string pathToOpen,
+                              const string pathToSave,
+                              const string url,
                               const bool checkMetaData)
 {
   return !(
@@ -207,45 +292,367 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
           .empty());
 }
 
-bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
+bool Manager::readBookFromPath(const string path, kiwix::Book* book)
 {
-  std::string tmp_path = path;
-  if (isRelativePath(path)) {
-    tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
-  }
   try {
-    kiwix::Reader reader(tmp_path);
-    book->update(reader);
-    book->setPathValid(true);
+    kiwix::Reader* reader = new kiwix::Reader(path);
+
+    if (book != NULL) {
+      book->path = path;
+      book->pathAbsolute = path;
+      book->id = reader->getId();
+      book->description = reader->getDescription();
+      book->language = reader->getLanguage();
+      book->date = reader->getDate();
+      book->creator = reader->getCreator();
+      book->publisher = reader->getPublisher();
+      book->title = reader->getTitle();
+      book->name = reader->getName();
+      book->tags = reader->getTags();
+      book->origId = reader->getOrigId();
+      std::ostringstream articleCountStream;
+      articleCountStream << reader->getArticleCount();
+      book->articleCount = articleCountStream.str();
+
+      std::ostringstream mediaCountStream;
+      mediaCountStream << reader->getMediaCount();
+      book->mediaCount = mediaCountStream.str();
+
+      ostringstream convert;
+      convert << reader->getFileSize();
+      book->size = convert.str();
+
+      string favicon;
+      string faviconMimeType;
+      if (reader->getFavicon(favicon, faviconMimeType)) {
+        book->favicon = base64_encode(
+            reinterpret_cast<const unsigned char*>(favicon.c_str()),
+            favicon.length());
+        book->faviconMimeType = faviconMimeType;
+      }
+    }
+
+    delete reader;
   } catch (const std::exception& e) {
-    book->setPathValid(false);
+    std::cerr << e.what() << std::endl;
     return false;
   }
 
   return true;
 }
 
-bool Manager::readBookmarkFile(const std::string& path)
+bool Manager::removeBookByIndex(const unsigned int bookIndex)
 {
-  pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(path.c_str());
+  return this->library.removeBookByIndex(bookIndex);
+}
 
-  if (!result) {
-    return false;
+bool Manager::removeBookById(const string id)
+{
+  unsigned int bookIndex = 0;
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (itr->id == id) {
+      return this->library.removeBookByIndex(bookIndex);
+    }
+    bookIndex++;
+  }
+  return false;
+}
+
+vector<string> Manager::getBooksLanguages()
+{
+  std::vector<string> booksLanguages;
+  std::vector<kiwix::Book>::iterator itr;
+  std::map<string, bool> booksLanguagesMap;
+
+  std::sort(
+      library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (booksLanguagesMap.find(itr->language) == booksLanguagesMap.end()) {
+      if (itr->origId.empty()) {
+        booksLanguagesMap[itr->language] = true;
+        booksLanguages.push_back(itr->language);
+      }
+    }
   }
 
-  pugi::xml_node libraryNode = doc.child("bookmarks");
+  return booksLanguages;
+}
 
-  for (pugi::xml_node node = libraryNode.child("bookmark"); node;
-       node = node.next_sibling("bookmark")) {
-    kiwix::Bookmark bookmark;
+vector<string> Manager::getBooksCreators()
+{
+  std::vector<string> booksCreators;
+  std::vector<kiwix::Book>::iterator itr;
+  std::map<string, bool> booksCreatorsMap;
 
-    bookmark.updateFromXml(node);
+  std::sort(
+      library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (booksCreatorsMap.find(itr->creator) == booksCreatorsMap.end()) {
+      if (itr->origId.empty()) {
+        booksCreatorsMap[itr->creator] = true;
+        booksCreators.push_back(itr->creator);
+      }
+    }
+  }
 
-    manipulator->addBookmarkToLibrary(bookmark);
+  return booksCreators;
+}
+
+vector<string> Manager::getBooksIds()
+{
+  std::vector<string> booksIds;
+  std::vector<kiwix::Book>::iterator itr;
+
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    booksIds.push_back(itr->id);
+  }
+
+  return booksIds;
+}
+
+vector<string> Manager::getBooksPublishers()
+{
+  std::vector<string> booksPublishers;
+  std::vector<kiwix::Book>::iterator itr;
+  std::map<string, bool> booksPublishersMap;
+
+  std::sort(
+      library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (booksPublishersMap.find(itr->publisher) == booksPublishersMap.end()) {
+      if (itr->origId.empty()) {
+        booksPublishersMap[itr->publisher] = true;
+        booksPublishers.push_back(itr->publisher);
+      }
+    }
+  }
+
+  return booksPublishers;
+}
+
+kiwix::Library Manager::cloneLibrary()
+{
+  return this->library;
+}
+bool Manager::getCurrentBook(Book& book)
+{
+  string currentBookId = getCurrentBookId();
+  if (currentBookId.empty()) {
+    return false;
+  } else {
+    getBookById(currentBookId, book);
+    return true;
+  }
+}
+
+bool Manager::getBookById(const string id, Book& book)
+{
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (itr->id == id) {
+      book = *itr;
+      return true;
+    }
+  }
+  return false;
+}
+
+bool Manager::updateBookLastOpenDateById(const string id)
+{
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (itr->id == id) {
+      char unixdate[12];
+      sprintf(unixdate, "%d", (int)time(NULL));
+      itr->last = unixdate;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool Manager::setBookIndex(const string id,
+                           const string path,
+                           const supportedIndexType type)
+{
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (itr->id == id) {
+      itr->indexPath = path;
+      itr->indexPathAbsolute
+          = isRelativePath(path)
+                ? computeAbsolutePath(
+                      removeLastPathElement(writableLibraryPath, true, false),
+                      path)
+                : path;
+      itr->indexType = type;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool Manager::setBookIndex(const string id, const string path)
+{
+  return this->setBookIndex(id, path, XAPIAN);
+}
+
+bool Manager::setBookPath(const string id, const string path)
+{
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if (itr->id == id) {
+      itr->path = path;
+      itr->pathAbsolute
+          = isRelativePath(path)
+                ? computeAbsolutePath(
+                      removeLastPathElement(writableLibraryPath, true, false),
+                      path)
+                : path;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void Manager::removeBookPaths()
+{
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    itr->path = "";
+    itr->pathAbsolute = "";
+  }
+}
+
+unsigned int Manager::getBookCount(const bool localBooks,
+                                   const bool remoteBooks)
+{
+  unsigned int result = 0;
+  std::vector<kiwix::Book>::iterator itr;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    if ((!itr->path.empty() && localBooks)
+        || (itr->path.empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
+
+bool Manager::listBooks(const supportedListMode mode,
+                        const supportedListSortBy sortBy,
+                        const unsigned int maxSize,
+                        const string language,
+                        const string creator,
+                        const string publisher,
+                        const string search)
+{
+  this->bookIdList.clear();
+  std::vector<kiwix::Book>::iterator itr;
+
+  /* Sort */
+  if (sortBy == TITLE) {
+    std::sort(
+        library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
+  } else if (sortBy == SIZE) {
+    std::sort(
+        library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
+  } else if (sortBy == DATE) {
+    std::sort(
+        library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
+  } else if (sortBy == CREATOR) {
+    std::sort(
+        library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
+  } else if (sortBy == PUBLISHER) {
+    std::sort(library.books.begin(),
+              library.books.end(),
+              kiwix::Book::sortByPublisher);
+  }
+
+  /* Special sort for LASTOPEN */
+  if (mode == LASTOPEN) {
+    std::sort(library.books.begin(),
+              library.books.end(),
+              kiwix::Book::sortByLastOpen);
+    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+      if (!itr->last.empty()) {
+        this->bookIdList.push_back(itr->id);
+      }
+    }
+  } else {
+    /* Generate the list of book id */
+    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+      bool ok = true;
+
+      if (mode == LOCAL && itr->path.empty()) {
+        ok = false;
+      }
+
+      if (ok == true && mode == REMOTE
+          && (!itr->path.empty() || itr->url.empty())) {
+        ok = false;
+      }
+
+      if (ok == true && maxSize != 0
+          && (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024) {
+        ok = false;
+      }
+
+      if (ok == true && !language.empty()
+          && !matchRegex(itr->language, language)) {
+        ok = false;
+      }
+
+      if (ok == true && !creator.empty() && itr->creator != creator) {
+        ok = false;
+      }
+
+      if (ok == true && !publisher.empty() && itr->publisher != publisher) {
+        ok = false;
+      }
+
+      if ((ok == true && !search.empty())
+          && !(matchRegex(itr->title, "\\Q" + search + "\\E")
+               || matchRegex(itr->description, "\\Q" + search + "\\E")
+               || matchRegex(itr->language, "\\Q" + search + "\\E"))) {
+        ok = false;
+      }
+
+      if (ok == true) {
+        this->bookIdList.push_back(itr->id);
+      }
+    }
   }
 
   return true;
 }
 
+void Manager::checkAndCleanBookPaths(Book& book, const string& libraryPath)
+{
+  if (!book.path.empty()) {
+    if (isRelativePath(book.path)) {
+      book.pathAbsolute = computeAbsolutePath(
+          removeLastPathElement(libraryPath, true, false), book.path);
+    } else {
+      book.pathAbsolute = book.path;
+      book.path = computeRelativePath(
+          removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
+    }
+  }
+
+  if (!book.indexPath.empty()) {
+    if (isRelativePath(book.indexPath)) {
+      book.indexPathAbsolute = computeAbsolutePath(
+          removeLastPathElement(libraryPath, true, false), book.indexPath);
+    } else {
+      book.indexPathAbsolute = book.indexPath;
+      book.indexPath
+          = computeRelativePath(removeLastPathElement(libraryPath, true, false),
+                                book.indexPathAbsolute);
+    }
+  }
+}
 }

src/meson.build

@@ -1,46 +1,33 @@
 kiwix_sources = [
-  'book.cpp',
-  'bookmark.cpp',
   'library.cpp',
   'manager.cpp',
-  'libxml_dumper.cpp',
-  'opds_dumper.cpp',
-  'downloader.cpp',
   'reader.cpp',
-  'entry.cpp',
-  'server.cpp',
   'searcher.cpp',
-  'search_renderer.cpp',
-  'subprocess.cpp',
-  'aria2.cpp',
-  'tools/base64.cpp',
-  'tools/pathTools.cpp',
-  'tools/regexTools.cpp',
-  'tools/stringTools.cpp',
-  'tools/networkTools.cpp',
-  'tools/otherTools.cpp',
-  'kiwixserve.cpp',
-  'name_mapper.cpp',
-  'server/etag.cpp',
-  'server/request_context.cpp',
-  'server/response.cpp'
+  'common/base64.cpp',
+  'common/pathTools.cpp',
+  'common/regexTools.cpp',
+  'common/stringTools.cpp',
+  'common/networkTools.cpp',
+  'common/otherTools.cpp',
+  'xapian/htmlparse.cc',
+  'xapian/myhtmlparse.cc'
 ]
 kiwix_sources += lib_resources
 
-if host_machine.system() == 'windows'
-  kiwix_sources += 'subprocess_windows.cpp'
-else
-  kiwix_sources += 'subprocess_unix.cpp'
+if xapian_dep.found()
+  kiwix_sources += ['xapianSearcher.cpp']
 endif
 
-if 'android' in wrapper
+if get_option('android')
+  subdir('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
-if 'android' in wrapper or 'java' in wrapper
-  subdir('wrapper/java')
+
+if has_ctpp2_dep
+  kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
 endif
 
 config_h = configure_file(output : 'kiwix_config.h',

src/name_mapper.cpp

@@ -1,62 +0,0 @@
-/*
- * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "name_mapper.h"
-#include "library.h"
-#include "tools/regexTools.h"
-#include <iostream>
-
-namespace kiwix {
-
-HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
-  for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = library.getBookById(bookId);
-    auto bookName = currentBook.getHumanReadableIdFromPath();
-    m_idToName[bookId] = bookName;
-    m_nameToId[bookName] = bookId;
-
-    if (!withAlias)
-      continue;
-
-    auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
-    if (aliasName == bookName) {
-      continue;
-    }
-    if (m_nameToId.find(aliasName) == m_nameToId.end()) {
-      m_nameToId[aliasName] = bookId;
-    } else {
-      auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
-      std::cerr << "Path collision: " << alreadyPresentPath
-                << " and " << currentBook.getPath()
-                << " can't share the same URL path '" << aliasName << "'."
-                << " Therefore, only " << alreadyPresentPath
-                << " will be served." << std::endl;
-    }
-  }
-}
-
-std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
-  return m_idToName.at(id);
-}
-
-std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
-  return m_nameToId.at(name);
-}
-
-}

src/opds_dumper.cpp

@@ -1,154 +0,0 @@
-/*
- * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "opds_dumper.h"
-#include "book.h"
-
-#include "tools/otherTools.h"
-#include <iomanip>
-
-namespace kiwix
-{
-/* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
-  : library(library)
-{
-}
-/* Destructor */
-OPDSDumper::~OPDSDumper()
-{
-}
-
-std::string gen_date_str()
-{
-  auto now = time(0);
-  auto tm = localtime(&now);
-
-  std::stringstream is;
-  is << std::setw(2) << std::setfill('0')
-     << 1900+tm->tm_year << "-"
-     << std::setw(2) << std::setfill('0') << tm->tm_mon << "-"
-     << std::setw(2) << std::setfill('0') << tm->tm_mday << "T"
-     << std::setw(2) << std::setfill('0') << tm->tm_hour << ":"
-     << std::setw(2) << std::setfill('0') << tm->tm_min << ":"
-     << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z";
-  return is.str();
-}
-
-static std::string gen_date_from_yyyy_mm_dd(const std::string& date)
-{
-  std::stringstream is;
-  is << date << "T00:00::00:Z";
-  return is.str();
-}
-
-void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
-{
-  m_totalResults = totalResults;
-  m_startIndex = startIndex,
-  m_count = count;
-  m_isSearchResult = true;
-}
-
-#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
-
-pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
-  auto entry_node = root_node.append_child("entry");
-  ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
-  ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
-  ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage());
-  ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
-  ADD_TEXT_ENTRY(entry_node, "name", book.getName());
-  ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
-  ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
-  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
-  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
-
-  auto content_node = entry_node.append_child("link");
-  content_node.append_attribute("type") = "text/html";
-  content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str();
-
-  auto author_node = entry_node.append_child("author");
-  ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
-
-  auto publisher_node = entry_node.append_child("publisher");
-  ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher());
-
-  if (! book.getUrl().empty()) {
-    auto acquisition_link = entry_node.append_child("link");
-    acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";
-    acquisition_link.append_attribute("type") = "application/x-zim";
-    acquisition_link.append_attribute("href") = book.getUrl().c_str();
-    acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str();
-  }
-
-  if (! book.getFaviconMimeType().empty() ) {
-    auto image_link = entry_node.append_child("link");
-    image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail";
-    image_link.append_attribute("type") = book.getFaviconMimeType().c_str();
-    image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str();
-  }
-  return entry_node;
-}
-
-string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds)
-{
-  date = gen_date_str();
-  pugi::xml_document doc;
-
-  auto root_node = doc.append_child("feed");
-  root_node.append_attribute("xmlns") = "http://www.w3.org/2005/Atom";
-  root_node.append_attribute("xmlns:opds") = "http://opds-spec.org/2010/catalog";
-
-  ADD_TEXT_ENTRY(root_node, "id", id);
-
-  ADD_TEXT_ENTRY(root_node, "title", title);
-  ADD_TEXT_ENTRY(root_node, "updated", date);
-
-  if (m_isSearchResult) {
-    ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults));
-    ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex));
-    ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count));
-  }
-
-  auto self_link_node = root_node.append_child("link");
-  self_link_node.append_attribute("rel") = "self";
-  self_link_node.append_attribute("href") = "";
-  self_link_node.append_attribute("type") = "application/atom+xml";
-
-
-  if (!searchDescriptionUrl.empty() ) {
-    auto search_link = root_node.append_child("link");
-    search_link.append_attribute("rel") = "search";
-    search_link.append_attribute("type") = "application/opensearchdescription+xml";
-    search_link.append_attribute("href") = searchDescriptionUrl.c_str();
-  }
-
-  if (library) {
-    for (auto& bookId: bookIds) {
-      handleBook(library->getBookById(bookId), root_node);
-    }
-  }
-
-  return nodeToString(root_node);
-}
-
-}

src/reader.cpp

@@ -22,8 +22,6 @@
 
 #include <zim/search.h>
 
-#include "tools/otherTools.h"
-
 inline char hi(char v)
 {
   char hex[] = "0123456789abcdef";
@@ -82,6 +80,7 @@ Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
     this->firstArticleOffset
         = this->zimFileHandler->getNamespaceBeginOffset('A');
     this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
+    this->currentArticleOffset = this->firstArticleOffset;
     this->nsACount = this->zimFileHandler->getNamespaceCount('A');
     this->nsICount = this->zimFileHandler->getNamespaceCount('I');
     this->zimFilePath = zimFilePath;
@@ -103,6 +102,11 @@ zim::File* Reader::getZimFileHandler() const
 {
   return this->zimFileHandler;
 }
+/* Reset the cursor for GetNextArticle() */
+void Reader::reset()
+{
+  this->currentArticleOffset = this->firstArticleOffset;
+}
 std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
 {
   std::map<const std::string, unsigned int> counters;
@@ -192,87 +196,79 @@ string Reader::getId() const
 /* Return a page url from a title */
 bool Reader::getPageUrlFromTitle(const string& title, string& url) const
 {
-  try {
-    auto entry = getEntryFromTitle(title);
-    entry = entry.getFinalEntry();
-    url = entry.getPath();
-    return true;
-  } catch (NoEntry& e) {
+  /* Extract the content from the zim file */
+  zim::Article article = this->zimFileHandler->getArticleByTitle('A', title);
+
+  if (!article.good()) {
     return false;
   }
+
+  unsigned int loopCounter = 0;
+  while (article.isRedirect() && loopCounter++ < 42) {
+    article = article.getRedirectArticle();
+  }
+
+  url = article.getLongUrl();
+  return true;
 }
 
 /* Return an URL from a title */
 string Reader::getRandomPageUrl() const
 {
-  return getRandomPage().getPath();
-}
-
-Entry Reader::getRandomPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
   zim::Article article;
-  std::string mainPagePath = this->getMainPage().getPath();
-  int watchdog = 42;
+  zim::size_type idx;
+  std::string mainPageUrl = this->getMainPageUrl();
 
   do {
-    auto idx = this->firstArticleOffset
+    idx = this->firstArticleOffset
           + (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
                              * this->nsACount);
     article = zimFileHandler->getArticle(idx);
-    if (!watchdog--) {
-      throw NoEntry();
-    }
-  } while (!article.good() && article.getLongUrl() == mainPagePath);
+  } while (article.getLongUrl() == mainPageUrl);
 
-  return article;
+  return article.getLongUrl();
 }
 
 /* Return the welcome page URL */
 string Reader::getMainPageUrl() const
 {
-  return getMainPage().getPath();
-}
+  string url = "";
 
-Entry Reader::getMainPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  zim::Article article;
-  if (this->zimFileHandler->getFileheader().hasMainPage())
-  {
-    article = zimFileHandler->getArticle(
+  if (this->zimFileHandler->getFileheader().hasMainPage()) {
+    zim::Article article = zimFileHandler->getArticle(
         this->zimFileHandler->getFileheader().getMainPage());
+    url = article.getLongUrl();
+
+    if (url.empty()) {
+      url = getFirstPageUrl();
+    }
+  } else {
+    url = getFirstPageUrl();
   }
 
-  if (!article.good())
-  {
-    return getFirstPage();
-  }
-
-  return article;
+  return url;
 }
 
 bool Reader::getFavicon(string& content, string& mimeType) const
 {
-  static const char* const paths[] = {"-/favicon", "-/favicon.png", "I/favicon.png", "I/favicon"};
+  unsigned int contentLength = 0;
+  string title;
 
-  for (auto &path: paths) {
-    try {
-      auto entry = getEntryFromPath(path);
-      entry = entry.getFinalEntry();
-      content = entry.getContent();
-      mimeType = entry.getMimetype();
-      return true;
-    } catch(NoEntry& e) {};
+  this->getContentByUrl("/-/favicon.png", content, title, contentLength, mimeType);
+
+  if (content.empty()) {
+    this->getContentByUrl("/I/favicon.png", content, title, contentLength, mimeType);
+
+    if (content.empty()) {
+      this->getContentByUrl("/I/favicon", content, title, contentLength, mimeType);
+
+      if (content.empty()) {
+        this->getContentByUrl("/-/favicon", content, title, contentLength, mimeType);
+      }
+    }
   }
 
-  return false;
+  return content.empty() ? false : true;
 }
 
 string Reader::getZimFilePath() const
@@ -280,28 +276,19 @@ string Reader::getZimFilePath() const
   return this->zimFilePath;
 }
 /* Return a metatag value */
-bool Reader::getMetadata(const string& name, string& value) const
+bool Reader::getMetatag(const string& name, string& value) const
 {
-  try {
-    auto entry = getEntryFromPath("M/"+name);
-    value = entry.getContent();
-    return true;
-  } catch(NoEntry& e) {
-    return false;
-  }
-}
+  unsigned int contentLength = 0;
+  string contentType = "";
+  string title;
 
-#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
-
-string Reader::getName() const
-{
-  METADATA("Name")
+  return this->getContentByUrl("/M/" + name, value, title, contentLength, contentType);
 }
 
 string Reader::getTitle() const
 {
   string value;
-  this->getMetadata("Title", value);
+  this->getMetatag("Title", value);
   if (value.empty()) {
     value = getLastPathElement(zimFileHandler->getFilename());
     std::replace(value.begin(), value.end(), '_', ' ');
@@ -311,98 +298,65 @@ string Reader::getTitle() const
   return value;
 }
 
-string Reader::getCreator() const
+string Reader::getName() const
 {
-  METADATA("Creator")
+  string value;
+  this->getMetatag("Name", value);
+  return value;
 }
 
-string Reader::getPublisher() const
+string Reader::getTags() const
 {
-  METADATA("Publisher")
-}
-
-string Reader::getDate() const
-{
-  METADATA("Date")
+  string value;
+  this->getMetatag("Tags", value);
+  return value;
 }
 
 string Reader::getDescription() const
 {
   string value;
-  this->getMetadata("Description", value);
+  this->getMetatag("Description", value);
 
   /* Mediawiki Collection tends to use the "Subtitle" name */
   if (value.empty()) {
-    this->getMetadata("Subtitle", value);
+    this->getMetatag("Subtitle", value);
   }
 
   return value;
 }
 
-string Reader::getLongDescription() const
-{
-  METADATA("LongDescription")
-}
-
 string Reader::getLanguage() const
 {
-  METADATA("Language")
+  string value;
+  this->getMetatag("Language", value);
+  return value;
 }
 
-string Reader::getLicense() const
+string Reader::getDate() const
 {
-  METADATA("License")
+  string value;
+  this->getMetatag("Date", value);
+  return value;
 }
 
-string Reader::getTags(bool original) const
+string Reader::getCreator() const
 {
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  if (original) {
-    return tags_str;
-  }
-  auto tags = convertTags(tags_str);
-  return join(tags, ";");
+  string value;
+  this->getMetatag("Creator", value);
+  return value;
 }
 
-
-string Reader::getTagStr(const std::string& tagName) const
+string Reader::getPublisher() const
 {
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  return getTagValueFromTagList(convertTags(tags_str), tagName);
+  string value;
+  this->getMetatag("Publisher", value);
+  return value;
 }
 
-bool Reader::getTagBool(const std::string& tagName) const
-{
-  return convertStrToBool(getTagStr(tagName));
-}
-
-string Reader::getRelation() const
-{
-  METADATA("Relation")
-}
-
-string Reader::getFlavour() const
-{
-  METADATA("Flavour")
-}
-
-string Reader::getSource() const
-{
-  METADATA("Source")
-}
-
-string Reader::getScraper() const
-{
-  METADATA("Scraper")
-}
-#undef METADATA
-
 string Reader::getOrigId() const
 {
   string value;
-  this->getMetadata("startfileuid", value);
+  this->getMetatag("startfileuid", value);
   if (value.empty()) {
     return "";
   }
@@ -427,101 +381,45 @@ string Reader::getOrigId() const
 /* Return the first page URL */
 string Reader::getFirstPageUrl() const
 {
-  return getFirstPage().getPath();
+  zim::size_type firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
+  zim::Article article = zimFileHandler->getArticle(firstPageOffset);
+  return article.getLongUrl();
 }
 
-Entry Reader::getFirstPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  auto firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
-  auto article = zimFileHandler->getArticle(firstPageOffset);
-
-  if (! article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-bool _parseUrl(const string& url, char* ns, string& title)
+bool Reader::parseUrl(const string& url, char* ns, string& title) const
 {
   /* Offset to visit the url */
   unsigned int urlLength = url.size();
   unsigned int offset = 0;
 
-  /* Ignore the first '/' */
-  if (url[offset] == '/')
+  /* Ignore the '/' */
+  while ((offset < urlLength) && (url[offset] == '/')) {
     offset++;
-
-  if (url[offset] == '/' || offset >= urlLength)
-    return false;
+  }
 
   /* Get namespace */
-  *ns = url[offset++];
+  while ((offset < urlLength) && (url[offset] != '/')) {
+    *ns = url[offset];
+    offset++;
+  }
 
-  if (url[offset] != '/' || offset >= urlLength)
-    return false;
-
-  offset++;
-
-  if ( offset >= urlLength)
-    return false;
+  /* Ignore the '/' */
+  while ((offset < urlLength) && (url[offset] == '/')) {
+    offset++;
+  }
 
   /* Get content title */
-  title = url.substr(offset, urlLength - offset);
+  unsigned int titleOffset = offset;
+  while (offset < urlLength) {
+    offset++;
+  }
+
+  /* unescape title */
+  title = url.substr(titleOffset, offset - titleOffset);
 
   return true;
 }
 
-bool Reader::parseUrl(const string& url, char* ns, string& title) const
-{
-  return _parseUrl(url, ns, title);
-}
-
-Entry Reader::getEntryFromPath(const std::string& path) const
-{
-  char ns = 0;
-  std::string short_url;
-
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-  _parseUrl(path, &ns, short_url);
-
-  if (short_url.empty() && ns == 0) {
-    return getMainPage();
-  }
-
-  auto article = zimFileHandler->getArticle(ns, short_url);
-  if (!article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-Entry Reader::getEntryFromEncodedPath(const std::string& path) const
-{
-  return getEntryFromPath(urlDecode(path, true));
-}
-
-Entry Reader::getEntryFromTitle(const std::string& title) const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  auto article = this->zimFileHandler->getArticleByTitle('A', title);
-  if (!article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
 /* Return article by url */
 bool Reader::getArticleObjectByDecodedUrl(const string& url,
                                           zim::Article& article) const
@@ -533,11 +431,11 @@ bool Reader::getArticleObjectByDecodedUrl(const string& url,
   /* Parse the url */
   char ns = 0;
   string urlStr;
-  _parseUrl(url, &ns, urlStr);
+  this->parseUrl(url, &ns, urlStr);
 
   /* Main page */
   if (urlStr.empty() && ns == 0) {
-    _parseUrl(this->getMainPage().getPath(), &ns, urlStr);
+    this->parseUrl(this->getMainPageUrl(), &ns, urlStr);
   }
 
   /* Extract the content from the zim file */
@@ -548,53 +446,26 @@ bool Reader::getArticleObjectByDecodedUrl(const string& url,
 /* Return the mimeType without the content */
 bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
 {
-  try {
-    auto entry = getEntryFromPath(url);
-    mimeType = entry.getMimetype();
+  if (this->zimFileHandler == NULL) {
+    return false;
+  }
+
+  zim::Article article;
+  if (this->getArticleObjectByDecodedUrl(url, article)) {
+    try {
+      mimeType = article.getMimeType();
+    } catch (exception& e) {
+      cerr << "Unable to get the mimetype for " << url << ":" << e.what()
+           << endl;
+      mimeType = "application/octet-stream";
+    }
     return true;
-  } catch (NoEntry& e) {
+  } else {
     mimeType = "";
     return false;
   }
 }
 
-bool get_content_by_decoded_url(const Reader& reader,
-                                const string& url,
-                                string& content,
-                                string& title,
-                                unsigned int& contentLength,
-                                string& contentType,
-                                string& baseUrl)
-{
-  content = "";
-  contentType = "";
-  contentLength = 0;
-
-  try {
-    auto entry = reader.getEntryFromPath(url);
-    entry = entry.getFinalEntry();
-    baseUrl = entry.getPath();
-    contentType = entry.getMimetype();
-    content = entry.getContent();
-    contentLength = entry.getSize();
-    title = entry.getTitle();
-
-    /* Try to set a stub HTML header/footer if necesssary */
-    if (contentType.find("text/html") != string::npos
-      && content.find("<body") == std::string::npos
-      && content.find("<BODY") == std::string::npos) {
-      content = "<html><head><title>" + title +
-              "</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
-              "charset=utf-8\" /></head><body>" +
-              content + "</body></html>";
-    }
-    return true;
-  } catch (NoEntry& e) {
-    return false;
-  }
-}
-
-
 /* Get a content from a zim file */
 bool Reader::getContentByUrl(const string& url,
                              string& content,
@@ -602,14 +473,7 @@ bool Reader::getContentByUrl(const string& url,
                              unsigned int& contentLength,
                              string& contentType) const
 {
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
+  return this->getContentByEncodedUrl(url, content, title, contentLength, contentType);
 }
 
 bool Reader::getContentByEncodedUrl(const string& url,
@@ -619,13 +483,8 @@ bool Reader::getContentByEncodedUrl(const string& url,
                                     string& contentType,
                                     string& baseUrl) const
 {
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
+  return this->getContentByDecodedUrl(
+      kiwix::urlDecode(url), content, title, contentLength, contentType, baseUrl);
 }
 
 bool Reader::getContentByEncodedUrl(const string& url,
@@ -635,13 +494,12 @@ bool Reader::getContentByEncodedUrl(const string& url,
                                     string& contentType) const
 {
   std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
+  return this->getContentByEncodedUrl(kiwix::urlDecode(url),
+                                      content,
+                                      title,
+                                      contentLength,
+                                      contentType,
+                                      stubRedirectUrl);
 }
 
 bool Reader::getContentByDecodedUrl(const string& url,
@@ -651,13 +509,12 @@ bool Reader::getContentByDecodedUrl(const string& url,
                                     string& contentType) const
 {
   std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
+  return this->getContentByDecodedUrl(kiwix::urlDecode(url),
+                                      content,
+                                      title,
+                                      contentLength,
+                                      contentType,
+                                      stubRedirectUrl);
 }
 
 bool Reader::getContentByDecodedUrl(const string& url,
@@ -667,31 +524,64 @@ bool Reader::getContentByDecodedUrl(const string& url,
                                     string& contentType,
                                     string& baseUrl) const
 {
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
+  content = "";
+  contentType = "";
+  contentLength = 0;
+
+  zim::Article article;
+  if (!this->getArticleObjectByDecodedUrl(url, article)) {
+    return false;
+  }
+
+  /* If redirect */
+  unsigned int loopCounter = 0;
+  while (article.isRedirect() && loopCounter++ < 42) {
+    article = article.getRedirectArticle();
+  }
+
+  if (loopCounter < 42) {
+    /* Compute base url (might be different from the url if redirects */
+    baseUrl
+        = "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl();
+
+    /* Get the content mime-type */
+    try {
+      contentType
+          = string(article.getMimeType().data(), article.getMimeType().size());
+    } catch (exception& e) {
+      cerr << "Unable to get the mimetype for " << baseUrl << ":" << e.what()
+           << endl;
+      contentType = "application/octet-stream";
+    }
+
+    /* Get the data */
+    content = string(article.getData().data(), article.getArticleSize());
+    title = article.getTitle();
+  }
+
+  /* Try to set a stub HTML header/footer if necesssary */
+  if (contentType.find("text/html") != string::npos
+      && content.find("<body") == std::string::npos
+      && content.find("<BODY") == std::string::npos) {
+    content = "<html><head><title>" + article.getTitle() +
+              "</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
+              "charset=utf-8\" /></head><body>" +
+              content + "</body></html>";
+  }
+
+  /* Get the data length */
+  contentLength = article.getArticleSize();
+
+  return true;
 }
 
 /* Check if an article exists */
 bool Reader::urlExists(const string& url) const
 {
-  return pathExists(url);
-}
-
-bool Reader::pathExists(const string& path) const
-{
-  if (!zimFileHandler)
-  {
-    return false;
-  }
-
   char ns = 0;
   string titleStr;
-  _parseUrl(path, &ns, titleStr);
+  this->parseUrl(url, &ns, titleStr);
+  titleStr = "/" + titleStr;
   zim::File::const_iterator findItr = zimFileHandler->find(ns, titleStr);
   return findItr != zimFileHandler->end() && findItr->getUrl() == titleStr;
 }
@@ -699,13 +589,8 @@ bool Reader::pathExists(const string& path) const
 /* Does the ZIM file has a fulltext index */
 bool Reader::hasFulltextIndex() const
 {
-  if (!zimFileHandler || zimFileHandler->is_multiPart() )
-  {
-    return false;
-  }
-
-  return ( pathExists("Z//fulltextIndex/xapian")
-        || pathExists("X/fulltext/xapian"));
+  return ( this->urlExists("/Z/fulltextIndex/xapian")
+        && !zimFileHandler->is_multiPart() );
 }
 
 /* Search titles by prefix */
@@ -714,6 +599,7 @@ bool Reader::searchSuggestions(const string& prefix,
                                const bool reset)
 {
   bool retVal = false;
+  zim::File::const_iterator articleItr;
 
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
@@ -731,7 +617,7 @@ bool Reader::searchSuggestions(const string& prefix,
     return false;
   }
 
-  for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
+  for (articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
        && this->suggestions.size() < suggestionsCount;
@@ -801,21 +687,18 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal = false;
+  bool retVal;
 
   this->suggestions.clear();
   this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
-  const auto suggestionSearch
+  const zim::Search* suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
 
   if (suggestionSearch->get_matches_estimated()) {
     for (auto current = suggestionSearch->begin();
          current != suggestionSearch->end();
          current++) {
-      if (!current->good()) {
-          continue;
-      }
       std::vector<std::string> suggestion;
       suggestion.push_back(current->getTitle());
       suggestion.push_back("/A/" + current->getUrl());
@@ -893,7 +776,7 @@ bool Reader::isCorrupted() const
 unsigned int Reader::getFileSize() const
 {
   zim::File* file = this->getZimFileHandler();
-  zim::size_type size = 0;
+  zim::offset_type size = 0;
 
   if (file != NULL) {
     size = file->getFilesize();

src/search_renderer.cpp

@@ -1,152 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <cmath>
-
-#include "search_renderer.h"
-#include "searcher.h"
-#include "reader.h"
-#include "library.h"
-#include "name_mapper.h"
-
-#include <zim/search.h>
-
-#include <mustache.hpp>
-#include "kiwixlib-resources.h"
-
-
-namespace kiwix
-{
-
-/* Constructor */
-SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : mp_searcher(searcher),
-      mp_nameMapper(mapper),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?")
-{}
-
-/* Destructor */
-SearchRenderer::~SearchRenderer() = default;
-
-void SearchRenderer::setSearchPattern(const std::string& pattern)
-{
-  this->searchPattern = pattern;
-}
-
-void SearchRenderer::setSearchContent(const std::string& name)
-{
-  this->searchContent = name;
-}
-
-void SearchRenderer::setProtocolPrefix(const std::string& prefix)
-{
-  this->protocolPrefix = prefix;
-}
-
-void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
-{
-  this->searchProtocolPrefix = prefix;
-}
-
-std::string SearchRenderer::getHtml()
-{
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
-
-  mp_searcher->restart_search();
-  Result* p_result = NULL;
-  while ((p_result = mp_searcher->getNextResult())) {
-    kainjow::mustache::data result;
-    result.set("title", p_result->get_title());
-    result.set("url", p_result->get_url());
-    result.set("snippet", p_result->get_snippet());
-    auto readerIndex = p_result->get_readerIndex();
-    auto reader = mp_searcher->get_reader(readerIndex);
-    result.set("resultContentId", mp_nameMapper->getNameForId(reader->getId()));
-
-    if (p_result->get_wordCount() >= 0) {
-      result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
-    }
-
-    results.push_back(result);
-    delete p_result;
-  }
-
-  // pages
-  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
-
-  auto resultStart = mp_searcher->getResultStart();
-  auto resultEnd = mp_searcher->getResultEnd();
-  auto resultCountPerPage = resultEnd - resultStart;
-  auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
-  auto currentPage = 0U;
-  auto pageStart = 0U;
-  auto pageEnd = 0U;
-  auto lastPageStart = 0U;
-  if (resultCountPerPage) {
-    currentPage = resultStart/resultCountPerPage;
-    pageStart = currentPage > 4 ? currentPage-4 : 0;
-    pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / resultCountPerPage) {
-      pageEnd = estimatedResultCount / resultCountPerPage;
-    }
-    if (estimatedResultCount > resultCountPerPage) {
-      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
-    }
-  }
-
-  for (unsigned int i = pageStart; i < pageEnd; i++) {
-    kainjow::mustache::data page;
-    page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * resultCountPerPage));
-    page.set("end", to_string((i + 1) * resultCountPerPage));
-
-    if (i == currentPage) {
-      page.set("selected", true);
-    }
-    pages.push_back(page);
-  }
-
-  std::string template_str = RESOURCE::templates::search_result_html;
-  kainjow::mustache::mustache tmpl(template_str);
-
-  kainjow::mustache::data allData;
-  allData.set("results", results);
-  allData.set("pages", pages);
-  allData.set("hasResults", estimatedResultCount != 0);
-  allData.set("hasPages", pageStart != pageEnd);
-  allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
-  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
-  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
-  allData.set("resultStart", to_string(resultStart + 1));
-  allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("resultRange", to_string(resultCountPerPage));
-  allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("lastResult", to_string(estimatedResultCount));
-  allData.set("protocolPrefix", this->protocolPrefix);
-  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
-  allData.set("contentId", this->searchContent);
-
-  std::stringstream ss;
-  tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
-
-}

src/searcher.cpp

@@ -17,25 +17,28 @@
  * MA 02110-1301, USA.
  */
 
-
-#include <cmath>
-
 #include "searcher.h"
+#include "kiwixlib-resources.h"
 #include "reader.h"
+#include "xapianSearcher.h"
 
 #include <zim/search.h>
 
-#include <mustache.hpp>
-#include "kiwixlib-resources.h"
+#ifdef ENABLE_CTPP2
+#include <ctpp2/CDT.hpp>
+#include <ctpp2/CTPP2FileLogger.hpp>
+#include <ctpp2/CTPP2SimpleVM.hpp>
+#include "ctpp2/CTPP2VMStringLoader.hpp"
 
-#define MAX_SEARCH_LEN 140
+using namespace CTPP;
+#endif
 
 namespace kiwix
 {
 class _Result : public Result
 {
  public:
-  _Result(zim::Search::iterator& iterator);
+  _Result(Searcher* searcher, zim::Search::iterator& iterator);
   virtual ~_Result(){};
 
   virtual std::string get_url();
@@ -48,26 +51,54 @@ class _Result : public Result
   virtual int get_readerIndex();
 
  private:
+  Searcher* searcher;
   zim::Search::iterator iterator;
 };
 
 struct SearcherInternal {
   const zim::Search* _search;
+  XapianSearcher* _xapianSearcher;
   zim::Search::iterator current_iterator;
 
-  SearcherInternal() : _search(NULL) {}
+  SearcherInternal() : _search(NULL), _xapianSearcher(NULL) {}
   ~SearcherInternal()
   {
     if (_search != NULL) {
       delete _search;
     }
+    if (_xapianSearcher != NULL) {
+      delete _xapianSearcher;
+    }
   }
 };
 
 /* Constructor */
+Searcher::Searcher(const string& xapianDirectoryPath,
+                   Reader* reader,
+                   const string& humanReadableName)
+    : internal(new SearcherInternal()),
+      searchPattern(""),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://?"),
+      resultCountPerPage(0),
+      estimatedResultCount(0),
+      resultStart(0),
+      resultEnd(0)
+{
+  loadICUExternalTables();
+  if (!reader || !reader->hasFulltextIndex()) {
+    internal->_xapianSearcher = new XapianSearcher(xapianDirectoryPath, reader);
+  }
+  this->contentHumanReadableId = humanReadableName;
+  this->humanReaderNames.push_back(humanReadableName);
+}
+
 Searcher::Searcher()
     : internal(new SearcherInternal()),
       searchPattern(""),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://?"),
+      resultCountPerPage(0),
       estimatedResultCount(0),
       resultStart(0),
       resultEnd(0)
@@ -81,23 +112,14 @@ Searcher::~Searcher()
   delete internal;
 }
 
-bool Searcher::add_reader(Reader* reader)
+void Searcher::add_reader(Reader* reader, const std::string& humanReadableName)
 {
-  if (!reader->hasFulltextIndex()) {
-      return false;
-  }
   this->readers.push_back(reader);
-  return true;
-}
-
-
-Reader* Searcher::get_reader(int readerIndex)
-{
-  return readers.at(readerIndex);
+  this->humanReaderNames.push_back(humanReadableName);
 }
 
 /* Search strings in the database */
-void Searcher::search(const std::string& search,
+void Searcher::search(std::string& search,
                       unsigned int resultStart,
                       unsigned int resultEnd,
                       const bool verbose)
@@ -108,84 +130,65 @@ void Searcher::search(const std::string& search,
     cout << "Performing query `" << search << "'" << endl;
   }
 
-  this->searchPattern = search;
-  this->resultStart = resultStart;
-  this->resultEnd = resultEnd;
+  /* If resultEnd & resultStart inverted */
+  if (resultStart > resultEnd) {
+    resultEnd += resultStart;
+    resultStart = resultEnd - resultStart;
+    resultEnd -= resultStart;
+  }
+
   /* Try to find results */
   if (resultStart != resultEnd) {
-    /* Perform the search */
-    string unaccentedSearch = removeAccents(search);
-    std::vector<const zim::File*> zims;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      if ( (*current)->hasFulltextIndex() ) {
-          zims.push_back((*current)->getZimFileHandler());
-      }
+    /* Avoid big researches */
+    this->resultCountPerPage = resultEnd - resultStart;
+    if (this->resultCountPerPage > 70) {
+      resultEnd = resultStart + 70;
+      this->resultCountPerPage = 70;
+    }
+
+    /* Perform the search */
+    this->searchPattern = search;
+    this->resultStart = resultStart;
+    this->resultEnd = resultEnd;
+    string unaccentedSearch = removeAccents(search);
+    if (internal->_xapianSearcher) {
+      internal->_xapianSearcher->searchInIndex(
+          unaccentedSearch, resultStart, resultEnd, verbose);
+      this->estimatedResultCount
+          = internal->_xapianSearcher->results.get_matches_estimated();
+    } else {
+      std::vector<const zim::File*> zims;
+      for (auto current = this->readers.begin(); current != this->readers.end();
+           current++) {
+        zims.push_back((*current)->getZimFileHandler());
+      }
+      zim::Search* search = new zim::Search(zims);
+      search->set_query(unaccentedSearch);
+      search->set_range(resultStart, resultEnd);
+      internal->_search = search;
+      internal->current_iterator = internal->_search->begin();
+      this->estimatedResultCount = internal->_search->get_matches_estimated();
     }
-    zim::Search* search = new zim::Search(zims);
-    search->set_verbose(verbose);
-    search->set_query(unaccentedSearch);
-    search->set_range(resultStart, resultEnd);
-    internal->_search = search;
-    internal->current_iterator = internal->_search->begin();
-    this->estimatedResultCount = internal->_search->get_matches_estimated();
   }
 
   return;
 }
 
-
-void Searcher::geo_search(float latitude, float longitude, float distance,
-                          unsigned int resultStart,
-                          unsigned int resultEnd,
-                          const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-  }
-
-  /* Perform the search */
-  std::ostringstream oss;
-  oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
-  this->searchPattern = oss.str();
-  this->resultStart = resultStart;
-  this->resultEnd = resultEnd;
-
-  /* Try to find results */
-  if (resultStart == resultEnd) {
-    return;
-  }
-
-  std::vector<const zim::File*> zims;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    zims.push_back((*current)->getZimFileHandler());
-  }
-  zim::Search* search = new zim::Search(zims);
-  search->set_verbose(verbose);
-  search->set_query("");
-  search->set_georange(latitude, longitude, distance);
-  search->set_range(resultStart, resultEnd);
-  internal->_search = search;
-  internal->current_iterator = internal->_search->begin();
-  this->estimatedResultCount = internal->_search->get_matches_estimated();
-}
-
-
 void Searcher::restart_search()
 {
-  if (internal->_search) {
+  if (internal->_xapianSearcher) {
+    internal->_xapianSearcher->restart_search();
+  } else {
     internal->current_iterator = internal->_search->begin();
   }
 }
 
 Result* Searcher::getNextResult()
 {
-  if (internal->_search &&
-             internal->current_iterator != internal->_search->end()) {
-    Result* result = new _Result(internal->current_iterator);
+  if (internal->_xapianSearcher) {
+    return internal->_xapianSearcher->getNextResult();
+  } else if (internal->current_iterator != internal->_search->end()) {
+    Result* result = new _Result(this, internal->current_iterator);
     internal->current_iterator++;
     return result;
   }
@@ -200,32 +203,37 @@ void Searcher::reset()
   return;
 }
 
-void Searcher::suggestions(std::string& searchPattern, const bool verbose)
+void Searcher::suggestions(std::string& search, const bool verbose)
 {
   this->reset();
 
   if (verbose == true) {
-    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
+    cout << "Performing suggestion query `" << search << "`" << endl;
   }
 
-  this->searchPattern = searchPattern;
+  this->searchPattern = search;
   this->resultStart = 0;
   this->resultEnd = 10;
-  string unaccentedSearch = removeAccents(searchPattern);
+  string unaccentedSearch = removeAccents(search);
 
-  std::vector<const zim::File*> zims;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    zims.push_back((*current)->getZimFileHandler());
+  if (internal->_xapianSearcher) {
+    /* [TODO] Suggestion on a external database ?
+     * We do not support that. */
+    this->estimatedResultCount = 0;
+  } else {
+    std::vector<const zim::File*> zims;
+    for (auto current = this->readers.begin(); current != this->readers.end();
+         current++) {
+      zims.push_back((*current)->getZimFileHandler());
+    }
+    zim::Search* search = new zim::Search(zims);
+    search->set_query(unaccentedSearch);
+    search->set_range(resultStart, resultEnd);
+    search->set_suggestion_mode(true);
+    internal->_search = search;
+    internal->current_iterator = internal->_search->begin();
+    this->estimatedResultCount = internal->_search->get_matches_estimated();
   }
-  zim::Search* search = new zim::Search(zims);
-  search->set_verbose(verbose);
-  search->set_query(unaccentedSearch);
-  search->set_range(resultStart, resultEnd);
-  search->set_suggestion_mode(true);
-  internal->_search = search;
-  internal->current_iterator = internal->_search->begin();
-  this->estimatedResultCount = internal->_search->get_matches_estimated();
 }
 
 /* Return the result count estimation */
@@ -234,8 +242,20 @@ unsigned int Searcher::getEstimatedResultCount()
   return this->estimatedResultCount;
 }
 
-_Result::_Result(zim::Search::iterator& iterator)
-    : iterator(iterator)
+bool Searcher::setProtocolPrefix(const std::string prefix)
+{
+  this->protocolPrefix = prefix;
+  return true;
+}
+
+bool Searcher::setSearchProtocolPrefix(const std::string prefix)
+{
+  this->searchProtocolPrefix = prefix;
+  return true;
+}
+
+_Result::_Result(Searcher* searcher, zim::Search::iterator& iterator)
+    : searcher(searcher), iterator(iterator)
 {
 }
 
@@ -274,6 +294,98 @@ int _Result::get_readerIndex()
 {
   return iterator.get_fileIndex();
 }
+#ifdef ENABLE_CTPP2
 
+string Searcher::getHtml()
+{
+  SimpleVM oSimpleVM;
 
+  // Fill data
+  CDT oData;
+  CDT resultsCDT(CDT::ARRAY_VAL);
+
+  this->restart_search();
+  Result* p_result = NULL;
+  while ((p_result = this->getNextResult())) {
+    CDT result;
+    result["title"] = p_result->get_title();
+    result["url"] = p_result->get_url();
+    result["snippet"] = p_result->get_snippet();
+    result["contentId"] = humanReaderNames[p_result->get_readerIndex()];
+
+    if (p_result->get_size() >= 0) {
+      result["size"] = kiwix::beautifyInteger(p_result->get_size());
+    }
+
+    if (p_result->get_wordCount() >= 0) {
+      result["wordCount"] = kiwix::beautifyInteger(p_result->get_wordCount());
+    }
+
+    resultsCDT.PushBack(result);
+    delete p_result;
+  }
+  this->restart_search();
+  oData["results"] = resultsCDT;
+
+  // pages
+  CDT pagesCDT(CDT::ARRAY_VAL);
+
+  unsigned int pageStart
+      = this->resultStart / this->resultCountPerPage >= 5
+            ? this->resultStart / this->resultCountPerPage - 4
+            : 0;
+  unsigned int pageCount
+      = this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
+
+  if (pageCount > 10) {
+    pageCount = 10;
+  } else if (pageCount == 1) {
+    pageCount = 0;
+  }
+
+  for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
+    CDT page;
+    page["label"] = i + 1;
+    page["start"] = i * this->resultCountPerPage;
+    page["end"] = (i + 1) * this->resultCountPerPage;
+
+    if (i * this->resultCountPerPage == this->resultStart) {
+      page["selected"] = true;
+    }
+
+    pagesCDT.PushBack(page);
+  }
+  oData["pages"] = pagesCDT;
+
+  oData["count"] = kiwix::beautifyInteger(this->estimatedResultCount);
+  oData["searchPattern"] = kiwix::encodeDiples(this->searchPattern);
+  oData["searchPatternEncoded"] = urlEncode(this->searchPattern);
+  oData["resultStart"] = this->resultStart + 1;
+  oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount
+                            ? this->estimatedResultCount
+                            : this->resultEnd);
+  oData["resultRange"] = this->resultCountPerPage;
+  oData["resultLastPageStart"]
+      = this->estimatedResultCount > this->resultCountPerPage
+            ? this->estimatedResultCount - this->resultCountPerPage
+            : 0;
+  oData["protocolPrefix"] = this->protocolPrefix;
+  oData["searchProtocolPrefix"] = this->searchProtocolPrefix;
+  oData["contentId"] = this->contentHumanReadableId;
+
+  std::string template_ct2 = RESOURCE::results_ct2;
+  VMStringLoader oLoader(template_ct2.c_str(), template_ct2.size());
+
+  FileLogger oLogger(stderr);
+
+  // DEBUG only (write output to stdout)
+  // oSimpleVM.Run(oData, oLoader, stdout, oLogger);
+
+  std::string sResult;
+  oSimpleVM.Run(oData, oLoader, sResult, oLogger);
+
+  return sResult;
+}
+
+#endif
 }

src/server.cpp

@@ -1,965 +0,0 @@
-/*
- * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "server.h"
-
-#ifdef _WIN32
-# if !defined(__MINGW32__) && (_MSC_VER < 1600)
-#   include "stdint4win.h"
-# endif
-# include <winsock2.h>
-# include <ws2tcpip.h>
-# ifdef __GNUC__
-  // inet_pton is not declared in mingw, even if the function exists.
-  extern "C" {
-    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
-  }
-# endif
-  typedef UINT64 uint64_t;
-  typedef UINT16 uint16_t;
-#endif
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
-#include "library.h"
-#include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
-#include "search_renderer.h"
-#include "opds_dumper.h"
-
-#include <zim/uuid.h>
-
-#include <mustache.hpp>
-
-#include <pthread.h>
-#include <atomic>
-#include <string>
-#include <vector>
-#include <chrono>
-#include "kiwixlib-resources.h"
-
-#ifndef _WIN32
-# include <arpa/inet.h>
-#endif
-
-#include "server/request_context.h"
-#include "server/response.h"
-
-#define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
-
-namespace kiwix {
-
-static IdNameMapper defaultNameMapper;
-
-typedef kainjow::mustache::data MustacheData;
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls);
-
-
-class InternalServer {
-  public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
-                   int port,
-                   std::string root,
-                   int nbThreads,
-                   bool verbose,
-                   bool withTaskbar,
-                   bool withLibraryButton,
-                   bool blockExternalLinks);
-    virtual ~InternalServer() = default;
-
-    int handlerCallback(struct MHD_Connection* connection,
-                        const char* url,
-                        const char* method,
-                        const char* version,
-                        const char* upload_data,
-                        size_t* upload_data_size,
-                        void** cont_cls);
-    bool start();
-    void stop();
-
-  private: // functions
-    Response handle_request(const RequestContext& request);
-    Response build_500(const std::string& msg);
-    Response build_404(const RequestContext& request, const std::string& zimName);
-    Response build_304(const RequestContext& request, const ETag& etag) const;
-    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
-    Response build_homepage(const RequestContext& request);
-    Response handle_skin(const RequestContext& request);
-    Response handle_catalog(const RequestContext& request);
-    Response handle_meta(const RequestContext& request);
-    Response handle_search(const RequestContext& request);
-    Response handle_suggest(const RequestContext& request);
-    Response handle_random(const RequestContext& request);
-    Response handle_captured_external(const RequestContext& request);
-    Response handle_content(const RequestContext& request);
-
-    MustacheData get_default_data() const;
-    MustacheData homepage_data() const;
-    Response get_default_response() const;
-
-    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
-
-  private: // data
-    std::string m_addr;
-    int m_port;
-    std::string m_root;
-    int m_nbThreads;
-    std::atomic_bool m_verbose;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    struct MHD_Daemon* mp_daemon;
-
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
-
-    std::string m_server_id;
-};
-
-
-Server::Server(Library* library, NameMapper* nameMapper) :
-  mp_library(library),
-  mp_nameMapper(nameMapper),
-  mp_server(nullptr)
-{
-}
-
-Server::~Server() = default;
-
-bool Server::start() {
-  mp_server.reset(new InternalServer(
-    mp_library,
-    mp_nameMapper,
-    m_addr,
-    m_port,
-    m_root,
-    m_nbThreads,
-    m_verbose,
-    m_withTaskbar,
-    m_withLibraryButton,
-    m_blockExternalLinks));
-  return mp_server->start();
-}
-
-void Server::stop() {
-  mp_server->stop();
-  mp_server.reset(nullptr);
-}
-
-void Server::setRoot(const std::string& root)
-{
-  m_root = root;
-  if (m_root[0] != '/') {
-    m_root = "/" + m_root;
-  }
-  if (m_root.back() == '/') {
-    m_root.erase(m_root.size() - 1);
-  }
-}
-
-
-InternalServer::InternalServer(Library* library,
-                               NameMapper* nameMapper,
-                               std::string addr,
-                               int port,
-                               std::string root,
-                               int nbThreads,
-                               bool verbose,
-                               bool withTaskbar,
-                               bool withLibraryButton,
-                               bool blockExternalLinks) :
-  m_addr(addr),
-  m_port(port),
-  m_root(root),
-  m_nbThreads(nbThreads),
-  m_verbose(verbose),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  mp_daemon(nullptr),
-  mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
-{}
-
-bool InternalServer::start() {
-#ifdef _WIN32
-  int flags = MHD_USE_SELECT_INTERNALLY;
-#else
-  int flags = MHD_USE_POLL_INTERNALLY;
-#endif
-  if (m_verbose.load())
-    flags |= MHD_USE_DEBUG;
-
-
-  struct sockaddr_in sockAddr;
-  memset(&sockAddr, 0, sizeof(sockAddr));
-  sockAddr.sin_family = AF_INET;
-  sockAddr.sin_port = htons(m_port);
-  if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
-      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-  } else {
-    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
-      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
-      return false;
-    }
-  }
-
-  mp_daemon = MHD_start_daemon(flags,
-                            m_port,
-                            NULL,
-                            NULL,
-                            &staticHandlerCallback,
-                            this,
-                            MHD_OPTION_SOCK_ADDR, &sockAddr,
-                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
-                            MHD_OPTION_END);
-  if (mp_daemon == nullptr) {
-    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
-              << " is maybe already occupied or need more permissions to be open. "
-                 "Please try as root or with a port number higher or equal to 1024."
-              << std::endl;
-    return false;
-  }
-  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
-  m_server_id = kiwix::to_string(server_start_time.count());
-  return true;
-}
-
-void InternalServer::stop()
-{
-  MHD_stop_daemon(mp_daemon);
-}
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls)
-{
-  InternalServer* _this = static_cast<InternalServer*>(cls);
-
-  return _this->handlerCallback(connection,
-                                url,
-                                method,
-                                version,
-                                upload_data,
-                                upload_data_size,
-                                cont_cls);
-}
-
-int InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                    const char* url,
-                                    const char* method,
-                                    const char* version,
-                                    const char* upload_data,
-                                    size_t* upload_data_size,
-                                    void** cont_cls)
-{
-  auto start_time = std::chrono::steady_clock::now();
-  if (m_verbose.load() ) {
-    printf("======================\n");
-    printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
-  }
-  RequestContext request(connection, m_root, url, method, version);
-
-  if (m_verbose.load() ) {
-    request.print_debug_info();
-  }
-  /* Unexpected method */
-  if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST
-   && request.get_method() != RequestMethod::HEAD) {
-    printf("Reject request because of unhandled request method.\n");
-    printf("----------------------\n");
-    return MHD_NO;
-  }
-
-  auto response = handle_request(request);
-
-  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
-    printf("========== INTERNAL ERROR !! ============\n");
-    if (!m_verbose.load()) {
-      printf("Requesting : \n");
-      printf("full_url : %s\n", url);
-      request.print_debug_info();
-    }
-  }
-
-  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response.set_server_id(m_server_id);
-
-  auto ret = response.send(request, connection);
-  auto end_time = std::chrono::steady_clock::now();
-  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
-  if (m_verbose.load()) {
-    printf("Request time : %fs\n", time_span.count());
-    printf("----------------------\n");
-  }
-  return ret;
-}
-
-Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
-{
-  auto response = get_default_response();
-  response.set_code(MHD_HTTP_NOT_MODIFIED);
-  response.set_etag(etag);
-  response.set_content("");
-  return response;
-}
-
-Response InternalServer::handle_request(const RequestContext& request)
-{
-  try {
-    if (! request.is_valid_url())
-      return build_404(request, "");
-
-    const ETag etag = get_matching_if_none_match_etag(request);
-    if ( etag )
-      return build_304(request, etag);
-
-    if (kiwix::startsWith(request.get_url(), "/skin/"))
-      return handle_skin(request);
-
-    if (startsWith(request.get_url(), "/catalog"))
-      return handle_catalog(request);
-
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
-
-    if (request.get_url() == "/search")
-      return handle_search(request);
-
-    if (request.get_url() == "/suggest")
-     return handle_suggest(request);
-
-    if (request.get_url() == "/random")
-      return handle_random(request);
-
-    if (request.get_url() == "/catch/external")
-      return handle_captured_external(request);
-
-    return handle_content(request);
-  } catch (std::exception& e) {
-    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return build_500(e.what());
-  } catch (...) {
-    fprintf(stderr, "===== Unhandled unknown error\n");
-    return build_500("Unknown error");
-  }
-}
-
-MustacheData InternalServer::get_default_data() const
-{
-  MustacheData data;
-  data.set("root", m_root);
-  return data;
-}
-
-Response InternalServer::get_default_response() const
-{
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
-}
-
-
-Response InternalServer::build_404(const RequestContext& request,
-                                   const std::string& bookName)
-{
-  MustacheData results;
-  results.set("url", request.get_full_url());
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::_404_html, results);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_NOT_FOUND);
-  response.set_compress(true);
-  response.set_taskbar(bookName, "");
-
-  return response;
-}
-
-Response InternalServer::build_500(const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  Response response(m_root, true, false, false, false);
-  response.set_template(RESOURCE::templates::_500_html, data);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
-MustacheData InternalServer::homepage_data() const
-{
-  auto data = get_default_data();
-
-  MustacheData books{MustacheData::type::list};
-  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = mp_library->getBookById(bookId);
-
-    MustacheData book;
-    book.set("name", mp_nameMapper->getNameForId(bookId));
-    book.set("title", currentBook.getTitle());
-    book.set("description", currentBook.getDescription());
-    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
-    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
-    books.push_back(book);
-  }
-
-  data.set("books", books);
-  return data;
-}
-
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
-Response InternalServer::build_homepage(const RequestContext& request)
-{
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, homepage_data());
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_meta(const RequestContext& request)
-{
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
-  } else {
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-  response.set_content(content);
-  response.set_mimeType(mimeType);
-  response.set_compress(false);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_suggest(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_suggest\n");
-  }
-
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-  std::string suggestion;
-
-  std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
-  }
-
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    reader->searchSuggestionsSmart(term, maxSuggestionCount);
-    while (reader->getNextSuggestion(suggestion)) {
-      MustacheData result;
-      result.set("label", suggestion);
-      result.set("value", suggestion);
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
-    }
-  }
-
-  /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
-    MustacheData result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term + " ");
-    result.set("first", first);
-    results.push_back(result);
-  }
-
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::suggestion_json, data);
-  response.set_mimeType("application/json; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_skin(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_skin\n");
-  }
-
-  auto response = get_default_response();
-  auto resourceName = request.get_url().substr(1);
-  try {
-    response.set_content(getResource(resourceName));
-  } catch (const ResourceNotFound& e) {
-    return build_404(request, "");
-  }
-  response.set_mimeType(getMimeTypeForFile(resourceName));
-  response.set_compress(true);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_search(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_search\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
-
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
-
-  std::shared_ptr<Reader> reader(nullptr);
-  try {
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Try first to load directly the article */
-  if (reader != nullptr && !patternString.empty()) {
-    std::string patternCorrespondingUrl;
-    auto variants = reader->getTitleVariants(patternString);
-    auto variantsItr = variants.begin();
-
-    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
-      try {
-        auto entry = reader->getEntryFromTitle(*variantsItr);
-        entry = entry.getFinalEntry();
-        patternCorrespondingUrl = entry.getPath();
-        break;
-      } catch(kiwix::NoEntry& e) {
-        variantsItr++;
-      }
-    }
-
-    /* If article found then redirect directly to it */
-    if (!patternCorrespondingUrl.empty()) {
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
-      return response;
-    }
-  }
-
-  /* Make the search */
-  auto response = get_default_response();
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
-  response.set_compress(true);
-
-  if ( (!reader && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    response.set_template(RESOURCE::templates::no_search_result_html, data);
-    response.set_code(MHD_HTTP_NOT_FOUND);
-    return response;
-  }
-
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-  auto end = 25;
-  try {
-    end = request.get_argument<unsigned int>("end");
-  } catch (const std::exception&) {}
-  if (start>end) {
-    auto tmp = start;
-    start = end;
-    end = tmp;
-  }
-  if (end > start + MAX_SEARCH_LEN) {
-    end = start + MAX_SEARCH_LEN;
-  }
-
-  /* Get the results */
-  try {
-    if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
-    } else {
-      searcher.search(patternString,
-                       start, end, m_verbose.load());
-    }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
-    response.set_content(renderer.getHtml());
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-  return response;
-}
-
-Response InternalServer::handle_random(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_random\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  try {
-    auto entry = reader->getRandomPage();
-    return build_redirect(bookName, entry.getFinalEntry());
-  } catch(kiwix::NoEntry& e) {
-    return build_404(request, bookName);
-  }
-}
-
-Response InternalServer::handle_captured_external(const RequestContext& request)
-{
-  std::string source = "";
-  try {
-    source = kiwix::urlDecode(request.get_argument("source"));
-  } catch (const std::out_of_range& e) {}
-
-  if (source.empty())
-    return build_404(request, "");
-
-  auto data = get_default_data();
-  data.set("source", source);
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::captured_external_html, data);
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return build_404(request, "");
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return build_404(request, "");
-  }
-
-  auto response = get_default_response();
-  response.set_compress(true);
-  if (url == "searchdescription.xml") {
-    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
-    response.set_mimeType("application/opensearchdescription+xml");
-    return response;
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper;
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    opdsDumper.setTitle("All zims");
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
-    string query("<Empty query>");
-    size_t count(10);
-    size_t startIndex(0);
-    try {
-      query = request.get_argument("q");
-      filter.query(query);
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      count = extractFromString<unsigned long>(request.get_argument("count"));
-    } catch (...) {}
-    try {
-      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
-    } catch (...) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    opdsDumper.setTitle("Search result for " + query);
-    uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(filter);
-    auto totalResults = bookIdsToDump.size();
-    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
-    if (count>0 && bookIdsToDump.size() > count) {
-      bookIdsToDump.resize(count);
-    }
-    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
-  }
-
-  opdsDumper.setId(kiwix::to_string(uuid));
-  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
-  return response;
-}
-
-namespace
-{
-
-std::string get_book_name(const RequestContext& request)
-{
-  try {
-    return request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return std::string();
-  }
-}
-
-} // unnamed namespace
-
-std::shared_ptr<Reader>
-InternalServer::get_reader(const std::string& bookName) const
-{
-  std::shared_ptr<Reader> reader;
-  try {
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-  }
-  return reader;
-}
-
-Response
-InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
-{
-  auto response = get_default_response();
-  response.set_redirection(m_root + "/" + bookName + "/" +
-                           kiwix::urlEncode(entry.getPath()));
-  return response;
-}
-
-Response InternalServer::handle_content(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_content\n");
-  }
-
-  const std::string bookName = get_book_name(request);
-  if (bookName.empty())
-    return build_homepage(request);
-
-  const std::shared_ptr<Reader> reader = get_reader(bookName);
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  auto urlStr = request.get_url().substr(bookName.size()+1);
-  if (urlStr[0] == '/') {
-    urlStr = urlStr.substr(1);
-  }
-
-  kiwix::Entry entry;
-
-  try {
-    entry = reader->getEntryFromPath(urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
-      return build_redirect(bookName, entry.getFinalEntry());
-    }
-  } catch(kiwix::NoEntry& e) {
-    if (m_verbose.load())
-      printf("Failed to find %s\n", urlStr.c_str());
-
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-
-  response.set_entry(entry, request);
-
-  if (m_verbose.load()) {
-    printf("Found %s\n", entry.getPath().c_str());
-    printf("mimeType: %s\n", response.get_mimeType().c_str());
-  }
-
-  if (response.get_mimeType().find("text/html") != string::npos)
-    response.set_taskbar(bookName, reader->getTitle());
-
-  return response;
-}
-
-}

src/server/etag.cpp

@@ -1,135 +0,0 @@
-/*
- * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include "etag.h"
-
-#include "tools/stringTools.h"
-
-#include <algorithm>
-#include <sstream>
-
-namespace kiwix {
-
-namespace {
-
-// Characters in the options part of the ETag could in principle be picked up
-// from the latin alphabet in natural order (the character corresponding to
-// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
-// this file). However it is better to have some mnemonics in the option names,
-// hence below variable: all_options[opt] corresponds to the character going
-// into the ETag for ETag::Option opt.
-// IMPORTANT: The characters in all_options must come in sorted order (so that
-// IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
-
-static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
-
-bool isValidServerId(const std::string& s)
-{
-  return !s.empty() && s.find_first_of("\"/") == std::string::npos;
-}
-
-bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
-{
-  std::string::size_type i = 0;
-  for ( const char c : s )
-  {
-    const std::string::size_type j = sortedString.find(c, i);
-    if ( j == std::string::npos )
-      return false;
-    i = j+1;
-  }
-  return true;
-}
-
-bool isValidOptionsString(const std::string& s)
-{
-  return isSubsequenceOf(s, all_options);
-}
-
-} // namespace
-
-
-void ETag::set_option(Option opt)
-{
-  if ( ! get_option(opt) )
-  {
-    m_options.push_back(all_options[opt]);
-    std::sort(m_options.begin(), m_options.end());
-  }
-}
-
-bool ETag::get_option(Option opt) const
-{
-  return m_options.find(all_options[opt]) != std::string::npos;
-}
-
-std::string ETag::get_etag() const
-{
-  if ( m_serverId.empty() )
-    return std::string();
-
-  return "\"" + m_serverId + "/" + m_options + "\"";
-}
-
-ETag::ETag(const std::string& serverId, const std::string& options)
-{
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
-  {
-    m_serverId = serverId;
-    m_options = options;
-  }
-}
-
-ETag ETag::parse(std::string s)
-{
-  if ( kiwix::startsWith("W/", s) )
-    s = s.substr(2);
-
-  if ( s.front() != '"' || s.back() != '"' )
-    return ETag();
-
-  s = s.substr(1, s.size()-2);
-
-  const std::string::size_type i = s.find('/');
-  if ( i == std::string::npos )
-    return ETag();
-
-  return ETag(s.substr(0, i), s.substr(i+1));
-}
-
-ETag ETag::match(const std::string& etags, const std::string& server_id)
-{
-  std::istringstream ss(etags);
-  std::string etag_str;
-  while ( ss >> etag_str )
-  {
-    if ( etag_str.back() == ',' )
-      etag_str.pop_back();
-
-    const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
-      return etag;
-  }
-
-  return ETag();
-}
-
-} // namespace kiwix

src/server/etag.h

@@ -1,85 +0,0 @@
-/*
- * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef KIWIXLIB_SERVER_ETAG_H
-#define KIWIXLIB_SERVER_ETAG_H
-
-#include <string>
-
-namespace kiwix {
-
-// The ETag string used by Kiwix server (more precisely, its value inside the
-// double quotes) consists of two parts:
-//
-// 1. ServerId - The string obtained on server start up
-//
-// 2. Options -  Zero or more characters encoding the values of some of the
-//               headers of the response
-//
-// The two parts are separated with a slash (/) symbol (which is always present,
-// even when the the options part is empty). Neither portion of a Kiwix ETag
-// may contain the slash symbol.
-// Examples of valid Kiwix server ETags (including the double quotes):
-//
-//   "abcdefghijklmn/"
-//   "1234567890/z"
-//   "1234567890/cz"
-//
-// The options part of the Kiwix ETag allows to correctly set the required
-// headers when responding to a conditional If-None-Match request with a 304
-// (Not Modified) response without following the full code path that would
-// discover the necessary options.
-
-class ETag
-{
-  public: // types
-    enum Option {
-      CACHEABLE_ENTITY,
-      COMPRESSED_CONTENT,
-      OPTION_COUNT
-    };
-
-  public: // functions
-    ETag() {}
-
-    void set_server_id(const std::string& id) { m_serverId = id; }
-    void set_option(Option opt);
-
-    explicit operator bool() const { return !m_serverId.empty(); }
-
-    bool get_option(Option opt) const;
-    std::string get_etag() const;
-
-
-    static ETag match(const std::string& etags, const std::string& server_id);
-
-  private: // functions
-    ETag(const std::string& serverId, const std::string& options);
-
-    static ETag parse(std::string s);
-
-  private: // data
-    std::string m_serverId;
-    std::string m_options;
-};
-
-} // namespace kiwix
-
-#endif // KIWIXLIB_SERVER_ETAG_H

src/server/request_context.cpp

@@ -1,208 +0,0 @@
-/*
- * Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include "request_context.h"
-#include <string.h>
-#include <stdexcept>
-#include <sstream>
-#include <cstdio>
-#include <atomic>
-
-namespace kiwix {
-
-static std::atomic_ullong s_requestIndex(0);
-
-namespace {
-
-RequestMethod str2RequestMethod(const std::string& method) {
-  if      (method == "GET")     return RequestMethod::GET;
-  else if (method == "HEAD")    return RequestMethod::HEAD;
-  else if (method == "POST")    return RequestMethod::POST;
-  else if (method == "PUT")     return RequestMethod::PUT;
-  else if (method == "DELETE")  return RequestMethod::DELETE_;
-  else if (method == "CONNECT") return RequestMethod::CONNECT;
-  else if (method == "OPTIONS") return RequestMethod::OPTIONS;
-  else if (method == "TRACE")   return RequestMethod::TRACE;
-  else if (method == "PATCH")   return RequestMethod::PATCH;
-  else                          return RequestMethod::OTHER;
-}
-
-std::string
-fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
-{
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    return full_url;
-  } else if (full_url == rootLocation) {
-    return "/";
-  } else if (full_url.size() > rootLocation.size() &&
-             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-    return full_url.substr(rootLocation.size());
-  } else {
-    return "";
-  }
-}
-
-} // unnamed namespace
-
-RequestContext::RequestContext(struct MHD_Connection* connection,
-                               std::string rootLocation,
-                               const std::string& _url,
-                               const std::string& _method,
-                               const std::string& version) :
-  full_url(_url),
-  url(fullURL2LocalURL(_url, rootLocation)),
-  method(str2RequestMethod(_method)),
-  version(version),
-  requestIndex(s_requestIndex++),
-  acceptEncodingDeflate(false),
-  accept_range(false),
-  range_pair(0, -1)
-{
-  MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
-  MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
-
-  try {
-    acceptEncodingDeflate =
-        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
-  } catch (const std::out_of_range&) {}
-
-  /*Check if range is requested. */
-  try {
-    auto range = get_header(MHD_HTTP_HEADER_RANGE);
-    int start = 0;
-    int end = -1;
-    std::istringstream iss(range);
-    char c;
-
-    iss >> start >> c;
-    if (iss.good() && c=='-') {
-      iss >> end;
-      if (iss.fail()) {
-        // Something went wrong will extracting.
-        end = -1;
-      }
-      if (iss.eof()) {
-        accept_range = true;
-        range_pair = std::pair<int, int>(start, end);
-      }
-    }
-  } catch (const std::out_of_range&) {}
-}
-
-RequestContext::~RequestContext()
-{}
-
-
-int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                 const char *key, const char *value)
-{
-  RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[key] = value;
-  return MHD_YES;
-}
-
-int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                   const char *key, const char* value)
-{
-  RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->arguments[key] = value == nullptr ? "" : value;
-  return MHD_YES;
-}
-
-void RequestContext::print_debug_info() const {
-  printf("method    : %s (%d)\n", method==RequestMethod::GET ? "GET" :
-                                  method==RequestMethod::POST ? "POST" :
-                                  "OTHER", (int)method);
-  printf("version   : %s\n", version.c_str());
-  printf("request#  : %lld\n", requestIndex);
-  printf("headers   :\n");
-  for (auto it=headers.begin(); it!=headers.end(); it++) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
-  }
-  printf("arguments :\n");
-  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
-  }
-  printf("Parsed : \n");
-  printf("full_url: %s\n", full_url.c_str());
-  printf("url   : %s\n", url.c_str());
-  printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
-  printf("has_range : %d\n", accept_range);
-  printf("is_valid_url : %d\n", is_valid_url());
-  printf(".............\n");
-}
-
-
-RequestMethod RequestContext::get_method() const {
-  return method;
-}
-
-std::string RequestContext::get_url() const {
-  return url;
-}
-
-std::string RequestContext::get_url_part(int number) const {
-  size_t start = 1;
-  while(true) {
-    auto found = url.find('/', start);
-    if (number == 0) {
-      if (found == std::string::npos) {
-        return url.substr(start);
-      } else {
-        return url.substr(start, found-start);
-      }
-    } else {
-      if (found == std::string::npos) {
-        throw std::out_of_range("No parts");
-      }
-      start = found + 1;
-      number -= 1;
-    }
-  }
-}
-
-std::string RequestContext::get_full_url() const {
-  return full_url;
-}
-
-bool RequestContext::is_valid_url() const {
-  return !url.empty();
-}
-
-bool RequestContext::has_range() const {
-  return accept_range;
-}
-
-std::pair<int, int> RequestContext::get_range() const {
-  return range_pair;
-}
-
-template<>
-std::string RequestContext::get_argument(const std::string& name) const {
-  return arguments.at(name);
-}
-
-std::string RequestContext::get_header(const std::string& name) const {
-  return headers.at(name);
-}
-
-}

src/server/request_context.h

@@ -1,112 +0,0 @@
-/*
- * Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef REQUEST_CONTEXT_H
-#define REQUEST_CONTEXT_H
-
-#include <string>
-#include <sstream>
-#include <map>
-#include <stdexcept>
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-namespace kiwix {
-
-enum class RequestMethod {
-    GET,
-    HEAD,
-    POST,
-    PUT,
-    DELETE_,
-    CONNECT,
-    OPTIONS,
-    TRACE,
-    PATCH,
-    OTHER
-};
-
-class KeyError : public std::runtime_error {};
-class IndexError: public std::runtime_error {};
-
-
-class RequestContext {
-  public: // types
-  typedef std::pair<int, int> ByteRange;
-
-  public: // functions
-    RequestContext(struct MHD_Connection* connection,
-                   std::string rootLocation,
-                   const std::string& url,
-                   const std::string& method,
-                   const std::string& version);
-    ~RequestContext();
-
-    void print_debug_info() const;
-
-    bool is_valid_url() const;
-
-    std::string get_header(const std::string& name) const;
-    template<typename T=std::string>
-    T get_argument(const std::string& name) const {
-        std::istringstream stream(arguments.at(name));
-        T v;
-        stream >> v;
-        return v;
-    }
-
-
-    RequestMethod get_method() const;
-    std::string get_url() const;
-    std::string get_url_part(int part) const;
-    std::string get_full_url() const;
-
-    bool has_range() const;
-    ByteRange get_range() const;
-
-    bool can_compress() const { return acceptEncodingDeflate; }
-
-  private: // data
-    std::string full_url;
-    std::string url;
-    RequestMethod method;
-    std::string version;
-    unsigned long long requestIndex;
-
-    bool acceptEncodingDeflate;
-
-    bool accept_range;
-    ByteRange range_pair;
-    std::map<std::string, std::string> headers;
-    std::map<std::string, std::string> arguments;
-
-  private: // functions
-    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
-};
-
-template<> std::string RequestContext::get_argument(const std::string& name) const;
-
-}
-
-#endif //REQUEST_CONTEXT_H

src/server/response.cpp

@@ -1,345 +0,0 @@
-
-
-
-#include "response.h"
-#include "request_context.h"
-#include "kiwixlib-resources.h"
-
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
-
-#include "string.h"
-#include <mustache.hpp>
-#include <zlib.h>
-
-
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
-
-namespace kiwix {
-
-namespace
-{
-// some utilities
-
-std::string get_mime_type(const kiwix::Entry& entry)
-{
-  try {
-    return entry.getMimetype();
-  } catch (exception& e) {
-    return "application/octet-stream";
-  }
-}
-
-bool is_compressible_mime_type(const std::string& mimeType)
-{
-  return mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/atom") != string::npos
-      || mimeType.find("application/opensearchdescription") != string::npos
-      || mimeType.find("application/json") != string::npos;
-}
-
-int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
-{
-  return range.second == -1
-       ? entry.getSize() - range.first
-       : range.second - range.first;
-}
-
-} // unnamed namespace
-
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
-  : m_verbose(verbose),
-    m_root(root),
-    m_content(""),
-    m_mimeType(""),
-    m_returnCode(MHD_HTTP_OK),
-    m_withTaskbar(withTaskbar),
-    m_withLibraryButton(withLibraryButton),
-    m_blockExternalLinks(blockExternalLinks),
-    m_addTaskbar(false),
-    m_bookName(""),
-    m_startRange(0),
-    m_lenRange(0)
-{
-}
-
-
-static int print_key_value (void *cls, enum MHD_ValueKind kind,
-                            const char *key, const char *value)
-{
-  printf (" - %s: '%s'\n", key, value);
-  return MHD_YES;
-}
-
-
-struct RunningResponse {
-   kiwix::Entry entry;
-   int range_start;
-
-   RunningResponse(kiwix::Entry entry,
-                   int range_start) :
-     entry(entry),
-     range_start(range_start)
-   {}
-};
-
-static ssize_t callback_reader_from_entry(void* cls,
-                                  uint64_t pos,
-                                  char* buf,
-                                  size_t max)
-{
-  RunningResponse* response = static_cast<RunningResponse*>(cls);
-
-  size_t max_size_to_set = min<size_t>(
-    max,
-    response->entry.getSize() - pos - response->range_start);
-
-  if (max_size_to_set <= 0) {
-    return MHD_CONTENT_READER_END_WITH_ERROR;
-  }
-
-  zim::Blob blob = response->entry.getBlob(response->range_start+pos, max_size_to_set);
-  memcpy(buf, blob.data(), max_size_to_set);
-  return max_size_to_set;
-}
-
-static void callback_free_response(void* cls)
-{
-  RunningResponse* response = static_cast<RunningResponse*>(cls);
-  delete response;
-}
-
-
-
-void print_response_info(int retCode, MHD_Response* response)
-{
-  printf("Response :\n");
-  printf("httpResponseCode : %d\n", retCode);
-  printf("headers :\n");
-  MHD_get_response_headers(response, print_key_value, nullptr);
-}
-
-
-std::string render_template(const std::string& template_str, kainjow::mustache::data data)
-{
-  kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
-  std::stringstream ss;
-  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
-
-
-void Response::introduce_taskbar()
-{
-  if (! m_withTaskbar)
-    // Taskbar is globally disabled.
-    return;
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  data.set("content", m_bookName);
-  data.set("hascontent", !m_bookName.empty());
-  data.set("title", m_bookTitle);
-  data.set("withlibrarybutton", m_withLibraryButton);
-  auto head_content = render_template(RESOURCE::templates::head_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<head>",
-    head_content);
-
-  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<body[^>]*>",
-    taskbar_part);
-}
-
-
-void Response::inject_externallinks_blocker()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<head>",
-    script_tag);
-}
-
-bool
-Response::can_compress(const RequestContext& request) const
-{
-  return request.can_compress()
-      && is_compressible_mime_type(m_mimeType)
-      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
-}
-
-MHD_Response*
-Response::create_raw_content_mhd_response(const RequestContext& request)
-{
-  if (m_addTaskbar) {
-    introduce_taskbar();
-  }
-  if ( m_blockExternalLinks ) {
-    inject_externallinks_blocker();
-  }
-
-  bool shouldCompress = m_compress && can_compress(request);
-  if (shouldCompress) {
-    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-    uLongf comprLen = compr_buffer.capacity();
-    int err = compress(&compr_buffer[0],
-                       &comprLen,
-                       (const Bytef*)(m_content.data()),
-                       m_content.size());
-    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-      /* /!\ Internet Explorer has a bug with deflate compression.
-         It can not handle the first two bytes (compression headers)
-         We need to chunk them off (move the content 2bytes)
-         It has no incidence on other browsers
-         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-      m_content = string((char*)&compr_buffer[2], comprLen - 2);
-      m_etag.set_option(ETag::COMPRESSED_CONTENT);
-    } else {
-      shouldCompress = false;
-    }
-  }
-
-  MHD_Response* response = MHD_create_response_from_buffer(
-    m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
-
-  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
-  // shouldCompress can have different values. This can happen for a 304 (Not
-  // Modified) response generated while handling a conditional If-None-Match
-  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
-  // option) is obtained from the ETag list of the If-None-Match header and the
-  // response has no body (which shouldn't be compressed).
-  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
-    MHD_add_response_header(
-        response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-  }
-  if (shouldCompress) {
-    MHD_add_response_header(
-        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
-  }
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_redirection_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_entry_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
-                                               16384,
-                                               callback_reader_from_entry,
-                                               new RunningResponse(m_entry, m_startRange),
-                                               callback_free_response);
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-  std::ostringstream oss;
-  oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-      << "/" << m_entry.getSize();
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_mhd_response(const RequestContext& request)
-{
-  switch (m_mode) {
-    case ResponseMode::RAW_CONTENT :
-      return create_raw_content_mhd_response(request);
-
-    case ResponseMode::REDIRECTION :
-      return create_redirection_mhd_response();
-
-    case ResponseMode::ENTRY :
-      return create_entry_mhd_response();
-  }
-  return nullptr;
-}
-
-int Response::send(const RequestContext& request, MHD_Connection* connection)
-{
-  MHD_Response* response = create_mhd_response(request);
-
-  MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
-  const std::string etag = m_etag.get_etag();
-  if ( ! etag.empty() )
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
-
-  if (m_returnCode == MHD_HTTP_OK && request.has_range())
-    m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
-
-  if (m_verbose)
-    print_response_info(m_returnCode, response);
-
-  auto ret = MHD_queue_response(connection, m_returnCode, response);
-  MHD_destroy_response(response);
-  return ret;
-}
-
-void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
-  set_content(render_template(template_str, data));
-}
-
-void Response::set_content(const std::string& content) {
-  m_content = content;
-  m_mode = ResponseMode::RAW_CONTENT;
-}
-
-void Response::set_redirection(const std::string& url) {
-  m_content = url;
-  m_mode = ResponseMode::REDIRECTION;
-  m_returnCode = MHD_HTTP_FOUND;
-}
-
-void Response::set_entry(const Entry& entry, const RequestContext& request) {
-  m_entry = entry;
-  m_mode = ResponseMode::ENTRY;
-
-  const std::string mimeType = get_mime_type(entry);
-  set_mimeType(mimeType);
-  set_cacheable();
-
-  if ( is_compressible_mime_type(mimeType) ) {
-    zim::Blob raw_content = entry.getBlob();
-    const std::string content = string(raw_content.data(), raw_content.size());
-
-    set_content(content);
-    set_compress(true);
-  } else {
-    const int range_len = get_range_len(entry, request.get_range());
-    set_range_first(request.get_range().first);
-    set_range_len(range_len);
-  }
-}
-
-void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
-{
-  m_addTaskbar = true;
-  m_bookName = bookName;
-  m_bookTitle = bookTitle;
-}
-
-
-}

src/server/response.h

@@ -1,103 +0,0 @@
-/*
- * Copyright 2019 Matthieu Gautier<mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef KIWIXLIB_SERVER_RESPONSE_H
-#define KIWIXLIB_SERVER_RESPONSE_H
-
-#include <string>
-
-#include <mustache.hpp>
-#include "entry.h"
-#include "etag.h"
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-namespace kiwix {
-
-enum class ResponseMode {
-  RAW_CONTENT,
-  REDIRECTION,
-  ENTRY
-};
-
-class RequestContext;
-
-class Response {
-  public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
-    ~Response() = default;
-
-    int send(const RequestContext& request, MHD_Connection* connection);
-
-    void set_template(const std::string& template_str, kainjow::mustache::data data);
-    void set_content(const std::string& content);
-    void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry, const RequestContext& request);
-
-
-    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
-    void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
-    void set_etag(const ETag& etag) { m_etag = etag; }
-    void set_compress(bool compress) { m_compress = compress; }
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
-    void set_range_first(uint64_t start) { m_startRange = start; }
-    void set_range_len(uint64_t len) { m_lenRange = len; }
-
-    int getReturnCode() const { return m_returnCode; }
-    std::string get_mimeType() const { return m_mimeType; }
-
-    void introduce_taskbar();
-    void inject_externallinks_blocker();
-
-    bool can_compress(const RequestContext& request) const;
-
-  private: // functions
-    MHD_Response* create_mhd_response(const RequestContext& request);
-    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
-    MHD_Response* create_redirection_mhd_response() const;
-    MHD_Response* create_entry_mhd_response() const;
-
-  private: // data
-    bool m_verbose;
-    ResponseMode m_mode;
-    std::string m_root;
-    std::string m_content;
-    Entry m_entry;
-    std::string m_mimeType;
-    int m_returnCode;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    bool m_compress;
-    bool m_addTaskbar;
-    std::string m_bookName;
-    std::string m_bookTitle;
-    uint64_t m_startRange;
-    uint64_t m_lenRange;
-    ETag m_etag;
-};
-
-}
-
-#endif //KIWIXLIB_SERVER_RESPONSE_H

src/subprocess.cpp

@@ -1,40 +0,0 @@
-
-
-#include "subprocess.h"
-
-#ifdef _WIN32
-# include "subprocess_windows.h"
-#else
-# include "subprocess_unix.h"
-#endif
-
-Subprocess::Subprocess(std::unique_ptr<SubprocessImpl> impl, commandLine_t& commandLine) :
-  mp_impl(std::move(impl))
-{
-  mp_impl->run(commandLine);
-}
-
-Subprocess::~Subprocess()
-{
-  mp_impl->kill();
-}
-
-std::unique_ptr<Subprocess> Subprocess::run(commandLine_t& commandLine)
-{
-#ifdef _WIN32
-  auto impl = std::unique_ptr<SubprocessImpl>(new WinImpl);
-#else
-  auto impl = std::unique_ptr<UnixImpl>(new UnixImpl);
-#endif
-  return std::unique_ptr<Subprocess>(new Subprocess(std::move(impl), commandLine));
-}
-
-bool Subprocess::isRunning()
-{
-  return mp_impl->isRunning();
-}
-
-bool Subprocess::kill()
-{
-  return mp_impl->kill();
-}

src/subprocess.h

@@ -1,36 +0,0 @@
-
-#ifndef KIWIX_SUBPROCESS_H_
-#define KIWIX_SUBPROCESS_H_
-
-#include <string>
-#include <memory>
-#include <vector>
-
-typedef std::vector<const char *> commandLine_t;
-
-class SubprocessImpl
-{
-  public:
-    virtual void run(commandLine_t& commandLine) = 0;
-    virtual bool kill() = 0;
-    virtual bool isRunning() = 0;
-    virtual ~SubprocessImpl() = default;
-};
-
-class Subprocess
-{
-  private:
-    // Impl depends of the system (window, unix, ...)
-    std::unique_ptr<SubprocessImpl> mp_impl;
-    Subprocess(std::unique_ptr<SubprocessImpl> impl, commandLine_t& commandLine);
-
-  public:
-    static std::unique_ptr<Subprocess> run(commandLine_t& commandLine);
-    ~Subprocess();
-
-    bool isRunning();
-    bool kill();
-};
-
-
-#endif // KIWIX_SUBPROCESS_H_

src/subprocess_unix.cpp

@@ -1,93 +0,0 @@
-
-
-#include "subprocess_unix.h"
-
-#include <sys/types.h>
-#include <unistd.h>
-#include <iostream>
-#include <signal.h>
-#include <sys/wait.h>
-#include <stdlib.h>
-
-UnixImpl::UnixImpl():
-  m_pid(0),
-  m_running(false),
-  m_mutex(PTHREAD_MUTEX_INITIALIZER),
-  m_waitingThread()
-{
-}
-
-UnixImpl::~UnixImpl()
-{
-  kill();
-// Android has no pthread_cancel :(
-#ifdef __ANDROID__
-  pthread_kill(m_waitingThread, SIGUSR1);
-#else
-  pthread_cancel(m_waitingThread);
-#endif
-}
-
-#ifdef __ANDROID__
-void thread_exit_handler(int sig) {
-  pthread_exit(0);
-}
-#endif
-
-void* UnixImpl::waitForPID(void* _self)
-{
-#ifdef __ANDROID__
-  struct sigaction actions;
-  memset(&actions, 0, sizeof(actions));
-  sigemptyset(&actions.sa_mask);
-  actions.sa_flags = 0;
-  actions.sa_handler = thread_exit_handler;
-  sigaction(SIGUSR1, &actions, NULL);
-#endif
-
-  UnixImpl* self = static_cast<UnixImpl*>(_self);
-  waitpid(self->m_pid, NULL, 0);
-
-  pthread_mutex_lock(&self->m_mutex);
-  self->m_running = false;
-  pthread_mutex_unlock(&self->m_mutex);
-
-  return self;
-}
-
-void UnixImpl::run(commandLine_t& commandLine)
-{
-  const char* binary = commandLine[0];
-  int pid = fork();
-  switch(pid) {
-    case -1:
-      std::cerr << "cannot fork" << std::endl;
-      break;
-    case 0:
-      commandLine.push_back(NULL);
-      if (execvp(binary, const_cast<char* const*>(commandLine.data()))) {
-        perror("Cannot launch\n");
-        _exit(-1);
-      }
-
-      break;
-    default:
-      m_pid = pid;
-      m_running = true;
-      pthread_create(&m_waitingThread, NULL, waitForPID, this);
-      break;
-  }
-}
-
-bool UnixImpl::kill()
-{
-  return (::kill(m_pid, SIGKILL) == 0);
-}
-
-bool UnixImpl::isRunning()
-{
-  pthread_mutex_lock(&m_mutex);
-  bool ret = m_running;
-  pthread_mutex_unlock(&m_mutex);
-  return ret;
-}

src/subprocess_unix.h

@@ -1,28 +0,0 @@
-#ifndef KIWIX_SUBPROCESS_UNIX_H_
-#define KIWIX_SUBPROCESS_UNIX_H_
-
-#include "subprocess.h"
-
-#include <pthread.h>
-
-
-class UnixImpl : public SubprocessImpl
-{
-  private:
-    int m_pid;
-    bool m_running;
-    pthread_mutex_t m_mutex;
-    pthread_t m_waitingThread;
-
-  public:
-    UnixImpl();
-    virtual ~UnixImpl();
-
-    void run(commandLine_t& commandLine);
-    bool kill();
-    bool isRunning();
-
-    static void* waitForPID(void* self);
-};
-
-#endif //KIWIX_SUBPROCESS_UNIX_H_

src/subprocess_windows.cpp

@@ -1,100 +0,0 @@
-
-
-#include "subprocess_windows.h"
-
-#include <windows.h>
-#include <winbase.h>
-#include <shlwapi.h>
-#include <iostream>
-#include <sstream>
-
-WinImpl::WinImpl():
-  m_pid(0),
-  m_running(false),
-  m_handle(INVALID_HANDLE_VALUE)
-{
-  InitializeCriticalSection(&m_criticalSection);
-}
-
-WinImpl::~WinImpl()
-{
-  kill();
-  CloseHandle(m_handle);
-  DeleteCriticalSection(&m_criticalSection);
-}
-
-DWORD WINAPI WinImpl::waitForPID(void* _self)
-{
-  WinImpl* self = static_cast<WinImpl*>(_self);
-  WaitForSingleObject(self->m_handle, INFINITE);
-
-  EnterCriticalSection(&self->m_criticalSection);
-  self->m_running = false;
-  LeaveCriticalSection(&self->m_criticalSection);
-
-  return 0;
-}
-
-std::unique_ptr<wchar_t[]> toWideChar(const std::string& value, size_t min_size = 0)
-{
-  size_t size = MultiByteToWideChar(CP_UTF8, 0,
-                value.c_str(), -1, nullptr, 0);
-  auto wdata = std::unique_ptr<wchar_t[]>(new wchar_t[size>min_size?size:min_size]);
-  auto ret = MultiByteToWideChar(CP_UTF8, 0,
-                value.c_str(), -1, wdata.get(), size);
-  if (0 == ret) {
-    std::ostringstream oss;
-    oss << "Cannot convert to wchar : " << GetLastError();
-    throw std::runtime_error(oss.str());
-  }
-  if (size < min_size) {
-    memset(wdata.get() + size, 0, min_size-size);
-  }
-  return wdata;
-}
-
-
-void WinImpl::run(commandLine_t& commandLine)
-{
-  STARTUPINFOW startInfo = {0};
-  PROCESS_INFORMATION procInfo;
-  startInfo.cb = sizeof(startInfo);
-  std::wostringstream oss;
-  for(auto& item: commandLine) {
-    auto witem = toWideChar(item, MAX_PATH);
-    PathQuoteSpacesW(witem.get());
-    oss << witem.get() << " ";
-  }
-  auto wCommandLine = oss.str();
-  if (CreateProcessW(
-    NULL,
-    const_cast<wchar_t*>(wCommandLine.c_str()),
-    NULL,
-    NULL,
-    false,
-    CREATE_NO_WINDOW,
-    NULL,
-    NULL,
-    &startInfo,
-    &procInfo))
-  {
-    m_pid = procInfo.dwProcessId;
-    m_handle = procInfo.hProcess;
-    CloseHandle(procInfo.hThread);
-    m_running = true;
-    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
-  }
-}
-
-bool WinImpl::kill()
-{
-  return TerminateProcess(m_handle, 0);
-}
-
-bool WinImpl::isRunning()
-{
-  EnterCriticalSection(&m_criticalSection);
-  bool ret = m_running;
-  LeaveCriticalSection(&m_criticalSection);
-  return ret;
-}