Add documentation for widget

Added documentation for current widget usage, currently supported arguments, using custom CSS/JS

.codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+
+ignore:
+  - "test"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/

.github/move.yml

@@ -0,0 +1,27 @@
+# Configuration for Move Issues - https://github.com/dessant/move-issues
+
+# Delete the command comment when it contains no other content
+deleteCommand: true
+
+# Close the source issue after moving
+closeSourceIssue: true
+
+# Lock the source issue after moving
+lockSourceIssue: false
+
+# Mention issue and comment authors
+mentionAuthors: true
+
+# Preserve mentions in the issue content
+keepContentMentions: true
+
+# Move labels that also exist on the target repository
+moveLabels: true
+
+# Set custom aliases for targets
+# aliases:
+#   r: repo
+#   or: owner/repo
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

.github/workflows/ci.yml

@@ -0,0 +1,149 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  Macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+      - name: Install packages
+        run: |
+          brew update
+          brew install gcovr pkg-config ninja
+      - name: Install python modules
+        run: pip3 install meson==0.49.2 pytest
+      - name: Install deps
+        shell: bash
+        run: |
+          ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
+          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
+      - name: Compile
+        shell: bash
+        run: |
+          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
+          meson . build --default-library=shared -Db_coverage=true
+          cd build
+          ninja
+      - name: Test
+        shell: bash
+        run: |
+          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
+          cd build
+          meson test --verbose
+        env:
+          SKIP_BIG_MEMORY_TEST: 1
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - native_static
+          - native_dyn
+          - android_arm
+          - android_arm64
+          - win32_static
+          - win32_dyn
+        include:
+          - name: native_static
+            target: native_static
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: native_dyn
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
+            image_variant: bionic
+            lib_postfix: '/arm-linux-androideabi'
+          - name: android_arm64
+            target: android_arm64
+            image_variant: bionic
+            lib_postfix: '/aarch64-linux-android'
+          - name: win32_static
+            target: win32_static
+            image_variant: f35
+            lib_postfix: '64'
+          - name: win32_dyn
+            target: win32_dyn
+            image_variant: f35
+            lib_postfix: '64'
+    env:
+      HOME: /home/runner
+    runs-on: ubuntu-latest
+    container:
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+    steps:
+    - name: Checkout code
+      shell: python
+      run: |
+        from subprocess import check_call
+        from os import environ
+        command = [
+          'git', 'clone',
+          'https://github.com/${{github.repository}}',
+          '--depth=1',
+          '--branch', '${{ github.head_ref || github.ref_name }}'
+        ]
+        check_call(command, cwd=environ['HOME'])
+    - name: Install deps
+      shell: bash
+      run: |
+        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_libkiwix.tar.xz
+        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
+    - name: Compile
+      shell: bash
+      run: |
+        meson --version
+        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
+          MESON_OPTION="--default-library=shared"
+        else
+          MESON_OPTION="--default-library=static"
+        fi
+        if [[ "${{matrix.target}}" =~ native_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
+        else
+          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
+        fi
+        if [[ "${{matrix.target}}" =~ android_.* ]]; then
+          MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
+        fi
+        cd $HOME/libkiwix
+        meson . build ${MESON_OPTION}
+        cd build
+        ninja
+      env:
+        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
+    - name: Test
+      if: startsWith(matrix.target, 'native_')
+      shell: bash
+      run: |
+        cd $HOME/libkiwix/build
+        meson test --verbose
+        ninja coverage
+      env:
+        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
+        SKIP_BIG_MEMORY_TEST: 1
+    - name: Publish coverage
+      shell: bash
+      run: |
+        cd $HOME/libkiwix
+        curl https://codecov.io/bash -o codecov.sh
+        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
+        rm codecov.sh
+      if: startsWith(matrix.target, 'native_')
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -0,0 +1,91 @@
+name: Packages
+on: [push, pull_request]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+          - ubuntu-jammy
+          - ubuntu-impish
+          - ubuntu-focal
+          - ubuntu-bionic
+    steps:
+      - uses: actions/checkout@v2
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-impish
+        if: matrix.distro == 'ubuntu-impish'
+        name: Build package for ubuntu-impish
+        id: build-ubuntu-impish
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to master
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.gitignore

@@ -0,0 +1,9 @@
+.idea/
+*.swp
+subprojects/googletest-release*
+*.class
+build/
+.vscode/
+builddir/
+.cache/
+.clangd/

.travis.yml

@@ -1,13 +0,0 @@
-language: cpp
-dist: trusty
-sudo: required
-cache: ccache
-install: travis/install_deps.sh
-script: travis/compile.sh
-env:
- - PLATFORM="native_static"
- - PLATFORM="native_dyn"
- - PLATFORM="win32_static"
- - PLATFORM="win32_dyn"
- - PLATFORM="android_arm"
- - PLATFORM="android_arm64"

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,521 @@
+libkiwix 11.0.0
+===============
+
+ * [server] Add support for internationalization (@veloman-yunkan #679)
+ * [server] Use gzip compression instead of deflat (mgautierfr #757)
+ * [server] Version the static resources. This allow better invalidating
+   browser cache when resources are changed (@veloman-yunkan #712)
+ * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server] Improve multizim search API:
+   - Improvement of the cache system
+   - Better API to select on which books to search in.
+   - SysAdmin is now able to limit the number of book we search in for a multizim search
+ * [server] Introduce a opensearch API for multizim fulltext search
+ * [wrapper] Remove java wrapper
+ * Testing:
+   - Testing of search result pages content (@veloman-yunkan #765)
+   - Better testing structure of xml search result (@veloman-yunkan #780)
+
+libkiwix 10.1.1
+===============
+
+ * Correctly detect the number of article for older zims (<=6) (@mgautier #743)
+ * [server] Fix fulltext search (@mgautierfr #724)
+ * [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
+ * Fix CI (@mgautierfr #736)
+
+libkiwix 10.1.0
+===============
+
+ This release is an important one as it fixes a Xss vulnerability introduced
+ in libkiwix 10.0.0
+
+ * [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
+ * [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
+ * [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
+ * [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
+ * Add `dc:issued` to opds output stream (@veloman-yunkan #715)
+ * Add handling of several languages not provided by ICU (@juuz0 #701)
+ * [server] Add a caching system for search and suggestion (@maneeshpm #620)
+ * Fix cross-compilation (@kelson42 #703)
+ * Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
+ * Better testing system of html response (@veloman-yunkan #725)
+
+libkiwix 10.0.1
+===============
+
+ * [server] The catalog search interpret `count=0` as no limit.
+   This was the case for a long time. This was changed unintentionally
+   (@veloman-yunkan #686)
+ * [server] Correctly generere a human friendly title in the server frontend.
+   (@juuz0 #687, @kelson42 #689)
+ * [server] Fix download button if there is no url do download from.
+   (@juuz0 #691)
+ * Add non-minified isotope.pkdg.js
+   Needed for debian packaging as we need the source and minified version is
+   not the source (@legoktm #693)
+ * [server] Add a tooltip with the full language for the lang tag.
+ * CI fixes (@kelson42 @legoktm)
+
+libkiwix 10.0.0
+===============
+
+ This release is huge release.
+ The project has been renamed to libkiwix, it is more coherent with the library name.
+
+ * Server front page :
+  - Use js in the front page to display the available book,
+     using the OPDS stream as source. The front page is now populated only with
+     the visible books and user can search for books. (@MananJethwany #530, #541, #534)
+     (@kelson42 #628)
+  - Revamp css (@MananJethwany #559)
+  - Correctly Convert 3iso language code to 2iso (@juuz0 #672)
+
+ * Server suggestions search :
+  - Add pagination for suggestion search (@maneeshpm #591)
+  - Fix suggestion system (@MananJethwany #498)
+  - Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
+  - The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
+  - Properly handle html encoding of suggestions (@veloman-yunkan #458)
+
+ * Server improvements :
+  - Remove meta endpoints (@mgautier #669)
+  - Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
+  - Add details on 404 error pages (@soumyankar #490)
+  - Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
+  - Better headbar insertion (after charset definition) (@kelson42 #442)
+
+ * New OPDS Stream v2 :
+  - Add a list of categories (@veloman-yunkan)
+  - Support for partial entries (@veloman-yunkan #602)
+  - Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
+  - Add language endpoint to catalog (@veloman-yunkan #553)
+  - Add illustration API to get the illustration of a book (@mgautierfr #645)
+  - OPDS search can now filter books by category (@veloman-yunkan #459)
+
+ * Library improvements :
+  - Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
+  - Properly handle removing of book from the library (@veloman-yunkan #485)
+  - Use xapian to search for books in the library (@veloman-yunkan #460, #488)
+
+ * Added methods/functions :
+  - Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
+  - Add `getVersions` and `printVersions` functions (@kelson42 #665)
+  - Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
+  - Add `get_zimid()` method to the search result (@maneeshpm #510)
+
+ * Various improvements :
+  - Better secret value for aria2c rpc (@juuz0 #666)
+  - Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
+  - Add basic documentation (@mgautierfr #640)
+  - Do not use Reader internally (@maneeshpm #536 #576)
+  - Remove dependency headers from our public headers (@mgautierfr #574)
+  - Downloader now don't write metalink on the filesystem (@kelson42 #502)
+  - Support opening a zim file using a fd (@veloman-yukan #429)
+  - Use C++11 std::thread instead of pthread (@mgautierfr #445)
+  - [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
+  - Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
+  - Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
+  - Better parsing of the counterMap (@kelson42 #437)
+  - Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
+  - Remove deprecated methods (@mgautierfr)
+  - CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
+  - Fix compilation for FreeBSD (@swills g#432)
+  - Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
+
+kiwix-lib 9.4.1
+===============
+
+ * Fix `M/Counter` parsing.
+ * [SERVER] Adjust body padding-top for taskbar
+ * Fix potential crash when stoping a server not started.
+ * Various fix in build system and the CI.
+
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
+kiwix-lib 9.2
+=============
+
+ * Add tests
+ * Refactoring server code.
+ * [SERVER] Add HEAD, Etag and If-None-Match support.
+ * [SERVER] Compress opds catalog answers.
+
+kiwix-lib 9.1.2
+===============
+
+ * Do not use the pathToSave if it is empty.
+
+kiwix-lib 9.1.1
+===============
+
+ * Fix the detection of the dataDirectory on windows.
+
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
+kiwix-lib 9.0.1
+===============
+
+ * [JAVA] Use a long to store the offset of an article in the zim file instead
+    of an int.
+
+kiwix-lib 9.0.0
+===============
+
+ * [OPDS] Correctly set the id of the OPDS stream.
+ * [OPDS] Do not try to filter the catalog if no filter field is given in the
+   request.
+ * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
+ * [LIBRARY] Remove the function the read file using a native path.
+   All path must be utf8, no need to pass a native path along the utf8 path.
+ * [TEST] Fix tests using the main function of gtest instead of custom one.
+ * [CI] Move to github CI instead of Travis.
+ * The `Book::update` method always update the book's fields. Even if they are
+   not empty.
+ * [JAVA] Add wrapping around the library manager (opds parsing)
+ * [ARIA2] Add api option to start download with option (destination folder)
+ * [OPDS] Fixes about opds parsing, generation (missing attributes)
+   and requesting (server)
+ * Add methods on `Book` to get specific tag values (was on `Reader` only)
+ * Add flavour attribute to `Book`
+ * Fix opensearch description.
+ * Trust the given library.xml (by default) instead of reading the value from
+   the zim files.
+ * [OPDS] Be able to filter the content by name or size.
+ * [WINDOWS] Fix launching subcommand when there is spaces in the path.
+
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
+kiwix-lib 8.1.0
+===============
+
+ * Fix pathTools manipulation.
+ * Add missing implementation of getArticleCount and getMediaCount on android.
+ * Correctly convert windows path to utf8.
+ * Add code coverage in the CI
+
+kiwix-lib 8.0.1
+===============
+
+ * Fix join function
+
+kiwix-lib 8.0.0
+===============
+
+ * Add new methods to get all (and new) metadata from the zim file.
+ * Add methods to get the value of a specific tag.
+ * [API Change] Convert tags value to the new convention.
+ * [API Change] Rename `getMatatag` method to `getMetadata`
+ * [ABI Change] Correctly detect executable path in appimage.
+
+kiwix-lib 7.0.0
+===============
+
+ * [API break] Add a argument to kiwix-serve to specify the library to use.
+
+kiwix-lib 6.0.4
+===============
+
+ * Fix HTML rendering of the search result if there is no result.
+ * Do not crash at html rendering if request ask for 0 results (start == end)
+ * Correctly find the executable path if we are using AppImage
+
+kiwix-lib 6.0.3
+===============
+
+ * force one column suggestion in kiwix-serve suggestions
+ * fix fulltext search link in suggestions
+ * UI fixes in kiwix-serve rendering
+
+kiwix-lib 6.0.2
+===============
+
+ * Correctly set the groupId in the pom file.
+
+kiwix-lib 6.0.1
+===============
+
+ * Generate the pom file for android/maven
+
+kiwix-lib 6.0.0
+===============
+
+ * Move the server code in kiwix-lib (from kiwix-serve).
+ * Add unit test on regex functions.
+ * Fix computerAbsolutePath (thread safe, memory leak).
+ * Correctly set the book's path as valid if we construct the book from a
+   reader.
+ * [JNI] Add a method to know if a article is a redirection.
+ * Do not embed the gtest dependency.
+ * [JNI] Add a constructor to JNIKiwixString.
+ * Change order of search of the favicon urls.
+ * Clean a lot of unecessary includes in headers. (potential "API break")
+
+kiwix-lib 5.2.0
+===============
+
+ * kiwix-serve integration (as a seperated process).
+ * Fix crash in the suggestion search.
+ * Better API to filter the library books.
+ * New kiwix-lib application for android. (.aar)
+ * Use ReLinker to link with libkiwix.so in android.
+ * Correctly set the verbosity of zim search.
+
+kiwix-lib 5.1.0
+===============
+
+ * Add function to pause, resume and stop downloads.
+ * Add zim's tags in the opds stream.
+ * Addapt to new libzim 5.0.0 API.
+
+kiwix-lib 5.0.0
+===============
+
+ * Remove error message when trying to open an wrong zim file.
+ * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
+ * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
+ * Remove rpath
+ * Detect infinite (and too long) redirect loops.
+
+kiwix-lib 4.1.0
+===============
+
+ * Allow the library to be filtered by tags.
+ * Fix language mapping.
+ * Update README about mustache dependency.
+
+kiwix-lib 4.0.1
+===============
+
+ * Fix "maybe uninitialize variable" issue.
+ * Ensure path are stored correctly (absolute path) in the library.
+ * [CI] Use the new deps archive xz
+
+kiwix-lib 4.0.0
+===============
+
+ * [API break] Remove support for external index.
+ * Move to the mustache templating system instead of ctpp2.
+ * Make meson.build works for meson>=0.43.0
+ * [API break] Move the basic tools from the `common` directory to `tools`.
+
+kiwix-lib 3.1.1
+===============
+
+ * The OPDS feed book's date must be the date of the book, not the date of the
+ feed generation.
+ * Convert the standard opds date to our format (YYYY-MM-DD)
+ * Remove duplicate language attribute in the libxml dumper.
+ * Create the datadirectory to not fail to write a file in a non-existent
+   directory
+
+kiwix-lib 3.1.0
+===============
+
+ * Add a method to get the favicon url of book (if available).
+ * Move dump code of library.xml in a specific class.
+ * Add a first support to bookmarks
+
+kiwix-lib 3.0.3
+===============
+
+ * Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
+ ('eng').
+ * Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
+ * Correctly fill the book size for the zim file size.
+ * Fix launch of aria2c.
+
+kiwix-lib 3.0.2
+===============
+
+ * Use the correct path separator when computing relativePath on Windows.
+
+kiwix-lib 3.0.1
+===============
+
+ * Small fix about parsing the opdsStream.
+
+kiwix-lib 3.0.0
+===============
+
+ * Change the downloader to use aria2 using a separated process (with rpc)
+   instead of using the libaria2. This simplify a lot the link process to
+   libaria2 on Windows.
+    - kiwix-lib doesn't depend on libaria2 anymore.
+    - kiwix-lib now depends on libcurl.
+ * [API break] Library class API has been updated :
+    - Books are referenced by id, not index. A lot of methods have been
+      updated this way.
+    - Books "list" is now private.
+    - There is no more "current" book.
+    - listBooksIds's filters have been updated.
+ * [API break] Book class API has been updated :
+    - Move the definition of Book in `book.h`.
+    - Use getter/setter methods instead public members.
+    - Size (getSize/setSize) is now returned in bytes, not kB.
+    - Dependending of how the book has been initialized (opdsfeed), the
+      faviconUrl may be stored in the book, the favicon being downloaded when
+      using `getFavicon`.
+    - The path (and indexPath) are always absolute path.
+    - Book has now a downloadId, corresponding to the aria2 download id (if
+      exists)
+  * [API break] Manager class API has been updated :
+    - The manager is mainly use to fill a Libray from a "library.xml" file or
+      opds feed. Other operations (has removeBookById, setBookPath, filter, ...)
+      have been removed.
+    - The manager use a intermediate class (LibraryManipulator) to add book to
+      the library. This dependency injection allow caller code to hook the add
+      of a book to the library.
+    - The manager work on a existing Library. It doesn't how a internal
+      Library.
+  * [API break] OpdsDumper class API has been updated :
+    - dumpOPDSFeed method now take the list of bookIds to dump instead of
+      dumping all books in the library.
+    - OpdsDumper can now dump openSearch result information (total result
+      count, start index, ...).
+  * [API break] Common tools API has been updated :
+    - `base64_encode` and `base64_decode` take std::string as arguments.
+    - New `download` function in networkTools.h using libcurl.
+    - New `getDataDirectory` function in pathTools.
+    - Better `beautifyInteger` and `beautifyFileSize` functions.
+    - New `nodeToString` function serializing a pugi::xml_node to a string.
+    - New `converta2toa3` function to convert alpha2 language code to aplha3
+      language code.
+
+
+kiwix-lib 2.0.2
+===============
+
+ * [Android] Forward c++ errors message de Java world.
+ * Follow redirection of favicon.
+ * Make aria2 dependency optional.
+ * Inculde unistd.h only on unix platform.
+
+kiwix-lib 2.0.1
+===============
+
+ * Fix parsing of url.
+ * Remove unused static resources.
+ * Correctly decode reserved characters in URLs.
+ * Explicitly use icu namespace to allow use of packaged icu lib.
+
+kiwix-lib 2.0.0
+===============
+
+ * Introduce a new API to retrive content from a reader.
+   * Introduce the `Entry` class.
+   * Reader's methods return an `Entry`.
+   * Content and other information can be retrieved from the `Entry`.
+   * Older Reader's methods are depreciated.
+ * Add an `OPDSDumper` class to dump a whole `Library` as an OPDS feed.
+ * Add a tool function to get the content of a file.
+ * Add a tool function to create a tempory directory.
+ * Add a `Downloader` class to download a file.
+ * Allow the manager to populate a `Library` from an OPDS feed.
+ * Try to locate libctpp2 in default system libdir and then fallback in 'lib'
+   directory.
+ * Build kiwix-lib setting RPATH.
+ * Build kiwix-lib without warning (werror=true)
+ * Build kiwix-lib on macos.
+
+kiwix-lib 1.1.1
+===============
+
+ * Correct the name of kiwix-lib (from `kiwixlib`) in meson.build to generate
+   dist archive with the correct name.
+ * Libzim version need to be at least 3.2.0
+
+kiwix-lib 1.1.0
+===============
+
+ * Allow for more than 70 search result per page in html results rendering
+   (kiwix/kiwix-tools#92)
+ * Add a small api to do geo queries.
+ * Add multi-search support in the JNI (#67)
+ * Add an API to get only one part of an article.
+ * Add an API to get direct location of an article content in the zim file.
+ * Improve urlencoding
+ * Fix pagination in html results rendering.
+ * Compile using gcc-5 on Travis.
+ * Allow JNI to access search snippets.
+ * JNI throw an exception instead of returning an invalid object if something
+   goes wrong.
+ * Add doctext documentation. (#116)
+ * Various bug fixes.
+
 kiwix-lib 1.0.0
 ===============
 

README.md

@@ -1,42 +1,55 @@
-Kiwix library
-=============
+Libkiwix
+========
 
-The Kiwix library provides the Kiwix software core. It contains the
-code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
+The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
+core. It contains the code shared by all Kiwix ports (Windows,
+GNU/Linux, macOS, Android, iOS, ...).
+
+[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
+[![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
+[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
 ----------
 
 This document assumes you have a little knowledge about software
 compilation. If you experience difficulties with the dependencies or
-with the Kiwix libary compilation itself, we recommend to have a look
-to [kiwix-build](https://github.com/kiwix/kiwix-build).
+with the Libkiwix compilation itself, we recommend to have a look to
+[kiwix-build](https://github.com/kiwix/kiwix-build).
 
 Preamble
 --------
 
-Although the Kiwix library can be compiled/cross-compiled on/for many
-sytems, the following documentation explains how to do it on POSIX
-ones. It is primarly though for GNU/Linux systems and has been tested
-on recent releases of Ubuntu and Fedora.
+Although the Libkiwix can be (cross-)compiled on/for many sytems, the
+following documentation explains how to do it on POSIX ones. It is
+primarly thought for GNU/Linux systems and has been tested on recent
+releases of Ubuntu and Fedora.
 
 Dependencies
 ------------
 
-The Kiwix library relies on many third parts software libraries. They
-are prerequisites to the Kiwix library compilation. Following
-libraries need to be available:
+The Libkiwix relies on many third party software libraries. They are
+prerequisites to the Libkiwix compilation. Following libraries need to
+be available:
+* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
+* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
+* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
+* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
+header `mustache.hpp` somewhere it can be found by the compiler and/or
+set CPPFLAGS with correct `-I` option). Use Mustache version 4.1 or above.
+* [Libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
+* [Microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
+* [Zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
 
-* ICU ................................... http://site.icu-project.org/
-(package libicu-dev on Ubuntu)
-* ZIM ........................................ http://www.openzim.org/
-(package libzim-dev on Ubuntu)
-* Pugixml ........................................ http://pugixml.org/
-(package libpugixml-dev on Ubuntu)
-* ctpp2 ........................................ http://ctpp.havoc.ru/
-(package libctpp2-dev on Ubuntu)
-* Xapian ......................................... https://xapian.org/
-(package libxapian-dev on Ubuntu)
+To test the code:
+* [Google Test](https://github.com/google/googletest) (package `googletest` on Ubuntu)
+
+The following dependency needs to be available at runtime:
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -45,76 +58,155 @@ In the worse case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
-kiwix-lib directory as install prefix.
+`libkiwix` directory as install prefix.
 
-If you compile ctpp2 from source and want to compile the Kiwix library
-statically then you will probably need to rename ctpp2 static library
-from ctpp2-st.a to ctpp2.a.
-
-Environnement
+Environment
 -------------
 
-The Kiwix library builds using [Meson](http://mesonbuild.com/) version
-0.34 or higher. Meson relies itself on Ninja, pkg-config and few other
+The Libkiwix builds using [Meson](https://mesonbuild.com/) version
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
-* Automake
-* Libtool
-* Virtualenv
-* Pkg-config
+* [Meson](https://mesonbuild.com/)
+* [Ninja](https://ninja-build.org/)
+* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
 
-Then install Meson itself:
-```
-virtualenv -p python3 ./ # Create virtualenv
-source bin/activate      # Activate the virtualenv
-pip install meson        # Install Meson
-hash -r                  # Refresh bash paths
-```
-
-Finally download and build Ninja locally:
-```
-git clone git://github.com/ninja-build/ninja.git
-cd ninja
-git checkout release
-./configure.py --bootstrap
-mkdir ../bin                      
-cp ninja ../bin
-cd ..
-```
+These tools should be packaged if you use a cutting edge operating
+system. If not, have a look to the [Troubleshooting](#Troubleshooting)
+section.
 
 Compilation
 -----------
 
-Once all dependencies are installed, you can compile kiwix-lib with:
-```
-mkdir build
+Once all dependencies are installed, you can compile the Libkiwix
+with:
+```bash
 meson . build
-cd build
-ninja
+ninja -C build
 ```
 
-By default, it will compile dynamic linked libraries. If you want
-statically linked libraries, you can add `--default-library=static`
-option to the Meson command.
+By default, it will compile dynamic linked libraries. All binary files
+will be created in the `build` directory created automatically by
+Meson. If you want statically linked libraries, you can add
+`--default-library=static` option to the Meson command.
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
+with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
+
+```bash
+meson . build -Dwrapper=android -Dwerror=false
+ninja -C build
+```
+
+Testing
+-------
+
+To run the automated tests:
+```bash
+cd build
+meson test
+```
+
 Installation
 ------------
 
-If you want to install the libraries you just have compiled on your
-system, here we go:
-
+If you want to install the Libkiwix and the headers you just have
+compiled on your system, here we go:
+```bash
+ninja -C build install
 ```
-ninja install
+
+You might need to run the command as root (or using `sudo`), depending
+where you want to install the libraries. After the installation
+succeeded, you may need to run `ldconfig` (as `root`).
+
+Uninstallation
+------------
+
+If you want to uninstall the Kiwix library:
+```bash
+ninja -C build uninstall
+```
+
+Like for the installation, you might need to run the command as `root`
+(or using `sudo`).
+
+Troubleshooting
+---------------
+
+If you need to install Meson "manually":
+```bash
+virtualenv -p python3 ./ # Create virtualenv
+source bin/activate      # Activate the virtualenv
+pip3 install meson       # Install Meson
+hash -r                  # Refresh bash paths
+```
+
+If you need to install Ninja "manually":
+```bash
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+mkdir ../bin
+cp ninja ../bin
 cd ..
 ```
 
-You might need to run the command as root, depending where you want to
-install the libraries.
+Custom Index Page
+-----------------
+
+to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
+(note - while using custom html file please mention all external links as absolute path.)
+
+to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
+
+To use JS provided by kiwix-serve you can use the following template to start with ->
+
+```
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title><-- Custom Tittle --></title>
+    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+  </head>
+  <body>
+  </body>
+</html>
+```
+
+- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
+- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
+- To add language select box add - `<select id="languageFilter"></select>` under body tag.
+- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add search box for books use following form -
+    ```
+        <form id='kiwixSearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
+        <input type="submit" class="kiwixButton" value="Search"/>
+        </form>
+    ```
+
+
+If you compile manually Libmicrohttpd, you might need to compile it
+without GNU TLS, a bug here will empeach further compilation
+otherwise.
+
+If the compilation still fails, you might need to get a more recent
+version of a dependency than the one packaged by your Linux
+distribution. Try then with a source tarball distributed by the
+problematic upstream project or even directly from the source code
+repository.
 
 License
 -------
 
-GPLv3 or later, see COPYING for more details.
+[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
+[COPYING](COPYING) for more details.

android-kiwix-lib-publisher/.gitignore

@@ -0,0 +1,13 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -0,0 +1,25 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    repositories {
+        google()
+        jcenter()
+
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.4.1'
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

android-kiwix-lib-publisher/gradle.properties

@@ -0,0 +1,15 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx1536m
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed Jun 19 15:28:39 BST 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -0,0 +1 @@
+/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -0,0 +1,64 @@
+apply plugin: 'com.android.library'
+apply plugin: 'maven'
+
+android {
+    compileSdkVersion 28
+    defaultConfig {
+        minSdkVersion 15
+        targetSdkVersion 28
+        versionCode 1
+        versionName "1.0"
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+dependencies {
+    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
+}
+
+task writePom {
+  pom {
+    project {
+        groupId 'org.kiwix.kiwixlib'
+        artifactId 'kiwixlib'
+        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        packaging 'aar'
+        name 'kiwixlib'
+        url 'https://github.com/kiwix/libkiwix'
+        licenses {
+            license {
+                name 'GPLv3'
+                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
+            }
+        }
+      developers {
+          developer {
+              id 'kiwix'
+              name 'kiwix'
+              email 'contact@kiwix.org'
+          }
+      }
+      scm {
+          connection 'https://github.com/kiwix/libkiwix.git'
+          developerConnection 'https://github.com/kiwix/libkiwix.git'
+          url 'https://github.com/kiwix/libkiwix'
+      }
+    }
+  }.withXml {
+    def dependenciesNode = asNode().appendNode('dependencies')
+
+    //Iterate over the implementation dependencies, adding a <dependency> node for each
+    configurations.implementation.allDependencies.each {
+      def dependencyNode = dependenciesNode.appendNode('dependency')
+      dependencyNode.appendNode('groupId', it.group)
+      dependencyNode.appendNode('artifactId', it.name)
+      dependencyNode.appendNode('version', it.version)
+    }
+  }.writeTo("$buildDir/pom.xml")
+}
+

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -0,0 +1,10 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.kiwix.kiwixlib">
+
+    <application
+        android:allowBackup="true"
+        android:supportsRtl="true">
+
+    </application>
+
+</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -0,0 +1 @@
+include ':kiwixLibAndroid'

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,46 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+ meson,
+ pkg-config,
+ libzim-dev (>= 7.2.0~),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/libkiwix
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 7.2.0~),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev,
+ libmicrohttpd-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix10
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
+Conflicts: libkiwix0, libkiwix3, libkiwix9
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix-dev.manpages

@@ -0,0 +1,2 @@
+usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

debian/libkiwix10.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

docs/.gitignore

@@ -0,0 +1,2 @@
+api
+xml

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'libkiwix'
+copyright = '2022, libkiwix-team'
+author = 'libkiwix-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+    'exhale'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+breathe_projects = {
+    "libkiwix": "./xml"
+}
+breathe_default_project = 'libkiwix'
+
+exhale_args = {
+    "containmentFolder":   "./api",
+    "rootFileName":        "ref_api.rst",
+    "rootFileTitle":       "Reference API",
+    "doxygenStripFromPath":"..",
+    "treeViewIsBootstrap": True,
+    "createTreeView" : True,
+    "exhaleExecutesDoxygen": True,
+    "exhaleDoxygenStdin":    "INPUT = ../include"
+}
+
+primary_domain = 'cpp'
+
+highlight_language = 'cpp'

docs/index.rst

@@ -0,0 +1,15 @@
+.. libkiwix documentation master file, created by
+   sphinx-quickstart on Fri Jul 24 15:40:50 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libkiwix's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage
+   api/ref_api
+   widget

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+breathe
+exhale

docs/usage.rst

@@ -0,0 +1,15 @@
+Libkiwix programming
+====================
+
+Introduction
+------------
+
+libkiwix is written in C++. To use the library, you need the include files of libkiwix have
+to link against libzim.
+
+Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
+which is always derived from std::exception.
+
+All classes are defined in the namespace kiwix.
+
+libkiwix is a set of tools to manage zim files and provide some common functionnality.

docs/widget.rst

@@ -0,0 +1,82 @@
+Kiwix serve widget
+====================
+
+Introduction
+------------
+
+The kiwix-serve widget provides an easy to embed way to show the `kiwix-serve` homepage.
+
+Usage
+-----
+
+To use the widget, simply add an iframe with its `src` attribute set to the `widget` endpoint.
+Example HTML Page ::
+
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+    <title>Widget Test</title>
+    </head>
+    <body>
+    <iframe src="http://192.168.18.8:8080/widget?disabledesc&disablefilter&disabledownload" width=1000 height=1000></iframe>
+    </body>
+    </html>
+
+This creates an iframe with the kiwix-serve homepage contents.
+
+Arguments are explained below.
+
+Possible Arguments
+-------------------
+
+Currently, the following arguments are supported.
+
+disabledesc (value = N/A)
+    Disables the description part of a tile.
+
+disablefilter (value = N/A)
+    Disables the search filters: language, category, tag and search function.
+
+disableclick (value = N/A)
+    Disables clicking the book to open it for reading.
+
+disabledownload (value = N/A)
+    Disables the download button (if avaialable at all) on the tile.
+
+
+Custom CSS and JS
+-----------------
+
+You can add your custom CSS rules and Javascript code to the widget.
+
+To do that, use the following code as template::
+
+    <iframe id="receiver" src="http://192.168.18.8:8080/widget?disabledesc=&disablefilter=&disabledownload=" width="1000" height="1000">
+    <p>Your browser does not support iframes.</p>
+    </iframe>
+
+    <script>
+    window.onload = function() {
+    var receiver = document.getElementById('receiver').contentWindow;
+    function sendMessage() {
+        let msg = {
+        css: `
+        .book__header {
+            color:red;
+        }`,
+        js: `
+        function widgetTest() {
+        console.log("Testing widget");
+        }
+        widgetTest();
+        `
+        }
+        receiver.postMessage(msg, 'http://192.168.18.8:8080/widget');
+    }
+    sendMessage();
+    }
+    </script>
+
+
+The CSS/JS fields are optional, you may send both or only one.
+

format_code.sh

@@ -7,6 +7,7 @@ files=(
 "include/common/otherTools.h"
 "include/common/regexTools.h"
 "include/common/networkTools.h"
+"include/common/archiveTools.h"
 "include/manager.h"
 "include/reader.h"
 "include/kiwix.h"
@@ -22,6 +23,7 @@ files=(
 "src/common/pathTools.cpp"
 "src/common/regexTools.cpp"
 "src/common/otherTools.cpp"
+"src/common/archiveTools.cpp"
 "src/common/networkTools.cpp"
 "src/common/stringTools.cpp"
 "src/xapianSearcher.cpp"

include/book.h

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOK_H
+#define KIWIX_BOOK_H
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include "common.h"
+
+namespace pugi {
+class xml_node;
+}
+
+namespace zim {
+class Archive;
+}
+
+namespace kiwix
+{
+
+class OPDSDumper;
+
+/**
+ * A class to store information about a book (a zim file)
+ */
+class Book
+{
+ public: // types
+  class Illustration
+  {
+    friend class Book;
+   public:
+    uint16_t width = 48;
+    uint16_t height = 48;
+    std::string mimeType;
+    std::string url;
+
+    const std::string& getData() const;
+
+   private:
+    mutable std::string data;
+    mutable std::mutex mutex;
+  };
+
+  typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
+
+ public: // functions
+  Book();
+  ~Book();
+
+  bool update(const Book& other);
+  void update(const zim::Archive& archive);
+  void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
+  void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
+  std::string getHumanReadableIdFromPath() const;
+
+  bool readOnly() const { return m_readOnly; }
+  const std::string& getId() const { return m_id; }
+  const std::string& getPath() const { return m_path; }
+  bool isPathValid() const { return m_pathValid; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getDescription() const { return m_description; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getCreator() const { return m_creator; }
+  const std::string& getPublisher() const { return m_publisher; }
+  const std::string& getDate() const { return m_date; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getName() const { return m_name; }
+  std::string getCategory() const;
+  const std::string& getTags() const { return m_tags; }
+  std::string getTagStr(const std::string& tagName) const;
+  bool getTagBool(const std::string& tagName) const;
+  const std::string& getFlavour() const { return m_flavour; }
+  const std::string& getOrigId() const { return m_origId; }
+  const uint64_t& getArticleCount() const { return m_articleCount; }
+  const uint64_t& getMediaCount() const { return m_mediaCount; }
+  const uint64_t& getSize() const { return m_size; }
+  DEPRECATED const std::string& getFavicon() const;
+  DEPRECATED const std::string& getFaviconUrl() const;
+  DEPRECATED const std::string& getFaviconMimeType() const;
+
+  Illustrations getIllustrations() const;
+  std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
+
+  const std::string& getDownloadId() const { return m_downloadId; }
+
+  void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
+  void setId(const std::string& id) { m_id = id; }
+  void setPath(const std::string& path);
+  void setPathValid(bool valid) { m_pathValid = valid; }
+  void setTitle(const std::string& title) { m_title = title; }
+  void setDescription(const std::string& description) { m_description = description; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setCreator(const std::string& creator) { m_creator = creator; }
+  void setPublisher(const std::string& publisher) { m_publisher = publisher; }
+  void setDate(const std::string& date) { m_date = date; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setName(const std::string& name) { m_name = name; }
+  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
+  void setTags(const std::string& tags) { m_tags = tags; }
+  void setOrigId(const std::string& origId) { m_origId = origId; }
+  void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
+  void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
+  void setSize(uint64_t size) { m_size = size; }
+  void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
+
+ private: // functions
+  std::string getCategoryFromTags() const;
+  const Illustration& getDefaultIllustration() const;
+
+ protected: // data
+  std::string m_id;
+  std::string m_downloadId;
+  std::string m_path;
+  bool m_pathValid = false;
+  std::string m_title;
+  std::string m_description;
+  std::string m_category;
+  std::string m_language;
+  std::string m_creator;
+  std::string m_publisher;
+  std::string m_date;
+  std::string m_url;
+  std::string m_name;
+  std::string m_flavour;
+  std::string m_tags;
+  std::string m_origId;
+  uint64_t m_articleCount = 0;
+  uint64_t m_mediaCount = 0;
+  bool m_readOnly = false;
+  uint64_t m_size = 0;
+  Illustrations m_illustrations;
+
+  // Used as the return value of getDefaultIllustration() when no default
+  // illustration is found in the book
+  static const Illustration missingDefaultIllustration;
+};
+
+}
+
+#endif

include/bookmark.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOKMARK_H
+#define KIWIX_BOOKMARK_H
+
+#include <string>
+
+namespace pugi {
+class xml_node;
+}
+
+namespace kiwix
+{
+
+/**
+ * A class to store information about a bookmark (an article in a book)
+ */
+class Bookmark
+{
+ public:
+  Bookmark();
+  ~Bookmark();
+
+  void updateFromXml(const pugi::xml_node& node);
+
+  const std::string& getBookId() const { return m_bookId; }
+  const std::string& getBookTitle()  const { return m_bookTitle; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getDate() const { return m_date; }
+
+  void setBookId(const std::string& bookId) { m_bookId = bookId; }
+  void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setTitle(const std::string& title) { m_title = title; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setDate(const std::string& date) { m_date = date; }
+
+ protected:
+  std::string m_bookId;
+  std::string m_bookTitle;
+  std::string m_url;
+  std::string m_title;
+  std::string m_language;
+  std::string m_date;
+};
+
+}
+
+#endif

include/common.h

@@ -0,0 +1,24 @@
+
+#ifndef _KIWIX_COMMON_H_
+#define _KIWIX_COMMON_H_
+
+#include <zim/zim.h>
+
+#ifdef __GNUC__
+#define DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+#define DEPRECATED __declspec(deprecated)
+#else
+#praga message("WARNING: You need to implement DEPRECATED for this compiler")
+#define DEPRECATED
+#endif
+
+
+namespace kiwix {
+
+typedef zim::size_type size_type;
+typedef zim::offset_type offset_type;
+
+}
+
+#endif //_KIWIX_COMMON_H_

include/common/base64.h

@@ -1,4 +0,0 @@
-#include <string>
-
-std::string base64_encode(unsigned char const* , unsigned int len);
-std::string base64_decode(std::string const& s);

include/common/pathTools.h

@@ -1,61 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_PATHTOOLS_H
-#define KIWIX_PATHTOOLS_H
-
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <fstream>
-#include <ios>
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#ifdef _WIN32
-#include <direct.h>
-#endif
-
-#include "stringTools.h"
-
-using namespace std;
-
-bool isRelativePath(const string& path);
-string computeAbsolutePath(const string path, const string relativePath);
-string computeRelativePath(const string path, const string absolutePath);
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator = false,
-                             const bool removePostSeparator = false);
-string appendToDirectory(const string& directoryPath, const string& filename);
-
-unsigned int getFileSize(const string& path);
-string getFileSizeAsString(const string& path);
-bool fileExists(const string& path);
-bool makeDirectory(const string& path);
-bool copyFile(const string& sourcePath, const string& destPath);
-string getLastPathElement(const string& path);
-string getExecutablePath();
-string getCurrentDirectory();
-bool writeTextFile(const string& path, const string& content);
-#endif

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -1,79 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef _CTPP2_VM_STRING_LOADER_HPP__
-#define _CTPP2_VM_STRING_LOADER_HPP__ 1
-
-#include <ctpp2/CTPP2VMLoader.hpp>
-#include <ctpp2/CTPP2Util.hpp>
-#include <ctpp2/CTPP2Exception.hpp>
-#include <ctpp2/CTPP2VMExecutable.hpp>
-#include <ctpp2/CTPP2VMInstruction.hpp>
-#include <ctpp2/CTPP2VMMemoryCore.hpp>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <iostream>
-#include <string>
-
-/**
-  @file VMStringLoader.hpp
-  @brief Load program core from file
-*/
-
-namespace CTPP // C++ Template Engine
-{
-// FWD
-struct VMExecutable;
-
-/**
-  @class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
-  @brief Load program core from file
-*/
-class CTPP2DECL VMStringLoader:
-  public VMLoader
-{
-public:
-	/**
-	*/
-	VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
-	/**
-	  @brief Get ready-to-run program
-	*/
-	const VMMemoryCore * GetCore() const;
-
-	/**
-	  @brief A destructor
-	*/
-	~VMStringLoader() throw();
-private:
-	/** Program core             */
-	VMExecutable  * oCore;
-	/** Ready-to-run program     */
-	VMMemoryCore  * pVMMemoryCore;
-};
-
-} // namespace CTPP
-#endif // _CTPP2_VM_STRING_LOADER_HPP__
-// End.

include/downloader.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_DOWNLOADER_H
+#define KIWIX_DOWNLOADER_H
+
+#include <string>
+#include <vector>
+#include <map>
+#include <memory>
+#include <stdexcept>
+
+namespace kiwix
+{
+
+class Aria2;
+struct DownloadedFile {
+  DownloadedFile()
+   : success(false) {}
+  bool success;
+  std::string path;
+};
+
+class AriaError : public std::runtime_error {
+ public:
+  AriaError(const std::string& message) : std::runtime_error(message) {}
+};
+
+
+class Download {
+ public:
+  typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
+
+  Download() :
+    m_status(K_UNKNOWN) {}
+  Download(std::shared_ptr<Aria2> p_aria, std::string did)
+    : mp_aria(p_aria),
+      m_status(K_UNKNOWN),
+      m_did(did) {};
+  void updateStatus(bool follow=false);
+  void pauseDownload();
+  void resumeDownload();
+  void cancelDownload();
+  StatusResult getStatus()          { return m_status; }
+  std::string  getDid()             { return m_did; }
+  std::string  getFollowedBy()      { return m_followedBy; }
+  uint64_t     getTotalLength()     { return m_totalLength; }
+  uint64_t     getCompletedLength() { return m_completedLength; }
+  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
+  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
+  std::string  getPath()            { return m_path; }
+  std::vector<std::string>&  getUris()             { return m_uris; }
+
+ protected:
+  std::shared_ptr<Aria2> mp_aria;
+  StatusResult m_status;
+  std::string m_did = "";
+  std::string m_followedBy = "";
+  uint64_t m_totalLength;
+  uint64_t m_completedLength;
+  uint64_t m_downloadSpeed;
+  uint64_t m_verifiedLength;
+  std::vector<std::string> m_uris;
+  std::string m_path;
+};
+
+/**
+ * A tool to download things.
+ *
+ */
+class Downloader
+{
+ public:
+  Downloader();
+  virtual ~Downloader();
+
+  void close();
+
+  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
+  Download* getDownload(const std::string& did);
+
+  size_t getNbDownload() { return m_knownDownloads.size(); }
+  std::vector<std::string> getDownloadIds();
+
+ private:
+  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
+  std::shared_ptr<Aria2> mp_aria;
+};
+}
+
+#endif

include/kiwix.h

@@ -22,4 +22,4 @@
 
 #include "library.h"
 
-#endif
+#endif

include/kiwixserve.h

@@ -0,0 +1,30 @@
+#ifndef KIWIXLIB_KIWIX_SERVE_H_
+#define KIWIXLIB_KIWIX_SERVE_H_
+
+#include <memory>
+#include <string>
+
+class Subprocess;
+namespace kiwix {
+
+class KiwixServe
+{
+  public:
+    KiwixServe(const std::string& libraryPath, int port = 8181);
+    ~KiwixServe();
+
+    void run();
+    void shutDown();
+    bool isRunning();
+    int getPort() { return m_port; }
+    int setPort(int port);
+
+  private:
+  std::unique_ptr<Subprocess> mp_kiwixServe;
+  int m_port;
+  std::string m_libraryPath;
+};
+
+}; //end namespace kiwix
+
+#endif // KIWIXLIB_KIWIX_SERVE_H_

include/library.h

@@ -20,87 +20,351 @@
 #ifndef KIWIX_LIBRARY_H
 #define KIWIX_LIBRARY_H
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stack>
 #include <string>
 #include <vector>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <zim/archive.h>
+#include <zim/search.h>
 
-#include "common/regexTools.h"
-#include "common/stringTools.h"
+#include "book.h"
+#include "bookmark.h"
+#include "common.h"
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
-using namespace std;
-
 namespace kiwix
 {
-enum supportedIndexType { UNKNOWN, XAPIAN };
 
-class Book
-{
- public:
-  Book();
-  ~Book();
+class OPDSDumper;
+class Library;
 
-  static bool sortByLastOpen(const Book& a, const Book& b);
-  static bool sortByTitle(const Book& a, const Book& b);
-  static bool sortBySize(const Book& a, const Book& b);
-  static bool sortByDate(const Book& a, const Book& b);
-  static bool sortByCreator(const Book& a, const Book& b);
-  static bool sortByPublisher(const Book& a, const Book& b);
-  static bool sortByLanguage(const Book& a, const Book& b);
-  string getHumanReadableIdFromPath();
-
-  string id;
-  string path;
-  string pathAbsolute;
-  string last;
-  string indexPath;
-  string indexPathAbsolute;
-  supportedIndexType indexType;
-  string title;
-  string description;
-  string language;
-  string creator;
-  string publisher;
-  string date;
-  string url;
-  string name;
-  string tags;
-  string origId;
-  string articleCount;
-  string mediaCount;
-  bool readOnly;
-  string size;
-  string favicon;
-  string faviconMimeType;
+enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
+enum supportedListMode {
+  ALL = 0,
+  LOCAL = 1,
+  REMOTE = 1 << 1,
+  NOLOCAL = 1 << 2,
+  NOREMOTE = 1 << 3,
+  VALID = 1 << 4,
+  NOVALID = 1 << 5
 };
 
+class Filter {
+  public: // types
+    using Tags = std::vector<std::string>;
+
+  private: // data
+    uint64_t activeFilters;
+    Tags _acceptTags;
+    Tags _rejectTags;
+    std::string _category;
+    std::string _lang;
+    std::string _publisher;
+    std::string _creator;
+    size_t _maxSize;
+    std::string _query;
+    bool _queryIsPartial;
+    std::string _name;
+
+  public: // functions
+    Filter();
+    ~Filter() = default;
+
+    /**
+     *  Set the filter to check local.
+     *
+     *  A local book is a book with a path.
+     *  If accept is true, only local book are accepted.
+     *  If accept is false, only non local book are accepted.
+     */
+    Filter& local(bool accept);
+
+    /**
+     *  Set the filter to check remote.
+     *
+     *  A remote book is a book with a url.
+     *  If accept is true, only remote book are accepted.
+     *  If accept is false, only non remote book are accepted.
+     */
+    Filter& remote(bool accept);
+
+    /**
+     *  Set the filter to check validity.
+     *
+     *  A valid book is a book with a path pointing to a existing zim file.
+     *  If accept is true, only valid book are accepted.
+     *  If accept is false, only non valid book are accepted.
+     */
+    Filter& valid(bool accept);
+
+    /**
+     * Set the filter to only accept book with corresponding tag.
+     */
+    Filter& acceptTags(const Tags& tags);
+    Filter& rejectTags(const Tags& tags);
+
+    Filter& category(std::string category);
+    Filter& lang(std::string lang);
+    Filter& publisher(std::string publisher);
+    Filter& creator(std::string creator);
+    Filter& maxSize(size_t size);
+    Filter& query(std::string query, bool partial=true);
+    Filter& name(std::string name);
+
+    bool hasQuery() const;
+    const std::string& getQuery() const { return _query; }
+    bool queryIsPartial() const { return _queryIsPartial; }
+
+    bool hasName() const;
+    const std::string& getName() const { return _name; }
+
+    bool hasCategory() const;
+    const std::string& getCategory() const { return _category; }
+
+    bool hasLang() const;
+    const std::string& getLang() const { return _lang; }
+
+    bool hasPublisher() const;
+    const std::string& getPublisher() const { return _publisher; }
+
+    bool hasCreator() const;
+    const std::string& getCreator() const { return _creator; }
+
+    const Tags& getAcceptTags() const { return _acceptTags; }
+    const Tags& getRejectTags() const { return _rejectTags; }
+
+private: // functions
+    friend class Library;
+
+    bool accept(const Book& book) const;
+};
+
+
+class ZimSearcher : public zim::Searcher
+{
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
+
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
+
+/**
+ * A Library store several books.
+ */
 class Library
 {
+  // all data fields must be added in LibraryBase
+  mutable std::mutex m_mutex;
+
+ public:
+  typedef uint64_t Revision;
+  typedef std::vector<std::string> BookIdCollection;
+  typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
+
  public:
   Library();
   ~Library();
 
-  string version;
-  bool addBook(const Book& book);
-  bool removeBookByIndex(const unsigned int bookIndex);
-  vector<kiwix::Book> books;
-
-  /*
-   * 'current' is the variable storing the current content/book id
-   * in the library. This is used to be able to load per default a
-   * content. As Kiwix may work with many library XML files, you may
-   * have "current" defined many time with different values. The
-   * last XML file read has the priority, Although we do not have an
-   * library object for each file, we want to be able to fallback to
-   * an 'old' current book if the one which should be load
-   * failed. That is the reason why we need a stack here
+  /**
+   * Library is not a copiable object. However it can be moved.
    */
-  stack<string> current;
+  Library(const Library& ) = delete;
+  Library(Library&& );
+  void operator=(const Library& ) = delete;
+  Library& operator=(Library&& );
+
+  /**
+   * Add a book to the library.
+   *
+   * If a book already exist in the library with the same id, update
+   * the existing book instead of adding a new one.
+   *
+   * @param book The book to add.
+   * @return True if the book has been added.
+   *         False if a book has been updated.
+   */
+  bool addBook(const Book& book);
+
+  /**
+   * A self-explanatory alias for addBook()
+   */
+  bool addOrUpdateBook(const Book& book) { return addBook(book); }
+
+  /**
+   * Add a bookmark to the library.
+   *
+   * @param bookmark the book to add.
+   */
+  void addBookmark(const Bookmark& bookmark);
+
+  /**
+   * Remove a bookmarkk
+   *
+   * @param zimId The zimId of the bookmark.
+   * @param url The url of the bookmark.
+   * @return True if the bookmark has been removed.
+   */
+  bool removeBookmark(const std::string& zimId, const std::string& url);
+
+  // XXX: This is a non-thread-safe operation
+  const Book& getBookById(const std::string& id) const;
+  // XXX: This is a non-thread-safe operation
+  const Book& getBookByPath(const std::string& path) const;
+
+  Book getBookByIdThreadSafe(const std::string& id) const;
+
+  std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
+
+  /**
+   * Remove a book from the library.
+   *
+   * @param id the id of the book to remove.
+   * @return True if the book were in the lirbrary and has been removed.
+   */
+  bool removeBookById(const std::string& id);
+
+  /**
+   * Write the library to a file.
+   *
+   * @param path the path of the file to write to.
+   * @return True if the library has been correctly saved.
+   */
+  bool writeToFile(const std::string& path) const;
+
+  /**
+   * Write the library bookmarks to a file.
+   *
+   * @param path the path of the file to write to.
+   * @return True if the library has been correctly saved.
+   */
+  bool writeBookmarksToFile(const std::string& path) const;
+
+  /**
+   * Get the number of book in the library.
+   *
+   * @param localBooks If we must count local books (books with a path).
+   * @param remoteBooks If we must count remote books (books with an url)
+   * @return The number of books.
+   */
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
+
+  /**
+   * Get all languagues of the books in the library.
+   *
+   * @return A list of languages.
+   */
+  std::vector<std::string> getBooksLanguages() const;
+
+  /**
+   * Get all languagues of the books in the library with counts.
+   *
+   * @return A list of languages with the count of books in each language.
+   */
+  AttributeCounts getBooksLanguagesWithCounts() const;
+
+  /**
+   * Get all categories of the books in the library.
+   *
+   * @return A list of categories.
+   */
+  std::vector<std::string> getBooksCategories() const;
+
+  /**
+   * Get all book creators of the books in the library.
+   *
+   * @return A list of book creators.
+   */
+  std::vector<std::string> getBooksCreators() const;
+
+  /**
+   * Get all book publishers of the books in the library.
+   *
+   * @return A list of book publishers.
+   */
+  std::vector<std::string> getBooksPublishers() const;
+
+  /**
+   * Get all bookmarks.
+   *
+   * @return A list of bookmarks
+   */
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true) const;
+
+  /**
+   * Get all book ids of the books in the library.
+   *
+   * @return A list of book ids.
+   */
+  BookIdCollection getBooksIds() const;
+
+  /**
+   * Filter the library and return the id of the keep elements.
+   *
+   * @param filter The filter to use.
+   * @return The list of bookIds corresponding to the filter.
+   */
+  BookIdCollection filter(const Filter& filter) const;
+
+
+  /**
+   * Sort (in place) bookIds using the given comparator.
+   *
+   * @param bookIds the list of book Ids to sort
+   * @param comparator how to sort the books
+   * @return The sorted list of books
+   */
+  void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
+
+  /**
+   * Return the current revision of the library.
+   *
+   * The revision of the library is updated (incremented by one) only by
+   * the addBook() operation.
+   *
+   * @return Current revision of the library.
+   */
+  Revision getRevision() const;
+
+  /**
+   * Remove books that have not been updated since the specified revision.
+   *
+   * @param  rev the library revision to use
+   * @return Count of books that were removed by this operation.
+   */
+  uint32_t removeBooksNotUpdatedSince(Revision rev);
+
+  friend class OPDSDumper;
+  friend class libXMLDumper;
+
+private: // types
+  typedef const std::string& (Book::*BookStrPropMemFn)() const;
+  struct Impl;
+
+private: // functions
+  AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
+  std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
+  BookIdCollection filterViaBookDB(const Filter& filter) const;
+  void updateBookDB(const Book& book);
+  void dropCache(const std::string& bookId);
+
+private: //data
+  std::unique_ptr<Impl> mp_impl;
 };
+
 }
 
 #endif

include/libxml_dumper.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBXML_DUMPER_H
+#define KIWIX_LIBXML_DUMPER_H
+
+#include <string>
+#include <vector>
+
+#include <pugixml.hpp>
+
+#include "library.h"
+
+namespace kiwix
+{
+
+/**
+ * A tool to dump a `Library` into a basic library.xml
+ *
+ */
+class LibXMLDumper
+{
+ public:
+  LibXMLDumper() = default;
+  LibXMLDumper(const Library* library);
+  ~LibXMLDumper();
+
+  /**
+   * Dump the library.xml
+   *
+   * @param id The id of the library.
+   * @return The library.xml content.
+   */
+  std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
+
+
+  /**
+   * Dump the bookmark of the library.
+   *
+   * @return The bookmark.xml content.
+   */
+  std::string dumpLibXMLBookmark();
+
+  /**
+   * Set the base directory used.
+   *
+   * @param baseDir the base directory to use.
+   */
+  void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
+
+  /**
+   * Set the library to dump.
+   *
+   * @param library The library to dump.
+   */
+  void setLibrary(const Library* library) { this->library = library; }
+
+ protected:
+  const kiwix::Library* library;
+  std::string baseDir;
+ private:
+  void handleBook(Book book, pugi::xml_node root_node);
+  void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
+};
+}
+
+#endif // KIWIX_OPDS_DUMPER_H

include/manager.h

@@ -20,88 +20,159 @@
 #ifndef KIWIX_MANAGER_H
 #define KIWIX_MANAGER_H
 
-#include <time.h>
-#include <sstream>
-#include <string>
-
-#include <pugixml.hpp>
-
-#include "common/base64.h"
-#include "common/pathTools.h"
-#include "common/regexTools.h"
+#include "book.h"
 #include "library.h"
-#include "reader.h"
 
-using namespace std;
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace pugi {
+class xml_document;
+}
 
 namespace kiwix
 {
-enum supportedListMode { LASTOPEN, REMOTE, LOCAL };
-enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
 
+class LibraryManipulator
+{
+ public: // functions
+  explicit LibraryManipulator(Library* library);
+  virtual ~LibraryManipulator();
+
+  Library& getLibrary() const { return library; }
+
+  bool addBookToLibrary(const Book& book);
+  void addBookmarkToLibrary(const Bookmark& bookmark);
+  uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
+
+ protected: // overrides
+  virtual void bookWasAddedToLibrary(const Book& book);
+  virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
+  virtual void booksWereRemovedFromLibrary();
+
+ private: // data
+  kiwix::Library& library;
+};
+
+/**
+ * A tool to manage a `Library`.
+ */
 class Manager
 {
- public:
-  Manager();
-  ~Manager();
+ public: // types
+  typedef std::vector<std::string> Paths;
 
-  bool readFile(const string path, const bool readOnly = true);
-  bool readFile(const string nativePath,
-                const string UTF8Path,
-                const bool readOnly = true);
-  bool readXml(const string xml,
+ public: // functions
+  explicit Manager(LibraryManipulator* manipulator);
+  explicit Manager(Library* library);
+
+  /**
+   * Read a `library.xml` and add book in the file to the library.
+   *
+   * @param path The (utf8) path to the `library.xml`.
+   * @param readOnly Set if the libray path could be overwritten latter with
+   *                 updated content.
+   * @param trustLibrary use book metadata coming from XML.
+   * @return True if file has been properly parsed.
+   */
+  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
+
+  /**
+   * Sync the contents of the library with one or more `library.xml` files.
+   *
+   * The metadata of the library files is trusted unconditionally.
+   * Any books not present in the input library.xml files are removed
+   * from the library.
+   *
+   * @param paths The (utf8) paths to the `library.xml` files.
+   */
+  void reload(const Paths& paths);
+
+  /**
+   * Load a library content store in the string.
+   *
+   * @param xml The content corresponding of the library xml
+   * @param readOnly Set if the libray path could be overwritten latter with
+   *                 updated content.
+   * @param libraryPath The library path (used to resolve relative path)
+   * @return True if the content has been properly parsed.
+   */
+  bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const string libraryPath = "");
-  bool writeFile(const string path);
-  bool removeBookByIndex(const unsigned int bookIndex);
-  bool removeBookById(const string id);
-  bool setCurrentBookId(const string id);
-  string getCurrentBookId();
-  bool setBookIndex(const string id,
-                    const string path,
-                    const supportedIndexType type);
-  bool setBookIndex(const string id, const string path);
-  bool setBookPath(const string id, const string path);
-  string addBookFromPathAndGetId(const string pathToOpen,
-                                 const string pathToSave = "",
-                                 const string url = "",
+               const std::string& libraryPath = "",
+               bool trustLibrary = true);
+
+  /**
+   * Load a library content stored in a OPDS stream.
+   *
+   * @param content The content of the OPDS stream.
+   * @param readOnly Set if the library path could be overwritten later with
+   *                 updated content.
+   * @param libraryPath The library path (used to resolve relative path)
+   * @return True if the content has been properly parsed.
+   */
+  bool readOpds(const std::string& content, const std::string& urlHost);
+
+
+  /**
+   * Load a bookmark file.
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed.
+   */
+  bool readBookmarkFile(const std::string& path);
+
+  /**
+   * Add a book to the library.
+   *
+   * @param pathToOpen The path to the zim file to add.
+   * @param pathToSave The path to store in the library in place of pathToOpen.
+   * @param url        The url of the book to store in the library.
+   * @param checMetaData Tell if we check metadata before adding book to the
+   *                     library.
+   * @return The id of the book if the book has been added to the library.
+   *         Else, an empty string.
+   */
+  std::string addBookFromPathAndGetId(const std::string& pathToOpen,
+                                 const std::string& pathToSave = "",
+                                 const std::string& url = "",
                                  const bool checkMetaData = false);
-  bool addBookFromPath(const string pathToOpen,
-                       const string pathToSave = "",
-                       const string url = "",
+
+  /**
+   * Add a book to the library.
+   *
+   * @param pathToOpen The path to the zim file to add.
+   * @param pathToSave The path to store in the library in place of pathToOpen.
+   * @param url        The url of the book to store in the library.
+   * @param checMetaData Tell if we check metadata before adding book to the
+   *                     library.
+   * @return True if the book has been added to the library.
+   */
+
+  bool addBookFromPath(const std::string& pathToOpen,
+                       const std::string& pathToSave = "",
+                       const std::string& url = "",
                        const bool checkMetaData = false);
-  Library cloneLibrary();
-  bool getBookById(const string id, Book& book);
-  bool getCurrentBook(Book& book);
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
-  bool updateBookLastOpenDateById(const string id);
-  void removeBookPaths();
-  bool listBooks(const supportedListMode mode,
-                 const supportedListSortBy sortBy,
-                 const unsigned int maxSize,
-                 const string language,
-                 const string creator,
-                 const string publisher,
-                 const string search);
-  vector<string> getBooksLanguages();
-  vector<string> getBooksCreators();
-  vector<string> getBooksPublishers();
-  vector<string> getBooksIds();
 
-  string writableLibraryPath;
+  std::string writableLibraryPath;
 
-  vector<std::string> bookIdList;
+  bool m_hasSearchResult = false;
+  uint64_t m_totalBooks = 0;
+  uint64_t m_startIndex = 0;
+  uint64_t m_itemsPerPage = 0;
 
  protected:
-  kiwix::Library library;
+  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
 
-  bool readBookFromPath(const string path, Book* book = NULL);
+  bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   const bool readOnly,
-                   const string libraryPath);
+                   bool readOnly,
+                   const std::string& libraryPath,
+                   bool trustLibrary);
+  bool parseOpdsDom(const pugi::xml_document& doc,
+                    const std::string& urlHost);
 
- private:
-  void checkAndCleanBookPaths(Book& book, const string& libraryPath);
 };
 }
 

include/meson.build

@@ -1,30 +1,18 @@
 headers = [
+  'book.h',
+  'bookmark.h',
+  'common.h',
   'library.h',
   'manager.h',
-  'reader.h',
-  'searcher.h'
+  'libxml_dumper.h',
+  'opds_dumper.h',
+  'downloader.h',
+  'search_renderer.h',
+  'server.h',
+  'kiwixserve.h',
+  'name_mapper.h',
+  'tools.h',
+  'version.h'
 ]
 
-if xapian_dep.found()
-  headers += ['xapianSearcher.h']
-endif
-
 install_headers(headers, subdir:'kiwix')
-
-install_headers(
-  'common/base64.h',
-  'common/networkTools.h',
-  'common/otherTools.h',
-  'common/pathTools.h',
-  'common/regexTools.h',
-  'common/stringTools.h',
-  subdir:'kiwix/common'
-)
-
-if has_ctpp2_dep
-  install_headers(
-    'ctpp2/CTPP2VMStringLoader.hpp',
-    subdir:'kiwix/ctpp2'
-  )
-endif
-

include/name_mapper.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_NAMEMAPPER_H
+#define KIWIX_NAMEMAPPER_H
+
+#include <string>
+#include <map>
+#include <memory>
+#include <mutex>
+
+namespace kiwix
+{
+
+class Library;
+
+class NameMapper {
+  public:
+    virtual ~NameMapper() = default;
+    virtual std::string getNameForId(const std::string& id) const = 0;
+    virtual std::string getIdForName(const std::string& name) const = 0;
+};
+
+
+class IdNameMapper : public NameMapper {
+  public:
+    virtual std::string getNameForId(const std::string& id) const { return id; };
+    virtual std::string getIdForName(const std::string& name) const { return name; };
+};
+
+class HumanReadableNameMapper : public NameMapper {
+  private:
+    std::map<std::string, std::string> m_idToName;
+    std::map<std::string, std::string> m_nameToId;
+
+  public:
+    HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
+    virtual ~HumanReadableNameMapper() = default;
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+};
+
+class UpdatableNameMapper : public NameMapper {
+    typedef std::shared_ptr<NameMapper> NameMapperHandle;
+  public:
+    UpdatableNameMapper(Library& library, bool withAlias);
+
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+
+    void update();
+
+  private:
+    NameMapperHandle currentNameMapper() const;
+
+  private:
+    mutable std::mutex mutex;
+    Library& library;
+    NameMapperHandle nameMapper;
+    const bool withAlias;
+};
+
+}
+
+#endif

include/opds_catalog.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,18 +17,17 @@
  * MA 02110-1301, USA.
  */
 
-#ifndef KIWIX_OTHERTOOLS_H
-#define KIWIX_OTHERTOOLS_H
+#ifndef KIWIX_OPDS_CATALOG_H
+#define KIWIX_OPDS_CATALOG_H
 
-#ifdef _WIN32
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif
+
+#include "library.h"
 
 namespace kiwix
 {
-void sleep(unsigned int milliseconds);
-}
 
-#endif
+std::string getSearchUrl(const Filter& f);
+
+} // namespace kiwix
+
+#endif // KIWIX_OPDS_CATALOG_H

include/opds_dumper.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_OPDS_DUMPER_H
+#define KIWIX_OPDS_DUMPER_H
+
+#include <time.h>
+#include <sstream>
+#include <string>
+
+#include <pugixml.hpp>
+
+#include "library.h"
+
+using namespace std;
+
+namespace kiwix
+{
+
+/**
+ * A tool to dump a `Library` into a opds stream.
+ *
+ */
+class OPDSDumper
+{
+ public:
+  OPDSDumper() = default;
+  OPDSDumper(Library* library);
+  ~OPDSDumper();
+
+  /**
+   * Dump the OPDS feed.
+   *
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
+   * @return The OPDS feed.
+   */
+  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const;
+
+  /**
+   * Dump the OPDS feed.
+   *
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
+   * @param partial whether the feed should include partial or complete entries
+   * @return The OPDS feed.
+   */
+  std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
+
+  /**
+   * Dump the OPDS complete entry document.
+   *
+   * @param bookId the id of the book
+   * @return The OPDS complete entry document.
+   */
+  std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
+
+  /**
+   * Dump the categories OPDS feed.
+   *
+   * @return The OPDS feed.
+   */
+  std::string categoriesOPDSFeed() const;
+
+  /**
+   * Dump the languages OPDS feed.
+   *
+   * @return The OPDS feed.
+   */
+  std::string languagesOPDSFeed() const;
+
+  /**
+   * Set the id of the library.
+   *
+   * @param id the id to use.
+   */
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
+
+  /**
+   * Set the root location used when generating url.
+   *
+   * @param rootLocation the root location to use.
+   */
+  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+
+ protected:
+  kiwix::Library* library;
+  std::string libraryId;
+  std::string rootLocation;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+};
+}
+
+#endif // KIWIX_OPDS_DUMPER_H

include/reader.h

@@ -1,129 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_READER_H
-#define KIWIX_READER_H
-
-#include <stdio.h>
-#include <zim/article.h>
-#include <zim/file.h>
-#include <zim/fileiterator.h>
-#include <zim/zim.h>
-#include <exception>
-#include <map>
-#include <sstream>
-#include <string>
-#include "common/pathTools.h"
-#include "common/stringTools.h"
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader
-{
- public:
-  Reader(const string zimFilePath);
-  ~Reader();
-
-  void reset();
-  unsigned int getArticleCount() const;
-  unsigned int getMediaCount() const;
-  unsigned int getGlobalCount() const;
-  string getZimFilePath() const;
-  string getId() const;
-  string getRandomPageUrl() const;
-  string getFirstPageUrl() const;
-  string getMainPageUrl() const;
-  bool getMetatag(const string& url, string& content) const;
-  string getTitle() const;
-  string getDescription() const;
-  string getLanguage() const;
-  string getName() const;
-  string getTags() const;
-  string getDate() const;
-  string getCreator() const;
-  string getPublisher() const;
-  string getOrigId() const;
-  bool getFavicon(string& content, string& mimeType) const;
-  bool getPageUrlFromTitle(const string& title, string& url) const;
-  bool getMimeTypeByUrl(const string& url, string& mimeType) const;
-  bool getContentByUrl(const string& url,
-                       string& content,
-                       string& title,
-                       unsigned int& contentLength,
-                       string& contentType) const;
-  bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-  bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-  bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-  bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-  bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         const bool reset = true);
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
-  bool urlExists(const string& url) const;
-  bool hasFulltextIndex() const;
-  std::vector<std::string> getTitleVariants(const std::string& title) const;
-  bool getNextSuggestion(string& title);
-  bool getNextSuggestion(string& title, string& url);
-  bool canCheckIntegrity() const;
-  bool isCorrupted() const;
-  bool parseUrl(const string& url, char* ns, string& title) const;
-  unsigned int getFileSize() const;
-  zim::File* getZimFileHandler() const;
-  bool getArticleObjectByDecodedUrl(const string& url,
-                                    zim::Article& article) const;
-
- protected:
-  zim::File* zimFileHandler;
-  zim::size_type firstArticleOffset;
-  zim::size_type lastArticleOffset;
-  zim::size_type currentArticleOffset;
-  zim::size_type nsACount;
-  zim::size_type nsICount;
-  std::string zimFilePath;
-
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
-
- private:
-  std::map<const std::string, unsigned int> parseCounterMetadata() const;
-};
-}
-
-#endif

include/search_renderer.h

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SEARCH_RENDERER_H
+#define KIWIX_SEARCH_RENDERER_H
+
+#include <string>
+#include <zim/search.h>
+#include "library.h"
+
+namespace kiwix
+{
+
+class NameMapper;
+/**
+ * The SearcherRenderer class is used to render a search result to a html page.
+ */
+class SearchRenderer
+{
+ public:
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * The constructed version of the SearchRenderer will not introduce
+   * the book name for each result. It is better to use the other constructor
+   * with a Library pointer to have a better html page.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                 unsigned int start, unsigned int estimatedResultCount);
+
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                 unsigned int start, unsigned int estimatedResultCount);
+
+  ~SearchRenderer();
+
+  /**
+   * Set the search pattern used to do the search
+   */
+  void setSearchPattern(const std::string& pattern);
+
+  /**
+   * Set the querystring used to select books
+   */
+  void setSearchBookQuery(const std::string& bookQuery);
+
+  /**
+   * Set protocol prefix.
+   */
+  void setProtocolPrefix(const std::string& prefix);
+
+  /**
+   * Set search protocol prefix.
+   */
+  void setSearchProtocolPrefix(const std::string& prefix);
+
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
+  std::string renderTemplate(const std::string& tmpl_str);
+
+  /**
+   * Generate the html page with the resutls of the search.
+   */
+  std::string getHtml();
+
+    /**
+   * Generate the xml page with the resutls of the search.
+   */
+  std::string getXml();
+
+
+ protected:
+  std::string beautifyInteger(const unsigned int number);
+  zim::SearchResultSet m_srs;
+  NameMapper* mp_nameMapper;
+  Library* mp_library;
+  std::string searchBookQuery;
+  std::string searchPattern;
+  std::string protocolPrefix;
+  std::string searchProtocolPrefix;
+  unsigned int pageLength;
+  unsigned int estimatedResultCount;
+  unsigned int resultStart;
+};
+
+
+}
+
+#endif

include/searcher.h

@@ -1,104 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCHER_H
-#define KIWIX_SEARCHER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unicode/putil.h>
-#include <algorithm>
-#include <cctype>
-#include <locale>
-#include <string>
-#include <vector>
-#include <vector>
-#include "common/pathTools.h"
-#include "common/stringTools.h"
-#include "kiwix_config.h"
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader;
-class Result
-{
- public:
-  virtual ~Result(){};
-  virtual std::string get_url() = 0;
-  virtual std::string get_title() = 0;
-  virtual int get_score() = 0;
-  virtual std::string get_snippet() = 0;
-  virtual std::string get_content() = 0;
-  virtual int get_wordCount() = 0;
-  virtual int get_size() = 0;
-  virtual int get_readerIndex() = 0;
-};
-
-struct SearcherInternal;
-class Searcher
-{
- public:
-  Searcher();
-  Searcher(const string& xapianDirectoryPath,
-           Reader* reader,
-           const string& humanReadableName);
-  ~Searcher();
-
-  void add_reader(Reader* reader, const std::string& humanReaderName);
-  void search(std::string& search,
-              unsigned int resultStart,
-              unsigned int resultEnd,
-              const bool verbose = false);
-  void suggestions(std::string& search, const bool verbose = false);
-  Result* getNextResult();
-  void restart_search();
-  unsigned int getEstimatedResultCount();
-  bool setProtocolPrefix(const std::string prefix);
-  bool setSearchProtocolPrefix(const std::string prefix);
-  void reset();
-
-#ifdef ENABLE_CTPP2
-  string getHtml();
-#endif
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  void closeIndex();
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-
-  std::vector<Reader*> readers;
-  std::vector<std::string> humanReaderNames;
-  SearcherInternal* internal;
-  std::string searchPattern;
-  std::string protocolPrefix;
-  std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int resultEnd;
-  std::string contentHumanReadableId;
-};
-}
-
-#endif

include/server.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_H
+#define KIWIX_SERVER_H
+
+#include <string>
+#include <memory>
+
+namespace kiwix
+{
+  class Library;
+  class NameMapper;
+  class InternalServer;
+
+  class Server {
+     public:
+       /**
+        * The default constructor.
+        *
+        * @param library The library to serve.
+        */
+       Server(Library* library, NameMapper* nameMapper=nullptr);
+
+       virtual ~Server();
+
+       /**
+        * Serve the content.
+        */
+       bool start();
+
+       /**
+        * Stop the daemon.
+        */
+       void stop();
+
+       void setRoot(const std::string& root);
+       void setAddress(const std::string& addr) { m_addr = addr; }
+       void setPort(int port) { m_port = port; }
+       void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
+       void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
+       void setVerbose(bool verbose) { m_verbose = verbose; }
+       void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
+       void setTaskbar(bool withTaskbar, bool withLibraryButton)
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
+       int getPort();
+       std::string getAddress();
+
+     protected:
+       Library* mp_library;
+       NameMapper* mp_nameMapper;
+       std::string m_root = "";
+       std::string m_addr = "";
+       std::string m_indexTemplateString = "";
+       int m_port = 80;
+       int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
+       bool m_verbose = false;
+       bool m_withTaskbar = true;
+       bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
+       int m_ipConnectionLimit = 0;
+       std::unique_ptr<InternalServer> mp_server;
+  };
+}
+
+#endif

include/tools.h

@@ -0,0 +1,220 @@
+/*
+ * Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_TOOLS_H
+#define KIWIX_TOOLS_H
+
+#include <string>
+#include <vector>
+#include <map>
+
+namespace kiwix {
+
+/**
+ * Return the current directory.
+ *
+ * @return the current directory (utf8 encoded)
+ */
+std::string getCurrentDirectory();
+
+/**
+ * Return the data directory.
+ *
+ * The data directory is a directory where to put data (zim files, ...)
+ * It depends of the platform and it may be changed by user using environment variable.
+ *
+ * The resolution order is :
+ * - `KIWIX_DATA_DIR` env variable (if set).
+ * - On Windows :
+ *    . `$APPDATA/kiwix` if $APPDATA is set
+ *    . `$USERPROFILE/kiwix` if $USERPROFILE is set
+ * - Else :
+ *    . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
+ *    . `$HOME/.local/share/kiwx` if $HOWE is set
+ * - current directory
+ *
+ * @return the path of the data directory (utf8 encoded)
+ */
+std::string getDataDirectory();
+
+/** Return the path of the executable
+ *
+ * Some application may be packaged in auto extractible archive (Appimage) and the
+ * real executable is different of the path of the archive.
+ * If `realPathOnly` is true, return the path of the real executable instead of the
+ * archive launched by the user.
+ *
+ * @param realPathOnly If we must return the real path of the executable.
+ * @return the path of the executable (utf8 encoded)
+ */
+std::string getExecutablePath(bool realPathOnly = false);
+
+/** Tell if the path is a relative path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path A utf8 encoded path.
+ * @return true if the path is relative.
+ */
+bool isRelativePath(const std::string& path);
+
+/** Append a path to another one.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param basePath the base path.
+ * @param relativePath a path to add to the base path, must be a relative path.
+ * @return The concatenation of the paths, using the right separator.
+ */
+std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
+
+/** Remove the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The parent directory (or empty string if none).
+ */
+std::string removeLastPathElement(const std::string& path);
+
+/** Get the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The base name of the path or empty string if none (ending with a separator).
+ */
+std::string getLastPathElement(const std::string& path);
+
+/** Compute the absolute path of a relative path based on another one
+ *
+ * Equivalent to appendToDirectory followed by a normalization of the path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path (if empty, current directory is taken).
+ * @param relativePath the relative path.
+ * @return a absolute path.
+ */
+std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
+
+/** Compute the relative path of a path relative to another one
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path.
+ * @param absolutePath the absolute path to find the relative path for.
+ * @return a relative path (pointing to absolutePath, relative to path).
+ */
+std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
+
+/** Sleep the current thread.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * @param milliseconds The number of milliseconds to wait for.
+ */
+void sleep(unsigned int milliseconds);
+
+/** Split a string
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * Assuming text = "foo:;bar;baz,oups;"
+ *
+ * split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
+ * split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
+ * split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
+ * split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
+ *
+ * @param str The string to split.
+ * @param delims A string of potential delimiters.
+ *               Each charater in the string can be a individual delimiters.
+ * @param dropEmpty true if empty part must be dropped from the result.
+ * @param keepDelim true if delimiter must be included from the result.
+ * @return a list of part (potentially containing delimiters)
+ */
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
+
+/** Convert language code from iso2 code to iso3
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate locales.
+ *
+ * @param a2code a iso2 code string.
+ * @return the corresponding iso3 code.
+ * @throw std::out_of_range if iso2 code is not known.
+ */
+std::string converta2toa3(const std::string& a2code);
+
+/** Extracts content from given file.
+ *
+ * This function provides content of a file provided it's path.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Content of corresponding file in string format.
+ */
+std::string getFileContent(const std::string& path);
+
+/** Checks if file exists.
+ *
+ * This function returns boolean stating if file exists.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file exists or not.
+ */
+bool fileExists(const std::string& path);
+
+/** Checks if file is readable.
+ *
+ * This function returns boolean stating if file is readable.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file is readale or not.
+ */
+bool fileReadable(const std::string& path);
+
+/** Provides mimetype from filename.
+ *
+ * This function provides mimetype from file-name.
+ *
+ * @param filename string containing filename.
+ * @return mimetype from filename in string format.
+ */
+std::string getMimeTypeForFile(const std::string& filename);
+
+/** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 network interfaces
+ */
+std::map<std::string, std::string> getNetworkInterfaces();
+
+/** Provides the best IP address
+ * This function provides the best IP address from the list given by getNetworkInterfaces
+ */
+std::string getBestPublicIp();
+
+}
+#endif // KIWIX_TOOLS_H

include/version.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,9 +17,18 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_VERSION_H
+#define KIWIX_VERSION_H
 
-public class JNIKiwixString
+#include <string>
+#include <vector>
+#include <iostream>
+
+namespace kiwix
 {
-  public String value;
+  typedef std::vector<std::pair<std::string, std::string>> LibVersions;
+  LibVersions getVersions();
+  void printVersions(std::ostream& out = std::cout);
 }
+
+#endif // KIWIX_VERSION_H

include/xapianSearcher.h

@@ -1,98 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_XAPIAN_SEARCHER_H
-#define KIWIX_XAPIAN_SEARCHER_H
-
-#include <xapian.h>
-#include "reader.h"
-#include "searcher.h"
-
-#include <map>
-#include <string>
-
-using namespace std;
-
-namespace kiwix
-{
-class XapianSearcher;
-
-class XapianResult : public Result
-{
- public:
-  XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
-  virtual ~XapianResult(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual int get_readerIndex() { return 0; };
-
- private:
-  XapianSearcher* searcher;
-  Xapian::MSetIterator iterator;
-  Xapian::Document document;
-};
-
-class NoXapianIndexInZim : public exception
-{
-  virtual const char* what() const throw()
-  {
-    return "There is no fulltext index in the zim file";
-  }
-};
-
-class XapianSearcher
-{
-  friend class XapianResult;
-
- public:
-  XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
-  virtual ~XapianSearcher(){};
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-  virtual Result* getNextResult();
-  void restart_search();
-
-  Xapian::MSet results;
-
- protected:
-  void closeIndex();
-  void openIndex(const string& xapianDirectoryPath);
-  void setup_queryParser();
-
-  Reader* reader;
-  Xapian::Database readableDatabase;
-  std::string language;
-  std::string stopwords;
-  Xapian::QueryParser queryParser;
-  Xapian::Stem stemmer;
-  Xapian::SimpleStopper stopper;
-  Xapian::MSetIterator current_result;
-  std::map<std::string, int> valuesmap;
-};
-}
-
-#endif

meson.build

@@ -1,100 +1,79 @@
-project('kiwixlib', 'cpp',
-  version : '1.0.1',
-  license : 'GPL',
-  default_options : ['c_std=c11', 'cpp_std=c++11'])
+project('libkiwix', 'cpp',
+  version : '11.0.0',
+  license : 'GPLv3+',
+  default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
-find_library_in_compiler = meson.version().version_compare('>=0.31.0')
 
-static_deps = get_option('android') or get_option('default_library') == 'static'
+static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
 
-thread_dep = dependency('threads')
-libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=3.0.0', static:static_deps)
-pugixml_dep = dependency('pugixml', static:static_deps)
-
-ctpp2_include_path = ''
-has_ctpp2_dep = false
-ctpp2_prefix_install = get_option('ctpp2-install-prefix')
-ctpp2_link_args = []
-if ctpp2_prefix_install == ''
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp')
-    if find_library_in_compiler
-      ctpp2_lib = compiler.find_library('ctpp2')
-    else
-      ctpp2_lib = find_library('ctpp2')
-    endif
-    ctpp2_link_args = ['-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      if find_library_in_compiler
-        iconv_lib = compiler.find_library('iconv', required:false)
-      else
-        iconv_lib = find_library('iconv', required:false)
-      endif
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
+# See https://github.com/kiwix/libkiwix/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
+  extra_libs = ['-latomic']
 else
-  if not find_library_in_compiler
-    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
-  endif
-  ctpp2_include_path = ctpp2_prefix_install + '/include'
-  ctpp2_include_args = ['-I'+ctpp2_include_path]
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
-    ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
-    ctpp2_lib_path = ctpp2_prefix_install+'/lib'
-    ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
-    ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      iconv_lib = compiler.find_library('iconv', required:false)
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
+  extra_libs = []
 endif
 
-xapian_dep = dependency('xapian-core', required:false, static:static_deps)
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
+  # C++ std::thread is implemented using pthread on linux by gcc
+  thread_dep = dependency('threads')
+else
+  thread_dep = dependency('', required:false)
+endif
+libicu_dep = dependency('icu-i18n', static:static_deps)
+pugixml_dep = dependency('pugixml', static:static_deps)
+libcurl_dep = dependency('libcurl', static:static_deps)
+microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+zlib_dep = dependency('zlib', static:static_deps)
+xapian_dep = dependency('xapian-core', static:static_deps)
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep]
-if has_ctpp2_dep
-  all_deps += [ctpp2_dep]
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
+  error('Cannot found header mustache.hpp')
 endif
 
-inc = include_directories('include')
+libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
+if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
+  error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
+endif
+
+
+extra_cflags = ''
+if host_machine.system() == 'windows' and static_deps
+  add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
+  extra_cflags += '-DCURL_STATICLIB'
+endif
+
+if host_machine.system() == 'windows'
+  add_project_arguments('-DNOMINMAX', language: 'cpp')
+endif
+
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
+
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
-conf.set('VERSION', '"@0@"'.format(meson.project_version()))
-conf.set('ENABLE_CTPP2', has_ctpp2_dep)
+conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
+
+if build_machine.system() == 'windows'
+     extra_link_args = ['-lshlwapi', '-lwinmm']
+else
+     extra_link_args = []
+endif
 
 subdir('include')
 subdir('scripts')
 subdir('static')
 subdir('src')
-
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml']
-if xapian_dep.found()
-    pkg_requires += ['xapian-core']
+subdir('test')
+if get_option('doc')
+  subdir('docs')
 endif
 
-extra_libs = []
-extra_cflags = ''
-if has_ctpp2_dep
-  extra_libs += ctpp2_link_args
-  if ctpp2_include_path != ''
-    extra_cflags = '-I'+ctpp2_include_path
-  endif
-endif
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,4 +1,4 @@
-option('ctpp2-install-prefix', type : 'string', value : '',
-  description : 'Prefix where ctpp libs has been installed')
-option('android', type : 'boolean', value : false,
-  description : 'Do we make a kiwix-lib for android')
+option('static-linkage', type : 'boolean', value : false,
+  description : 'Link statically with the dependencies.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')

scripts/ctpp2c.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-
-ctpp2c=$1
-SOURCE=$(pwd)/$2
-DEST=$3
-
-$ctpp2c $SOURCE $DEST

scripts/kiwix-compile-i18n

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.data = f.read()
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_files', nargs='+',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    resources = [Resource(filename) for filename in args.i18n_resource_files]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_files ...
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,14 +95,14 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/kiwix-resources

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_file(src_path, dst_path):
+    with open(src_path, 'rb') as src:
+        with open(dst_path, 'wb') as dst:
+            dst.write(src.read())
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        preprocess_resource(resource)
+    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,5 +1,13 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
-intermediate_ctpp2c = find_program('ctpp2c.sh')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/android/AndroidManifest.xml

@@ -1,13 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-
-  package="kiwix.org.kiwixlib"
->
-
-  <application android:allowBackup="true"
-    android:label="@string/app_name"
-    android:supportsRtl="true"
-  >
-
-  </application>
-
-</manifest>

src/android/gen_kiwix.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-BUILD_PATH=$(pwd)
-
-javac -d $BUILD_PATH/src/android $1 $2 $3 $4
-
-cd $BUILD_PATH/src/android
-javah -jni org.kiwix.kiwixlib.JNIKiwix
-cd $BUILD_PATH

src/android/kiwix.cpp

@@ -1,559 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwix.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include <iostream>
-#include <string>
-
-#include "common/base64.h"
-#include "reader.h"
-#include "searcher.h"
-#include "unicode/putil.h"
-
-#include <android/log.h>
-#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, "kiwix", __VA_ARGS__)
-
-#include <xapian.h>
-#include <zim/article.h>
-#include <zim/error.h>
-#include <zim/file.h>
-#include <zim/zim.h>
-
-/* global variables */
-kiwix::Reader* reader = NULL;
-kiwix::Searcher* searcher = NULL;
-
-static pthread_mutex_t readerLock = PTHREAD_MUTEX_INITIALIZER;
-static pthread_mutex_t searcherLock = PTHREAD_MUTEX_INITIALIZER;
-
-/* c2jni type conversion functions */
-jboolean c2jni(const bool& val)
-{
-  return val ? JNI_TRUE : JNI_FALSE;
-}
-jstring c2jni(const std::string& val, JNIEnv* env)
-{
-  return env->NewStringUTF(val.c_str());
-}
-
-jint c2jni(const int val)
-{
-  return (jint)val;
-}
-jint c2jni(const unsigned val)
-{
-  return (unsigned)val;
-}
-/* jni2c type conversion functions */
-bool jni2c(const jboolean& val)
-{
-  return val == JNI_TRUE;
-}
-std::string jni2c(const jstring& val, JNIEnv* env)
-{
-  return std::string(env->GetStringUTFChars(val, 0));
-}
-
-int jni2c(const jint val)
-{
-  return (int)val;
-}
-/* Method to deal with variable passed by reference */
-void setStringObjValue(const std::string& value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
-  env->SetObjectField(obj, objFid, c2jni(value, env));
-}
-
-void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "I");
-  env->SetIntField(obj, objFid, value);
-}
-
-void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
-  env->SetIntField(obj, objFid, c2jni(value));
-}
-
-/* Kiwix library functions */
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv* env, jobject obj)
-{
-  jstring url;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cUrl = reader->getMainPageUrl();
-      url = c2jni(cUrl, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM main page" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return url;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv* env,
-                                                                 jobject obj)
-{
-  jstring id;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cId = reader->getId();
-      id = c2jni(cId, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM id" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return id;
-}
-
-JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv* env,
-                                                                    jobject obj)
-{
-  jint size;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      int cSize = reader->getFileSize();
-      size = c2jni(cSize);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM file size" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return size;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv* env, jobject obj)
-{
-  jstring creator;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cCreator = reader->getCreator();
-      creator = c2jni(cCreator, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM creator" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return creator;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv* env, jobject obj)
-{
-  jstring publisher;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cPublisher = reader->getPublisher();
-      publisher = c2jni(cPublisher, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM creator" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return publisher;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv* env,
-                                                                   jobject obj)
-{
-  jstring name;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cName = reader->getName();
-      name = c2jni(cName, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM name" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return name;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv* env, jobject obj)
-{
-  jstring favicon;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cContent;
-      std::string cMime;
-      reader->getFavicon(cContent, cMime);
-      favicon
-          = c2jni(base64_encode(
-                      reinterpret_cast<const unsigned char*>(cContent.c_str()),
-                      cContent.length()),
-                  env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM favicon" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return favicon;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv* env,
-                                                                   jobject obj)
-{
-  jstring date;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cDate = reader->getDate();
-      date = c2jni(cDate, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM date" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return date;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv* env, jobject obj)
-{
-  jstring language;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cLanguage = reader->getLanguage();
-      language = c2jni(cLanguage, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM language" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return language;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  jstring mimeType;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    std::string cUrl = jni2c(url, env);
-    try {
-      std::string cMimeType;
-      reader->getMimeTypeByUrl(cUrl, cMimeType);
-      mimeType = c2jni(cMimeType, env);
-    } catch (...) {
-      std::cerr << "Unable to get mime-type for url " << cUrl << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return mimeType;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv* env, jobject obj, jstring path)
-{
-  jboolean retVal = JNI_TRUE;
-  std::string cPath = jni2c(path, env);
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      delete reader;
-    }
-    reader = new kiwix::Reader(cPath);
-  } catch (...) {
-    std::cerr << "Unable to load ZIM " << cPath << std::endl;
-    reader = NULL;
-    retVal = JNI_FALSE;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(
-    JNIEnv* env, jobject obj, jstring url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
-{
-  /* Default values */
-  setStringObjValue("", titleObj, env);
-  setStringObjValue("", mimeTypeObj, env);
-  setIntObjValue(0, sizeObj, env);
-  jbyteArray data = env->NewByteArray(0);
-
-  /* Retrieve the content */
-  if (reader != NULL) {
-    std::string cUrl = jni2c(url, env);
-    std::string cData;
-    std::string cTitle;
-    std::string cMimeType;
-    unsigned int cSize = 0;
-
-    pthread_mutex_lock(&readerLock);
-    try {
-      if (reader->getContentByUrl(cUrl, cData, cTitle, cSize, cMimeType)) {
-        data = env->NewByteArray(cSize);
-        env->SetByteArrayRegion(
-            data, 0, cSize, reinterpret_cast<const jbyte*>(cData.c_str()));
-        setStringObjValue(cMimeType, mimeTypeObj, env);
-        setStringObjValue(cTitle, titleObj, env);
-        setIntObjValue(cSize, sizeObj, env);
-      }
-    } catch (...) {
-      std::cerr << "Unable to get content for url " << cUrl << std::endl;
-    }
-    pthread_mutex_unlock(&readerLock);
-  }
-
-  return data;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions(
-    JNIEnv* env, jobject obj, jstring prefix, jint count)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count);
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      if (reader->searchSuggestionsSmart(cPrefix, cCount)) {
-        retVal = JNI_TRUE;
-      }
-    }
-  } catch (...) {
-    std::cerr << "Unable to search suggestions for pattern " << cPrefix
-              << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion(
-    JNIEnv* env, jobject obj, jobject titleObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      if (reader->getNextSuggestion(cTitle)) {
-        setStringObjValue(cTitle, titleObj, env);
-        retVal = JNI_TRUE;
-      }
-    }
-  } catch (...) {
-    std::cerr << "Unable to get next suggestion" << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle(
-    JNIEnv* env, jobject obj, jstring title, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle = jni2c(title, env);
-  std::string cUrl;
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      if (reader->getPageUrlFromTitle(cTitle, cUrl)) {
-        setStringObjValue(cUrl, urlObj, env);
-        retVal = JNI_TRUE;
-      }
-    }
-  } catch (...) {
-    std::cerr << "Unable to get URL for title " << cTitle << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle(
-    JNIEnv* env, jobject obj, jobject titleObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      std::string cTitle = reader->getTitle();
-      setStringObjValue(cTitle, titleObj, env);
-      retVal = JNI_TRUE;
-    }
-  } catch (...) {
-    std::cerr << "Unable to get ZIM title" << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv* env, jobject obj)
-{
-  jstring description;
-
-  pthread_mutex_lock(&readerLock);
-  if (reader != NULL) {
-    try {
-      std::string cDescription = reader->getDescription();
-      description = c2jni(cDescription, env);
-    } catch (...) {
-      std::cerr << "Unable to get ZIM description" << std::endl;
-    }
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return description;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage(
-    JNIEnv* env, jobject obj, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cUrl;
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    if (reader != NULL) {
-      std::string cUrl = reader->getRandomPageUrl();
-      setStringObjValue(cUrl, urlObj, env);
-      retVal = JNI_TRUE;
-    }
-  } catch (...) {
-    std::cerr << "Unable to get random page" << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-
-  return retVal;
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
-    JNIEnv* env, jobject obj, jstring dirStr)
-{
-  std::string cPath = jni2c(dirStr, env);
-
-  pthread_mutex_lock(&readerLock);
-  try {
-    u_setDataDirectory(cPath.c_str());
-  } catch (...) {
-    std::cerr << "Unable to set data directory " << cPath << std::endl;
-  }
-  pthread_mutex_unlock(&readerLock);
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(
-    JNIEnv* env, jobject obj, jstring path)
-{
-  jboolean retVal = JNI_TRUE;
-  std::string cPath = jni2c(path, env);
-
-  pthread_mutex_lock(&searcherLock);
-  try {
-    if (searcher != NULL) {
-      delete searcher;
-    }
-    if (!reader || !reader->hasFulltextIndex()) {
-      // Use old API (no embedded full text index).
-      searcher = new kiwix::Searcher(cPath, reader, "");
-    } else {
-      // Use the new API. We don't care about the human readable name as
-      // we don't use it (in android).
-      searcher = new kiwix::Searcher();
-      searcher->add_reader(reader, "");
-    }
-  } catch (...) {
-    searcher = NULL;
-    retVal = JNI_FALSE;
-    std::cerr << "Unable to load full text index " << cPath << std::endl;
-  }
-  pthread_mutex_unlock(&searcherLock);
-
-  return retVal;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery(
-    JNIEnv* env, jclass obj, jstring query, jint count)
-{
-  std::string cQuery = jni2c(query, env);
-  unsigned int cCount = jni2c(count);
-  kiwix::Result* p_result;
-  std::string result;
-
-  pthread_mutex_lock(&searcherLock);
-  try {
-    if (searcher != NULL) {
-      searcher->search(cQuery, 0, count);
-      while ((p_result = searcher->getNextResult())
-             && !(p_result->get_title().empty())
-             && !(p_result->get_url().empty())) {
-        result += p_result->get_title() + "\n";
-        delete p_result;
-      }
-    }
-  } catch (...) {
-    std::cerr << "Unable to make indexed query " << cQuery << std::endl;
-  }
-  pthread_mutex_unlock(&searcherLock);
-
-  return env->NewStringUTF(result.c_str());
-}

src/android/meson.build

@@ -1,17 +0,0 @@
-
-jni_generator = find_program('gen_kiwix.sh')
-
-kiwix_jni = custom_target('jni',
-  input: ['org/kiwix/kiwixlib/JNIKiwix.java',
-          'org/kiwix/kiwixlib/JNIKiwixInt.java',
-          'org/kiwix/kiwixlib/JNIKiwixString.java',
-          'org/kiwix/kiwixlib/JNIKiwixBool.java'],
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h'],
-  command:[jni_generator, '@INPUT@']
-)
-
-kiwix_sources += ['android/kiwix.cpp', kiwix_jni]
-
-install_subdir('org', install_dir: 'kiwix-lib/java')
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/android/org/kiwix/kiwixlib/JNIKiwix.java

@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixString;
-import org.kiwix.kiwixlib.JNIKiwixBool;
-import org.kiwix.kiwixlib.JNIKiwixInt;
-
-public class JNIKiwix
-{
-  static { System.loadLibrary("kiwix"); }
-  public native String getMainPage();
-
-  public native String getId();
-
-  public native String getLanguage();
-
-  public native String getMimeType(String url);
-
-  public native boolean loadZIM(String path);
-
-  public native boolean loadFulltextIndex(String path);
-
-  public native byte[] getContent(String url, JNIKiwixString title, JNIKiwixString mimeType, JNIKiwixInt size);
-
-  public native boolean searchSuggestions(String prefix, int count);
-
-  public native boolean getNextSuggestion(JNIKiwixString title);
-
-  public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
-
-  public native boolean getTitle(JNIKiwixString title);
-
-  public native String getDescription();
-
-  public native String getDate();
-
-  public native String getFavicon();
-
-  public native String getCreator();
-
-  public native String getPublisher();
-
-  public native String getName();
-
-  public native int getFileSize();
-
-  public native int getArticleCount();
-
-  public native int getMediaCount();
-
-  public native boolean getRandomPage(JNIKiwixString url);
-
-  public native void setDataDirectory(String icuDataDir);
-
-  public static native String indexedQuery(String db, int count);
-}

src/android/res/values/strings.xml

@@ -1,3 +0,0 @@
-<resources>
-  <string name="app_name">Kiwix Lib</string>
-</resources>

src/aria2.cpp

@@ -0,0 +1,281 @@
+
+
+#include "aria2.h"
+#include "xmlrpc.h"
+#include <iostream>
+#include <algorithm>
+#include <sstream>
+#include <thread>
+#include <chrono>
+#include "tools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "downloader.h" // For AriaError
+
+#ifdef _WIN32
+# define ARIA2_CMD "aria2c.exe"
+#else
+# define ARIA2_CMD "aria2c"
+# include <unistd.h>
+#endif
+
+
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+  }
+
+namespace kiwix {
+
+Aria2::Aria2():
+  mp_aria(nullptr),
+  m_port(42042),
+  m_secret(getNewRpcSecret()),
+  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
+  mp_curl(nullptr)
+{
+  m_downloadDir = getDataDirectory();
+  makeDirectory(m_downloadDir);
+  std::vector<const char*> callCmd;
+
+  std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
+  std::string download_dir = "--dir=" + getDataDirectory();
+  std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
+  std::string session = "--save-session=" + session_file;
+  std::string inputFile = "--input-file=" + session_file;
+//  std::string log_dir = "--log=\"" + logDir + "\"";
+#ifdef _WIN32
+  int pid = GetCurrentProcessId();
+#else
+  pid_t pid = getpid();
+#endif
+  std::string stop_with_pid = "--stop-with-process=" + to_string(pid);
+  std::string rpc_secret = "--rpc-secret=" + m_secret;
+  m_secret = "token:"+m_secret;
+
+  std::string aria2cmd = appendToDirectory(
+    removeLastPathElement(getExecutablePath(true)),
+    ARIA2_CMD);
+  if (fileExists(aria2cmd)) {
+    // A local aria2c exe exists (packaged with kiwix-desktop), use it.
+    callCmd.push_back(aria2cmd.c_str());
+  } else {
+    // Try to use a potential installed aria2c.
+    callCmd.push_back(ARIA2_CMD);
+  }
+  callCmd.push_back("--follow-metalink=mem");
+  callCmd.push_back("--enable-rpc");
+  callCmd.push_back(rpc_secret.c_str());
+  callCmd.push_back(rpc_port.c_str());
+  callCmd.push_back(download_dir.c_str());
+  if (fileReadable(session_file)) {
+    callCmd.push_back(inputFile.c_str());
+  }
+  callCmd.push_back(session.c_str());
+//  callCmd.push_back(log_dir.c_str());
+  callCmd.push_back("--auto-save-interval=10");
+  callCmd.push_back(stop_with_pid.c_str());
+  callCmd.push_back("--allow-overwrite=true");
+  callCmd.push_back("--dht-entry-point=router.bittorrent.com:6881");
+  callCmd.push_back("--dht-entry-point6=router.bittorrent.com:6881");
+  callCmd.push_back("--quiet=true");
+  callCmd.push_back("--bt-enable-lpd=true");
+  callCmd.push_back("--always-resume=true");
+  callCmd.push_back("--max-concurrent-downloads=42");
+  callCmd.push_back("--rpc-max-request-size=6M");
+  callCmd.push_back("--file-allocation=none");
+  std::string launchCmd;
+  for (auto &cmd : callCmd) {
+      launchCmd.append(cmd).append(" ");
+  }
+  mp_aria = Subprocess::run(callCmd);
+  mp_curl = curl_easy_init();
+
+  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
+
+  int watchdog = 50;
+  while(--watchdog) {
+    sleep(10);
+    m_curlErrorBuffer[0] = 0;
+    auto res = curl_easy_perform(mp_curl);
+    if (res == CURLE_OK) {
+      break;
+    } else if (watchdog == 1) {
+      LOG_ARIA_ERROR();
+    }
+  }
+  if (!watchdog) {
+    curl_easy_cleanup(mp_curl);
+    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
+  }
+}
+
+Aria2::~Aria2()
+{
+  std::unique_lock<std::mutex> lock(m_lock);
+  curl_easy_cleanup(mp_curl);
+}
+
+void Aria2::close()
+{
+  saveSession();
+  shutdown();
+}
+
+size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+  auto outStream = static_cast<std::stringstream*>(userdata);
+  outStream->write(ptr, nmemb);
+  return nmemb;
+}
+
+std::string Aria2::doRequest(const MethodCall& methodCall)
+{
+  auto requestContent = methodCall.toString();
+  std::stringstream outStream;
+  CURLcode res;
+  long response_code;
+  {
+    std::unique_lock<std::mutex> lock(m_lock);
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
+    m_curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(mp_curl);
+    if (res != CURLE_OK) {
+      LOG_ARIA_ERROR();
+      throw std::runtime_error("Cannot perform request");
+    }
+    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  }
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
+}
+
+std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
+{
+  MethodCall methodCall("aria2.addUri", m_secret);
+  auto uriParams = methodCall.newParamValue().getArray();
+  for (auto& uri : uris) {
+    uriParams.addValue().set(uri);
+  }
+  for (auto& option : options) {
+    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
+  }
+  auto ret = doRequest(methodCall);
+  MethodResponse response(ret);
+  return response.getParamValue(0).getAsS();
+}
+
+std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::string>& statusKey)
+{
+  MethodCall methodCall("aria2.tellStatus", m_secret);
+  methodCall.newParamValue().set(gid);
+  if (!statusKey.empty()) {
+    auto statusArray = methodCall.newParamValue().getArray();
+    for (auto& key : statusKey) {
+      statusArray.addValue().set(key);
+    }
+  }
+  return doRequest(methodCall);
+}
+
+std::string Aria2::getNewRpcSecret()
+{
+  std::string uuid = gen_uuid("");
+  uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
+  return uuid.substr(0, 9);
+}
+
+std::vector<std::string> Aria2::tellActive()
+{
+  MethodCall methodCall("aria2.tellActive", m_secret);
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> activeGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      activeGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return activeGID;
+}
+
+std::vector<std::string> Aria2::tellWaiting()
+{
+  MethodCall methodCall("aria2.tellWaiting", m_secret);
+  methodCall.newParamValue().set(0);
+  methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> waitingGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      waitingGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return waitingGID;
+}
+
+void Aria2::saveSession()
+{
+  MethodCall methodCall("aria2.saveSession", m_secret);
+  doRequest(methodCall);
+  std::cout << "session saved" << std::endl;
+}
+
+void Aria2::shutdown()
+{
+  MethodCall methodCall("aria2.shutdown", m_secret);
+  doRequest(methodCall);
+}
+
+void Aria2::pause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.pause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::unpause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.unpause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::remove(const std::string& gid)
+{
+    MethodCall methodCall("aria2.remove", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+} // end namespace kiwix

src/aria2.h

@@ -0,0 +1,52 @@
+
+
+#ifndef KIWIXLIB_ARIA2_H_
+#define KIWIXLIB_ARIA2_H_
+
+#ifdef _WIN32
+// winsock2.h need to be included before windows.h (included by curl.h)
+# include <winsock2.h>
+#endif
+
+#include "subprocess.h"
+#include "xmlrpc.h"
+
+#include <memory>
+#include <mutex>
+#include <curl/curl.h>
+
+namespace kiwix {
+
+class Aria2
+{
+  private:
+    std::unique_ptr<Subprocess> mp_aria;
+    int m_port;
+    std::string m_secret;
+    std::string m_downloadDir;
+    std::unique_ptr<char[]> m_curlErrorBuffer;
+    CURL* mp_curl;
+    std::mutex m_lock;
+
+    std::string doRequest(const MethodCall& methodCall);
+
+  public:
+    Aria2();
+    virtual ~Aria2();
+    void close();
+
+    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
+    std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    static std::string getNewRpcSecret();
+    std::vector<std::string> tellActive();
+    std::vector<std::string> tellWaiting();
+    void saveSession();
+    void shutdown();
+    void pause(const std::string& gid);
+    void unpause(const std::string& gid);
+    void remove(const std::string& gid);
+};
+
+}; //end namespace kiwix
+
+#endif // KIWIXLIB_ARIA2_H_

src/book.cpp

@@ -0,0 +1,289 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "book.h"
+
+#include "tools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/networkTools.h"
+#include "tools/otherTools.h"
+#include "tools/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/archiveTools.h"
+
+#include <zim/archive.h>
+#include <zim/item.h>
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Book::Book() :
+  m_pathValid(false),
+  m_readOnly(false)
+{
+}
+
+/* Destructor */
+Book::~Book()
+{
+}
+
+Book::Illustrations Book::getIllustrations() const
+{
+  return m_illustrations;
+}
+
+bool Book::update(const kiwix::Book& other)
+{
+  if (m_readOnly)
+    return false;
+
+  if (m_id != other.m_id)
+    return false;
+
+  *this = other;
+  return true;
+}
+
+void Book::update(const zim::Archive& archive) {
+  m_path = archive.getFilename();
+  m_pathValid = true;
+  m_id = getArchiveId(archive);
+  m_title = getArchiveTitle(archive);
+  m_description = getMetaDescription(archive);
+  m_language = getMetaLanguage(archive);
+  m_creator = getMetaCreator(archive);
+  m_publisher = getMetaPublisher(archive);
+  m_date = getMetaDate(archive);
+  m_name = getMetaName(archive);
+  m_flavour = getMetaFlavour(archive);
+  m_tags = getMetaTags(archive);
+  m_category = getCategoryFromTags();
+  m_articleCount = getArchiveArticleCount(archive);
+  m_mediaCount = getArchiveMediaCount(archive);
+  m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
+
+  m_illustrations.clear();
+  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+    const auto illustration = std::make_shared<Illustration>();
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
+    illustration->width = illustration->height = illustrationSize;
+    illustration->mimeType = illustrationItem.getMimetype();
+    illustration->data = illustrationItem.getData();
+    // NOTE: illustration->url is left uninitialized
+    m_illustrations.push_back(illustration);
+  }
+}
+
+#define ATTR(name) node.attribute(name).value()
+void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
+{
+  m_id = ATTR("id");
+  std::string path = ATTR("path");
+  if (isRelativePath(path)) {
+    path = computeAbsolutePath(baseDir, path);
+  }
+  m_path = path;
+  m_pathValid = fileReadable(path);
+  m_title = ATTR("title");
+  m_description = ATTR("description");
+  m_language = ATTR("language");
+  m_creator = ATTR("creator");
+  m_publisher = ATTR("publisher");
+  m_date = ATTR("date");
+  m_url = ATTR("url");
+  m_name = ATTR("name");
+  m_flavour = ATTR("flavour");
+  m_tags = ATTR("tags");
+  m_origId = ATTR("origId");
+  m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
+  m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
+  m_size = strtoull(ATTR("size"), 0, 0) << 10;
+  std::string favicon_mimetype = ATTR("faviconMimeType");
+  if (! favicon_mimetype.empty()) {
+    const auto favicon = std::make_shared<Illustration>();
+    favicon->data = base64_decode(ATTR("favicon"));
+    favicon->mimeType = favicon_mimetype;
+    favicon->url = ATTR("faviconUrl");
+    m_illustrations.assign(1, favicon);
+  }
+  try {
+    m_downloadId = ATTR("downloadId");
+  } catch(...) {}
+  const auto catattr = node.attribute("category");
+  m_category = catattr.empty() ? getCategoryFromTags() : catattr.value();
+}
+#undef ATTR
+
+
+static std::string fromOpdsDate(const std::string& date)
+{
+  //The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
+  //and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
+  return date.substr(0, 10);
+}
+
+
+#define VALUE(name) node.child(name).child_value()
+void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
+{
+  m_id = VALUE("id");
+  if (!m_id.compare(0, 9, "urn:uuid:")) {
+    m_id.erase(0, 9);
+  }
+  // No path on opds.
+  m_title = VALUE("title");
+  m_description = VALUE("summary");
+  m_language = VALUE("language");
+  m_creator = node.child("author").child("name").child_value();
+  m_publisher = node.child("publisher").child("name").child_value();
+  const std::string dcIssuedDate = VALUE("dc:issued");
+  m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
+  m_date = fromOpdsDate(m_date);
+  m_name = VALUE("name");
+  m_flavour = VALUE("flavour");
+  m_tags = VALUE("tags");
+  const auto catnode = node.child("category");
+  m_category = catnode.empty() ? getCategoryFromTags() : catnode.child_value();
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
+  for(auto linkNode = node.child("link"); linkNode;
+           linkNode = linkNode.next_sibling("link")) {
+    std::string rel = linkNode.attribute("rel").value();
+
+    if (rel == "http://opds-spec.org/acquisition/open-access") {
+      m_url = linkNode.attribute("href").value();
+      m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
+    }
+    if (rel == "http://opds-spec.org/image/thumbnail") {
+      const auto favicon = std::make_shared<Illustration>();
+      favicon->data.clear();
+      favicon->url = urlHost + linkNode.attribute("href").value();
+      favicon->mimeType = linkNode.attribute("type").value();
+      m_illustrations.assign(1, favicon);
+    }
+ }
+
+}
+#undef VALUE
+
+std::string Book::getHumanReadableIdFromPath() const
+{
+  std::string id = m_path;
+  if (!id.empty()) {
+    id = kiwix::removeAccents(id);
+
+#ifdef _WIN32
+    id = replaceRegex(id, "", "^.*\\\\");
+#else
+    id = replaceRegex(id, "", "^.*/");
+#endif
+
+    id = replaceRegex(id, "", "\\.zim[a-z]*$");
+    id = replaceRegex(id, "_", " ");
+    id = replaceRegex(id, "plus", "\\+");
+  }
+  return id;
+}
+
+void Book::setPath(const std::string& path)
+{
+ m_path = isRelativePath(path)
+   ? computeAbsolutePath(getCurrentDirectory(), path)
+   : path;
+}
+
+const Book::Illustration Book::missingDefaultIllustration;
+
+std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
+{
+  for ( const auto& ilPtr : m_illustrations ) {
+    if (ilPtr->width == size && ilPtr->height == size) {
+      return ilPtr;
+    }
+  }
+  throw std::runtime_error("Cannot find illustration");
+}
+
+const Book::Illustration& Book::getDefaultIllustration() const
+{
+  try {
+    return *getIllustration(48);
+  } catch (...) {
+    return missingDefaultIllustration;
+  }
+}
+
+const std::string& Book::Illustration::getData() const
+{
+  if (data.empty() && !url.empty()) {
+    const std::lock_guard<std::mutex> l(mutex);
+    if ( data.empty() ) {
+      try {
+        data = download(url);
+      } catch(...) {
+        std::cerr << "Cannot download favicon from " << url;
+      }
+    }
+  }
+  return data;
+}
+
+const std::string& Book::getFavicon() const {
+  return getDefaultIllustration().getData();
+}
+
+const std::string& Book::getFaviconUrl() const
+{
+  return getDefaultIllustration().url;
+}
+
+const std::string& Book::getFaviconMimeType() const
+{
+  return getDefaultIllustration().mimeType;
+}
+
+std::string Book::getTagStr(const std::string& tagName) const {
+  return getTagValueFromTagList(convertTags(m_tags), tagName);
+}
+
+bool Book::getTagBool(const std::string& tagName) const {
+  return convertStrToBool(getTagStr(tagName));
+}
+
+std::string Book::getCategory() const
+{
+  return m_category;
+}
+
+std::string Book::getCategoryFromTags() const
+{
+  try
+  {
+    return getTagStr("category");
+  }
+  catch ( const std::out_of_range& )
+  {
+    return "";
+  }
+}
+
+}

src/bookmark.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "bookmark.h"
+
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Bookmark::Bookmark()
+{
+}
+
+/* Destructor */
+Bookmark::~Bookmark()
+{
+}
+
+void Bookmark::updateFromXml(const pugi::xml_node& node)
+{
+  auto bookNode = node.child("book");
+  m_bookId = bookNode.child("id").child_value();
+  m_bookTitle = bookNode.child("title").child_value();
+  m_language = bookNode.child("language").child_value();
+  m_date = bookNode.child("date").child_value();
+  m_title = node.child("title").child_value();
+  m_url = node.child("url").child_value();
+}
+
+}

src/common/networkTools.cpp

@@ -1,160 +0,0 @@
-/*
- * Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/networkTools.h>
-
-std::map<std::string, std::string> kiwix::getNetworkInterfaces()
-{
-  std::map<std::string, std::string> interfaces;
-
-#ifdef _WIN32
-  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
-  if (sd == SOCKET_ERROR) {
-    std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  INTERFACE_INFO InterfaceList[20];
-  unsigned long nBytesReturned;
-  if (WSAIoctl(sd,
-               SIO_GET_INTERFACE_LIST,
-               0,
-               0,
-               &InterfaceList,
-               sizeof(InterfaceList),
-               &nBytesReturned,
-               0,
-               0)
-      == SOCKET_ERROR) {
-    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
-  for (int i = 0; i < nNumInterfaces; ++i) {
-    sockaddr_in* pAddress;
-    pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
-
-    /* Add to the map */
-    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
-    std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
-    interfaces.insert(
-        std::pair<std::string, std::string>(interfaceName, interfaceIp));
-  }
-#else
-  /* Get Network interfaces information */
-  char buf[16384];
-  struct ifconf ifconf;
-  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
-  ifconf.ifc_len = sizeof buf;
-  ifconf.ifc_buf = buf;
-  if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
-    perror("ioctl(SIOCGIFCONF)");
-    exit(EXIT_FAILURE);
-  }
-
-  /* Go through each interface */
-  int i;
-  size_t len;
-  struct ifreq* ifreq;
-  ifreq = ifconf.ifc_req;
-  for (i = 0; i < ifconf.ifc_len;) {
-    if (ifreq->ifr_addr.sa_family == AF_INET) {
-      /* Get the network interface ip */
-      char host[128] = {0};
-      const int error = getnameinfo(&(ifreq->ifr_addr),
-                                    sizeof ifreq->ifr_addr,
-                                    host,
-                                    sizeof host,
-                                    0,
-                                    0,
-                                    NI_NUMERICHOST);
-      if (!error) {
-        std::string interfaceName = std::string(ifreq->ifr_name);
-        std::string interfaceIp = std::string(host);
-        /* Add to the map */
-        interfaces.insert(
-            std::pair<std::string, std::string>(interfaceName, interfaceIp));
-      } else {
-        perror("getnameinfo()");
-      }
-    }
-
-/* some systems have ifr_addr.sa_len and adjust the length that
- * way, but not mine. weird */
-#ifndef __linux__
-    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
-#else
-    len = sizeof *ifreq;
-#endif
-    ifreq = (struct ifreq*)((char*)ifreq + len);
-    i += len;
-  }
-#endif
-  return interfaces;
-}
-
-std::string kiwix::getBestPublicIp()
-{
-  std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
-
-#ifndef _WIN32
-  const char* const prioritizedNames[]
-      = {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
-  const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
-  for (int i = 0; i < count; ++i) {
-    std::map<std::string, std::string>::const_iterator it
-        = interfaces.find(prioritizedNames[i]);
-    if (it != interfaces.end()) {
-      return it->second;
-    }
-  }
-#endif
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
-      return interfaceIp;
-    }
-  }
-
-  return "127.0.0.1";
-}

src/common/otherTools.cpp

@@ -1,29 +0,0 @@
-/*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/otherTools.h>
-
-void kiwix::sleep(unsigned int milliseconds)
-{
-#ifdef _WIN32
-  Sleep(milliseconds);
-#else
-  usleep(1000 * milliseconds);
-#endif
-}

src/common/pathTools.cpp

@@ -1,274 +0,0 @@
-/*
- * Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/pathTools.h>
-
-#ifdef __APPLE__
-#include <limits.h>
-#include <mach-o/dyld.h>
-#elif _WIN32
-#include <direct.h>
-#include <windows.h>
-#include "shlwapi.h"
-#define getcwd _getcwd  // stupid MSFT "deprecation" warning
-#endif
-
-#ifdef _WIN32
-#else
-#include <unistd.h>
-#endif
-
-#ifdef _WIN32
-#define SEPARATOR "\\"
-#else
-#define SEPARATOR "/"
-#include <unistd.h>
-#endif
-
-#include <stdlib.h>
-
-#ifndef PATH_MAX
-#define PATH_MAX 1024
-#endif
-
-bool isRelativePath(const string& path)
-{
-#ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
-#else
-  return path.empty() || path.substr(0, 1) == "/" ? false : true;
-#endif
-}
-
-string computeRelativePath(const string path, const string absolutePath)
-{
-  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
-  std::vector<std::string> absolutePathParts
-      = kiwix::split(absolutePath, SEPARATOR);
-
-  unsigned int commonCount = 0;
-  while (commonCount < pathParts.size()
-         && commonCount < absolutePathParts.size()
-         && pathParts[commonCount] == absolutePathParts[commonCount]) {
-    if (!pathParts[commonCount].empty()) {
-      commonCount++;
-    }
-  }
-
-  string relativePath;
-#ifdef _WIN32
-  /* On Windows you have a token more because the root is represented
-     by a letter */
-  if (commonCount == 0) {
-    relativePath = "../";
-  }
-#endif
-
-  for (unsigned int i = commonCount; i < pathParts.size(); i++) {
-    relativePath += "../";
-  }
-  for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
-    relativePath += absolutePathParts[i];
-    relativePath += i + 1 < absolutePathParts.size() ? "/" : "";
-  }
-
-  return relativePath;
-}
-
-/* Warning: the relative path must be with slashes */
-string computeAbsolutePath(const string path, const string relativePath)
-{
-  string absolutePath;
-
-  if (path.empty()) {
-    char* path = NULL;
-    size_t size = 0;
-
-#ifdef _WIN32
-    path = _getcwd(path, size);
-#else
-    path = getcwd(path, size);
-#endif
-
-    absolutePath = string(path) + SEPARATOR;
-  } else {
-    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
-                       ? path
-                       : path + SEPARATOR;
-  }
-
-#if _WIN32
-  char* cRelativePath = _strdup(relativePath.c_str());
-#else
-  char* cRelativePath = strdup(relativePath.c_str());
-#endif
-  char* token = strtok(cRelativePath, "/");
-
-  while (token != NULL) {
-    if (string(token) == "..") {
-      absolutePath = removeLastPathElement(absolutePath, true, false);
-      token = strtok(NULL, "/");
-    } else if (strcmp(token, ".") && strcmp(token, "")) {
-      absolutePath += string(token);
-      token = strtok(NULL, "/");
-      if (token != NULL) {
-        absolutePath += SEPARATOR;
-      }
-    } else {
-      token = strtok(NULL, "/");
-    }
-  }
-
-  return absolutePath;
-}
-
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator,
-                             const bool removePostSeparator)
-{
-  string newPath = path;
-  size_t offset = newPath.find_last_of(SEPARATOR);
-  if (removePreSeparator &&
-#ifndef _WIN32
-      offset != newPath.find_first_of(SEPARATOR) &&
-#endif
-      offset == newPath.length() - 1) {
-    newPath = newPath.substr(0, offset);
-    offset = newPath.find_last_of(SEPARATOR);
-  }
-  newPath = removePostSeparator ? newPath.substr(0, offset)
-                                : newPath.substr(0, offset + 1);
-  return newPath;
-}
-
-string appendToDirectory(const string& directoryPath, const string& filename)
-{
-  string newPath = directoryPath + SEPARATOR + filename;
-  return newPath;
-}
-
-string getLastPathElement(const string& path)
-{
-  return path.substr(path.find_last_of(SEPARATOR) + 1);
-}
-
-unsigned int getFileSize(const string& path)
-{
-#ifdef _WIN32
-  struct _stat filestatus;
-  _stat(path.c_str(), &filestatus);
-#else
-  struct stat filestatus;
-  stat(path.c_str(), &filestatus);
-#endif
-
-  return filestatus.st_size / 1024;
-}
-
-string getFileSizeAsString(const string& path)
-{
-  ostringstream convert;
-  convert << getFileSize(path);
-  return convert.str();
-}
-
-bool fileExists(const string& path)
-{
-#ifdef _WIN32
-  return PathFileExists(path.c_str());
-#else
-  bool flag = false;
-  fstream fin;
-  fin.open(path.c_str(), ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
-#endif
-}
-
-bool makeDirectory(const string& path)
-{
-#ifdef _WIN32
-  int status = _mkdir(path.c_str());
-#else
-  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-#endif
-  return status == 0;
-}
-
-/* Try to create a link and if does not work then make a copy */
-bool copyFile(const string& sourcePath, const string& destPath)
-{
-  try {
-#ifndef _WIN32
-    if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
-#endif
-      std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
-      std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
-      outfile << infile.rdbuf();
-#ifndef _WIN32
-    }
-#endif
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return false;
-  }
-
-  return true;
-}
-
-string getExecutablePath()
-{
-  char binRootPath[PATH_MAX];
-
-#ifdef _WIN32
-  GetModuleFileName(NULL, binRootPath, PATH_MAX);
-  return std::string(binRootPath);
-#elif __APPLE__
-  uint32_t max = (uint32_t)PATH_MAX;
-  _NSGetExecutablePath(binRootPath, &max);
-  return std::string(binRootPath);
-#else
-  ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
-  if (size != -1) {
-    return std::string(binRootPath, size);
-  }
-#endif
-
-  return "";
-}
-
-bool writeTextFile(const string& path, const string& content)
-{
-  std::ofstream file;
-  file.open(path.c_str());
-  file << content;
-  file.close();
-  return true;
-}
-
-string getCurrentDirectory()
-{
-  char* a_cwd = getcwd(NULL, 0);
-  string s_cwd(a_cwd);
-  free(a_cwd);
-  return s_cwd;
-}

src/common/regexTools.cpp

@@ -1,93 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/regexTools.h>
-
-std::map<std::string, RegexMatcher*> regexCache;
-
-RegexMatcher* buildRegex(const std::string& regex)
-{
-  RegexMatcher* matcher;
-  std::map<std::string, RegexMatcher*>::iterator itr = regexCache.find(regex);
-
-  /* Regex is in cache */
-  if (itr != regexCache.end()) {
-    matcher = itr->second;
-  }
-
-  /* Regex needs to be parsed (and cached) */
-  else {
-    UErrorCode status = U_ZERO_ERROR;
-    UnicodeString uregex = UnicodeString(regex.c_str());
-    matcher = new RegexMatcher(uregex, UREGEX_CASE_INSENSITIVE, status);
-    regexCache[regex] = matcher;
-  }
-
-  return matcher;
-}
-
-/* todo */
-void freeRegexCache()
-{
-}
-bool matchRegex(const std::string& content, const std::string& regex)
-{
-  ucnv_setDefaultName("UTF-8");
-  UnicodeString ucontent = UnicodeString(content.c_str());
-  RegexMatcher* matcher = buildRegex(regex);
-  matcher->reset(ucontent);
-  return matcher->find();
-}
-
-std::string replaceRegex(const std::string& content,
-                         const std::string& replacement,
-                         const std::string& regex)
-{
-  ucnv_setDefaultName("UTF-8");
-  UnicodeString ucontent = UnicodeString(content.c_str());
-  UnicodeString ureplacement = UnicodeString(replacement.c_str());
-  RegexMatcher* matcher = buildRegex(regex);
-  matcher->reset(ucontent);
-  UErrorCode status = U_ZERO_ERROR;
-  UnicodeString uresult = matcher->replaceAll(ureplacement, status);
-  std::string tmp;
-  uresult.toUTF8String(tmp);
-  return tmp;
-}
-
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  UnicodeString ucontent = UnicodeString(content.c_str());
-  UnicodeString ureplacement = UnicodeString(replacement.c_str());
-  RegexMatcher* matcher = buildRegex(regex);
-  matcher->reset(ucontent);
-
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->end(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}

src/common/stringTools.cpp

@@ -1,316 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/stringTools.h>
-
-#include <unicode/normlzr.h>
-#include <unicode/rep.h>
-#include <unicode/translit.h>
-#include <unicode/ucnv.h>
-#include <unicode/uniset.h>
-#include <unicode/ustring.h>
-
-/* tell ICU where to find its dat file (tables) */
-void kiwix::loadICUExternalTables()
-{
-#ifdef __APPLE__
-  std::string executablePath = getExecutablePath();
-  std::string executableDirectory = removeLastPathElement(executablePath);
-  std::string datPath
-      = computeAbsolutePath(executableDirectory, "icudt49l.dat");
-  try {
-    u_setDataDirectory(datPath.c_str());
-  } catch (exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-#endif
-}
-
-std::string kiwix::removeAccents(const std::string& text)
-{
-  loadICUExternalTables();
-  ucnv_setDefaultName("UTF-8");
-  UErrorCode status = U_ZERO_ERROR;
-  Transliterator* removeAccentsTrans = Transliterator::createInstance(
-      "Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
-  UnicodeString ustring = UnicodeString(text.c_str());
-  removeAccentsTrans->transliterate(ustring);
-  delete removeAccentsTrans;
-  std::string unaccentedText;
-  ustring.toUTF8String(unaccentedText);
-  return unaccentedText;
-}
-
-#ifndef __ANDROID__
-
-/* Prepare integer for display */
-std::string kiwix::beautifyInteger(const unsigned int number)
-{
-  std::stringstream numberStream;
-  numberStream << number;
-  std::string numberString = numberStream.str();
-
-  signed int offset = numberString.size() - 3;
-  while (offset > 0) {
-    numberString.insert(offset, ",");
-    offset -= 3;
-  }
-
-  return numberString;
-}
-
-std::string kiwix::beautifyFileSize(const unsigned int number)
-{
-  if (number > 1024 * 1024) {
-    return kiwix::beautifyInteger(number / (1024 * 1024)) + " GB";
-  } else {
-    return kiwix::beautifyInteger(number / 1024 != 0 ? number / 1024 : 1)
-           + " MB";
-  }
-}
-
-void kiwix::printStringInHexadecimal(UnicodeString s)
-{
-  std::cout << std::showbase << std::hex;
-  for (int i = 0; i < s.length(); i++) {
-    char c = (char)((s.getTerminatedBuffer())[i]);
-    if (c & 0x80) {
-      std::cout << (c & 0xffff) << " ";
-    } else {
-      std::cout << c << " ";
-    }
-  }
-  std::cout << std::endl;
-}
-
-void kiwix::printStringInHexadecimal(const char* s)
-{
-  std::cout << std::showbase << std::hex;
-  for (char const* pc = s; *pc; ++pc) {
-    if (*pc & 0x80) {
-      std::cout << (*pc & 0xffff);
-    } else {
-      std::cout << *pc;
-    }
-    std::cout << ' ';
-  }
-  std::cout << std::endl;
-}
-
-void kiwix::stringReplacement(std::string& str,
-                              const std::string& oldStr,
-                              const std::string& newStr)
-{
-  size_t pos = 0;
-  while ((pos = str.find(oldStr, pos)) != std::string::npos) {
-    str.replace(pos, oldStr.length(), newStr);
-    pos += newStr.length();
-  }
-}
-
-/* Encode string to avoid XSS attacks */
-std::string kiwix::encodeDiples(const std::string& str)
-{
-  std::string result = str;
-  kiwix::stringReplacement(result, "<", "&lt;");
-  kiwix::stringReplacement(result, ">", "&gt;");
-  return result;
-}
-
-// Urlencode
-// based on javascript encodeURIComponent()
-
-std::string char2hex(char dec)
-{
-  char dig1 = (dec & 0xF0) >> 4;
-  char dig2 = (dec & 0x0F);
-  if (0 <= dig1 && dig1 <= 9) {
-    dig1 += 48;  // 0,48inascii
-  }
-  if (10 <= dig1 && dig1 <= 15) {
-    dig1 += 97 - 10;  // a,97inascii
-  }
-  if (0 <= dig2 && dig2 <= 9) {
-    dig2 += 48;
-  }
-  if (10 <= dig2 && dig2 <= 15) {
-    dig2 += 97 - 10;
-  }
-
-  std::string r;
-  r.append(&dig1, 1);
-  r.append(&dig2, 1);
-  return r;
-}
-
-std::string kiwix::urlEncode(const std::string& c)
-{
-  std::string escaped = "";
-  int max = c.length();
-  for (int i = 0; i < max; i++) {
-    if ((48 <= c[i] && c[i] <= 57) ||  // 0-9
-        (65 <= c[i] && c[i] <= 90)
-        ||  // abc...xyz
-        (97 <= c[i] && c[i] <= 122)
-        ||  // ABC...XYZ
-        (c[i] == '~' || c[i] == '!' || c[i] == '*' || c[i] == '(' || c[i] == ')'
-         || c[i] == '\'')) {
-      escaped.append(&c[i], 1);
-    } else {
-      escaped.append("%");
-      escaped.append(char2hex(c[i]));  // converts char 255 to string "ff"
-    }
-  }
-  return escaped;
-}
-
-#endif
-
-static char charFromHex(std::string a)
-{
-  std::istringstream Blat(a);
-  int Z;
-  Blat >> std::hex >> Z;
-  return char(Z);
-}
-
-std::string kiwix::urlDecode(const std::string& originalUrl)
-{
-  std::string url = originalUrl;
-  std::string::size_type pos = 0;
-  while ((pos = url.find('%', pos)) != std::string::npos
-         && pos + 2 < url.length()) {
-    url.replace(pos, 3, 1, charFromHex(url.substr(pos + 1, 2)));
-    ++pos;
-  }
-  return url;
-}
-
-/* Split string in a token array */
-std::vector<std::string> kiwix::split(const std::string& str,
-                                      const std::string& delims = " *-")
-{
-  std::string::size_type lastPos = str.find_first_not_of(delims, 0);
-  std::string::size_type pos = str.find_first_of(delims, lastPos);
-  std::vector<std::string> tokens;
-
-  while (std::string::npos != pos || std::string::npos != lastPos) {
-    tokens.push_back(str.substr(lastPos, pos - lastPos));
-    lastPos = str.find_first_not_of(delims, pos);
-    pos = str.find_first_of(delims, lastPos);
-  }
-
-  return tokens;
-}
-
-std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
-{
-  const std::string m1(lhs), m2(rhs);
-  return split(m1, m2);
-}
-
-std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
-{
-  return split(lhs, rhs.c_str());
-}
-
-std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
-{
-  return split(lhs.c_str(), rhs);
-}
-
-std::string kiwix::ucFirst(const std::string& word)
-{
-  if (word.empty()) {
-    return "";
-  }
-
-  std::string result;
-
-  UnicodeString unicodeWord(word.c_str());
-  UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toUpper();
-  unicodeWord.replace(0, 1, unicodeFirstLetter);
-  unicodeWord.toUTF8String(result);
-
-  return result;
-}
-
-std::string kiwix::ucAll(const std::string& word)
-{
-  if (word.empty()) {
-    return "";
-  }
-
-  std::string result;
-
-  UnicodeString unicodeWord(word.c_str());
-  unicodeWord.toUpper().toUTF8String(result);
-
-  return result;
-}
-
-std::string kiwix::lcFirst(const std::string& word)
-{
-  if (word.empty()) {
-    return "";
-  }
-
-  std::string result;
-
-  UnicodeString unicodeWord(word.c_str());
-  UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toLower();
-  unicodeWord.replace(0, 1, unicodeFirstLetter);
-  unicodeWord.toUTF8String(result);
-
-  return result;
-}
-
-std::string kiwix::lcAll(const std::string& word)
-{
-  if (word.empty()) {
-    return "";
-  }
-
-  std::string result;
-
-  UnicodeString unicodeWord(word.c_str());
-  unicodeWord.toLower().toUTF8String(result);
-
-  return result;
-}
-
-std::string kiwix::toTitle(const std::string& word)
-{
-  if (word.empty()) {
-    return "";
-  }
-
-  std::string result;
-
-  UnicodeString unicodeWord(word.c_str());
-  unicodeWord = unicodeWord.toTitle(0);
-  unicodeWord.toUTF8String(result);
-
-  return result;
-}
-
-std::string kiwix::normalize(const std::string& word)
-{
-  return kiwix::lcAll(word);
-}

src/config.h.in

@@ -1,4 +1,3 @@
 
-#mesondefine VERSION
+#mesondefine LIBKIWIX_VERSION
 
-#mesondefine ENABLE_CTPP2

src/ctpp2/CTPP2VMStringLoader.cpp

@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <ctpp2/CTPP2VMStringLoader.hpp>
-
-namespace CTPP // C++ Template Engine
-{
-
-//
-// Convert byte order
-//
-static void ConvertExecutable(VMExecutable * oCore)
-{
-	// Code entry point
-	oCore -> entry_point = Swap32(oCore -> entry_point);
-	// Offset of code segment
-	oCore -> code_offset = Swap32(oCore -> code_offset);
-
-	// Code segment size
-	oCore -> code_size   = Swap32(oCore -> code_size);
-
-	// Offset of static text segment
-	oCore -> syscalls_offset    = Swap32(oCore -> syscalls_offset);
-	// Static text segment size
-	oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
-
-	// Offset of static text index segment
-	oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
-	// Static text index segment size
-	oCore -> syscalls_index_size   = Swap32(oCore -> syscalls_index_size);
-
-	// Offset of static data segment
-	oCore -> static_data_offset    = Swap32(oCore -> static_data_offset);
-
-	// Static data segment size
-	oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
-
-	// Offset of static text segment
-	oCore -> static_text_offset    = Swap32(oCore -> static_text_offset);
-	// Static text segment size
-	oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
-
-	// Offset of static text index segment
-	oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
-	// Static text index segment size
-	oCore -> static_text_index_size   = Swap32(oCore -> static_text_index_size);
-
-	// Version 2.2+
-	// Offset of static data bit index
-	oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
-	/// Offset of static data bit index
-	oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
-
-	// Platform
-	oCore -> platform      = Swap64(oCore -> platform);
-
-	// Ugly-jolly hack!
-	// ... dereferencing type-punned pointer will break strict-aliasing rules ...
-	UINT_64 iTMP;
-	memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
-	iTMP = Swap64(iTMP);
-	memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
-
-	// Cyclic Redundancy Check
-	oCore -> crc = 0;
-
-	// Convert data structures
-
-	// Convert code segment
-	VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
-	UINT_32 iI = 0;
-	UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pInstructions -> instruction = Swap32(pInstructions -> instruction);
-		pInstructions -> argument    = Swap32(pInstructions -> argument);
-		pInstructions -> reserved    = Swap64(pInstructions -> reserved);
-		++pInstructions;
-	}
-
-	// Convert syscalls index
-	TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
-	iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static text index
-	pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
-	iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static data
-	StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
-	iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
-		++pStaticDataVar;
-	}
-}
-
-//
-// Constructor
-//
-VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
-{
-    oCore = (VMExecutable *)malloc(rawContentSize + 1);
-    memcpy(oCore, rawContent, rawContentSize);
-
-	if (oCore -> magic[0] == 'C' &&
-	    oCore -> magic[1] == 'T' &&
-	    oCore -> magic[2] == 'P' &&
-	    oCore -> magic[3] == 'P')
-	{
-		// Check version
-		if (oCore -> version[0] >= 1)
-		{
-			// Platform-dependent data (byte order)
-			if (oCore -> platform == 0x4142434445464748ull)
-			{
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
-#endif
-
-				// Nothing to do, only check crc
-				UINT_32 iCRC = oCore -> crc;
-				oCore -> crc = 0;
-
-				// Calculate CRC of file
-                // KELSON: next line used to refer to oStat.st_size
-                // changed it to rawContentSize
-				if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
-				{
-					free(oCore);
-					throw CTPPLogicError("CRC checksum invalid");
-				}
-			}
-			// Platform-dependent data (byte order)
-			else if (oCore -> platform == 0x4847464544434241ull)
-			{
-				// Need to reconvert data
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
-#endif
-				ConvertExecutable(oCore);
-			}
-			else
-			{
-				free(oCore);
-				throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
-			}
-
-			// Check IEEE 754 format
-			if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
-			{
-				free(oCore);
-				throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
-			}
-		}
-
-		pVMMemoryCore = new VMMemoryCore(oCore);
-	}
-	else
-	{
-		free(oCore);
-		throw CTPPLogicError("Not an CTPP bytecode file.");
-	}
-}
-
-//
-// Get ready-to-run program
-//
-const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
-
-//
-// A destructor
-//
-VMStringLoader::~VMStringLoader() throw()
-{
-	delete pVMMemoryCore;
-	free(oCore);
-}
-
-} // namespace CTPP
-// End.

src/downloader.cpp

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "downloader.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+
+#include <algorithm>
+#include <thread>
+#include <chrono>
+
+#include <iostream>
+
+#include "aria2.h"
+#include "xmlrpc.h"
+#include "tools/otherTools.h"
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+
+void Download::updateStatus(bool follow)
+{
+  if (m_status == Download::K_REMOVED)
+    return;
+  static std::vector<std::string> statusKey = {"status", "files", "totalLength",
+                                               "completedLength", "followedBy",
+                                               "downloadSpeed", "verifiedLength"};
+  std::string strStatus;
+  if(follow && !m_followedBy.empty()) {
+    strStatus = mp_aria->tellStatus(m_followedBy, statusKey);
+  } else {
+    strStatus = mp_aria->tellStatus(m_did, statusKey);
+  }
+//  std::cout << strStatus << std::endl;
+  MethodResponse response(strStatus);
+  if (response.isFault()) {
+    m_status = Download::K_UNKNOWN;
+    return;
+  }
+  auto structNode = response.getParams().getParam(0).getValue().getStruct();
+  auto _status = structNode.getMember("status").getValue().getAsS();
+  auto status = _status == "active" ? Download::K_ACTIVE
+              : _status == "waiting" ? Download::K_WAITING
+              : _status == "paused" ? Download::K_PAUSED
+              : _status == "error" ? Download::K_ERROR
+              : _status == "complete" ? Download::K_COMPLETE
+              : _status == "removed" ? Download::K_REMOVED
+              : Download::K_UNKNOWN;
+  if (status == K_COMPLETE) {
+    try {
+      auto followedByMember = structNode.getMember("followedBy");
+      m_followedBy = followedByMember.getValue().getArray().getValue(0).getAsS();
+      if (follow) {
+        status = K_ACTIVE;
+        updateStatus(true);
+        return;
+      }
+    } catch (InvalidRPCNode& e) { }
+  }
+  m_status = status;
+  m_totalLength = extractFromString<uint64_t>(structNode.getMember("totalLength").getValue().getAsS());
+  m_completedLength = extractFromString<uint64_t>(structNode.getMember("completedLength").getValue().getAsS());
+  m_downloadSpeed = extractFromString<uint64_t>(structNode.getMember("downloadSpeed").getValue().getAsS());
+  try {
+    auto verifiedLengthValue = structNode.getMember("verifiedLength").getValue();
+    m_verifiedLength = extractFromString<uint64_t>(verifiedLengthValue.getAsS());
+  } catch (InvalidRPCNode& e) { m_verifiedLength = 0; }
+  auto filesMember = structNode.getMember("files");
+  auto fileStruct = filesMember.getValue().getArray().getValue(0).getStruct();
+  m_path = fileStruct.getMember("path").getValue().getAsS();
+  auto urisArray = fileStruct.getMember("uris").getValue().getArray();
+  int index = 0;
+  m_uris.clear();
+  while(true) {
+    try {
+      auto uriNode = urisArray.getValue(index++).getStruct().getMember("uri");
+      m_uris.push_back(uriNode.getValue().getAsS());
+    } catch(InvalidRPCNode& e) { break; }
+  }
+}
+
+void Download::resumeDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->unpause(m_followedBy);
+    else
+        mp_aria->unpause(m_did);
+    updateStatus(true);
+}
+
+void Download::pauseDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->pause(m_followedBy);
+    else
+        mp_aria->pause(m_did);
+    updateStatus(true);
+}
+
+void Download::cancelDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->remove(m_followedBy);
+    else
+        mp_aria->remove(m_did);
+    m_status = Download::K_REMOVED;
+}
+
+/* Constructor */
+Downloader::Downloader() :
+  mp_aria(new Aria2())
+{
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
+  }
+  try {
+    for (auto gid : mp_aria->tellWaiting()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
+  }
+}
+
+/* Destructor */
+Downloader::~Downloader()
+{
+}
+
+void Downloader::close()
+{
+  mp_aria->close();
+}
+
+std::vector<std::string> Downloader::getDownloadIds() {
+  std::vector<std::string> ret;
+  for(auto& p:m_knownDownloads) {
+    ret.push_back(p.first);
+  }
+  return ret;
+}
+
+Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
+{
+  for (auto& p: m_knownDownloads) {
+    auto& d = p.second;
+    auto& uris = d->getUris();
+    if (std::find(uris.begin(), uris.end(), uri) != uris.end())
+      return d.get();
+  }
+  std::vector<std::string> uris = {uri};
+  auto gid = mp_aria->addUri(uris, options);
+  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+  return m_knownDownloads[gid].get();
+}
+
+Download* Downloader::getDownload(const std::string& did)
+{
+  try {
+    m_knownDownloads.at(did).get()->updateStatus(true);
+    return m_knownDownloads.at(did).get();
+  } catch(std::exception& e) {
+    for (auto gid : mp_aria->tellActive()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
+        return m_knownDownloads[gid].get();
+      }
+    }
+    for (auto gid : mp_aria->tellWaiting()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
+        return m_knownDownloads[gid].get();
+      }
+  }
+    throw e;
+  }
+}
+
+}

src/kiwixserve.cpp

@@ -0,0 +1,82 @@
+#include "kiwixserve.h"
+#include "subprocess.h"
+
+#ifdef _WIN32
+    # define KIWIXSERVE_CMD "kiwix-serve.exe"
+    # include <windows.h>
+#else
+    # define KIWIXSERVE_CMD "kiwix-serve"
+    # include <unistd.h>
+#endif
+
+#include "tools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+
+namespace kiwix {
+
+KiwixServe::KiwixServe(const std::string& libraryPath, int port)
+  : m_port(port),
+    m_libraryPath(libraryPath)
+{
+}
+
+KiwixServe::~KiwixServe()
+{
+    shutDown();
+}
+
+void KiwixServe::run()
+{
+    #ifdef _WIN32
+        int pid = GetCurrentProcessId();
+    #else
+        pid_t pid = getpid();
+
+    #endif
+
+    std::vector<const char*> callCmd;
+    std::string kiwixServeCmd = appendToDirectory(
+        removeLastPathElement(getExecutablePath(true)),
+        KIWIXSERVE_CMD);
+    if (fileExists(kiwixServeCmd)) {
+        // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
+        callCmd.push_back(kiwixServeCmd.c_str());
+    } else {
+        // Try to use a potential installed kiwix-serve.
+        callCmd.push_back(KIWIXSERVE_CMD);
+    }
+    std::string attachProcessOpt = "-a" + to_string(pid);
+    std::string portOpt = "-p" + to_string(m_port);
+    callCmd.push_back(attachProcessOpt.c_str());
+    callCmd.push_back(portOpt.c_str());
+    callCmd.push_back("-l");
+    callCmd.push_back(m_libraryPath.c_str());
+    mp_kiwixServe = Subprocess::run(callCmd);
+}
+
+void KiwixServe::shutDown()
+{
+    if (mp_kiwixServe)
+        mp_kiwixServe->kill();
+}
+
+bool KiwixServe::isRunning()
+{
+    if (mp_kiwixServe) {
+        return (mp_kiwixServe->isRunning());
+    }
+    return false;
+}
+
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
+}

src/library.cpp

@@ -18,137 +18,881 @@
  */
 
 #include "library.h"
+#include "book.h"
+#include "libxml_dumper.h"
+
+#include "tools.h"
+#include "tools/base64.h"
+#include "tools/regexTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
+
+#include <pugixml.hpp>
+#include <algorithm>
+#include <set>
+#include <cmath>
+#include <unicode/locid.h>
+#include <xapian.h>
 
 namespace kiwix
 {
-/* Constructor */
-Book::Book() : readOnly(false)
+
+namespace
 {
-}
-/* Destructor */
-Book::~Book()
+
+std::string iso639_3ToXapian(const std::string& lang) {
+  return icu::Locale(lang.c_str()).getLanguage();
+};
+
+std::string normalizeText(const std::string& text)
 {
-}
-/* Sort functions */
-bool Book::sortByLastOpen(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return atoi(a.last.c_str()) > atoi(b.last.c_str());
+  return removeAccents(text);
 }
 
-bool Book::sortByTitle(const kiwix::Book& a, const kiwix::Book& b)
+bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
 {
-  return strcmp(a.title.c_str(), b.title.c_str()) < 0;
+  return book1.isPathValid()
+      && book2.isPathValid()
+      && book1.getPath() == book2.getPath();
 }
 
-bool Book::sortByDate(const kiwix::Book& a, const kiwix::Book& b)
+template<typename Key, typename Value>
+class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
 {
-  return strcmp(a.date.c_str(), b.date.c_str()) > 0;
-}
+  public:
+    explicit MultiKeyCache(size_t maxEntries)
+      : ConcurrentCache<std::set<Key>, Value>(maxEntries)
+    {}
 
-bool Book::sortBySize(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return atoi(a.size.c_str()) < atoi(b.size.c_str());
-}
-
-bool Book::sortByPublisher(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.publisher.c_str(), b.publisher.c_str()) < 0;
-}
-
-bool Book::sortByCreator(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.creator.c_str(), b.creator.c_str()) < 0;
-}
-
-bool Book::sortByLanguage(const kiwix::Book& a, const kiwix::Book& b)
-{
-  return strcmp(a.language.c_str(), b.language.c_str()) < 0;
-}
-
-std::string Book::getHumanReadableIdFromPath()
-{
-  std::string id = pathAbsolute;
-  if (!id.empty()) {
-    kiwix::removeAccents(id);
-
-#ifdef _WIN32
-    id = replaceRegex(id, "", "^.*\\\\");
-#else
-    id = replaceRegex(id, "", "^.*/");
-#endif
-
-    id = replaceRegex(id, "", "\\.zim[a-z]*$");
-    id = replaceRegex(id, "_", " ");
-    id = replaceRegex(id, "plus", "\\+");
-  }
-  return id;
-}
-
-/* Constructor */
-Library::Library() : version(KIWIX_LIBRARY_VERSION)
-{
-}
-/* Destructor */
-Library::~Library()
-{
-}
-bool Library::addBook(const Book& book)
-{
-  /* Try to find it */
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = this->books.begin(); itr != this->books.end(); ++itr) {
-    if (itr->id == book.id) {
-      if (!itr->readOnly) {
-        itr->readOnly = book.readOnly;
-
-        if (itr->path.empty()) {
-          itr->path = book.path;
-        }
-
-        if (itr->pathAbsolute.empty()) {
-          itr->pathAbsolute = book.pathAbsolute;
-        }
-
-        if (itr->url.empty()) {
-          itr->url = book.url;
-        }
-
-        if (itr->tags.empty()) {
-          itr->tags = book.tags;
-        }
-
-        if (itr->name.empty()) {
-          itr->name = book.name;
-        }
-
-        if (itr->indexPath.empty()) {
-          itr->indexPath = book.indexPath;
-          itr->indexType = book.indexType;
-        }
-
-        if (itr->indexPathAbsolute.empty()) {
-          itr->indexPathAbsolute = book.indexPathAbsolute;
-          itr->indexType = book.indexType;
-        }
-
-        if (itr->faviconMimeType.empty()) {
-          itr->favicon = book.favicon;
-          itr->faviconMimeType = book.faviconMimeType;
+    bool drop(const Key& key)
+    {
+      std::unique_lock<std::mutex> l(this->lock_);
+      bool removed = false;
+      for(auto& cache_key: this->impl_.keys()) {
+        if(cache_key.find(key)!=cache_key.end()) {
+          removed |= this->impl_.drop(cache_key);
         }
       }
+      return removed;
+    }
+};
 
-      return false;
+} // unnamed namespace
+
+struct Library::Impl
+{
+  struct Entry : Book
+  {
+    Library::Revision lastUpdatedRevision = 0;
+  };
+
+  Library::Revision m_revision;
+  std::map<std::string, Entry> m_books;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+  Xapian::WritableDatabase m_bookDB;
+
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
+
+  Impl();
+  ~Impl();
+
+  Impl(Impl&& );
+  Impl& operator=(Impl&& );
+};
+
+Library::Impl::Impl()
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
+{
+}
+
+Library::Impl::~Impl()
+{
+}
+
+Library::Impl::Impl(Library::Impl&& ) = default;
+Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
+
+unsigned int
+Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
+{
+  unsigned int result = 0;
+  for (auto& pair: m_books) {
+    auto& book = pair.second;
+    if ((!book.getPath().empty() && localBooks)
+        || (!book.getUrl().empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
+
+/* Constructor */
+Library::Library()
+  : mp_impl(new Library::Impl)
+{
+}
+
+Library::Library(Library&& other)
+  : mp_impl(std::move(other.mp_impl))
+{
+}
+
+Library& Library::operator=(Library&& other)
+{
+  mp_impl = std::move(other.mp_impl);
+  return *this;
+}
+
+/* Destructor */
+Library::~Library() = default;
+
+bool Library::addBook(const Book& book)
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  ++mp_impl->m_revision;
+  /* Try to find it */
+  updateBookDB(book);
+  try {
+    auto& oldbook = mp_impl->m_books.at(book.getId());
+    if ( ! booksReferToTheSameArchive(oldbook, book) ) {
+      dropCache(book.getId());
+    }
+    oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
+                          // XXX: Then m_bookDB will become out-of-sync with
+                          // XXX: the real contents of the library.
+    oldbook.lastUpdatedRevision = mp_impl->m_revision;
+    return false;
+  } catch (std::out_of_range&) {
+    auto& newEntry = mp_impl->m_books[book.getId()];
+    static_cast<Book&>(newEntry) = book;
+    newEntry.lastUpdatedRevision = mp_impl->m_revision;
+    size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
+    if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+    }
+    if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+    }
+    return true;
+  }
+}
+
+void Library::addBookmark(const Bookmark& bookmark)
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  mp_impl->m_bookmarks.push_back(bookmark);
+}
+
+bool Library::removeBookmark(const std::string& zimId, const std::string& url)
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
+    if (it->getBookId() == zimId && it->getUrl() == url) {
+      mp_impl->m_bookmarks.erase(it);
+      return true;
+    }
+  }
+  return false;
+}
+
+
+void Library::dropCache(const std::string& id)
+{
+  mp_impl->mp_archiveCache->drop(id);
+  mp_impl->mp_searcherCache->drop(id);
+}
+
+bool Library::removeBookById(const std::string& id)
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  mp_impl->m_bookDB.delete_document("Q" + id);
+  dropCache(id);
+  // We do not change the cache size here
+  // Most of the time, the book is remove in case of library refresh, it is
+  // often associated with addBook calls (which will properly set the cache size)
+  // Having a too big cache is not a problem here (or it would have been before)
+  // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+  //  will be removed in put or getOrPut).
+  return mp_impl->m_books.erase(id) == 1;
+}
+
+Library::Revision Library::getRevision() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return mp_impl->m_revision;
+}
+
+uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
+{
+  BookIdCollection booksToRemove;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    for ( const auto& entry : mp_impl->m_books) {
+      if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
+        booksToRemove.push_back(entry.first);
+      }
     }
   }
 
-  /* otherwise */
-  this->books.push_back(book);
+  uint32_t countOfRemovedBooks = 0;
+  for ( const auto& id : booksToRemove ) {
+    if ( removeBookById(id) )
+      ++countOfRemovedBooks;
+  }
+  return countOfRemovedBooks;
+}
+
+const Book& Library::getBookById(const std::string& id) const
+{
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
+  return mp_impl->m_books.at(id);
+}
+
+Book Library::getBookByIdThreadSafe(const std::string& id) const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return getBookById(id);
+}
+
+const Book& Library::getBookByPath(const std::string& path) const
+{
+  // XXX: Doesn't make sense to lock this operation since it cannot
+  // XXX: guarantee thread-safety because of its return type
+  for(auto& it: mp_impl->m_books) {
+    auto& book = it.second;
+    if (book.getPath() == path)
+      return book;
+  }
+  std::ostringstream ss;
+  ss << "No book with path " << path << " in the library." << std::endl;
+  throw std::out_of_range(ss.str());
+}
+
+std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
+{
+  try {
+    return mp_impl->mp_archiveCache->getOrPut(id,
+    [&](){
+      auto book = getBookById(id);
+      if (!book.isPathValid()) {
+        throw std::invalid_argument("");
+      }
+      return std::make_shared<zim::Archive>(book.getPath());
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
+}
+
+std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
+{
+  assert(!ids.empty());
+  try {
+    return mp_impl->mp_searcherCache->getOrPut(ids,
+    [&](){
+      std::vector<zim::Archive> archives;
+      for(auto& id:ids) {
+        auto archive = getArchiveById(id);
+        if(!archive) {
+          throw std::invalid_argument("");
+        }
+        archives.push_back(*archive);
+      }
+      return std::make_shared<ZimSearcher>(zim::Searcher(archives));
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
+}
+
+unsigned int Library::getBookCount(const bool localBooks,
+                                   const bool remoteBooks) const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  return mp_impl->getBookCount(localBooks, remoteBooks);
+}
+
+bool Library::writeToFile(const std::string& path) const
+{
+  const auto allBookIds = getBooksIds();
+
+  auto baseDir = removeLastPathElement(path);
+  LibXMLDumper dumper(this);
+  dumper.setBaseDir(baseDir);
+  std::string xml;
+  {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    xml = dumper.dumpLibXMLContent(allBookIds);
+  };
+  return writeTextFile(path, xml);
+}
+
+bool Library::writeBookmarksToFile(const std::string& path) const
+{
+  LibXMLDumper dumper(this);
+  // NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
+  const std::string xml = dumper.dumpLibXMLBookmark();
+  return writeTextFile(path, xml);
+}
+
+Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  AttributeCounts propValueCounts;
+
+  for (const auto& pair: mp_impl->m_books) {
+    const auto& book = pair.second;
+    if (book.getOrigId().empty()) {
+      propValueCounts[(book.*p)()] += 1;
+    }
+  }
+  return propValueCounts;
+}
+
+std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
+{
+  std::vector<std::string> result;
+  for ( const auto& kv : getBookAttributeCounts(p) ) {
+    result.push_back(kv.first);
+  }
+  return result;
+}
+
+std::vector<std::string> Library::getBooksLanguages() const
+{
+  return getBookPropValueSet(&Book::getLanguage);
+}
+
+Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
+{
+  return getBookAttributeCounts(&Book::getLanguage);
+}
+
+std::vector<std::string> Library::getBooksCategories() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  std::set<std::string> categories;
+
+  for (const auto& pair: mp_impl->m_books) {
+    const auto& book = pair.second;
+    const auto& c = book.getCategory();
+    if ( !c.empty() ) {
+      categories.insert(c);
+    }
+  }
+
+  return std::vector<std::string>(categories.begin(), categories.end());
+}
+
+std::vector<std::string> Library::getBooksCreators() const
+{
+  return getBookPropValueSet(&Book::getCreator);
+}
+
+std::vector<std::string> Library::getBooksPublishers() const
+{
+  return getBookPropValueSet(&Book::getPublisher);
+}
+
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
+{
+  if (!onlyValidBookmarks) {
+    return mp_impl->m_bookmarks;
+  }
+  std::vector<kiwix::Bookmark> validBookmarks;
+  auto booksId = getBooksIds();
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto& bookmark:mp_impl->m_bookmarks) {
+    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
+      validBookmarks.push_back(bookmark);
+    }
+  }
+  return validBookmarks;
+}
+
+Library::BookIdCollection Library::getBooksIds() const
+{
+  std::lock_guard<std::mutex> lock(m_mutex);
+  BookIdCollection bookIds;
+
+  for (auto& pair: mp_impl->m_books) {
+    bookIds.push_back(pair.first);
+  }
+
+  return bookIds;
+}
+
+
+void Library::updateBookDB(const Book& book)
+{
+  Xapian::Stem stemmer;
+  Xapian::TermGenerator indexer;
+  const std::string lang = book.getLanguage();
+  try {
+    stemmer = Xapian::Stem(iso639_3ToXapian(lang));
+    indexer.set_stemmer(stemmer);
+    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
+  } catch (...) {}
+  Xapian::Document doc;
+  indexer.set_document(doc);
+
+  const std::string title = normalizeText(book.getTitle());
+  const std::string desc = normalizeText(book.getDescription());
+
+  // Index title and description without prefixes for general search
+  indexer.index_text(title);
+  indexer.increase_termpos();
+  indexer.index_text(desc);
+
+  // Index all fields for field-based search
+  indexer.index_text(title, 1, "S");
+  indexer.index_text(desc,  1, "XD");
+  indexer.index_text(lang,  1, "L");
+  indexer.index_text(normalizeText(book.getCreator()),   1, "A");
+  indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
+  indexer.index_text(normalizeText(book.getName()),      1, "XN");
+  indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
+
+  for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
+    doc.add_boolean_term("XT" + tag);
+    if ( tag[0] != '_' ) {
+      indexer.increase_termpos();
+      indexer.index_text(tag);
+    }
+  }
+
+  const std::string idterm = "Q" + book.getId();
+  doc.add_boolean_term(idterm);
+
+  doc.set_data(book.getId());
+
+  mp_impl->m_bookDB.replace_document(idterm, doc);
+}
+
+namespace
+{
+
+bool willSelectEverything(const Xapian::Query& query)
+{
+  return query.get_type() == Xapian::Query::LEAF_MATCH_ALL;
+}
+
+
+Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
+{
+  if ( !filter.hasQuery() || filter.getQuery().empty() ) {
+    // This is a thread-safe way to construct an equivalent of
+    // a Xapian::Query::MatchAll query
+    return Xapian::Query(std::string());
+  }
+
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_AND);
+  queryParser.add_prefix("title", "S");
+  queryParser.add_prefix("description", "XD");
+  queryParser.add_prefix("name", "XN");
+  queryParser.add_prefix("category", "XC");
+  queryParser.add_prefix("lang", "L");
+  queryParser.add_prefix("publisher", "XP");
+  queryParser.add_prefix("creator", "A");
+  queryParser.add_prefix("tag", "XT");
+  const auto partialQueryFlag = filter.queryIsPartial()
+                              ? Xapian::QueryParser::FLAG_PARTIAL
+                              : 0;
+  // Language assumed for the query is not known for sure so stemming
+  // is not applied
+  //queryParser.set_stemmer(Xapian::Stem(iso639_3ToXapian(???)));
+  //queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_SOME);
+  const auto flags = Xapian::QueryParser::FLAG_PHRASE
+                   | Xapian::QueryParser::FLAG_BOOLEAN
+                   | Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE
+                   | Xapian::QueryParser::FLAG_LOVEHATE
+                   | Xapian::QueryParser::FLAG_WILDCARD
+                   | partialQueryFlag;
+  return queryParser.parse_query(normalizeText(filter.getQuery()), flags);
+}
+
+Xapian::Query nameQuery(const std::string& name)
+{
+  return Xapian::Query("XN" + normalizeText(name));
+}
+
+Xapian::Query categoryQuery(const std::string& category)
+{
+  return Xapian::Query("XC" + normalizeText(category));
+}
+
+Xapian::Query langQuery(const std::string& lang)
+{
+  return Xapian::Query("L" + normalizeText(lang));
+}
+
+Xapian::Query publisherQuery(const std::string& publisher)
+{
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_OR);
+  queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_NONE);
+  const auto flags = 0;
+  const auto q = queryParser.parse_query(normalizeText(publisher), flags, "XP");
+  return Xapian::Query(Xapian::Query::OP_PHRASE, q.get_terms_begin(), q.get_terms_end(), q.get_length());
+}
+
+Xapian::Query creatorQuery(const std::string& creator)
+{
+  Xapian::QueryParser queryParser;
+  queryParser.set_default_op(Xapian::Query::OP_OR);
+  queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_NONE);
+  const auto flags = 0;
+  const auto q = queryParser.parse_query(normalizeText(creator), flags, "A");
+  return Xapian::Query(Xapian::Query::OP_PHRASE, q.get_terms_begin(), q.get_terms_end(), q.get_length());
+}
+
+Xapian::Query tagsQuery(const Filter::Tags& acceptTags, const Filter::Tags& rejectTags)
+{
+  Xapian::Query q = Xapian::Query(std::string());
+  if (!acceptTags.empty()) {
+    for ( const auto& tag : acceptTags )
+      q &= Xapian::Query("XT" + normalizeText(tag));
+  }
+
+  if (!rejectTags.empty()) {
+    for ( const auto& tag : rejectTags )
+      q = Xapian::Query(Xapian::Query::OP_AND_NOT, q, "XT" + normalizeText(tag));
+  }
+  return q;
+}
+
+Xapian::Query buildXapianQuery(const Filter& filter)
+{
+  auto q = buildXapianQueryFromFilterQuery(filter);
+  if ( filter.hasName() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, nameQuery(filter.getName()));
+  }
+  if ( filter.hasCategory() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, categoryQuery(filter.getCategory()));
+  }
+  if ( filter.hasLang() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, langQuery(filter.getLang()));
+  }
+  if ( filter.hasPublisher() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, publisherQuery(filter.getPublisher()));
+  }
+  if ( filter.hasCreator() ) {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, creatorQuery(filter.getCreator()));
+  }
+  if ( !filter.getAcceptTags().empty() || !filter.getRejectTags().empty() ) {
+    const auto tq = tagsQuery(filter.getAcceptTags(), filter.getRejectTags());
+    q = Xapian::Query(Xapian::Query::OP_AND, q, tq);;
+  }
+  return q;
+}
+
+} // unnamed namespace
+
+Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
+{
+  const auto query = buildXapianQuery(filter);
+
+  if ( willSelectEverything(query) )
+    return getBooksIds();
+
+  BookIdCollection bookIds;
+
+  std::lock_guard<std::mutex> lock(m_mutex);
+  Xapian::Enquire enquire(mp_impl->m_bookDB);
+  enquire.set_query(query);
+  const auto results = enquire.get_mset(0, mp_impl->m_books.size());
+  for ( auto it = results.begin(); it != results.end(); ++it  ) {
+    bookIds.push_back(it.get_document().get_data());
+  }
+
+  return bookIds;
+}
+
+Library::BookIdCollection Library::filter(const Filter& filter) const
+{
+  BookIdCollection result;
+  const auto preliminaryResult = filterViaBookDB(filter);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  for(auto id : preliminaryResult) {
+    if(filter.accept(mp_impl->m_books.at(id))) {
+      result.push_back(id);
+    }
+  }
+  return result;
+}
+
+template<supportedListSortBy SORT>
+struct KEY_TYPE {
+  typedef std::string TYPE;
+};
+
+template<>
+struct KEY_TYPE<SIZE> {
+  typedef size_t TYPE;
+};
+
+template<supportedListSortBy sort>
+class Comparator {
+  private:
+    const Library* const lib;
+    const bool     ascending;
+
+    inline typename KEY_TYPE<sort>::TYPE get_key(const std::string& id);
+
+  public:
+    Comparator(const Library* lib, bool ascending) : lib(lib), ascending(ascending) {}
+    inline bool operator() (const std::string& id1, const std::string& id2) {
+      if (ascending) {
+        return get_key(id1) < get_key(id2);
+      } else {
+        return get_key(id2) < get_key(id1);
+      }
+    }
+};
+
+template<>
+std::string Comparator<TITLE>::get_key(const std::string& id)
+{
+  return lib->getBookById(id).getTitle();
+}
+
+template<>
+size_t Comparator<SIZE>::get_key(const std::string& id)
+{
+  return lib->getBookById(id).getSize();
+}
+
+template<>
+std::string Comparator<DATE>::get_key(const std::string& id)
+{
+  return lib->getBookById(id).getDate();
+}
+
+template<>
+std::string Comparator<CREATOR>::get_key(const std::string& id)
+{
+  return lib->getBookById(id).getCreator();
+}
+
+template<>
+std::string Comparator<PUBLISHER>::get_key(const std::string& id)
+{
+  return lib->getBookById(id).getPublisher();
+}
+
+void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
+{
+  // NOTE: Can reimplement this method in a way that doesn't require locking
+  // NOTE: for the entire duration of the sort. Will need to obtain (under a
+  // NOTE: lock) the required atributes from the books once, and then the
+  // NOTE: sorting will run on a copy of data without locking.
+  std::lock_guard<std::mutex> lock(m_mutex);
+  switch(sort) {
+    case TITLE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
+      break;
+    case SIZE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<SIZE>(this, ascending));
+      break;
+    case DATE:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<DATE>(this, ascending));
+      break;
+    case CREATOR:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<CREATOR>(this, ascending));
+      break;
+    case PUBLISHER:
+      std::sort(bookIds.begin(), bookIds.end(), Comparator<PUBLISHER>(this, ascending));
+      break;
+    default:
+      break;
+  }
+}
+
+
+Filter::Filter()
+  : activeFilters(0),
+    _maxSize(0)
+{};
+
+#define FLAG(x) (1 << x)
+enum filterTypes {
+  NONE = 0,
+  _LOCAL = FLAG(0),
+  _REMOTE = FLAG(1),
+  _NOLOCAL = FLAG(2),
+  _NOREMOTE = FLAG(3),
+  _VALID = FLAG(4),
+  _NOVALID = FLAG(5),
+  ACCEPTTAGS = FLAG(6),
+  REJECTTAGS = FLAG(7),
+  LANG = FLAG(8),
+  _PUBLISHER = FLAG(9),
+  _CREATOR = FLAG(10),
+  MAXSIZE = FLAG(11),
+  QUERY = FLAG(12),
+  NAME = FLAG(13),
+  CATEGORY = FLAG(14),
+};
+
+Filter& Filter::local(bool accept)
+{
+  if (accept) {
+    activeFilters |= _LOCAL;
+    activeFilters &= ~_NOLOCAL;
+  } else {
+    activeFilters |= _NOLOCAL;
+    activeFilters &= ~_LOCAL;
+  }
+  return *this;
+}
+
+Filter& Filter::remote(bool accept)
+{
+  if (accept) {
+    activeFilters |= _REMOTE;
+    activeFilters &= ~_NOREMOTE;
+  } else {
+    activeFilters |= _NOREMOTE;
+    activeFilters &= ~_REMOTE;
+  }
+  return *this;
+}
+
+Filter& Filter::valid(bool accept)
+{
+  if (accept) {
+    activeFilters |= _VALID;
+    activeFilters &= ~_NOVALID;
+  } else {
+    activeFilters |= _NOVALID;
+    activeFilters &= ~_VALID;
+  }
+  return *this;
+}
+
+Filter& Filter::acceptTags(const Tags& tags)
+{
+  _acceptTags = tags;
+  activeFilters |= ACCEPTTAGS;
+  return *this;
+}
+
+Filter& Filter::rejectTags(const Tags& tags)
+{
+  _rejectTags = tags;
+  activeFilters |= REJECTTAGS;
+  return *this;
+}
+
+Filter& Filter::category(std::string category)
+{
+  _category = category;
+  activeFilters |= CATEGORY;
+  return *this;
+}
+
+Filter& Filter::lang(std::string lang)
+{
+  _lang = lang;
+  activeFilters |= LANG;
+  return *this;
+}
+
+Filter& Filter::publisher(std::string publisher)
+{
+  _publisher = publisher;
+  activeFilters |= _PUBLISHER;
+  return *this;
+}
+
+Filter& Filter::creator(std::string creator)
+{
+  _creator = creator;
+  activeFilters |= _CREATOR;
+  return *this;
+}
+
+Filter& Filter::maxSize(size_t maxSize)
+{
+  _maxSize = maxSize;
+  activeFilters |= MAXSIZE;
+  return *this;
+}
+
+Filter& Filter::query(std::string query, bool partial)
+{
+  _query = query;
+  _queryIsPartial = partial;
+  activeFilters |= QUERY;
+  return *this;
+}
+
+Filter& Filter::name(std::string name)
+{
+  _name = name;
+  activeFilters |= NAME;
+  return *this;
+}
+
+#define ACTIVE(X) (activeFilters & (X))
+#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
+bool Filter::hasQuery() const
+{
+  return ACTIVE(QUERY);
+}
+
+bool Filter::hasName() const
+{
+  return ACTIVE(NAME);
+}
+
+bool Filter::hasCategory() const
+{
+  return ACTIVE(CATEGORY);
+}
+
+bool Filter::hasLang() const
+{
+  return ACTIVE(LANG);
+}
+
+bool Filter::hasPublisher() const
+{
+  return ACTIVE(_PUBLISHER);
+}
+
+bool Filter::hasCreator() const
+{
+  return ACTIVE(_CREATOR);
+}
+
+bool Filter::accept(const Book& book) const
+{
+  auto local = !book.getPath().empty();
+  FILTER(_LOCAL, local)
+  FILTER(_NOLOCAL, !local)
+
+  auto valid = book.isPathValid();
+  FILTER(_VALID, valid)
+  FILTER(_NOVALID, !valid)
+
+  auto remote = !book.getUrl().empty();
+  FILTER(_REMOTE, remote)
+  FILTER(_NOREMOTE, !remote)
+
+  FILTER(MAXSIZE, book.getSize() <= _maxSize)
+
   return true;
 }
 
-bool Library::removeBookByIndex(const unsigned int bookIndex)
-{
-  books.erase(books.begin() + bookIndex);
-  return true;
-}
 }

src/libxml_dumper.cpp

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "libxml_dumper.h"
+#include "book.h"
+
+#include "tools.h"
+#include "tools/base64.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibXMLDumper::LibXMLDumper(const Library* library)
+  : library(library)
+{
+}
+/* Destructor */
+LibXMLDumper::~LibXMLDumper()
+{
+}
+
+#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
+#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
+
+void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
+  if (book.readOnly())
+    return;
+
+  auto entry_node = root_node.append_child("book");
+  ADD_ATTRIBUTE(entry_node, "id", book.getId());
+
+  if (!book.getPath().empty()) {
+    ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
+  }
+
+  if (book.getOrigId().empty()) {
+    ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
+    ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
+    ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
+    try {
+      auto defaultIllustration = book.getIllustration(48);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
+      if (!defaultIllustration->getData().empty())
+        ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
+    } catch(...) {}
+  } else {
+    ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
+  }
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
+  ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
+
+  if (book.getArticleCount())
+    ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
+
+  if (book.getMediaCount())
+    ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
+
+  if (book.getSize())
+    ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
+}
+
+#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
+
+void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
+
+  auto entry_node = root_node.append_child("bookmark");
+  auto book_node = entry_node.append_child("book");
+
+  try {
+    auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "id", book.getId());
+    ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
+    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", book.getDate());
+  } catch (...) {
+    ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
+    ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
+  }
+  ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
+  ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
+}
+
+
+std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node libraryNode = doc.append_child("library");
+
+  libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
+
+  if (library) {
+    for (auto& bookId: bookIds) {
+      handleBook(library->getBookById(bookId), libraryNode);
+    }
+  }
+  return nodeToString(libraryNode);
+}
+
+std::string LibXMLDumper::dumpLibXMLBookmark()
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
+
+  if (library) {
+    for (auto& bookmark: library->getBookmarks()) {
+      handleBookmark(bookmark, bookmarksNode);
+    }
+  }
+  return nodeToString(bookmarksNode);
+}
+
+}

src/manager.cpp

@@ -19,105 +19,191 @@
 
 #include "manager.h"
 
+#include "tools.h"
+#include "tools/pathTools.h"
+
+#include <pugixml.hpp>
+
 namespace kiwix
 {
+
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+////////////////////////////////////////////////////////////////////////////////
+// LibraryManipulator
+////////////////////////////////////////////////////////////////////////////////
+
+LibraryManipulator::LibraryManipulator(Library* library)
+  : library(*library)
+{}
+
+LibraryManipulator::~LibraryManipulator()
+{}
+
+bool LibraryManipulator::addBookToLibrary(const Book& book)
+{
+  const auto ret = library.addBook(book);
+  if ( ret ) {
+    bookWasAddedToLibrary(book);
+  }
+  return ret;
+}
+
+void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
+{
+  library.addBookmark(bookmark);
+  bookmarkWasAddedToLibrary(bookmark);
+}
+
+uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
+{
+  const auto n = library.removeBooksNotUpdatedSince(rev);
+  if ( n != 0 ) {
+    booksWereRemovedFromLibrary();
+  }
+  return n;
+}
+
+void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
+{
+}
+
+void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
+{
+}
+
+void LibraryManipulator::booksWereRemovedFromLibrary()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Manager
+////////////////////////////////////////////////////////////////////////////////
+
 /* Constructor */
-Manager::Manager() : writableLibraryPath("")
+Manager::Manager(LibraryManipulator* manipulator):
+  writableLibraryPath(""),
+  manipulator(manipulator, NoDelete())
 {
 }
-/* Destructor */
-Manager::~Manager()
+
+Manager::Manager(Library* library) :
+  writableLibraryPath(""),
+  manipulator(new LibraryManipulator(library))
 {
 }
+
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          const bool readOnly,
-                          const string libraryPath)
+                          bool readOnly,
+                          const std::string& libraryPath,
+                          bool trustLibrary)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
-  if (strlen(libraryNode.attribute("current").value()))
-    this->setCurrentBookId(libraryNode.attribute("current").value());
-
-  string libraryVersion = libraryNode.attribute("version").value();
+  std::string libraryVersion = libraryNode.attribute("version").value();
 
   for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode;
        bookNode = bookNode.next_sibling("book")) {
-    bool ok = true;
     kiwix::Book book;
 
-    book.readOnly = readOnly;
-    book.id = bookNode.attribute("id").value();
-    book.path = bookNode.attribute("path").value();
-    book.last = (std::string(bookNode.attribute("last").value()) != "undefined"
-                     ? bookNode.attribute("last").value()
-                     : "");
-    book.indexPath = bookNode.attribute("indexPath").value();
-    book.indexType = XAPIAN;
-    book.title = bookNode.attribute("title").value();
-    book.name = bookNode.attribute("name").value();
-    book.tags = bookNode.attribute("tags").value();
-    book.description = bookNode.attribute("description").value();
-    book.language = bookNode.attribute("language").value();
-    book.date = bookNode.attribute("date").value();
-    book.creator = bookNode.attribute("creator").value();
-    book.publisher = bookNode.attribute("publisher").value();
-    book.url = bookNode.attribute("url").value();
-    book.origId = bookNode.attribute("origId").value();
-    book.articleCount = bookNode.attribute("articleCount").value();
-    book.mediaCount = bookNode.attribute("mediaCount").value();
-    book.size = bookNode.attribute("size").value();
-    book.favicon = bookNode.attribute("favicon").value();
-    book.faviconMimeType = bookNode.attribute("faviconMimeType").value();
+    book.setReadOnly(readOnly);
+    book.updateFromXml(bookNode,
+                       removeLastPathElement(libraryPath));
 
-    /* Check absolute and relative paths */
-    this->checkAndCleanBookPaths(book, libraryPath);
-
-    /* Update the book properties with the new importer */
-    if (libraryVersion.empty()
-        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.path.empty()) {
-        ok = this->readBookFromPath(book.pathAbsolute);
-      }
-    }
-
-    if (ok) {
-      library.addBook(book);
+    if (!trustLibrary && !book.getPath().empty()) {
+      this->readBookFromPath(book.getPath(), &book);
     }
+    manipulator->addBookToLibrary(book);
   }
 
   return true;
 }
 
-bool Manager::readXml(const string xml,
-                      const bool readOnly,
-                      const string libraryPath)
+bool Manager::readXml(const std::string& xml,
+                      bool readOnly,
+                      const std::string& libraryPath,
+                      bool trustLibrary)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)xml.data(), xml.size());
+      = doc.load_buffer((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath);
+    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
   }
 
   return true;
 }
 
-bool Manager::readFile(const string path, const bool readOnly)
+
+
+bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& urlHost)
 {
-  return this->readFile(path, path, readOnly);
+  pugi::xml_node libraryNode = doc.child("feed");
+
+  try {
+    m_totalBooks = strtoull(libraryNode.child("totalResults").child_value(), 0, 0);
+    m_startIndex = strtoull(libraryNode.child("startIndex").child_value(), 0, 0);
+    m_itemsPerPage = strtoull(libraryNode.child("itemsPerPage").child_value(), 0, 0);
+    m_hasSearchResult = true;
+  } catch(...) {
+    m_hasSearchResult = false;
+  }
+
+  for (pugi::xml_node entryNode = libraryNode.child("entry"); entryNode;
+       entryNode = entryNode.next_sibling("entry")) {
+    kiwix::Book book;
+
+    book.setReadOnly(false);
+    book.updateFromOpds(entryNode, urlHost);
+
+    /* Update the book properties with the new importer */
+    manipulator->addBookToLibrary(book);
+  }
+
+  return true;
 }
 
-bool Manager::readFile(const string nativePath,
-                       const string UTF8Path,
-                       const bool readOnly)
+
+
+bool Manager::readOpds(const std::string& content, const std::string& urlHost)
+{
+  pugi::xml_document doc;
+  pugi::xml_parse_result result
+      = doc.load_buffer((void*)content.data(), content.size());
+
+  if (result) {
+    this->parseOpdsDom(doc, urlHost);
+    return true;
+  }
+
+  return false;
+}
+
+bool Manager::readFile(
+  const std::string& path,
+  bool readOnly,
+  bool trustLibrary)
 {
   bool retVal = true;
   pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
+
+#ifdef _WIN32
+  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
+#else
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+#endif
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, UTF8Path);
+    this->parseXmlDom(doc, readOnly, path, trustLibrary);
   } else {
     retVal = false;
   }
@@ -126,154 +212,37 @@ bool Manager::readFile(const string nativePath,
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = UTF8Path;
+    this->writableLibraryPath = path;
   }
 
   return retVal;
 }
 
-bool Manager::writeFile(const string path)
-{
-  pugi::xml_document doc;
-
-  /* Add the library node */
-  pugi::xml_node libraryNode = doc.append_child("library");
-
-  if (!getCurrentBookId().empty()) {
-    libraryNode.append_attribute("current") = getCurrentBookId().c_str();
-  }
-
-  if (!library.version.empty())
-    libraryNode.append_attribute("version") = library.version.c_str();
-
-  /* Add each book */
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (!itr->readOnly) {
-      this->checkAndCleanBookPaths(*itr, path);
-
-      pugi::xml_node bookNode = libraryNode.append_child("book");
-      bookNode.append_attribute("id") = itr->id.c_str();
-
-      if (!itr->path.empty()) {
-        bookNode.append_attribute("path") = itr->path.c_str();
-      }
-
-      if (!itr->last.empty() && itr->last != "undefined") {
-        bookNode.append_attribute("last") = itr->last.c_str();
-      }
-
-      if (!itr->indexPath.empty())
-        bookNode.append_attribute("indexPath") = itr->indexPath.c_str();
-
-      if (!itr->indexPath.empty() || !itr->indexPathAbsolute.empty()) {
-        if (itr->indexType == XAPIAN) {
-          bookNode.append_attribute("indexType") = "xapian";
-        }
-      }
-
-      if (itr->origId.empty()) {
-        if (!itr->title.empty())
-          bookNode.append_attribute("title") = itr->title.c_str();
-
-        if (!itr->name.empty())
-          bookNode.append_attribute("name") = itr->name.c_str();
-
-        if (!itr->tags.empty())
-          bookNode.append_attribute("tags") = itr->tags.c_str();
-
-        if (!itr->description.empty())
-          bookNode.append_attribute("description") = itr->description.c_str();
-
-        if (!itr->language.empty())
-          bookNode.append_attribute("language") = itr->language.c_str();
-
-        if (!itr->creator.empty())
-          bookNode.append_attribute("creator") = itr->creator.c_str();
-
-        if (!itr->publisher.empty())
-          bookNode.append_attribute("publisher") = itr->publisher.c_str();
-
-        if (!itr->favicon.empty())
-          bookNode.append_attribute("favicon") = itr->favicon.c_str();
-
-        if (!itr->faviconMimeType.empty())
-          bookNode.append_attribute("faviconMimeType")
-              = itr->faviconMimeType.c_str();
-      }
-
-      if (!itr->date.empty()) {
-        bookNode.append_attribute("date") = itr->date.c_str();
-      }
-
-      if (!itr->url.empty()) {
-        bookNode.append_attribute("url") = itr->url.c_str();
-      }
-
-      if (!itr->origId.empty())
-        bookNode.append_attribute("origId") = itr->origId.c_str();
-
-      if (!itr->articleCount.empty())
-        bookNode.append_attribute("articleCount") = itr->articleCount.c_str();
-
-      if (!itr->mediaCount.empty())
-        bookNode.append_attribute("mediaCount") = itr->mediaCount.c_str();
-
-      if (!itr->size.empty()) {
-        bookNode.append_attribute("size") = itr->size.c_str();
-      }
-    }
-  }
-
-  /* saving file */
-  doc.save_file(path.c_str());
-
-  return true;
-}
-
-bool Manager::setCurrentBookId(const string id)
-{
-  if (library.current.empty() || library.current.top() != id) {
-    if (id.empty() && !library.current.empty()) {
-      library.current.pop();
-    } else {
-      library.current.push(id);
-    }
-  }
-  return true;
-}
-
-string Manager::getCurrentBookId()
-{
-  return library.current.empty() ? "" : library.current.top();
-}
 
 /* Add a book to the library. Return empty string if failed, book id otherwise
  */
-string Manager::addBookFromPathAndGetId(const string pathToOpen,
-                                        const string pathToSave,
-                                        const string url,
-                                        const bool checkMetaData)
+std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
+                                             const std::string& pathToSave,
+                                             const std::string& url,
+                                             const bool checkMetaData)
 {
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (pathToSave != pathToOpen) {
-      book.path = pathToSave;
-      book.pathAbsolute
-          = isRelativePath(pathToSave)
+    if (!pathToSave.empty() && pathToSave != pathToOpen) {
+      book.setPath(isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
+                      removeLastPathElement(writableLibraryPath),
                       pathToSave)
-                : pathToSave;
+                : pathToSave);
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.title.empty() && !book.language.empty()
-            && !book.date.empty())) {
-      book.url = url;
-      library.addBook(book);
-      return book.id;
+        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
+            && !book.getDate().empty())) {
+      book.setUrl(url);
+      manipulator->addBookToLibrary(book);
+      return book.getId();
     }
   }
 
@@ -282,9 +251,9 @@ string Manager::addBookFromPathAndGetId(const string pathToOpen,
 
 /* Wrapper over Manager::addBookFromPath which return a bool instead of a string
  */
-bool Manager::addBookFromPath(const string pathToOpen,
-                              const string pathToSave,
-                              const string url,
+bool Manager::addBookFromPath(const std::string& pathToOpen,
+                              const std::string& pathToSave,
+                              const std::string& url,
                               const bool checkMetaData)
 {
   return !(
@@ -292,367 +261,62 @@ bool Manager::addBookFromPath(const string pathToOpen,
           .empty());
 }
 
-bool Manager::readBookFromPath(const string path, kiwix::Book* book)
+bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
+  std::string tmp_path = path;
+  if (isRelativePath(path)) {
+    tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
+  }
   try {
-    kiwix::Reader* reader = new kiwix::Reader(path);
-
-    if (book != NULL) {
-      book->path = path;
-      book->pathAbsolute = path;
-      book->id = reader->getId();
-      book->description = reader->getDescription();
-      book->language = reader->getLanguage();
-      book->date = reader->getDate();
-      book->creator = reader->getCreator();
-      book->publisher = reader->getPublisher();
-      book->title = reader->getTitle();
-      book->name = reader->getName();
-      book->tags = reader->getTags();
-      book->origId = reader->getOrigId();
-      std::ostringstream articleCountStream;
-      articleCountStream << reader->getArticleCount();
-      book->articleCount = articleCountStream.str();
-
-      std::ostringstream mediaCountStream;
-      mediaCountStream << reader->getMediaCount();
-      book->mediaCount = mediaCountStream.str();
-
-      ostringstream convert;
-      convert << reader->getFileSize();
-      book->size = convert.str();
-
-      string favicon;
-      string faviconMimeType;
-      if (reader->getFavicon(favicon, faviconMimeType)) {
-        book->favicon = base64_encode(
-            reinterpret_cast<const unsigned char*>(favicon.c_str()),
-            favicon.length());
-        book->faviconMimeType = faviconMimeType;
-      }
-    }
-
-    delete reader;
+    zim::Archive archive(tmp_path);
+    book->update(archive);
+    book->setPathValid(true);
   } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
+    book->setPathValid(false);
     return false;
   }
 
   return true;
 }
 
-bool Manager::removeBookByIndex(const unsigned int bookIndex)
+bool Manager::readBookmarkFile(const std::string& path)
 {
-  return this->library.removeBookByIndex(bookIndex);
-}
+  pugi::xml_document doc;
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
 
-bool Manager::removeBookById(const string id)
-{
-  unsigned int bookIndex = 0;
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      return this->library.removeBookByIndex(bookIndex);
-    }
-    bookIndex++;
-  }
-  return false;
-}
-
-vector<string> Manager::getBooksLanguages()
-{
-  std::vector<string> booksLanguages;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksLanguagesMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksLanguagesMap.find(itr->language) == booksLanguagesMap.end()) {
-      if (itr->origId.empty()) {
-        booksLanguagesMap[itr->language] = true;
-        booksLanguages.push_back(itr->language);
-      }
-    }
-  }
-
-  return booksLanguages;
-}
-
-vector<string> Manager::getBooksCreators()
-{
-  std::vector<string> booksCreators;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksCreatorsMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksCreatorsMap.find(itr->creator) == booksCreatorsMap.end()) {
-      if (itr->origId.empty()) {
-        booksCreatorsMap[itr->creator] = true;
-        booksCreators.push_back(itr->creator);
-      }
-    }
-  }
-
-  return booksCreators;
-}
-
-vector<string> Manager::getBooksIds()
-{
-  std::vector<string> booksIds;
-  std::vector<kiwix::Book>::iterator itr;
-
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    booksIds.push_back(itr->id);
-  }
-
-  return booksIds;
-}
-
-vector<string> Manager::getBooksPublishers()
-{
-  std::vector<string> booksPublishers;
-  std::vector<kiwix::Book>::iterator itr;
-  std::map<string, bool> booksPublishersMap;
-
-  std::sort(
-      library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (booksPublishersMap.find(itr->publisher) == booksPublishersMap.end()) {
-      if (itr->origId.empty()) {
-        booksPublishersMap[itr->publisher] = true;
-        booksPublishers.push_back(itr->publisher);
-      }
-    }
-  }
-
-  return booksPublishers;
-}
-
-kiwix::Library Manager::cloneLibrary()
-{
-  return this->library;
-}
-bool Manager::getCurrentBook(Book& book)
-{
-  string currentBookId = getCurrentBookId();
-  if (currentBookId.empty()) {
+  if (!result) {
     return false;
-  } else {
-    getBookById(currentBookId, book);
-    return true;
-  }
-}
-
-bool Manager::getBookById(const string id, Book& book)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      book = *itr;
-      return true;
-    }
-  }
-  return false;
-}
-
-bool Manager::updateBookLastOpenDateById(const string id)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      char unixdate[12];
-      sprintf(unixdate, "%d", (int)time(NULL));
-      itr->last = unixdate;
-      return true;
-    }
   }
 
-  return false;
-}
+  pugi::xml_node libraryNode = doc.child("bookmarks");
 
-bool Manager::setBookIndex(const string id,
-                           const string path,
-                           const supportedIndexType type)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      itr->indexPath = path;
-      itr->indexPathAbsolute
-          = isRelativePath(path)
-                ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
-                      path)
-                : path;
-      itr->indexType = type;
-      return true;
-    }
-  }
+  for (pugi::xml_node node = libraryNode.child("bookmark"); node;
+       node = node.next_sibling("bookmark")) {
+    kiwix::Bookmark bookmark;
 
-  return false;
-}
+    bookmark.updateFromXml(node);
 
-bool Manager::setBookIndex(const string id, const string path)
-{
-  return this->setBookIndex(id, path, XAPIAN);
-}
-
-bool Manager::setBookPath(const string id, const string path)
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if (itr->id == id) {
-      itr->path = path;
-      itr->pathAbsolute
-          = isRelativePath(path)
-                ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
-                      path)
-                : path;
-      return true;
-    }
-  }
-
-  return false;
-}
-
-void Manager::removeBookPaths()
-{
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    itr->path = "";
-    itr->pathAbsolute = "";
-  }
-}
-
-unsigned int Manager::getBookCount(const bool localBooks,
-                                   const bool remoteBooks)
-{
-  unsigned int result = 0;
-  std::vector<kiwix::Book>::iterator itr;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-    if ((!itr->path.empty() && localBooks)
-        || (itr->path.empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
-}
-
-bool Manager::listBooks(const supportedListMode mode,
-                        const supportedListSortBy sortBy,
-                        const unsigned int maxSize,
-                        const string language,
-                        const string creator,
-                        const string publisher,
-                        const string search)
-{
-  this->bookIdList.clear();
-  std::vector<kiwix::Book>::iterator itr;
-
-  /* Sort */
-  if (sortBy == TITLE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
-  } else if (sortBy == SIZE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
-  } else if (sortBy == DATE) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
-  } else if (sortBy == CREATOR) {
-    std::sort(
-        library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
-  } else if (sortBy == PUBLISHER) {
-    std::sort(library.books.begin(),
-              library.books.end(),
-              kiwix::Book::sortByPublisher);
-  }
-
-  /* Special sort for LASTOPEN */
-  if (mode == LASTOPEN) {
-    std::sort(library.books.begin(),
-              library.books.end(),
-              kiwix::Book::sortByLastOpen);
-    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-      if (!itr->last.empty()) {
-        this->bookIdList.push_back(itr->id);
-      }
-    }
-  } else {
-    /* Generate the list of book id */
-    for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
-      bool ok = true;
-
-      if (mode == LOCAL && itr->path.empty()) {
-        ok = false;
-      }
-
-      if (ok == true && mode == REMOTE
-          && (!itr->path.empty() || itr->url.empty())) {
-        ok = false;
-      }
-
-      if (ok == true && maxSize != 0
-          && (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024) {
-        ok = false;
-      }
-
-      if (ok == true && !language.empty()
-          && !matchRegex(itr->language, language)) {
-        ok = false;
-      }
-
-      if (ok == true && !creator.empty() && itr->creator != creator) {
-        ok = false;
-      }
-
-      if (ok == true && !publisher.empty() && itr->publisher != publisher) {
-        ok = false;
-      }
-
-      if ((ok == true && !search.empty())
-          && !(matchRegex(itr->title, "\\Q" + search + "\\E")
-               || matchRegex(itr->description, "\\Q" + search + "\\E")
-               || matchRegex(itr->language, "\\Q" + search + "\\E"))) {
-        ok = false;
-      }
-
-      if (ok == true) {
-        this->bookIdList.push_back(itr->id);
-      }
-    }
+    manipulator->addBookmarkToLibrary(bookmark);
   }
 
   return true;
 }
 
-void Manager::checkAndCleanBookPaths(Book& book, const string& libraryPath)
+void Manager::reload(const Paths& paths)
 {
-  if (!book.path.empty()) {
-    if (isRelativePath(book.path)) {
-      book.pathAbsolute = computeAbsolutePath(
-          removeLastPathElement(libraryPath, true, false), book.path);
-    } else {
-      book.pathAbsolute = book.path;
-      book.path = computeRelativePath(
-          removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
+  const auto libRevision = manipulator->getLibrary().getRevision();
+  for (std::string path : paths) {
+    if (!path.empty()) {
+      if ( kiwix::isRelativePath(path) )
+        path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
+
+      if (!readFile(path, false, true)) {
+        throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
+      }
     }
   }
 
-  if (!book.indexPath.empty()) {
-    if (isRelativePath(book.indexPath)) {
-      book.indexPathAbsolute = computeAbsolutePath(
-          removeLastPathElement(libraryPath, true, false), book.indexPath);
-    } else {
-      book.indexPathAbsolute = book.indexPath;
-      book.indexPath
-          = computeRelativePath(removeLastPathElement(libraryPath, true, false),
-                                book.indexPathAbsolute);
-    }
-  }
+  manipulator->removeBooksNotUpdatedSince(libRevision);
 }
+
 }

src/meson.build

@@ -1,34 +1,44 @@
 kiwix_sources = [
+  'book.cpp',
+  'bookmark.cpp',
   'library.cpp',
   'manager.cpp',
-  'reader.cpp',
-  'searcher.cpp',
-  'common/base64.cpp',
-  'common/pathTools.cpp',
-  'common/regexTools.cpp',
-  'common/stringTools.cpp',
-  'common/networkTools.cpp',
-  'common/otherTools.cpp',
-  'xapian/htmlparse.cc',
-  'xapian/myhtmlparse.cc'
+  'libxml_dumper.cpp',
+  'opds_dumper.cpp',
+  'downloader.cpp',
+  'server.cpp',
+  'search_renderer.cpp',
+  'subprocess.cpp',
+  'aria2.cpp',
+  'tools/base64.cpp',
+  'tools/pathTools.cpp',
+  'tools/regexTools.cpp',
+  'tools/stringTools.cpp',
+  'tools/networkTools.cpp',
+  'tools/otherTools.cpp',
+  'tools/archiveTools.cpp',
+  'kiwixserve.cpp',
+  'name_mapper.cpp',
+  'server/byte_range.cpp',
+  'server/etag.cpp',
+  'server/request_context.cpp',
+  'server/response.cpp',
+  'server/internalServer.cpp',
+  'server/internalServer_catalog_v2.cpp',
+  'server/i18n.cpp',
+  'opds_catalog.cpp',
+  'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
-if xapian_dep.found()
-  kiwix_sources += ['xapianSearcher.cpp']
-endif
-
-if get_option('android')
-  subdir('android')
-  install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
+if host_machine.system() == 'windows'
+  kiwix_sources += 'subprocess_windows.cpp'
 else
-  install_dir = get_option('libdir')
+  kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-
-if has_ctpp2_dep
-  kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
-endif
+install_dir = get_option('libdir')
 
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
@@ -39,6 +49,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
                    install_dir: install_dir)

src/microhttpd_wrapper.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,9 +17,8 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#include <microhttpd.h>
 
-public class JNIKiwixInt
-{
-  public int value;
-}
+#if MHD_VERSION < 0x00097002
+typedef int MHD_Result;
+#endif

src/name_mapper.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "name_mapper.h"
+#include "library.h"
+#include "tools/regexTools.h"
+#include <iostream>
+
+namespace kiwix {
+
+HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
+  for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = library.getBookById(bookId);
+    auto bookName = currentBook.getHumanReadableIdFromPath();
+    m_idToName[bookId] = bookName;
+    m_nameToId[bookName] = bookId;
+
+    if (!withAlias)
+      continue;
+
+    auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
+    if (aliasName == bookName) {
+      continue;
+    }
+    if (m_nameToId.find(aliasName) == m_nameToId.end()) {
+      m_nameToId[aliasName] = bookId;
+    } else {
+      auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
+      std::cerr << "Path collision: " << alreadyPresentPath
+                << " and " << currentBook.getPath()
+                << " can't share the same URL path '" << aliasName << "'."
+                << " Therefore, only " << alreadyPresentPath
+                << " will be served." << std::endl;
+    }
+  }
+}
+
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
+  return m_idToName.at(id);
+}
+
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
+  return m_nameToId.at(name);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// UpdatableNameMapper
+////////////////////////////////////////////////////////////////////////////////
+
+UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+  : library(lib)
+  , withAlias(withAlias)
+{
+  update();
+}
+
+void UpdatableNameMapper::update()
+{
+  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  std::lock_guard<std::mutex> lock(mutex);
+  nameMapper.reset(newNameMapper);
+}
+
+UpdatableNameMapper::NameMapperHandle
+UpdatableNameMapper::currentNameMapper() const
+{
+  // Return a copy of the handle to the current NameMapper object. It will
+  // ensure that the object survives any call to UpdatableNameMapper::update()
+  // made before the completion of any pending operation on that object.
+  std::lock_guard<std::mutex> lock(mutex);
+  return nameMapper;
+}
+
+std::string UpdatableNameMapper::getNameForId(const std::string& id) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getNameForId(id);
+}
+
+std::string UpdatableNameMapper::getIdForName(const std::string& name) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getIdForName(name);
+}
+
+}

src/opds_catalog.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "opds_catalog.h"
+#include "tools/stringTools.h"
+
+#include <sstream>
+
+namespace kiwix
+{
+
+namespace
+{
+const char opdsSearchEndpoint[] = "/catalog/v2/entries";
+
+enum Separator { AMP };
+
+std::ostringstream& operator<<(std::ostringstream& oss, Separator sep)
+{
+  if ( oss.tellp() > 0 )
+    oss << "&";
+  return oss;
+}
+
+std::string buildSearchString(const Filter& f)
+{
+  std::ostringstream oss;
+  if ( f.hasQuery() )
+    oss << AMP << "q=" << urlEncode(f.getQuery());
+
+  if ( f.hasCategory() )
+    oss << AMP << "category=" << urlEncode(f.getCategory());
+
+  if ( f.hasLang() )
+    oss << AMP << "lang=" << urlEncode(f.getLang());
+
+  if ( f.hasName() )
+    oss << AMP << "name=" << urlEncode(f.getName());
+
+  if ( !f.getAcceptTags().empty() )
+    oss << AMP << "tag=" << urlEncode(join(f.getAcceptTags(), ";"));
+
+  return oss.str();
+}
+
+} // unnamed namespace
+
+std::string getSearchUrl(const Filter& f)
+{
+  const std::string searchString = buildSearchString(f);
+
+  if ( searchString.empty() )
+    return opdsSearchEndpoint;
+  else
+    return opdsSearchEndpoint + ("?" + searchString);
+}
+
+} // namespace kiwix

src/opds_dumper.cpp

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "opds_dumper.h"
+#include "book.h"
+
+#include "kiwixlib-resources.h"
+#include <mustache.hpp>
+
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+OPDSDumper::OPDSDumper(Library* library)
+  : library(library)
+{
+}
+/* Destructor */
+OPDSDumper::~OPDSDumper()
+{
+}
+
+void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+}
+
+namespace
+{
+
+typedef kainjow::mustache::data MustacheData;
+typedef kainjow::mustache::list BooksData;
+typedef kainjow::mustache::list IllustrationInfo;
+
+IllustrationInfo getBookIllustrationInfo(const Book& book)
+{
+    kainjow::mustache::list illustrations;
+    if ( book.isPathValid() ) {
+      for ( const auto& illustration : book.getIllustrations() ) {
+        // For now, we are handling only sizexsize@1 illustration.
+        // So we can simply pass one size to mustache.
+        illustrations.push_back(kainjow::mustache::object{
+          {"icon_size", to_string(illustration->width)},
+          {"icon_mimetype", illustration->mimeType}
+        });
+      }
+    }
+    return illustrations;
+}
+
+kainjow::mustache::object getSingleBookData(const Book& book)
+{
+    const auto bookDate = book.getDate() + "T00:00:00Z";
+    return kainjow::mustache::object{
+      {"id", book.getId()},
+      {"name", book.getName()},
+      {"title", book.getTitle()},
+      {"description", book.getDescription()},
+      {"language", book.getLanguage()},
+      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+      {"book_date", bookDate},
+      {"category", book.getCategory()},
+      {"flavour", book.getFlavour()},
+      {"tags", book.getTags()},
+      {"article_count", to_string(book.getArticleCount())},
+      {"media_count", to_string(book.getMediaCount())},
+      {"author_name", book.getCreator()},
+      {"publisher_name", book.getPublisher()},
+      {"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
+      {"size", to_string(book.getSize())},
+      {"icons", getBookIllustrationInfo(book)},
+    };
+}
+
+std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+{
+  auto data = getSingleBookData(book);
+  data["with_xml_header"] = MustacheData(withXMLHeader);
+  data["dump_partial_entries"] = MustacheData(partial);
+  data["endpoint_root"] = endpointRoot;
+  data["root"] = rootLocation;
+  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+}
+
+BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+{
+  BooksData booksData;
+  for ( const auto& bookId : bookIds ) {
+    try {
+      const Book book = library->getBookByIdThreadSafe(bookId);
+      booksData.push_back(kainjow::mustache::object{
+          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
+      });
+    } catch ( const std::out_of_range& ) {
+      // the book was removed from the library since its id was obtained
+      // ignore it
+    }
+  }
+
+  return booksData;
+}
+
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"}
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
+  }
+}
+
+std::string getLanguageSelfName(const std::string& lang) {
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
+};
+
+} // unnamed namespace
+
+string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
+{
+  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"root", rootLocation},
+     {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", booksData }
+  };
+
+  return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
+}
+
+string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
+{
+  const auto endpointRoot = rootLocation + "/catalog/v2";
+  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
+
+  const char* const endpoint = partial ? "/partial_entries" : "/entries";
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"endpoint_root", endpointRoot},
+     {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
+     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", booksData },
+     {"dump_partial_entries", MustacheData(partial)}
+  };
+
+  return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
+}
+
+std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
+{
+  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+}
+
+std::string OPDSDumper::categoriesOPDSFeed() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : library->getBooksCategories() ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
+  }
+
+  return render_template(
+             RESOURCE::templates::catalog_v2_categories_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/categories")},
+               {"categories", categoryData }
+             }
+  );
+}
+
+std::string OPDSDumper::languagesOPDSFeed() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+
+  return render_template(
+             RESOURCE::templates::catalog_v2_languages_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/languages")},
+               {"languages", languageData }
+             }
+  );
+}
+
+}

src/reader.cpp

@@ -1,787 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-
-inline char hi(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[(v >> 4) & 0xf];
-}
-
-inline char lo(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[v & 0xf];
-}
-
-std::string hexUUID(std::string in)
-{
-  std::ostringstream out;
-  for (unsigned n = 0; n < 4; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 4; n < 6; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 6; n < 8; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 8; n < 10; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 10; n < 16; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  std::string op = out.str();
-  return op;
-}
-
-namespace kiwix
-{
-/* Constructor */
-Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
-{
-  string tmpZimFilePath = zimFilePath;
-
-  /* Remove potential trailing zimaa */
-  size_t found = tmpZimFilePath.rfind("zimaa");
-  if (found != string::npos && tmpZimFilePath.size() > 5
-      && found == tmpZimFilePath.size() - 5) {
-    tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
-  }
-
-  this->zimFileHandler = new zim::File(tmpZimFilePath);
-
-  if (this->zimFileHandler != NULL) {
-    this->firstArticleOffset
-        = this->zimFileHandler->getNamespaceBeginOffset('A');
-    this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
-    this->currentArticleOffset = this->firstArticleOffset;
-    this->nsACount = this->zimFileHandler->getNamespaceCount('A');
-    this->nsICount = this->zimFileHandler->getNamespaceCount('I');
-    this->zimFilePath = zimFilePath;
-  }
-
-  /* initialize random seed: */
-  srand(time(NULL));
-}
-
-/* Destructor */
-Reader::~Reader()
-{
-  if (this->zimFileHandler != NULL) {
-    delete this->zimFileHandler;
-  }
-}
-
-zim::File* Reader::getZimFileHandler() const
-{
-  return this->zimFileHandler;
-}
-/* Reset the cursor for GetNextArticle() */
-void Reader::reset()
-{
-  this->currentArticleOffset = this->firstArticleOffset;
-}
-std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
-{
-  std::map<const std::string, unsigned int> counters;
-  string mimeType, item, counterString;
-  unsigned int counter;
-
-  zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
-
-  if (article.good()) {
-    stringstream ssContent(article.getData());
-
-    while (getline(ssContent, item, ';')) {
-      stringstream ssItem(item);
-      getline(ssItem, mimeType, '=');
-      getline(ssItem, counterString, '=');
-      if (!counterString.empty() && !mimeType.empty()) {
-        sscanf(counterString.c_str(), "%u", &counter);
-        counters.insert(pair<string, int>(mimeType, counter));
-      }
-    }
-  }
-
-  return counters;
-}
-
-/* Get the count of articles which can be indexed/displayed */
-unsigned int Reader::getArticleCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  if (counterMap.empty()) {
-    counter = this->nsACount;
-  } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
-    }
-  }
-
-  return counter;
-}
-
-/* Get the count of medias content in the ZIM file */
-unsigned int Reader::getMediaCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  if (counterMap.empty()) {
-    counter = this->nsICount;
-  } else {
-    auto it = counterMap.find("image/jpeg");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/gif");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/png");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-  }
-  return counter;
-}
-
-/* Get the total of all items of a ZIM file, redirects included */
-unsigned int Reader::getGlobalCount() const
-{
-  return this->zimFileHandler->getCountArticles();
-}
-
-/* Return the UID of the ZIM file */
-string Reader::getId() const
-{
-  std::ostringstream s;
-  s << this->zimFileHandler->getFileheader().getUuid();
-  return s.str();
-}
-
-/* Return a page url from a title */
-bool Reader::getPageUrlFromTitle(const string& title, string& url) const
-{
-  /* Extract the content from the zim file */
-  zim::Article article = this->zimFileHandler->getArticleByTitle('A', title);
-
-  if (!article.good()) {
-    return false;
-  }
-
-  unsigned int loopCounter = 0;
-  while (article.isRedirect() && loopCounter++ < 42) {
-    article = article.getRedirectArticle();
-  }
-
-  url = article.getLongUrl();
-  return true;
-}
-
-/* Return an URL from a title */
-string Reader::getRandomPageUrl() const
-{
-  zim::Article article;
-  zim::size_type idx;
-  std::string mainPageUrl = this->getMainPageUrl();
-
-  do {
-    idx = this->firstArticleOffset
-          + (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
-                             * this->nsACount);
-    article = zimFileHandler->getArticle(idx);
-  } while (article.getLongUrl() == mainPageUrl);
-
-  return article.getLongUrl();
-}
-
-/* Return the welcome page URL */
-string Reader::getMainPageUrl() const
-{
-  string url = "";
-
-  if (this->zimFileHandler->getFileheader().hasMainPage()) {
-    zim::Article article = zimFileHandler->getArticle(
-        this->zimFileHandler->getFileheader().getMainPage());
-    url = article.getLongUrl();
-
-    if (url.empty()) {
-      url = getFirstPageUrl();
-    }
-  } else {
-    url = getFirstPageUrl();
-  }
-
-  return url;
-}
-
-bool Reader::getFavicon(string& content, string& mimeType) const
-{
-  unsigned int contentLength = 0;
-  string title;
-
-  this->getContentByUrl("/-/favicon.png", content, title, contentLength, mimeType);
-
-  if (content.empty()) {
-    this->getContentByUrl("/I/favicon.png", content, title, contentLength, mimeType);
-
-    if (content.empty()) {
-      this->getContentByUrl("/I/favicon", content, title, contentLength, mimeType);
-
-      if (content.empty()) {
-        this->getContentByUrl("/-/favicon", content, title, contentLength, mimeType);
-      }
-    }
-  }
-
-  return content.empty() ? false : true;
-}
-
-string Reader::getZimFilePath() const
-{
-  return this->zimFilePath;
-}
-/* Return a metatag value */
-bool Reader::getMetatag(const string& name, string& value) const
-{
-  unsigned int contentLength = 0;
-  string contentType = "";
-  string title;
-
-  return this->getContentByUrl("/M/" + name, value, title, contentLength, contentType);
-}
-
-string Reader::getTitle() const
-{
-  string value;
-  this->getMetatag("Title", value);
-  if (value.empty()) {
-    value = getLastPathElement(zimFileHandler->getFilename());
-    std::replace(value.begin(), value.end(), '_', ' ');
-    size_t pos = value.find(".zim");
-    value = value.substr(0, pos);
-  }
-  return value;
-}
-
-string Reader::getName() const
-{
-  string value;
-  this->getMetatag("Name", value);
-  return value;
-}
-
-string Reader::getTags() const
-{
-  string value;
-  this->getMetatag("Tags", value);
-  return value;
-}
-
-string Reader::getDescription() const
-{
-  string value;
-  this->getMetatag("Description", value);
-
-  /* Mediawiki Collection tends to use the "Subtitle" name */
-  if (value.empty()) {
-    this->getMetatag("Subtitle", value);
-  }
-
-  return value;
-}
-
-string Reader::getLanguage() const
-{
-  string value;
-  this->getMetatag("Language", value);
-  return value;
-}
-
-string Reader::getDate() const
-{
-  string value;
-  this->getMetatag("Date", value);
-  return value;
-}
-
-string Reader::getCreator() const
-{
-  string value;
-  this->getMetatag("Creator", value);
-  return value;
-}
-
-string Reader::getPublisher() const
-{
-  string value;
-  this->getMetatag("Publisher", value);
-  return value;
-}
-
-string Reader::getOrigId() const
-{
-  string value;
-  this->getMetatag("startfileuid", value);
-  if (value.empty()) {
-    return "";
-  }
-  std::string id = value;
-  std::string origID;
-  std::string temp = "";
-  unsigned int k = 0;
-  char tempArray[16] = "";
-  for (unsigned int i = 0; i < id.size(); i++) {
-    if (id[i] == '\n') {
-      tempArray[k] = atoi(temp.c_str());
-      temp = "";
-      k++;
-    } else {
-      temp += id[i];
-    }
-  }
-  origID = hexUUID(tempArray);
-  return origID;
-}
-
-/* Return the first page URL */
-string Reader::getFirstPageUrl() const
-{
-  zim::size_type firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
-  zim::Article article = zimFileHandler->getArticle(firstPageOffset);
-  return article.getLongUrl();
-}
-
-bool Reader::parseUrl(const string& url, char* ns, string& title) const
-{
-  /* Offset to visit the url */
-  unsigned int urlLength = url.size();
-  unsigned int offset = 0;
-
-  /* Ignore the '/' */
-  while ((offset < urlLength) && (url[offset] == '/')) {
-    offset++;
-  }
-
-  /* Get namespace */
-  while ((offset < urlLength) && (url[offset] != '/')) {
-    *ns = url[offset];
-    offset++;
-  }
-
-  /* Ignore the '/' */
-  while ((offset < urlLength) && (url[offset] == '/')) {
-    offset++;
-  }
-
-  /* Get content title */
-  unsigned int titleOffset = offset;
-  while (offset < urlLength) {
-    offset++;
-  }
-
-  /* unescape title */
-  title = url.substr(titleOffset, offset - titleOffset);
-
-  return true;
-}
-
-/* Return article by url */
-bool Reader::getArticleObjectByDecodedUrl(const string& url,
-                                          zim::Article& article) const
-{
-  if (this->zimFileHandler == NULL) {
-    return false;
-  }
-
-  /* Parse the url */
-  char ns = 0;
-  string urlStr;
-  this->parseUrl(url, &ns, urlStr);
-
-  /* Main page */
-  if (urlStr.empty() && ns == 0) {
-    this->parseUrl(this->getMainPageUrl(), &ns, urlStr);
-  }
-
-  /* Extract the content from the zim file */
-  article = zimFileHandler->getArticle(ns, urlStr);
-  return article.good();
-}
-
-/* Return the mimeType without the content */
-bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
-{
-  if (this->zimFileHandler == NULL) {
-    return false;
-  }
-
-  zim::Article article;
-  if (this->getArticleObjectByDecodedUrl(url, article)) {
-    try {
-      mimeType = article.getMimeType();
-    } catch (exception& e) {
-      cerr << "Unable to get the mimetype for " << url << ":" << e.what()
-           << endl;
-      mimeType = "application/octet-stream";
-    }
-    return true;
-  } else {
-    mimeType = "";
-    return false;
-  }
-}
-
-/* Get a content from a zim file */
-bool Reader::getContentByUrl(const string& url,
-                             string& content,
-                             string& title,
-                             unsigned int& contentLength,
-                             string& contentType) const
-{
-  return this->getContentByEncodedUrl(url, content, title, contentLength, contentType);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  return this->getContentByDecodedUrl(
-      kiwix::urlDecode(url), content, title, contentLength, contentType, baseUrl);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return this->getContentByEncodedUrl(kiwix::urlDecode(url),
-                                      content,
-                                      title,
-                                      contentLength,
-                                      contentType,
-                                      stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return this->getContentByDecodedUrl(kiwix::urlDecode(url),
-                                      content,
-                                      title,
-                                      contentLength,
-                                      contentType,
-                                      stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  content = "";
-  contentType = "";
-  contentLength = 0;
-
-  zim::Article article;
-  if (!this->getArticleObjectByDecodedUrl(url, article)) {
-    return false;
-  }
-
-  /* If redirect */
-  unsigned int loopCounter = 0;
-  while (article.isRedirect() && loopCounter++ < 42) {
-    article = article.getRedirectArticle();
-  }
-
-  if (loopCounter < 42) {
-    /* Compute base url (might be different from the url if redirects */
-    baseUrl
-        = "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl();
-
-    /* Get the content mime-type */
-    try {
-      contentType
-          = string(article.getMimeType().data(), article.getMimeType().size());
-    } catch (exception& e) {
-      cerr << "Unable to get the mimetype for " << baseUrl << ":" << e.what()
-           << endl;
-      contentType = "application/octet-stream";
-    }
-
-    /* Get the data */
-    content = string(article.getData().data(), article.getArticleSize());
-    title = article.getTitle();
-  }
-
-  /* Try to set a stub HTML header/footer if necesssary */
-  if (contentType.find("text/html") != string::npos
-      && content.find("<body") == std::string::npos
-      && content.find("<BODY") == std::string::npos) {
-    content = "<html><head><title>" + article.getTitle() +
-              "</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
-              "charset=utf-8\" /></head><body>" +
-              content + "</body></html>";
-  }
-
-  /* Get the data length */
-  contentLength = article.getArticleSize();
-
-  return true;
-}
-
-/* Check if an article exists */
-bool Reader::urlExists(const string& url) const
-{
-  char ns = 0;
-  string titleStr;
-  this->parseUrl(url, &ns, titleStr);
-  titleStr = "/" + titleStr;
-  zim::File::const_iterator findItr = zimFileHandler->find(ns, titleStr);
-  return findItr != zimFileHandler->end() && findItr->getUrl() == titleStr;
-}
-
-/* Does the ZIM file has a fulltext index */
-bool Reader::hasFulltextIndex() const
-{
-  return ( this->urlExists("/Z/fulltextIndex/xapian")
-        && !zimFileHandler->is_multiPart() );
-}
-
-/* Search titles by prefix */
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               const bool reset)
-{
-  bool retVal = false;
-  zim::File::const_iterator articleItr;
-
-  /* Reset the suggestions otherwise check if the suggestions number is less
-   * than the suggestionsCount */
-  if (reset) {
-    this->suggestions.clear();
-    this->suggestionsOffset = this->suggestions.begin();
-  } else {
-    if (this->suggestions.size() > suggestionsCount) {
-      return false;
-    }
-  }
-
-  /* Return if no prefix */
-  if (prefix.size() == 0) {
-    return false;
-  }
-
-  for (articleItr = zimFileHandler->findByTitle('A', prefix);
-       articleItr != zimFileHandler->end()
-       && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
-       ++articleItr) {
-    /* Extract the interesting part of article title & url */
-    std::string normalizedArticleTitle
-        = kiwix::normalize(articleItr->getTitle());
-    std::string articleFinalUrl = "/A/" + articleItr->getUrl();
-    if (articleItr->isRedirect()) {
-      zim::Article article = *articleItr;
-      unsigned int loopCounter = 0;
-      while (article.isRedirect() && loopCounter++ < 42) {
-        article = article.getRedirectArticle();
-      }
-      articleFinalUrl = "/A/" + article.getUrl();
-    }
-
-    /* Go through all already found suggestions and skip if this
-       article is already in the suggestions list (with an other
-       title) */
-    bool insert = true;
-    std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
-         suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
-        insert = false;
-        break;
-      } else if (result < 0) {
-        break;
-      }
-    }
-
-    /* Insert if possible */
-    if (insert) {
-      std::vector<std::string> suggestion;
-      suggestion.push_back(articleItr->getTitle());
-      suggestion.push_back(articleFinalUrl);
-      suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
-    }
-
-    /* Suggestions where found */
-    retVal = true;
-  }
-
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return retVal;
-}
-
-std::vector<std::string> Reader::getTitleVariants(
-    const std::string& title) const
-{
-  std::vector<std::string> variants;
-  variants.push_back(title);
-  variants.push_back(kiwix::ucFirst(title));
-  variants.push_back(kiwix::lcFirst(title));
-  variants.push_back(kiwix::toTitle(title));
-  return variants;
-}
-
-/* Try also a few variations of the prefix to have better results */
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount)
-{
-  std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal;
-
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
-  /* Try to search in the title using fulltext search database */
-  const zim::Search* suggestionSearch
-      = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
-
-  if (suggestionSearch->get_matches_estimated()) {
-    for (auto current = suggestionSearch->begin();
-         current != suggestionSearch->end();
-         current++) {
-      std::vector<std::string> suggestion;
-      suggestion.push_back(current->getTitle());
-      suggestion.push_back("/A/" + current->getUrl());
-      suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
-    }
-    this->suggestionsOffset = this->suggestions.begin();
-    retVal = true;
-  } else {
-    for (std::vector<std::string>::iterator variantsItr = variants.begin();
-         variantsItr != variants.end();
-         variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
-               || retVal;
-    }
-  }
-
-  return retVal;
-}
-
-/* Get next suggestion */
-bool Reader::getNextSuggestion(string& title)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset))[0];
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-bool Reader::getNextSuggestion(string& title, string& url)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset))[0];
-    url = (*(this->suggestionsOffset))[1];
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-/* Check if the file has as checksum */
-bool Reader::canCheckIntegrity() const
-{
-  return this->zimFileHandler->getChecksum() != "";
-}
-
-/* Return true if corrupted, false otherwise */
-bool Reader::isCorrupted() const
-{
-  try {
-    if (this->zimFileHandler->verify() == true) {
-      return false;
-    }
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return true;
-  }
-
-  return true;
-}
-
-/* Return the file size, works also for splitted files */
-unsigned int Reader::getFileSize() const
-{
-  zim::File* file = this->getZimFileHandler();
-  zim::offset_type size = 0;
-
-  if (file != NULL) {
-    size = file->getFilesize();
-  }
-
-  return (size / 1024);
-}
-}

src/search_renderer.cpp

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <cmath>
+
+#include "search_renderer.h"
+#include "library.h"
+#include "name_mapper.h"
+
+#include "tools/archiveTools.h"
+
+#include <zim/search.h>
+
+#include <mustache.hpp>
+#include "kiwixlib-resources.h"
+#include "tools/stringTools.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
+{}
+
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : m_srs(srs),
+      mp_nameMapper(mapper),
+      mp_library(library),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://"),
+      estimatedResultCount(estimatedResultCount),
+      resultStart(start)
+{}
+
+/* Destructor */
+SearchRenderer::~SearchRenderer() = default;
+
+void SearchRenderer::setSearchPattern(const std::string& pattern)
+{
+  searchPattern = pattern;
+}
+
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
+{
+  searchBookQuery = bookQuery;
+}
+
+void SearchRenderer::setProtocolPrefix(const std::string& prefix)
+{
+  this->protocolPrefix = prefix;
+}
+
+void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
+{
+  this->searchProtocolPrefix = prefix;
+}
+
+std::string extractValueFromQuery(const std::string& query, const std::string& key) {
+  const std::string p = key + "=";
+  const size_t i = query.find(p);
+  if (i == std::string::npos) {
+    return "";
+  }
+  std::string r = query.substr(i + p.size());
+  return r.substr(0, r.find("&"));
+}
+
+kainjow::mustache::data buildQueryData
+(
+  const std::string& searchProtocolPrefix,
+  const std::string& pattern,
+  const std::string& bookQuery
+) {
+  kainjow::mustache::data query;
+  query.set("pattern", kiwix::encodeDiples(pattern));
+  std::ostringstream ss;
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << "&" << bookQuery;
+  query.set("unpaginatedQuery", ss.str());
+  auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
+  if(!lang.empty()) {
+    query.set("lang", lang);
+  }
+  return query;
+}
+
+kainjow::mustache::data buildPagination(
+  unsigned int pageLength,
+  unsigned int resultsCount,
+  unsigned int resultsStart
+)
+{
+  assert(pageLength!=0);
+  kainjow::mustache::data pagination;
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  if (resultsCount == 0) {
+    // Easy case
+    pagination.set("itemsPerPage", to_string(pageLength));
+    pagination.set("hasPages", false);
+    pagination.set("pages", pages);
+    return pagination;
+  }
+
+  // First we want to display pages starting at a multiple of `pageLength`
+  // so, let's calculate the start index of the current page.
+  auto currentPage = resultsStart/pageLength;
+  auto lastPage = ((resultsCount-1)/pageLength);
+  auto lastPageStart = lastPage*pageLength;
+  auto nbPages = lastPage + 1;
+
+  auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
+  auto lastPageGenerated = std::min(currentPage+4, lastPage);
+
+  if (nbPages != 1) {
+    if (firstPageGenerated!=0) {
+      kainjow::mustache::data page;
+      page.set("label", "◀");
+      page.set("start", to_string(0));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+
+    for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
+      kainjow::mustache::data page;
+      page.set("label", to_string(i+1));
+      page.set("start", to_string(i*pageLength));
+      page.set("current", bool(i == currentPage));
+      pages.push_back(page);
+    }
+
+    if (lastPageGenerated!=lastPage) {
+      kainjow::mustache::data page;
+      page.set("label", "▶");
+      page.set("start", to_string(lastPageStart));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+  }
+
+  pagination.set("itemsPerPage", to_string(pageLength));
+  pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
+  pagination.set("pages", pages);
+  return pagination;
+}
+
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+{
+  const std::string absPathPrefix = protocolPrefix + "content/";
+  // Build the results list
+  kainjow::mustache::data items{kainjow::mustache::data::type::list};
+  for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
+    kainjow::mustache::data result;
+    std::string zim_id(it.getZimId());
+    result.set("title", it.getTitle());
+    result.set("absolutePath", absPathPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
+    result.set("snippet", it.getSnippet());
+    if (mp_library) {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
+    if (it.getWordCount() >= 0) {
+      result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
+    }
+
+    items.push_back(result);
+  }
+  kainjow::mustache::data results;
+  results.set("items", items);
+  results.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  results.set("hasResults", estimatedResultCount != 0);
+  results.set("start", kiwix::beautifyInteger(resultStart));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
+
+  // pagination
+  auto pagination = buildPagination(
+    pageLength,
+    estimatedResultCount,
+    resultStart
+  );
+
+  kainjow::mustache::data query = buildQueryData(
+    searchProtocolPrefix,
+    searchPattern,
+    searchBookQuery
+  );
+
+
+  kainjow::mustache::data allData;
+  allData.set("protocolPrefix", protocolPrefix);
+  allData.set("results", results);
+  allData.set("pagination", pagination);
+  allData.set("query", query);
+
+  kainjow::mustache::mustache tmpl(tmpl_str);
+
+  std::stringstream ss;
+  tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  if (!tmpl.is_valid()) {
+    throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
+  }
+  return ss.str();
+}
+
+std::string SearchRenderer::getHtml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_html);
+}
+
+std::string SearchRenderer::getXml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_xml);
+}
+
+
+}