trigger here

This reverts commit 95bde675ef.

.codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+
+ignore:
+  - "test"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: kiwix
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # https://kiwix.org/support-us/

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - can-revert
+  pull_request:
+
+jobs:
+  macOS:
+    runs-on: macos-12
+    env:
+      HOME: /Users/runner
+    steps:
+      - name: Retrieve source code
+        uses: actions/checkout@v3
+
+      - name: Install packages
+        run: |
+          brew update
+          brew install pkg-config ninja meson
+
+      - name: Install dependencies
+        env:
+          ARCHIVE_NAME: deps2_osx_native_dyn_libkiwix.tar.xz
+        run: |
+          wget -O- https://tmp.kiwix.org/ci/${{env.ARCHIVE_NAME}} | tar -xJ -C ${{env.HOME}}
+
+      - name: Compile source code
+        env:
+          PKG_CONFIG_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          CPPFLAGS: -I${{env.HOME}}/BUILD_native_dyn/INSTALL/include
+        run: |
+          meson . build --default-library=shared -Db_coverage=true
+          ninja -C build
+
+      - name: Test libkiwix
+        env:
+          SKIP_BIG_MEMORY_TEST: 1
+          LD_LIBRARY_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib:${{env.HOME}}/BUILD_native_dyn/INSTALL/lib64
+        run: meson test -C build --verbose

.github/workflows/package.yml

@@ -0,0 +1,96 @@
+name: Packages
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+          - ubuntu-kinetic
+          - ubuntu-jammy
+          - ubuntu-focal
+          - ubuntu-bionic
+    steps:
+      - uses: actions/checkout@v3
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-kinetic
+        if: matrix.distro == 'ubuntu-kinetic'
+        name: Build package for ubuntu-kinetic
+        id: build-ubuntu-kinetic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.gitignore

@@ -0,0 +1,9 @@
+.idea/
+*.swp
+subprojects/googletest-release*
+*.class
+build/
+.vscode/
+builddir/
+.cache/
+.clangd/

.travis.yml

@@ -1,36 +0,0 @@
-language: cpp
-dist: trusty
-sudo: false
-cache: ccache
-before_install:
-- PATH=$PATH:$HOME/bin
-install: travis/install_deps.sh
-script: travis/compile.sh
-env:
-  matrix:
-    - PLATFORM="native_static"
-    - PLATFORM="native_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-addons:
-  apt:
-    packages:
-      - cmake
-      - python3-pip
-      - libbz2-dev
-      - ccache
-      - zlib1g-dev
-      - uuid-dev
-      - libctpp2-dev
-      - ctpp2-utils
-      - libmicrohttpd-dev
-      - g++-mingw-w64-i686
-      - gcc-mingw-w64-i686
-      - gcc-mingw-w64-base
-      - mingw-w64-tools
-matrix:
-  include:
-    - env: PLATFORM="native_dyn"
-      os: osx

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,453 @@
+libkiwix 12.0.0
+===============
+
+ * [API Break] Remove wrapper around libzim (@mgautierfr #789)
+ * Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
+ * Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
+ * Prevent search on multi language content (@veloman-yunkan #838)
+ * Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
+ * Catalog:
+  - Include tags in free text catalog search (@veloman-yunkan #802)
+  - Illustration's url is based on book's uuid (@veloman-yunkan #804)
+  - Cleanup of the opds-dumper (@veloman-yunkan #829)
+  - Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
+  - Make opds-dumper respect the namemapper (@mgautierfr #837)
+ * Server:
+  - Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
+  - Better http caching (@veloman-yunkan #833)
+  - Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
+  - Better redirection of main page (@veloman-yunkan #827)
+  - Remove jquery (@mgautierfr @juuz0 #796)
+  - Better Viewer of zim content :
+    . Introduce `/content` endpoints (@veloman-yunkan #806)
+    . Switch to iframe based content viewer (@veloman-yunkan #716)
+  - Optimised design of the welcome page:
+    . Alignement (@juuz0 @kelson42 #786)
+    . Exit download modal on pressing escape key (@juzz0 #800)
+    . Add favicon for different devices (@juzz0 #805)
+    . Fix auto hidding of the toolbar (@veloman-yunkan #821)
+    . Allow user to filter books by tags in the front page (@juuz0 #711)
+ * CI :
+   - Trigger CI on pull_request (@kelson42 #791)
+   - Drop Ubuntu Impish packaging (@legoktm #825)
+   - Add Ubuntu Kinetic packaging (@legoktm #801)
+ * Testing:
+   - Test ICULanguageInfo (@veloman-yunkan #795)
+   - Introduce fake `test` language to test i18n (@veloman-yunkan #848)
+ * Fix documentation (@kelson42 #816)
+ * Udpate translation (#787 #839 #847)
+
+
+libkiwix 11.0.0
+===============
+
+ * [server] Add support for internationalization (@veloman-yunkan #679)
+ * [server] Use gzip compression instead of deflat (mgautierfr #757)
+ * [server] Version the static resources. This allow better invalidating
+   browser cache when resources are changed (@veloman-yunkan #712)
+ * [server|front] Use integer to query the host for page length (@juuz0 #772)
+ * [server] Improve multizim search API:
+   - Improvement of the cache system
+   - Better API to select on which books to search in.
+   - SysAdmin is now able to limit the number of book we search in for a multizim search
+ * [server] Introduce a opensearch API for multizim fulltext search
+ * [wrapper] Remove java wrapper
+ * Testing:
+   - Testing of search result pages content (@veloman-yunkan #765)
+   - Better testing structure of xml search result (@veloman-yunkan #780)
+
+libkiwix 10.1.1
+===============
+
+ * Correctly detect the number of article for older zims (<=6) (@mgautier #743)
+ * [server] Fix fulltext search (@mgautierfr #724)
+ * [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
+ * Fix CI (@mgautierfr #736)
+
+libkiwix 10.1.0
+===============
+
+ This release is an important one as it fixes a Xss vulnerability introduced
+ in libkiwix 10.0.0
+
+ * [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
+ * [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
+ * [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
+ * [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
+ * Add `dc:issued` to opds output stream (@veloman-yunkan #715)
+ * Add handling of several languages not provided by ICU (@juuz0 #701)
+ * [server] Add a caching system for search and suggestion (@maneeshpm #620)
+ * Fix cross-compilation (@kelson42 #703)
+ * Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
+ * Better testing system of html response (@veloman-yunkan #725)
+
+libkiwix 10.0.1
+===============
+
+ * [server] The catalog search interpret `count=0` as no limit.
+   This was the case for a long time. This was changed unintentionally
+   (@veloman-yunkan #686)
+ * [server] Correctly generere a human friendly title in the server frontend.
+   (@juuz0 #687, @kelson42 #689)
+ * [server] Fix download button if there is no url do download from.
+   (@juuz0 #691)
+ * Add non-minified isotope.pkdg.js
+   Needed for debian packaging as we need the source and minified version is
+   not the source (@legoktm #693)
+ * [server] Add a tooltip with the full language for the lang tag.
+ * CI fixes (@kelson42 @legoktm)
+
+libkiwix 10.0.0
+===============
+
+ This release is huge release.
+ The project has been renamed to libkiwix, it is more coherent with the library name.
+
+ * Server front page :
+  - Use js in the front page to display the available book,
+     using the OPDS stream as source. The front page is now populated only with
+     the visible books and user can search for books. (@MananJethwany #530, #541, #534)
+     (@kelson42 #628)
+  - Revamp css (@MananJethwany #559)
+  - Correctly Convert 3iso language code to 2iso (@juuz0 #672)
+
+ * Server suggestions search :
+  - Add pagination for suggestion search (@maneeshpm #591)
+  - Fix suggestion system (@MananJethwany #498)
+  - Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
+  - The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
+  - Properly handle html encoding of suggestions (@veloman-yunkan #458)
+
+ * Server improvements :
+  - Remove meta endpoints (@mgautier #669)
+  - Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
+  - Add details on 404 error pages (@soumyankar #490)
+  - Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
+  - Better headbar insertion (after charset definition) (@kelson42 #442)
+
+ * New OPDS Stream v2 :
+  - Add a list of categories (@veloman-yunkan)
+  - Support for partial entries (@veloman-yunkan #602)
+  - Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
+  - Add language endpoint to catalog (@veloman-yunkan #553)
+  - Add illustration API to get the illustration of a book (@mgautierfr #645)
+  - OPDS search can now filter books by category (@veloman-yunkan #459)
+
+ * Library improvements :
+  - Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
+  - Properly handle removing of book from the library (@veloman-yunkan #485)
+  - Use xapian to search for books in the library (@veloman-yunkan #460, #488)
+
+ * Added methods/functions :
+  - Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
+  - Add `getVersions` and `printVersions` functions (@kelson42 #665)
+  - Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
+  - Add `get_zimid()` method to the search result (@maneeshpm #510)
+
+ * Various improvements :
+  - Better secret value for aria2c rpc (@juuz0 #666)
+  - Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
+  - Add basic documentation (@mgautierfr #640)
+  - Do not use Reader internally (@maneeshpm #536 #576)
+  - Remove dependency headers from our public headers (@mgautierfr #574)
+  - Downloader now don't write metalink on the filesystem (@kelson42 #502)
+  - Support opening a zim file using a fd (@veloman-yukan #429)
+  - Use C++11 std::thread instead of pthread (@mgautierfr #445)
+  - [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
+  - Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
+  - Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
+  - Better parsing of the counterMap (@kelson42 #437)
+  - Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
+  - Remove deprecated methods (@mgautierfr)
+  - CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
+  - Fix compilation for FreeBSD (@swills g#432)
+  - Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
+
+kiwix-lib 9.4.1
+===============
+
+ * Fix `M/Counter` parsing.
+ * [SERVER] Adjust body padding-top for taskbar
+ * Fix potential crash when stoping a server not started.
+ * Various fix in build system and the CI.
+
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
+kiwix-lib 9.2
+=============
+
+ * Add tests
+ * Refactoring server code.
+ * [SERVER] Add HEAD, Etag and If-None-Match support.
+ * [SERVER] Compress opds catalog answers.
+
+kiwix-lib 9.1.2
+===============
+
+ * Do not use the pathToSave if it is empty.
+
+kiwix-lib 9.1.1
+===============
+
+ * Fix the detection of the dataDirectory on windows.
+
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
+kiwix-lib 9.0.1
+===============
+
+ * [JAVA] Use a long to store the offset of an article in the zim file instead
+    of an int.
+
+kiwix-lib 9.0.0
+===============
+
+ * [OPDS] Correctly set the id of the OPDS stream.
+ * [OPDS] Do not try to filter the catalog if no filter field is given in the
+   request.
+ * [WINDOWS] Correctly convert path to wide chars when opening the library.xml
+ * [LIBRARY] Remove the function the read file using a native path.
+   All path must be utf8, no need to pass a native path along the utf8 path.
+ * [TEST] Fix tests using the main function of gtest instead of custom one.
+ * [CI] Move to github CI instead of Travis.
+ * The `Book::update` method always update the book's fields. Even if they are
+   not empty.
+ * [JAVA] Add wrapping around the library manager (opds parsing)
+ * [ARIA2] Add api option to start download with option (destination folder)
+ * [OPDS] Fixes about opds parsing, generation (missing attributes)
+   and requesting (server)
+ * Add methods on `Book` to get specific tag values (was on `Reader` only)
+ * Add flavour attribute to `Book`
+ * Fix opensearch description.
+ * Trust the given library.xml (by default) instead of reading the value from
+   the zim files.
+ * [OPDS] Be able to filter the content by name or size.
+ * [WINDOWS] Fix launching subcommand when there is spaces in the path.
+
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
+kiwix-lib 8.1.0
+===============
+
+ * Fix pathTools manipulation.
+ * Add missing implementation of getArticleCount and getMediaCount on android.
+ * Correctly convert windows path to utf8.
+ * Add code coverage in the CI
+
+kiwix-lib 8.0.1
+===============
+
+ * Fix join function
+
+kiwix-lib 8.0.0
+===============
+
+ * Add new methods to get all (and new) metadata from the zim file.
+ * Add methods to get the value of a specific tag.
+ * [API Change] Convert tags value to the new convention.
+ * [API Change] Rename `getMatatag` method to `getMetadata`
+ * [ABI Change] Correctly detect executable path in appimage.
+
+kiwix-lib 7.0.0
+===============
+
+ * [API break] Add a argument to kiwix-serve to specify the library to use.
+
+kiwix-lib 6.0.4
+===============
+
+ * Fix HTML rendering of the search result if there is no result.
+ * Do not crash at html rendering if request ask for 0 results (start == end)
+ * Correctly find the executable path if we are using AppImage
+
+kiwix-lib 6.0.3
+===============
+
+ * force one column suggestion in kiwix-serve suggestions
+ * fix fulltext search link in suggestions
+ * UI fixes in kiwix-serve rendering
+
+kiwix-lib 6.0.2
+===============
+
+ * Correctly set the groupId in the pom file.
+
+kiwix-lib 6.0.1
+===============
+
+ * Generate the pom file for android/maven
+
+kiwix-lib 6.0.0
+===============
+
+ * Move the server code in kiwix-lib (from kiwix-serve).
+ * Add unit test on regex functions.
+ * Fix computerAbsolutePath (thread safe, memory leak).
+ * Correctly set the book's path as valid if we construct the book from a
+   reader.
+ * [JNI] Add a method to know if a article is a redirection.
+ * Do not embed the gtest dependency.
+ * [JNI] Add a constructor to JNIKiwixString.
+ * Change order of search of the favicon urls.
+ * Clean a lot of unecessary includes in headers. (potential "API break")
+
+kiwix-lib 5.2.0
+===============
+
+ * kiwix-serve integration (as a seperated process).
+ * Fix crash in the suggestion search.
+ * Better API to filter the library books.
+ * New kiwix-lib application for android. (.aar)
+ * Use ReLinker to link with libkiwix.so in android.
+ * Correctly set the verbosity of zim search.
+
+kiwix-lib 5.1.0
+===============
+
+ * Add function to pause, resume and stop downloads.
+ * Add zim's tags in the opds stream.
+ * Addapt to new libzim 5.0.0 API.
+
+kiwix-lib 5.0.0
+===============
+
+ * Remove error message when trying to open an wrong zim file.
+ * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
+ * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
+ * Remove rpath
+ * Detect infinite (and too long) redirect loops.
+
+kiwix-lib 4.1.0
+===============
+
+ * Allow the library to be filtered by tags.
+ * Fix language mapping.
+ * Update README about mustache dependency.
+
+kiwix-lib 4.0.1
+===============
+
+ * Fix "maybe uninitialize variable" issue.
+ * Ensure path are stored correctly (absolute path) in the library.
+ * [CI] Use the new deps archive xz
+
+kiwix-lib 4.0.0
+===============
+
+ * [API break] Remove support for external index.
+ * Move to the mustache templating system instead of ctpp2.
+ * Make meson.build works for meson>=0.43.0
+ * [API break] Move the basic tools from the `common` directory to `tools`.
+
+kiwix-lib 3.1.1
+===============
+
+ * The OPDS feed book's date must be the date of the book, not the date of the
+ feed generation.
+ * Convert the standard opds date to our format (YYYY-MM-DD)
+ * Remove duplicate language attribute in the libxml dumper.
+ * Create the datadirectory to not fail to write a file in a non-existent
+   directory
+
+kiwix-lib 3.1.0
+===============
+
+ * Add a method to get the favicon url of book (if available).
+ * Move dump code of library.xml in a specific class.
+ * Add a first support to bookmarks
+
+kiwix-lib 3.0.3
+===============
+
+ * Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
+ ('eng').
+ * Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
+ * Correctly fill the book size for the zim file size.
+ * Fix launch of aria2c.
+
+kiwix-lib 3.0.2
+===============
+
+ * Use the correct path separator when computing relativePath on Windows.
+
+kiwix-lib 3.0.1
+===============
+
+ * Small fix about parsing the opdsStream.
+
 kiwix-lib 3.0.0
 ===============
 

README.md

@@ -1,44 +1,55 @@
-Kiwix library
+Libkiwix
-=============
+========
 
-The Kiwix library provides the Kiwix software core. It contains the
+The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
-code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
+core. It contains the code shared by all Kiwix ports (Windows,
+GNU/Linux, macOS, Android, iOS, ...).
+
+[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
+[![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
+[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
+[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
 ----------
 
 This document assumes you have a little knowledge about software
 compilation. If you experience difficulties with the dependencies or
-with the Kiwix libary compilation itself, we recommend to have a look
+with the Libkiwix compilation itself, we recommend to have a look to
-to [kiwix-build](https://github.com/kiwix/kiwix-build).
+[kiwix-build](https://github.com/kiwix/kiwix-build).
 
 Preamble
 --------
 
-Although the Kiwix library can be (cross-)compiled on/for many
+Although the Libkiwix can be (cross-)compiled on/for many sytems, the
-sytems, the following documentation explains how to do it on POSIX
+following documentation explains how to do it on POSIX ones. It is
-ones. It is primarly thought for GNU/Linux systems and has been tested
+primarly thought for GNU/Linux systems and has been tested on recent
-on recent releases of Ubuntu and Fedora.
+releases of Ubuntu and Fedora.
 
 Dependencies
 ------------
 
-The Kiwix library relies on many third parts software libraries. They
+The Libkiwix relies on many third party software libraries. They are
-are prerequisites to the Kiwix library compilation. Following
+prerequisites to the Libkiwix compilation. Following libraries need to
-libraries need to be available:
+be available:
+* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
+* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
+* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
+* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
+header `mustache.hpp` somewhere it can be found by the compiler and/or
+set CPPFLAGS with correct `-I` option). Use Mustache version 4.1 or above.
+* [Libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
+* [Microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
+* [Zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
 
-* ICU ................................... http://site.icu-project.org/
+To test the code:
-(package libicu-dev on Ubuntu)
+* [Google Test](https://github.com/google/googletest) (package `googletest` on Ubuntu)
-* ZIM ........................................ http://www.openzim.org/
+
-(package libzim-dev on Ubuntu)
+The following dependency needs to be available at runtime:
-* Pugixml ........................................ http://pugixml.org/
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
-(package libpugixml-dev on Ubuntu)
-* ctpp2 ........................................ http://ctpp.havoc.ru/
-(package libctpp2-dev on Ubuntu)
-* Xapian ......................................... https://xapian.org/
-(package libxapian-dev on Ubuntu)
-* libaria2 .................................. https://aria2.github.io/
-(no package on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -47,75 +58,163 @@ In the worse case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
-kiwix-lib directory as install prefix.
+`libkiwix` directory as install prefix.
-
-If you compile ctpp2 from source and want to compile the Kiwix library
-statically then you will probably need to rename ctpp2 static library
-from ctpp2-st.a to ctpp2.a.
 
 Environment
 -------------
 
-The Kiwix library builds using [Meson](http://mesonbuild.com/) version
+The Libkiwix builds using [Meson](https://mesonbuild.com/) version
-0.39 or higher. Meson relies itself on Ninja, pkg-config and few other
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:
-* Meson
+* [Meson](https://mesonbuild.com/)
-* Ninja
+* [Ninja](https://ninja-build.org/)
-* Pkg-config
+* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
 
 These tools should be packaged if you use a cutting edge operating
-system. If not, have a look to the "Troubleshooting" section.
+system. If not, have a look to the [Troubleshooting](#Troubleshooting)
+section.
 
 Compilation
 -----------
 
-Once all dependencies are installed, you can compile the Kiwix library
+Once all dependencies are installed, you can compile the Libkiwix
 with:
-```
+```bash
 meson . build
 ninja -C build
 ```
 
 By default, it will compile dynamic linked libraries. All binary files
-will be created in the "build" directory created automatically by
+will be created in the `build` directory created automatically by
 Meson. If you want statically linked libraries, you can add
 `--default-library=static` option to the Meson command.
 
 Depending of you system, `ninja` may be called `ninja-build`.
 
+The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
+with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
+
+```bash
+meson . build -Dwrapper=android -Dwerror=false
+ninja -C build
+```
+
+Static files compilation
+------------------------
+
+Libkiwix has a few static files 'compiled' within the binary
+code. This is mostly Javascript/HTML/pictures necessary for the HTTP
+daemon.
+
+These static files are available in the `static` directory and are
+compiled by custom Python code available in this repository `scripts`
+directory. This happens automatically at compilation time without any
+additional command to run.
+
+To avoid HTTP caching issues, the URLs (to the static content) are
+appended with a `cacheid` parameter (this is called "cache
+busting"). This `cacheid` value derived from the
+[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
+static file. As a consequence, each time you change a static file, the
+corresponding `cacheid` value will change.
+
+To properly test this feature, this `cacheid` needs to be added
+manually to the automated tests and has to be commited. After
+modifying the needed static file, [run the automated
+tests](#Testing). They will fail, but the inspection of the testing
+log will give you the new `cacheid` value(s). Finally update
+`test/server.cpp` with the appropriate `cacheid` value(s) which have
+changed.
+
+Testing
+-------
+
+To run the automated tests:
+```bash
+cd build
+meson test
+```
+
 Installation
 ------------
 
-If you want to install the Kiwix library and the headers you just have
+If you want to install the Libkiwix and the headers you just have
 compiled on your system, here we go:
-
+```bash
-```
 ninja -C build install
 ```
 
-You might need to run the command as root (or using 'sudo'), depending
+You might need to run the command as root (or using `sudo`), depending
 where you want to install the libraries. After the installation
-succeeded, you may need to run ldconfig (as root).
+succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
-------------
+--------------
 
 If you want to uninstall the Kiwix library:
-
+```bash
-```
 ninja -C build uninstall
 ```
 
-Like for the installation, you might need to run the command as root
+Like for the installation, you might need to run the command as `root`
-(or using 'sudo').
+(or using `sudo`).
+
+Custom Index Page
+-----------------
+
+to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
+(note - while using custom html file please mention all external links as absolute path.)
+
+to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
+
+To use JS provided by kiwix-serve you can use the following template to start with ->
+
+```
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title><-- Custom Tittle --></title>
+    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+  </head>
+  <body>
+  </body>
+</html>
+```
+
+- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
+- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
+- To add language select box add - `<select id="languageFilter"></select>` under body tag.
+- To add category select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add search box for books use following form -
+    ```
+        <form id='kiwixSearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
+        <input type="submit" class="kiwixButton" value="Search"/>
+        </form>
+    ```
+
+
+If you compile manually Libmicrohttpd, you might need to compile it
+without GNU TLS, a bug here will empeach further compilation
+otherwise.
+
+If the compilation still fails, you might need to get a more recent
+version of a dependency than the one packaged by your Linux
+distribution. Try then with a source tarball distributed by the
+problematic upstream project or even directly from the source code
+repository.
 
 Troubleshooting
 ---------------
 
 If you need to install Meson "manually":
-```
+```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
 pip3 install meson       # Install Meson
@@ -123,7 +222,7 @@ hash -r                  # Refresh bash paths
 ```
 
 If you need to install Ninja "manually":
-```
+```bash
 git clone git://github.com/ninja-build/ninja.git
 cd ninja
 git checkout release
@@ -133,13 +232,8 @@ cp ninja ../bin
 cd ..
 ```
 
-If the compilation still fails, you might need to get a more recent
-version of a dependency than the one packaged by your Linux
-distribution. Try then with a source tarball distributed by the
-problematic upstream project or even directly from the source code
-repository.
-
 License
 -------
 
-GPLv3 or later, see COPYING for more details.
+[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
+[COPYING](COPYING) for more details.

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,46 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+ meson,
+ pkg-config,
+ libzim-dev (>= 7.2.0~),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/libkiwix
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 7.2.0~),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev,
+ libmicrohttpd-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix10
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
+Conflicts: libkiwix0, libkiwix3, libkiwix9
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix-dev.manpages

@@ -0,0 +1,2 @@
+usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

debian/libkiwix10.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

docs/.gitignore

@@ -0,0 +1,2 @@
+api
+xml

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'libkiwix'
+copyright = '2022, libkiwix-team'
+author = 'libkiwix-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+    'exhale'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+breathe_projects = {
+    "libkiwix": "./xml"
+}
+breathe_default_project = 'libkiwix'
+
+exhale_args = {
+    "containmentFolder":   "./api",
+    "rootFileName":        "ref_api.rst",
+    "rootFileTitle":       "Reference API",
+    "doxygenStripFromPath":"..",
+    "treeViewIsBootstrap": True,
+    "createTreeView" : True,
+    "exhaleExecutesDoxygen": True,
+    "exhaleDoxygenStdin":    "INPUT = ../include"
+}
+
+primary_domain = 'cpp'
+
+highlight_language = 'cpp'

docs/index.rst

@@ -0,0 +1,14 @@
+.. libkiwix documentation master file, created by
+   sphinx-quickstart on Fri Jul 24 15:40:50 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libkiwix's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage
+   api/ref_api

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+breathe
+exhale

docs/usage.rst

@@ -0,0 +1,15 @@
+Libkiwix programming
+====================
+
+Introduction
+------------
+
+libkiwix is written in C++. To use the library, you need the include files of libkiwix have
+to link against libzim.
+
+Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
+which is always derived from std::exception.
+
+All classes are defined in the namespace kiwix.
+
+libkiwix is a set of tools to manage zim files and provide some common functionnality.

format_code.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/bash
-
-files=(
-"include/library.h"
-"include/common/stringTools.h"
-"include/common/pathTools.h"
-"include/common/otherTools.h"
-"include/common/regexTools.h"
-"include/common/networkTools.h"
-"include/manager.h"
-"include/reader.h"
-"include/kiwix.h"
-"include/xapianSearcher.h"
-"include/searcher.h"
-"src/library.cpp"
-"src/android/kiwix.cpp"
-"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
-"src/searcher.cpp"
-"src/common/pathTools.cpp"
-"src/common/regexTools.cpp"
-"src/common/otherTools.cpp"
-"src/common/networkTools.cpp"
-"src/common/stringTools.cpp"
-"src/xapianSearcher.cpp"
-"src/manager.cpp"
-"src/reader.cpp"
-)
-
-for i in "${files[@]}"
-do
-  echo $i
-  clang-format -i -style=file $i
-done

include/book.h

@@ -21,62 +21,94 @@
 #define KIWIX_BOOK_H
 
 #include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include "common.h"
 
 namespace pugi {
 class xml_node;
 }
 
+namespace zim {
+class Archive;
+}
+
 namespace kiwix
 {
-enum supportedIndexType { UNKNOWN, XAPIAN };
 
 class OPDSDumper;
-class Reader;
 
 /**
  * A class to store information about a book (a zim file)
  */
 class Book
 {
- public:
+ public: // types
+  class Illustration
+  {
+    friend class Book;
+   public:
+    uint16_t width = 48;
+    uint16_t height = 48;
+    std::string mimeType;
+    std::string url;
+
+    const std::string& getData() const;
+
+   private:
+    mutable std::string data;
+    mutable std::mutex mutex;
+  };
+
+  typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
+
+ public: // functions
   Book();
   ~Book();
 
   bool update(const Book& other);
-  void update(const Reader& reader);
+  void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
-  std::string getHumanReadableIdFromPath();
+  std::string getHumanReadableIdFromPath() const;
 
   bool readOnly() const { return m_readOnly; }
   const std::string& getId() const { return m_id; }
   const std::string& getPath() const { return m_path; }
   bool isPathValid() const { return m_pathValid; }
-  const std::string& getIndexPath() const { return m_indexPath; }
-  const supportedIndexType& getIndexType() const { return m_indexType; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
-  const std::string& getLanguage() const { return m_language; }
+  DEPRECATED const std::string& getLanguage() const { return m_language; }
+  const std::string& getCommaSeparatedLanguages() const { return m_language; }
+  const std::vector<std::string> getLanguages() const;
   const std::string& getCreator() const { return m_creator; }
   const std::string& getPublisher() const { return m_publisher; }
   const std::string& getDate() const { return m_date; }
   const std::string& getUrl() const { return m_url; }
   const std::string& getName() const { return m_name; }
+  std::string getCategory() const;
   const std::string& getTags() const { return m_tags; }
+  std::string getTagStr(const std::string& tagName) const;
+  bool getTagBool(const std::string& tagName) const;
+  const std::string& getFlavour() const { return m_flavour; }
   const std::string& getOrigId() const { return m_origId; }
   const uint64_t& getArticleCount() const { return m_articleCount; }
   const uint64_t& getMediaCount() const { return m_mediaCount; }
   const uint64_t& getSize() const { return m_size; }
-  const std::string& getFavicon() const;
+  DEPRECATED const std::string& getFavicon() const;
-  const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
+  DEPRECATED const std::string& getFaviconUrl() const;
+  DEPRECATED const std::string& getFaviconMimeType() const;
+
+  Illustrations getIllustrations() const;
+  std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
+
   const std::string& getDownloadId() const { return m_downloadId; }
 
   void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
   void setId(const std::string& id) { m_id = id; }
   void setPath(const std::string& path);
   void setPathValid(bool valid) { m_pathValid = valid; }
-  void setIndexPath(const std::string& indexPath);
-  void setIndexType(supportedIndexType indexType) { m_indexType = indexType;}
   void setTitle(const std::string& title) { m_title = title; }
   void setDescription(const std::string& description) { m_description = description; }
   void setLanguage(const std::string& language) { m_language = language; }
@@ -85,39 +117,44 @@ class Book
   void setDate(const std::string& date) { m_date = date; }
   void setUrl(const std::string& url) { m_url = url; }
   void setName(const std::string& name) { m_name = name; }
+  void setFlavour(const std::string& flavour) { m_flavour = flavour; }
   void setTags(const std::string& tags) { m_tags = tags; }
   void setOrigId(const std::string& origId) { m_origId = origId; }
   void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
   void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
   void setSize(uint64_t size) { m_size = size; }
-  void setFavicon(const std::string& favicon) { m_favicon = favicon; }
-  void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
   void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
 
- protected:
+ private: // functions
+  std::string getCategoryFromTags() const;
+  const Illustration& getDefaultIllustration() const;
+
+ protected: // data
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
-  bool m_pathValid;
+  bool m_pathValid = false;
-  std::string m_indexPath;
-  supportedIndexType m_indexType;
   std::string m_title;
   std::string m_description;
+  std::string m_category;
   std::string m_language;
   std::string m_creator;
   std::string m_publisher;
   std::string m_date;
   std::string m_url;
   std::string m_name;
+  std::string m_flavour;
   std::string m_tags;
   std::string m_origId;
-  uint64_t m_articleCount;
+  uint64_t m_articleCount = 0;
-  uint64_t m_mediaCount;
+  uint64_t m_mediaCount = 0;
-  bool m_readOnly;
+  bool m_readOnly = false;
-  uint64_t m_size;
+  uint64_t m_size = 0;
-  mutable std::string m_favicon;
+  Illustrations m_illustrations;
-  std::string m_faviconUrl;
+
-  std::string m_faviconMimeType;
+  // Used as the return value of getDefaultIllustration() when no default
+  // illustration is found in the book
+  static const Illustration missingDefaultIllustration;
 };
 
 }

include/bookmark.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_BOOKMARK_H
+#define KIWIX_BOOKMARK_H
+
+#include <string>
+
+namespace pugi {
+class xml_node;
+}
+
+namespace kiwix
+{
+
+/**
+ * A class to store information about a bookmark (an article in a book)
+ */
+class Bookmark
+{
+ public:
+  Bookmark();
+  ~Bookmark();
+
+  void updateFromXml(const pugi::xml_node& node);
+
+  const std::string& getBookId() const { return m_bookId; }
+  const std::string& getBookTitle()  const { return m_bookTitle; }
+  const std::string& getUrl() const { return m_url; }
+  const std::string& getTitle() const { return m_title; }
+  const std::string& getLanguage() const { return m_language; }
+  const std::string& getDate() const { return m_date; }
+
+  void setBookId(const std::string& bookId) { m_bookId = bookId; }
+  void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
+  void setUrl(const std::string& url) { m_url = url; }
+  void setTitle(const std::string& title) { m_title = title; }
+  void setLanguage(const std::string& language) { m_language = language; }
+  void setDate(const std::string& date) { m_date = date; }
+
+ protected:
+  std::string m_bookId;
+  std::string m_bookTitle;
+  std::string m_url;
+  std::string m_title;
+  std::string m_language;
+  std::string m_date;
+};
+
+}
+
+#endif

include/common/pathTools.h

@@ -1,64 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_PATHTOOLS_H
-#define KIWIX_PATHTOOLS_H
-
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <fstream>
-#include <ios>
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#ifdef _WIN32
-#include <direct.h>
-#endif
-
-#include "stringTools.h"
-
-using namespace std;
-
-bool isRelativePath(const string& path);
-string computeAbsolutePath(const string path, const string relativePath);
-string computeRelativePath(const string path, const string absolutePath);
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator = false,
-                             const bool removePostSeparator = false);
-string appendToDirectory(const string& directoryPath, const string& filename);
-
-unsigned int getFileSize(const string& path);
-string getFileSizeAsString(const string& path);
-string getFileContent(const string& path);
-bool fileExists(const string& path);
-bool makeDirectory(const string& path);
-string makeTmpDirectory();
-bool copyFile(const string& sourcePath, const string& destPath);
-string getLastPathElement(const string& path);
-string getExecutablePath();
-string getCurrentDirectory();
-string getDataDirectory();
-bool writeTextFile(const string& path, const string& content);
-#endif

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -1,79 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef _CTPP2_VM_STRING_LOADER_HPP__
-#define _CTPP2_VM_STRING_LOADER_HPP__ 1
-
-#include <ctpp2/CTPP2VMLoader.hpp>
-#include <ctpp2/CTPP2Util.hpp>
-#include <ctpp2/CTPP2Exception.hpp>
-#include <ctpp2/CTPP2VMExecutable.hpp>
-#include <ctpp2/CTPP2VMInstruction.hpp>
-#include <ctpp2/CTPP2VMMemoryCore.hpp>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <iostream>
-#include <string>
-
-/**
-  @file VMStringLoader.hpp
-  @brief Load program core from file
-*/
-
-namespace CTPP // C++ Template Engine
-{
-// FWD
-struct VMExecutable;
-
-/**
-  @class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
-  @brief Load program core from file
-*/
-class CTPP2DECL VMStringLoader:
-  public VMLoader
-{
-public:
-	/**
-	*/
-	VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
-	/**
-	  @brief Get ready-to-run program
-	*/
-	const VMMemoryCore * GetCore() const;
-
-	/**
-	  @brief A destructor
-	*/
-	~VMStringLoader() throw();
-private:
-	/** Program core             */
-	VMExecutable  * oCore;
-	/** Ready-to-run program     */
-	VMMemoryCore  * pVMMemoryCore;
-};
-
-} // namespace CTPP
-#endif // _CTPP2_VM_STRING_LOADER_HPP__
-// End.

include/downloader.h

@@ -23,8 +23,9 @@
 #include <string>
 #include <vector>
 #include <map>
-#include <pthread.h>
 #include <memory>
+#include <stdexcept>
+#include <mutex>
 
 namespace kiwix
 {
@@ -43,6 +44,14 @@ class AriaError : public std::runtime_error {
 };
 
 
+/**
+ * A representation of a current download.
+ *
+ * `Download` is not thread safe. User must care to not call method on a
+ * same download from different threads.
+ * However, it is safe to use different `Download`s from different threads.
+ */
+
 class Download {
  public:
   typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
@@ -53,16 +62,89 @@ class Download {
     : mp_aria(p_aria),
       m_status(K_UNKNOWN),
       m_did(did) {};
-  void updateStatus(bool follow=false);
+
-  StatusResult getStatus()          { return m_status; }
+  /**
-  std::string  getDid()             { return m_did; }
+   * Update the status of the download.
-  std::string  getFollowedBy()      { return m_followedBy; }
+   *
-  uint64_t     getTotalLength()     { return m_totalLength; }
+   * This call make an aria rpc call and is blocking.
-  uint64_t     getCompletedLength() { return m_completedLength; }
+   * Some download (started with a metalink) are in fact several downloads.
-  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
+   * - A first one to download the metadlink.
-  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
+   * - A second one to download the real file.
-  std::string  getPath()            { return m_path; }
+   *
-  std::vector<std::string>&  getUris()             { return m_uris; }
+   * If `follow` is true, updateStatus tries to detect that and tracks
+   * the second download when the first one is finished.
+   * By passing false to `follow`, `Download` will only track the first download.
+   *
+   * `getFoo` methods are based on the last statusUpdate.
+   *
+   * @param follow: Do we have to follow following downloads.
+   */
+  void updateStatus(bool follow);
+
+  /**
+   * Pause the download (and call updateStatus)
+   */
+  void pauseDownload();
+
+  /**
+   * Resume the download (and call updateStatus)
+   */
+  void resumeDownload();
+
+  /**
+   * Cancel the download.
+   *
+   * A canceled downlod cannot be resume and updateStatus does nothing.
+   * However, you can still get information based on the last known information.
+   */
+  void cancelDownload();
+
+  /*
+   * Get the status of the download.
+   */
+  StatusResult getStatus() const             { return m_status; }
+
+  /*
+   * Get the id of the download.
+   */
+  const std::string&  getDid() const         { return m_did; }
+
+  /*
+   * Get the id of the "second" download.
+   *
+   * Set only if the "first" download is a metalink and is complete.
+   */
+  const std::string&  getFollowedBy() const  { return m_followedBy; }
+
+  /*
+   * Get the total length of the download.
+   */
+  uint64_t     getTotalLength() const        { return m_totalLength; }
+
+  /*
+   * Get the completed length of the download.
+   */
+  uint64_t     getCompletedLength() const    { return m_completedLength; }
+
+  /*
+   * Get the download speed of the download.
+   */
+  uint64_t     getDownloadSpeed() const      { return m_downloadSpeed; }
+
+  /*
+   * Get the verified length of the download.
+   */
+  uint64_t     getVerifiedLength() const     { return m_verifiedLength; }
+
+  /*
+   * Get the path (local file) of the download.
+   */
+  const std::string&  getPath() const        { return m_path; }
+
+  /*
+   * Get the download uris of the download.
+   */
+  const std::vector<std::string>&  getUris() const { return m_uris; }
 
  protected:
   std::shared_ptr<Aria2> mp_aria;
@@ -80,6 +162,9 @@ class Download {
 /**
  * A tool to download things.
  *
+ * A Downloader manages `Download` using aria2 in the background.
+ * `Downloader` is threadsafe.
+ * However, the returned `Download`s are NOT threadsafe.
  */
 class Downloader
 {
@@ -89,14 +174,41 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri);
+  /**
-  Download* getDownload(const std::string& did);
+   * Start a new download.
+   *
+   * This method is thread safe and return a pointer to a newly created `Download`.
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param uri: The uri of the thing to download.
+   * @param options: A series of pair <option_name, option_value> to pass to aria.
+   * @return: The newly created Download.
+   */
+  std::shared_ptr<Download> startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
 
-  size_t getNbDownload() { return m_knownDownloads.size(); }
+  /**
-  std::vector<std::string> getDownloadIds();
+   * Get a download corrsponding to a download id (did)
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param did: The download id to search for.
+   * @return: The Download corresponding to did.
+   * @throw: Throw std::out_of_range if did is not found.
+   */
+  std::shared_ptr<Download> getDownload(const std::string& did);
+
+  /**
+   * Get the number of downloads currently managed.
+   */
+  size_t getNbDownload() const;
+
+  /**
+   * Get the ids of the managed downloads.
+   */
+  std::vector<std::string> getDownloadIds() const;
 
  private:
-  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
+  mutable std::mutex m_lock;
+  std::map<std::string, std::shared_ptr<Download>> m_knownDownloads;
   std::shared_ptr<Aria2> mp_aria;
 };
 }

include/entry.h

@@ -1,191 +0,0 @@
-/*
- * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_ENTRY_H
-#define KIWIX_ENTRY_H
-
-#include <stdio.h>
-#include <zim/article.h>
-#include <exception>
-#include <string>
-#include "common.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-
-class NoEntry : public std::exception {};
-
-/**
- * A entry represent an.. entry in a zim file.
- */
-class Entry
-{
-  public:
-    /**
-     * Default constructor.
-     *
-     * Construct an invalid entry.
-     */
-    Entry() = default;
-
-    /**
-     * Construct an entry making reference to an zim article.
-     *
-     * @param article a zim::Article object
-     */
-    Entry(zim::Article article);
-    virtual ~Entry() = default;
-
-    /**
-     * Get the path of the entry.
-     *
-     * The path is the "key" of an entry.
-     *
-     * @return the path of the entry.
-     */
-    std::string getPath() const;
-    
-    /**
-     * Get the title of the entry.
-     *
-     * @return the title of the entry.
-     */
-    std::string getTitle() const;
-        
-    /**
-     * Get the content of the entry.
-     *
-     * The string is a copy of the content.
-     * If you don't want to do a copy, use get_blob.
-     *
-     * @return the content of the entry.
-     */
-    std::string getContent() const;
-        
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset = 0) const;
-        
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @param size The size of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset, size_type size) const;
-        
-    /**
-     * Get the info for direct access to the content of the entry.
-     *
-     * Some entry (ie binary ones) have their content plain stored
-     * in the zim file. Knowing the offset where the content is stored
-     * an user can directly read the content in the zim file bypassing the
-     * kiwix-lib/libzim.
-     *
-     * @return A pair specifying where to read the content.
-     *         The string is the real file to read (may be different that .zim
-     *         file if zim is cut).
-     *         The offset is the offset to read in the file.
-     *         Return <"",0> if is not possible to read directly.
-     */
-    std::pair<std::string, offset_type> getDirectAccessInfo() const;
-        
-    /**
-     * Get the size of the entry.
-     *
-     * @return the size of the entry.
-     */
-    size_type   getSize() const;
-
-    /**
-     * Get the mime_type of the entry.
-     *
-     * @return the mime_type of the entry.
-     */
-    std::string getMimetype() const;
-    
-    
-    /**
-     * Get if the entry is a redirect entry.
-     *
-     * @return True if the entry is a redirect.
-     */
-    bool isRedirect() const;
-
-    /**
-     * Get if the entry is a link target entry.
-     *
-     * @return True if the entry is a link target.
-     */
-    bool isLinkTarget() const;
-
-    /**
-     * Get if the entry is a deleted entry.
-     *
-     * @return True if the entry is a deleted entry.
-     */
-    bool isDeleted() const;
-
-    /**
-     * Get the entry pointed by this entry.
-     *
-     * @return the entry pointed.
-     * @throw NoEntry if the entry is not a redirected entry.
-     */
-    Entry getRedirectEntry() const;
-
-    /**
-     * Get the final entry pointed by this entry.
-     *
-     * Follow the redirection until a "not redirecting" entry is found.
-     * If the entry is not a redirected entry, return the entry itself.
-     *
-     * @return the final entry.
-     */
-    Entry getFinalEntry() const;
-
-    /**
-     * Convert the entry to a boolean value.
-     *
-     * @return True if the entry is valid.
-     */
-    explicit operator bool() const { return good(); }
-
-  private:
-    zim::Article article;
-    mutable zim::Article final_article;
-
-    bool good() const { return article.good(); }
-};
-
-}
-
-#endif // KIWIX_ENTRY_H

include/html_dumper.h

@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -18,14 +17,34 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_HTML_DUMPER_H
+#define KIWIX_HTML_DUMPER_H
 
-import org.kiwix.kiwixlib.JNIKiwixReader;
+#include <string>
-import org.kiwix.kiwixlib.JNIKiwixString;
 
-public class JNIKiwix
+#include "library_dumper.h"
+
+namespace kiwix
 {
-  static { System.loadLibrary("kiwix"); }
 
-  public native void setDataDirectory(String icuDataDir);
+/**
+ * A class to dump Library in HTML format.
+ */
+class HTMLDumper : public LibraryDumper
+{
+ public:
+  HTMLDumper(const Library* library, const NameMapper* NameMapper);
+  ~HTMLDumper();
+
+
+  /**
+   * Dump library in HTML
+   *
+   * @return HTML content
+   */
+  std::string dumpPlainHTML(kiwix::Filter filter) const;
+};
+
 }
+
+#endif // KIWIX_HTML_DUMPER_H

include/kiwix.h

@@ -22,4 +22,4 @@
 
 #include "library.h"
 
-#endif
+#endif

include/kiwixserve.h

@@ -0,0 +1,30 @@
+#ifndef KIWIXLIB_KIWIX_SERVE_H_
+#define KIWIXLIB_KIWIX_SERVE_H_
+
+#include <memory>
+#include <string>
+
+class Subprocess;
+namespace kiwix {
+
+class KiwixServe
+{
+  public:
+    KiwixServe(const std::string& libraryPath, int port = 8181);
+    ~KiwixServe();
+
+    void run();
+    void shutDown();
+    bool isRunning();
+    int getPort() { return m_port; }
+    int setPort(int port);
+
+  private:
+  std::unique_ptr<Subprocess> mp_kiwixServe;
+  int m_port;
+  std::string m_libraryPath;
+};
+
+}; //end namespace kiwix
+
+#endif // KIWIXLIB_KIWIX_SERVE_H_

include/library.h

@@ -23,14 +23,22 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <memory>
+#include <mutex>
+#include <zim/archive.h>
+#include <zim/search.h>
+
+#include "book.h"
+#include "bookmark.h"
+#include "common.h"
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
 namespace kiwix
 {
 
-class Book;
 class OPDSDumper;
+class Library;
 
 enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
 enum supportedListMode {
@@ -42,17 +50,149 @@ enum supportedListMode {
   VALID = 1 << 4,
   NOVALID = 1 << 5
 };
+
+class Filter {
+  public: // types
+    using Tags = std::vector<std::string>;
+
+  private: // data
+    uint64_t activeFilters;
+    Tags _acceptTags;
+    Tags _rejectTags;
+    std::string _category;
+    std::string _lang;
+    std::string _publisher;
+    std::string _creator;
+    size_t _maxSize;
+    std::string _query;
+    bool _queryIsPartial;
+    std::string _name;
+
+  public: // functions
+    Filter();
+    ~Filter() = default;
+
+    /**
+     *  Set the filter to check local.
+     *
+     *  A local book is a book with a path.
+     *  If accept is true, only local book are accepted.
+     *  If accept is false, only non local book are accepted.
+     */
+    Filter& local(bool accept);
+
+    /**
+     *  Set the filter to check remote.
+     *
+     *  A remote book is a book with a url.
+     *  If accept is true, only remote book are accepted.
+     *  If accept is false, only non remote book are accepted.
+     */
+    Filter& remote(bool accept);
+
+    /**
+     *  Set the filter to check validity.
+     *
+     *  A valid book is a book with a path pointing to a existing zim file.
+     *  If accept is true, only valid book are accepted.
+     *  If accept is false, only non valid book are accepted.
+     */
+    Filter& valid(bool accept);
+
+    /**
+     * Set the filter to only accept book with corresponding tag.
+     */
+    Filter& acceptTags(const Tags& tags);
+    Filter& rejectTags(const Tags& tags);
+
+    Filter& category(std::string category);
+
+    /**
+     *  Set the filter to only accept books in the specified language.
+     *
+     *  Multiple languages can be specified as a comma-separated list (in
+     *  which case a book in any of those languages will match).
+     */
+    Filter& lang(std::string lang);
+
+    Filter& publisher(std::string publisher);
+    Filter& creator(std::string creator);
+    Filter& maxSize(size_t size);
+    Filter& query(std::string query, bool partial=true);
+    Filter& name(std::string name);
+    Filter& clearLang();
+    Filter& clearCategory();
+
+    bool hasQuery() const;
+    const std::string& getQuery() const { return _query; }
+    bool queryIsPartial() const { return _queryIsPartial; }
+
+    bool hasName() const;
+    const std::string& getName() const { return _name; }
+
+    bool hasCategory() const;
+    const std::string& getCategory() const { return _category; }
+
+    bool hasLang() const;
+    const std::string& getLang() const { return _lang; }
+
+    bool hasPublisher() const;
+    const std::string& getPublisher() const { return _publisher; }
+
+    bool hasCreator() const;
+    const std::string& getCreator() const { return _creator; }
+
+    const Tags& getAcceptTags() const { return _acceptTags; }
+    const Tags& getRejectTags() const { return _rejectTags; }
+
+private: // functions
+    friend class Library;
+
+    bool accept(const Book& book) const;
+};
+
+
+class ZimSearcher : public zim::Searcher
+{
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
+
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
+
 /**
  * A Library store several books.
  */
 class Library
 {
-  std::map<std::string, kiwix::Book> books;
+  // all data fields must be added in LibraryBase
+  mutable std::mutex m_mutex;
+
+ public:
+  typedef uint64_t Revision;
+  typedef std::vector<std::string> BookIdCollection;
+  typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
+
  public:
   Library();
   ~Library();
 
-  std::string version;
+  /**
+   * Library is not a copiable object. However it can be moved.
+   */
+  Library(const Library& ) = delete;
+  Library(Library&& );
+  void operator=(const Library& ) = delete;
+  Library& operator=(Library&& );
+
   /**
    * Add a book to the library.
    *
@@ -65,7 +205,39 @@ class Library
    */
   bool addBook(const Book& book);
 
-  Book& getBookById(const std::string& id);
+  /**
+   * A self-explanatory alias for addBook()
+   */
+  bool addOrUpdateBook(const Book& book) { return addBook(book); }
+
+  /**
+   * Add a bookmark to the library.
+   *
+   * @param bookmark the book to add.
+   */
+  void addBookmark(const Bookmark& bookmark);
+
+  /**
+   * Remove a bookmarkk
+   *
+   * @param zimId The zimId of the bookmark.
+   * @param url The url of the bookmark.
+   * @return True if the bookmark has been removed.
+   */
+  bool removeBookmark(const std::string& zimId, const std::string& url);
+
+  // XXX: This is a non-thread-safe operation
+  const Book& getBookById(const std::string& id) const;
+  // XXX: This is a non-thread-safe operation
+  const Book& getBookByPath(const std::string& path) const;
+
+  Book getBookByIdThreadSafe(const std::string& id) const;
+
+  std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
 
   /**
    * Remove a book from the library.
@@ -79,9 +251,17 @@ class Library
    * Write the library to a file.
    *
    * @param path the path of the file to write to.
-   * @return True if the library has been correctly save.
+   * @return True if the library has been correctly saved.
    */
-  bool writeToFile(const std::string& path);
+  bool writeToFile(const std::string& path) const;
+
+  /**
+   * Write the library bookmarks to a file.
+   *
+   * @param path the path of the file to write to.
+   * @return True if the library has been correctly saved.
+   */
+  bool writeBookmarksToFile(const std::string& path) const;
 
   /**
    * Get the number of book in the library.
@@ -90,80 +270,111 @@ class Library
    * @param remoteBooks If we must count remote books (books with an url)
    * @return The number of books.
    */
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
 
   /**
-   * Get all langagues of the books in the library.
+   * Get all languagues of the books in the library.
    *
    * @return A list of languages.
    */
-  std::vector<std::string> getBooksLanguages();
+  std::vector<std::string> getBooksLanguages() const;
+
+  /**
+   * Get all languagues of the books in the library with counts.
+   *
+   * @return A list of languages with the count of books in each language.
+   */
+  AttributeCounts getBooksLanguagesWithCounts() const;
+
+  /**
+   * Get all categories of the books in the library.
+   *
+   * @return A list of categories.
+   */
+  std::vector<std::string> getBooksCategories() const;
 
   /**
    * Get all book creators of the books in the library.
    *
    * @return A list of book creators.
    */
-  std::vector<std::string> getBooksCreators();
+  std::vector<std::string> getBooksCreators() const;
 
   /**
    * Get all book publishers of the books in the library.
    *
    * @return A list of book publishers.
    */
-  std::vector<std::string> getBooksPublishers();
+  std::vector<std::string> getBooksPublishers() const;
+
+  /**
+   * Get all bookmarks.
+   *
+   * @return A list of bookmarks
+   */
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true) const;
 
   /**
    * Get all book ids of the books in the library.
    *
    * @return A list of book ids.
    */
-  std::vector<std::string> getBooksIds();
+  BookIdCollection getBooksIds() const;
 
   /**
-   * Filter the library and generate a new one with the keep elements.
+   * Filter the library and return the id of the keep elements.
    *
-   * This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
+   * @param filter The filter to use.
-   *
+   * @return The list of bookIds corresponding to the filter.
-   * @param search List only books with search in the title or description.
-   * @return The list of bookIds corresponding to the query.
    */
-  std::vector<std::string> filter(const std::string& search);
+  BookIdCollection filter(const Filter& filter) const;
 
 
   /**
-   * List books in the library.
+   * Sort (in place) bookIds using the given comparator.
    *
-   * @param mode The mode of listing :
+   * @param bookIds the list of book Ids to sort
-   *             - LOCAL    : list only local books (with a path).
+   * @param comparator how to sort the books
-   *             - REMOTE   : list only remote books (with an url).
+   * @return The sorted list of books
-   *             - VALID    : list only valid books (without a path or with a
-   *                          path pointing to a valid zim file).
-   *             - NOLOCAL  : list only books without valid path.
-   *             - NOREMOTE : list only books without url.
-   *             - NOVALID  : list only books not valid.
-   *             - ALL : Do not do any filter (LOCAL or REMOTE)
-   *             - Flags can be combined.
-   * @param sortBy Attribute to sort by the book list.
-   * @param search List only books with search in the title, description.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param maxSize Do not list book bigger than maxSize.
-   *                Set to 0 to cancel this filter.
-   * @return The list of bookIds corresponding to the query.
    */
-  std::vector<std::string> listBooksIds(
+  void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
-    int supportedListMode = ALL,
+
-    supportedListSortBy sortBy = UNSORTED,
+  /**
-    const std::string& search = "",
+   * Return the current revision of the library.
-    const std::string& language = "",
+   *
-    const std::string& creator = "",
+   * The revision of the library is updated (incremented by one) by
-    const std::string& publisher = "",
+   * the addBook() and removeBookById() operations.
-    size_t maxSize = 0);
+   *
+   * @return Current revision of the library.
+   */
+  Revision getRevision() const;
+
+  /**
+   * Remove books that have not been updated since the specified revision.
+   *
+   * @param  rev the library revision to use
+   * @return Count of books that were removed by this operation.
+   */
+  uint32_t removeBooksNotUpdatedSince(Revision rev);
 
   friend class OPDSDumper;
+  friend class libXMLDumper;
+
+private: // types
+  typedef const std::string& (Book::*BookStrPropMemFn)() const;
+  struct Impl;
+
+private: // functions
+  AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
+  std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
+  BookIdCollection filterViaBookDB(const Filter& filter) const;
+  void updateBookDB(const Book& book);
+  void dropCache(const std::string& bookId);
+
+private: //data
+  std::unique_ptr<Impl> mp_impl;
 };
+
 }
 
 #endif

include/library_dumper.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBRARY_DUMPER_H
+#define KIWIX_LIBRARY_DUMPER_H
+
+#include <string>
+
+
+#include "library.h"
+#include "name_mapper.h"
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+/**
+ * A base class to dump Library in various formats.
+ *
+ */
+class LibraryDumper
+{
+ public:
+  LibraryDumper(const Library* library, const NameMapper* NameMapper);
+  ~LibraryDumper();
+
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
+
+  /**
+   * Set the root location used when generating url.
+   *
+   * @param rootLocation the root location to use.
+   */
+  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+  
+  /**
+   * Sets user default language
+   * 
+   * @param userLang the user language to be set
+   */
+  void setUserLanguage(std::string userLang) { this->m_userLang = userLang; }
+
+  /**
+   * Get the data of categories
+   */
+  kainjow::mustache::list getCategoryData() const;
+
+  /**
+   * Get the data of languages
+   */
+  kainjow::mustache::list getLanguageData() const;
+
+ protected:
+  const kiwix::Library* const library;
+  const kiwix::NameMapper* const nameMapper;
+  std::string libraryId;
+  std::string rootLocation;
+  std::string m_userLang;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+};
+}
+
+#endif // KIWIX_LIBRARY_DUMPER_H

include/libxml_dumper.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBXML_DUMPER_H
+#define KIWIX_LIBXML_DUMPER_H
+
+#include <string>
+#include <vector>
+
+#include <pugixml.hpp>
+
+#include "library.h"
+
+namespace kiwix
+{
+
+/**
+ * A tool to dump a `Library` into a basic library.xml
+ *
+ */
+class LibXMLDumper
+{
+ public:
+  LibXMLDumper() = default;
+  LibXMLDumper(const Library* library);
+  ~LibXMLDumper();
+
+  /**
+   * Dump the library.xml
+   *
+   * @param id The id of the library.
+   * @return The library.xml content.
+   */
+  std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
+
+
+  /**
+   * Dump the bookmark of the library.
+   *
+   * @return The bookmark.xml content.
+   */
+  std::string dumpLibXMLBookmark();
+
+  /**
+   * Set the base directory used.
+   *
+   * @param baseDir the base directory to use.
+   */
+  void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
+
+  /**
+   * Set the library to dump.
+   *
+   * @param library The library to dump.
+   */
+  void setLibrary(const Library* library) { this->library = library; }
+
+ protected:
+  const kiwix::Library* library;
+  std::string baseDir;
+ private:
+  void handleBook(Book book, pugi::xml_node root_node);
+  void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
+};
+}
+
+#endif // KIWIX_OPDS_DUMPER_H

include/manager.h

@@ -22,10 +22,10 @@
 
 #include "book.h"
 #include "library.h"
-#include "reader.h"
 
 #include <string>
 #include <vector>
+#include <memory>
 
 namespace pugi {
 class xml_document;
@@ -34,60 +34,60 @@ class xml_document;
 namespace kiwix
 {
 
-class LibraryManipulator {
+class LibraryManipulator
- public:
+{
-  virtual ~LibraryManipulator() {}
+ public: // functions
-  virtual bool addBookToLibrary(Book book) = 0;
+  explicit LibraryManipulator(Library* library);
-};
+  virtual ~LibraryManipulator();
 
-class DefaultLibraryManipulator : public LibraryManipulator {
+  Library& getLibrary() const { return library; }
- public:
+
-  DefaultLibraryManipulator(Library* library) :
+  bool addBookToLibrary(const Book& book);
-    library(library) {}
+  void addBookmarkToLibrary(const Bookmark& bookmark);
-  virtual ~DefaultLibraryManipulator() {}
+  uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
-  bool addBookToLibrary(Book book) {
+
-    return library->addBook(book);
+ protected: // overrides
-  }
+  virtual void bookWasAddedToLibrary(const Book& book);
- private:
+  virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
-   kiwix::Library* library;
+  virtual void booksWereRemovedFromLibrary();
+
+ private: // data
+  kiwix::Library& library;
 };
 
 /**
  * A tool to manage a `Library`.
- *
- * A `Manager` handle a internal `Library`.
- * This `Library` can be retrived with `cloneLibrary` method.
  */
 class Manager
 {
- public:
+ public: // types
-  Manager(LibraryManipulator* manipulator);
+  typedef std::vector<std::string> Paths;
-  Manager(Library* library);
+
-  ~Manager();
+ public: // functions
+  explicit Manager(LibraryManipulator* manipulator);
+  explicit Manager(Library* library);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
    *
-   * @param path The path to the `library.xml`.
+   * @param path The (utf8) path to the `library.xml`.
    * @param readOnly Set if the libray path could be overwritten latter with
    *                 updated content.
+   * @param trustLibrary use book metadata coming from XML.
    * @return True if file has been properly parsed.
    */
-  bool readFile(const std::string& path, const bool readOnly = true);
+  bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
 
   /**
-   * Read a `library.xml` and add book in the file to the library.
+   * Sync the contents of the library with one or more `library.xml` files.
    *
-   * @param nativePath The path of the `library.xml`
+   * The metadata of the library files is trusted unconditionally.
-   * @param UTF8Path The utf8 version (?) of the path. Also the path where the
+   * Any books not present in the input library.xml files are removed
-   *                 library will be writen i readOnly is False.
+   * from the library.
-   * @param readOnly Set if the libray path could be overwritten latter with
+   *
-   *                 updated content.
+   * @param paths The (utf8) paths to the `library.xml` files.
-   * @return True if file has been properly parsed.
    */
-  bool readFile(const std::string& nativePath,
+  void reload(const Paths& paths);
-                const std::string& UTF8Path,
-                const bool readOnly = true);
 
   /**
    * Load a library content store in the string.
@@ -100,7 +100,8 @@ class Manager
    */
   bool readXml(const std::string& xml,
                const bool readOnly = true,
-               const std::string& libraryPath = "");
+               const std::string& libraryPath = "",
+               bool trustLibrary = true);
 
   /**
    * Load a library content stored in a OPDS stream.
@@ -113,6 +114,15 @@ class Manager
    */
   bool readOpds(const std::string& content, const std::string& urlHost);
 
+
+  /**
+   * Load a bookmark file.
+   *
+   * @param path The path of the file to read.
+   * @return True if the content has been properly parsed.
+   */
+  bool readBookmarkFile(const std::string& path);
+
   /**
    * Add a book to the library.
    *
@@ -145,72 +155,24 @@ class Manager
                        const std::string& url = "",
                        const bool checkMetaData = false);
 
-  /**
-   * Get the book corresponding to an id.
-   *
-   * @param[in] id The id of the book
-   * @param[out] book The book corresponding to the id.
-   * @return True if the book has been found.
-   */
-  bool getBookById(const std::string& id, Book& book);
-
-  /**
-   * Update the "last open date" of a book
-   *
-   * @param id the id of the book.
-   * @return True if the book is in the library.
-   */
-  bool updateBookLastOpenDateById(const std::string& id);
-
-  /**
-   * Remove (set to empty) paths of all books in the library.
-   */
-  void removeBookPaths();
-
-  /**
-   * List books in the library.
-   *
-   * The books list will be available in public vector member `bookIdList`.
-   *
-   * @param mode The mode of listing :
-   *             - LASTOPEN sort by last opened book.
-   *             - LOCAL list only local file.
-   *             - REMOTE list only remote file.
-   * @param sortBy Attribute to sort by the book list.
-   * @param maxSize Do not list book bigger than maxSize MiB.
-   *                Set to 0 to cancel this filter.
-   * @param language List only books in this language.
-   * @param creator List only books of this creator.
-   * @param publisher List only books of this publisher.
-   * @param search List only books with search in the title, description or
-   *               language.
-   * @return True
-   */
-  bool listBooks(const supportedListMode mode,
-                 const supportedListSortBy sortBy,
-                 const unsigned int maxSize,
-                 const std::string& language,
-                 const std::string& creator,
-                 const std::string& publisher,
-                 const std::string& search);
-
   std::string writableLibraryPath;
 
-  std::vector<std::string> bookIdList;
+  bool m_hasSearchResult = false;
+  uint64_t m_totalBooks = 0;
+  uint64_t m_startIndex = 0;
+  uint64_t m_itemsPerPage = 0;
 
  protected:
-  kiwix::LibraryManipulator* manipulator;
+  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
-  bool mustDeleteManipulator;
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,
-                   const bool readOnly,
+                   bool readOnly,
-                   const std::string& libraryPath);
+                   const std::string& libraryPath,
+                   bool trustLibrary);
   bool parseOpdsDom(const pugi::xml_document& doc,
                     const std::string& urlHost);
 
- private:
-  void checkAndCleanBookPaths(Book& book, const std::string& libraryPath);
 };
 }
 

include/meson.build

@@ -1,35 +1,18 @@
 headers = [
   'book.h',
+  'bookmark.h',
   'common.h',
   'library.h',
   'manager.h',
+  'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
-  'reader.h',
+  'search_renderer.h',
-  'entry.h',
+  'server.h',
-  'searcher.h'
+  'kiwixserve.h',
+  'name_mapper.h',
+  'tools.h',
+  'version.h'
 ]
 
-if xapian_dep.found()
-  headers += ['xapianSearcher.h']
-endif
-
 install_headers(headers, subdir:'kiwix')
-
-install_headers(
-  'common/base64.h',
-  'common/networkTools.h',
-  'common/otherTools.h',
-  'common/pathTools.h',
-  'common/regexTools.h',
-  'common/stringTools.h',
-  subdir:'kiwix/common'
-)
-
-if has_ctpp2_dep
-  install_headers(
-    'ctpp2/CTPP2VMStringLoader.hpp',
-    subdir:'kiwix/ctpp2'
-  )
-endif
-

include/name_mapper.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_NAMEMAPPER_H
+#define KIWIX_NAMEMAPPER_H
+
+#include <string>
+#include <map>
+#include <memory>
+#include <mutex>
+
+namespace kiwix
+{
+
+class Library;
+
+class NameMapper {
+  public:
+    virtual ~NameMapper() = default;
+    virtual std::string getNameForId(const std::string& id) const = 0;
+    virtual std::string getIdForName(const std::string& name) const = 0;
+};
+
+
+class IdNameMapper : public NameMapper {
+  public:
+    virtual std::string getNameForId(const std::string& id) const { return id; };
+    virtual std::string getIdForName(const std::string& name) const { return name; };
+};
+
+class HumanReadableNameMapper : public NameMapper {
+  private:
+    std::map<std::string, std::string> m_idToName;
+    std::map<std::string, std::string> m_nameToId;
+
+  public:
+    HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
+    virtual ~HumanReadableNameMapper() = default;
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+};
+
+class UpdatableNameMapper : public NameMapper {
+    typedef std::shared_ptr<NameMapper> NameMapperHandle;
+  public:
+    UpdatableNameMapper(Library& library, bool withAlias);
+
+    virtual std::string getNameForId(const std::string& id) const;
+    virtual std::string getIdForName(const std::string& name) const;
+
+    void update();
+
+  private:
+    NameMapperHandle currentNameMapper() const;
+
+  private:
+    mutable std::mutex mutex;
+    Library& library;
+    NameMapperHandle nameMapper;
+    const bool withAlias;
+};
+
+}
+
+#endif

include/opds_catalog.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,9 +17,17 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#ifndef KIWIX_OPDS_CATALOG_H
+#define KIWIX_OPDS_CATALOG_H
 
-public class JNIKiwixBool
+
+#include "library.h"
+
+namespace kiwix
 {
-  public boolean value;
+
-}
+std::string getSearchUrl(const Filter& f);
+
+} // namespace kiwix
+
+#endif // KIWIX_OPDS_CATALOG_H

include/opds_dumper.h

@@ -26,11 +26,9 @@
 
 #include <pugixml.hpp>
 
-#include "common/base64.h"
-#include "common/pathTools.h"
-#include "common/regexTools.h"
 #include "library.h"
-#include "reader.h"
+#include "name_mapper.h"
+#include "library_dumper.h"
 
 using namespace std;
 
@@ -41,79 +39,52 @@ namespace kiwix
  * A tool to dump a `Library` into a opds stream.
  *
  */
-class OPDSDumper
+class OPDSDumper : public LibraryDumper
 {
  public:
-  OPDSDumper() = default;
+  OPDSDumper(const Library* library, const NameMapper* NameMapper);
-  OPDSDumper(Library* library);
   ~OPDSDumper();
 
   /**
    * Dump the OPDS feed.
    *
-   * @param id The id of the library.
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
    * @return The OPDS feed.
    */
-  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds);
+  std::string dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const;
 
   /**
-   * Set the id of the opds stream.
+   * Dump the OPDS feed.
    *
-   * @param id the id to use.
+   * @param bookIds the ids of the books to include in the feed
+   * @param query the query used to obtain the list of book ids
+   * @param partial whether the feed should include partial or complete entries
+   * @return The OPDS feed.
    */
-  void setId(const std::string& id) { this->id = id;}
+  std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
 
   /**
-   * Set the title oft the opds stream.
+   * Dump the OPDS complete entry document.
    *
-   * @param title the title to use.
+   * @param bookId the id of the book
+   * @return The OPDS complete entry document.
    */
-  void setTitle(const std::string& title) { this->title = title; }
+  std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
 
   /**
-   * Set the root location used when generating url.
+   * Dump the categories OPDS feed.
    *
-   * @param rootLocation the root location to use.
+   * @return The OPDS feed.
    */
-  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+  std::string categoriesOPDSFeed() const;
 
   /**
-   * Set the search url.
+   * Dump the languages OPDS feed.
    *
-   * @param searchUrl the search url to use.
+   * @return The OPDS feed.
    */
-  void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; }
+  std::string languagesOPDSFeed() const;
-
-  /**
-   * Set some informations about the search results.
-   *
-   * @param totalResult the total number of results of the search.
-   * @param startIndex the start index of the result.
-   * @param count the number of result of the current set (or page).
-   */
-  void setOpenSearchInfo(int totalResult, int startIndex, int count);
-
-  /**
-   * Set the library to dump.
-   *
-   * @param library The library to dump.
-   */
-  void setLibrary(Library* library) { this->library = library; }
-
- protected:
-  kiwix::Library* library;
-  std::string id;
-  std::string title;
-  std::string date;
-  std::string rootLocation;
-  std::string searchDescriptionUrl;
-  int m_totalResults;
-  int m_startIndex;
-  int m_count;
-  bool m_isSearchResult = false;
-
- private:
-  pugi::xml_node handleBook(Book book, pugi::xml_node root_node);
 };
 }
 

include/reader.h

@@ -1,505 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_READER_H
-#define KIWIX_READER_H
-
-#include <stdio.h>
-#include <zim/article.h>
-#include <zim/file.h>
-#include <zim/fileiterator.h>
-#include <zim/zim.h>
-#include <exception>
-#include <map>
-#include <sstream>
-#include <string>
-#include "common.h"
-#include "entry.h"
-#include "common/pathTools.h"
-#include "common/stringTools.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-/**
- * The Reader class is the class who allow to get an entry content from a zim
- * file.
- */
-class Reader
-{
- public:
-  /**
-   * Create a Reader to read a zim file specified by zimFilePath.
-   *
-   * @param zimFilePath The path to the zim file to read.
-   *                    The zim file can be splitted (.zimaa, .zimab, ...).
-   *                    In this case, the file path must still point to the
-   *                    unsplitted path as if the file were not splitted
-   *                    (.zim extesion).
-   */
-  Reader(const string zimFilePath);
-  ~Reader();
-
-  /**
-   * Get the number of "displayable" entries in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'text/html' MIMEtype specified in the metadata.
-   *         Else return the number of entries in the 'A' namespace.
-   */
-  unsigned int getArticleCount() const;
-
-  /**
-   * Get the number of media in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'image/jpeg', 'image/gif' and 'image/png' in
-   *         the metadata.
-   *         Else return the number of entries in the 'I' namespace.
-   */
-  unsigned int getMediaCount() const;
-
-  /**
-   * Get the number of all entries in the zim file.
-   *
-   * @return Return the number of all the entries, whatever their MIMEtype or
-   *         their namespace.
-   */
-  unsigned int getGlobalCount() const;
-
-  /**
-   * Get the path of the zim file.
-   *
-   * @return the path of the zim file as given in the constructor.
-   */
-  string getZimFilePath() const;
-
-  /**
-   * Get the Id of the zim file.
-   *
-   * @return The uuid stored in the zim file.
-   */
-  string getId() const;
-
-  /**
-   * Get the url of a random page.
-   *
-   * Deprecated : Use `getRandomPage` instead.
-   *
-   * @return Url of a random page. The page is picked from all entries in
-   *         the 'A' namespace.
-   *         The main page is excluded from the potential results.
-   */
-  DEPRECATED string getRandomPageUrl() const;
-
-  /**
-   * Get a random page.
-   *
-   * @return A random Entry. The entry is picked from all entries in
-   *         the 'A' namespace.
-   *         The main entry is excluded from the potential results.
-   */
-  Entry getRandomPage() const;
-
-  /**
-   * Get the url of the first page.
-   *
-   * Deprecated : Use `getFirstPage` instead.
-   *
-   * @return Url of the first entry in the 'A' namespace.
-   */
-  DEPRECATED string getFirstPageUrl() const;
-
-  /**
-   * Get the entry of the first page.
-   *
-   * @return The first entry in the 'A' namespace.
-   */
-  Entry getFirstPage() const;
-
-  /**
-   * Get the url of the main page.
-   *
-   * Deprecated : Use `getMainPage` instead.
-   *
-   * @return Url of the main page as specified in the zim file.
-   */
-  DEPRECATED string getMainPageUrl() const;
-
-  /**
-   * Get the entry of the main page.
-   *
-   * @return Entry of the main page as specified in the zim file.
-   */
-  Entry getMainPage() const;
-
-  /**
-   * Get the content of a metadata.
-   *
-   * @param[in]  name The name of the metadata.
-   * @param[out] value The value will be set to the content of the metadata.
-   * @return True if it was possible to get the content of the metadata.
-   */
-  bool getMetatag(const string& name, string& value) const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
-
-  /**
-   * Get the name of the zim file.
-   *
-   * @return The name of the zim file as specified in the zim metadata.
-   */
-  string getName() const;
-
-  /**
-   * Get the tags of the zim file.
-   *
-   * @return The tags of the zim file as specified in the zim metadata.
-   */
-  string getTags() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
-
-  /**
-   * Get the creator of the zim file.
-   *
-   * @return The creator of the zim file as specified in the zim metadata.
-   */
-  string getCreator() const;
-
-  /**
-   * Get the publisher of the zim file.
-   *
-   * @return The publisher of the zim file as specified in the zim metadata.
-   */
-  string getPublisher() const;
-
-  /**
-   * Get the origId of the zim file.
-   *
-   * The origId is only used in the case of patch zim file and is the Id
-   * of the original zim file.
-   *
-   * @return The origId of the zim file as specified in the zim metadata.
-   */
-  string getOrigId() const;
-
-  /**
-   * Get the favicon of the zim file.
-   *
-   * @param[out] content The content of the favicon.
-   * @param[out] mimeType The mimeType of the favicon.
-   * @return True if a favicon has been found.
-   */
-  bool getFavicon(string& content, string& mimeType) const;
-
-  /**
-   * Get an entry associated to an path.
-   *
-   * @param path The path of the entry.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromPath(const std::string& path) const;
-
-  /**
-   * Get an entry associated to an url encoded path.
-   *
-   * Equivalent to `getEntryFromPath(urlDecode(path));`
-   *
-   * @param path The url encoded path.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromEncodedPath(const std::string& path) const;
-
-  /**
-   * Get un entry associated to a title.
-   *
-   * @param title The title.
-   * @return The entry
-   * throw NoEntry If no entry correspond to the url.
-   */
-  Entry getEntryFromTitle(const std::string& title) const;
-
-  /**
-   * Get the url of a page specified by a title.
-   *
-   * @param[in] title the title of the page.
-   * @param[out] url the url of the page.
-   * @return True if the page can be found.
-   */
-  DEPRECATED bool getPageUrlFromTitle(const string& title, string& url) const;
-
-  /**
-   * Get the mimetype of a entry specified by a url.
-   *
-   * @param[in] url the url of the entry.
-   * @param[out] mimeType the mimeType of the entry.
-   * @return True if the mimeType has been found.
-   */
-  DEPRECATED bool getMimeTypeByUrl(const string& url, string& mimeType) const;
-
-  /**
-   * Get the content of an entry specifed by a url.
-   *
-   * Alias to `getContentByEncodedUrl`
-   */
-  DEPRECATED bool getContentByUrl(const string& url,
-                       string& content,
-                       string& title,
-                       unsigned int& contentLength,
-                       string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url encoded url.
-   *
-   * Equivalent to getContentByDecodedUrl(urlDecode(url), ...).
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-
-  /**
-   * Get the content of an entry specified by an url encoded url.
-   *
-   * Equivalent to getContentByEncodedUrl but without baseUrl.
-   */
-  DEPRECATED bool getContentByEncodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * @param[in] url The url of the entry.
-   * @param[out] content The content of the entry.
-   * @param[out] title the title of the entry.
-   * @param[out] contentLength The size of the entry (size of content).
-   * @param[out] contentType The mimeType of the entry.
-   * @param[out] baseUrl Return the true url of the entry.
-   *                     If the specified entry is a redirection, contains
-   *                     the url of the targeted entry.
-   * @return True if the entry has been found.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType,
-                              string& baseUrl) const;
-  /**
-   * Get the content of an entry specified by a url.
-   *
-   * Equivalent to getContentByDecodedUrl but withou the baseUrl.
-   */
-  DEPRECATED bool getContentByDecodedUrl(const string& url,
-                              string& content,
-                              string& title,
-                              unsigned int& contentLength,
-                              string& contentType) const;
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are stored in an internal vector and can be retrieved using
-   * `getNextSuggestion` method.
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param reset If true, remove previous suggestions in the internal vector.
-   *              If false, add suggestions to the internal vector
-   *              (until internal vector size is suggestionCount (or no more
-   *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
-   */
-  bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         const bool reset = true);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   */
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
-
-  /**
-   * Check if the url exists in the zim file.
-   *
-   * Deprecated : Use `pathExists` instead.
-   *
-   * @param url the url to check.
-   * @return True if the url exits in the zim file.
-   */
-  DEPRECATED bool urlExists(const string& url) const;
-
-  /**
-   * Check if the path exists in the zim file.
-   *
-   * @param path the path to check.
-   * @return True if the path exists in the zim file.
-   */
-  bool pathExists(const string& path) const;
-
-  /**
-   * Check if the zim file has a embedded fulltext index.
-   *
-   * @return True if the zim file has a embedded fulltext index
-   *         and is not split (else the fulltext is not accessible).
-   */
-  bool hasFulltextIndex() const;
-
-  /**
-   * Get potential case title variations for a title.
-   *
-   * @param title a title.
-   * @return the list of variantions.
-   */
-  std::vector<std::string> getTitleVariants(const std::string& title) const;
-
-  /**
-   * Get the next suggestion title.
-   *
-   * @param[out] title the title of the suggestion.
-   * @return True if title has been set.
-   */
-  bool getNextSuggestion(string& title);
-
-  /**
-   * Get the next suggestion title and url.
-   *
-   * @param[out] title the title of the suggestion.
-   * @param[out] url the url of the suggestion.
-   * @return True if title and url have been set.
-   */
-  bool getNextSuggestion(string& title, string& url);
-
-  /**
-   * Get if we can check zim file integrity (has a checksum).
-   *
-   * @return True if zim file have a checksum.
-   */
-  bool canCheckIntegrity() const;
-
-  /**
-   * Check is zim file is corrupted.
-   *
-   * @return True if zim file is corrupted.
-   */
-  bool isCorrupted() const;
-
-  /**
-   * Parse a full url into a namespace and url.
-   *
-   * @param[in] url The full url ("/N/url").
-   * @param[out] ns The namespace (N).
-   * @param[out] title The url (url).
-   * @return True
-   */
-  DEPRECATED bool parseUrl(const string& url, char* ns, string& title) const;
-
-  /**
-   * Return the total size of the zim file.
-   *
-   * If zim file is split, return the sum of all parts' size.
-   *
-   * @return Size of the size file is KiB.
-   */
-  unsigned int getFileSize() const;
-
-  /**
-   * Get the zim file handler.
-   *
-   * @return The libzim file handler.
-   */
-  zim::File* getZimFileHandler() const;
-
-  /**
-   * Get the zim article object associated to a url.
-   *
-   * @param[in] url The url of the article.
-   * @param[out] article The libzim article object.
-   * @return True if the url is good (article.good()).
-   */
-  DEPRECATED bool getArticleObjectByDecodedUrl(const string& url,
-                                    zim::Article& article) const;
-
- protected:
-  zim::File* zimFileHandler;
-  zim::size_type firstArticleOffset;
-  zim::size_type lastArticleOffset;
-  zim::size_type nsACount;
-  zim::size_type nsICount;
-  std::string zimFilePath;
-
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
-
- private:
-  std::map<const std::string, unsigned int> parseCounterMetadata() const;
-};
-}
-
-#endif

include/search_renderer.h

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SEARCH_RENDERER_H
+#define KIWIX_SEARCH_RENDERER_H
+
+#include <string>
+#include <zim/search.h>
+#include "library.h"
+
+namespace kiwix
+{
+
+class NameMapper;
+/**
+ * The SearcherRenderer class is used to render a search result to a html page.
+ */
+class SearchRenderer
+{
+ public:
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * The constructed version of the SearchRenderer will not introduce
+   * the book name for each result. It is better to use the other constructor
+   * with a Library pointer to have a better html page.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                 unsigned int start, unsigned int estimatedResultCount);
+
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                 unsigned int start, unsigned int estimatedResultCount);
+
+  ~SearchRenderer();
+
+  /**
+   * Set the search pattern used to do the search
+   */
+  void setSearchPattern(const std::string& pattern);
+
+  /**
+   * Set the querystring used to select books
+   */
+  void setSearchBookQuery(const std::string& bookQuery);
+
+  /**
+   * Set protocol prefix.
+   */
+  void setProtocolPrefix(const std::string& prefix);
+
+  /**
+   * Set search protocol prefix.
+   */
+  void setSearchProtocolPrefix(const std::string& prefix);
+
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
+  std::string renderTemplate(const std::string& tmpl_str);
+
+  /**
+   * Generate the html page with the resutls of the search.
+   */
+  std::string getHtml();
+
+    /**
+   * Generate the xml page with the resutls of the search.
+   */
+  std::string getXml();
+
+
+ protected:
+  std::string beautifyInteger(const unsigned int number);
+  zim::SearchResultSet m_srs;
+  NameMapper* mp_nameMapper;
+  Library* mp_library;
+  std::string searchBookQuery;
+  std::string searchPattern;
+  std::string protocolPrefix;
+  std::string searchProtocolPrefix;
+  unsigned int pageLength;
+  unsigned int estimatedResultCount;
+  unsigned int resultStart;
+};
+
+
+}
+
+#endif

include/searcher.h

@@ -1,230 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCHER_H
-#define KIWIX_SEARCHER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unicode/putil.h>
-#include <algorithm>
-#include <cctype>
-#include <locale>
-#include <string>
-#include <vector>
-#include <vector>
-#include "common/pathTools.h"
-#include "common/stringTools.h"
-#include "kiwix_config.h"
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader;
-class Result
-{
- public:
-  virtual ~Result(){};
-  virtual std::string get_url() = 0;
-  virtual std::string get_title() = 0;
-  virtual int get_score() = 0;
-  virtual std::string get_snippet() = 0;
-  virtual std::string get_content() = 0;
-  virtual int get_wordCount() = 0;
-  virtual int get_size() = 0;
-  virtual int get_readerIndex() = 0;
-};
-
-struct SearcherInternal;
-/**
- * The Searcher class is reponsible to do different kind of search using the
- * fulltext index.
- *
- * Historically, there are two kind of fulltext index :
- *  - The legacy one, is the external fulltext index. A directory stored outside
- *    of the zim file.
- *  - The new one, a embedded fulltext index in the zim file.
- *
- * Legacy external fulltext index has to be considered as obsolet format with
- * less functionnalities:
- *  - No multi zim search ;
- *  - No geo_search ;
- *  - No suggestions search ;
- *
- * To reflect this, there is two Search creation "API":
- *  - One for the external fulltext index, using the constructor taking a
- *    xapianDirectoryPath) ;
- *  - One for the embedded fulltext index, using a "empty" constructor and the
- *  `add_reader` method".
- *
- *  On top of that, the Searcher may (if compiled with ctpp2) be used to
- *  generate a html page for the search result. This use a template that need a
- *  humanReaderName. This feature is only used by kiwix-serve and this should be
- *  move outside of Searcher (and with a better API). If you don't use the html
- *  rendering (getHtml method), you better should simply ignore the different
- *  humanReadeableName attributes (or give an empty string).
- */
-class Searcher
-{
- public:
-  /**
-   * The default constructor.
-   *
-   * @param humanReadableName The global zim's humanReadableName.
-   *                          Used to generate pagination links.
-   */
-  Searcher(const string& humanReadableName = "");
-
-  /**
-   * The constructor for legacy external fulltext index.
-   *
-   * @param xapianDirectoryPath The path to the external index directory.
-   * @param reader The reader associated to the external index.
-   *               It will be used retrive the article content or generate
-   *               the snippet.
-   * @param humanReadableName The humanReadableName for the zim.
-   */
-  Searcher(const string& xapianDirectoryPath,
-           Reader* reader,
-           const string& humanReadableName);
-  ~Searcher();
-
-  /**
-   * Add a reader (containing embedded fulltext index) to the search.
-   *
-   * @param reader The Reader for the zim containing the fulltext index.
-   * @param humanReaderName The human readable name of the reader.
-   * @return true if the reader has been added.
-   *         false if the reader cannot be added (no embedded fulltext index present)
-   */
-  bool add_reader(Reader* reader, const std::string& humanReaderName);
-
-  /**
-   * Start a search on the zim associated to the Searcher.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param search The search query.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void search(std::string& search,
-              unsigned int resultStart,
-              unsigned int resultEnd,
-              const bool verbose = false);
-
-  /**
-   * Start a geographique search.
-   * The search return result for entry in a disc of center latitude/longitude
-   * and radius distance.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param latitude The latitude of the center point.
-   * @param longitude The longitude of the center point.
-   * @param distance The radius of the disc.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param resultEnd the end offset of the search results (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void geo_search(float latitude, float longitude, float distance,
-                  unsigned int resultStart,
-                  unsigned int resultEnd,
-                  const bool verbose = false);
-
-  /**
-   * Start a suggestion search.
-   * The search made depend of the "version" of the embedded index.
-   *  - If the index is newer enough and have a title namespace, the search is
-   *    made in the titles only.
-   *  - Else the search is made on the whole article content.
-   * In any case, the search is made "partial" (as adding '*' at the end of the query)
-   *
-   * @param search The search query.
-   * @param verbose print some info on stdout if true.
-   */
-  void suggestions(std::string& search, const bool verbose = false);
-
-  /**
-   * Get the next result of a started search.
-   * This is the method to use to loop hover the search results.
-   */
-  Result* getNextResult();
-
-  /**
-   * Restart the previous search.
-   * Next call to getNextResult will return the first result.
-   */
-  void restart_search();
-
-  /**
-   * Get a estimation of the result count.
-   */
-  unsigned int getEstimatedResultCount();
-
-  /**
-   * Set protocol prefix.
-   * Only used by getHtml.
-   */
-  bool setProtocolPrefix(const std::string prefix);
-
-  /**
-   * Set search protocol prefix.
-   * Only used by getHtml.
-   */
-  bool setSearchProtocolPrefix(const std::string prefix);
-
-#ifdef ENABLE_CTPP2
-  /**
-   * Generate the html page with the resutls of the search.
-   */
-  string getHtml();
-#endif
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  void closeIndex();
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-
-  std::vector<Reader*> readers;
-  std::vector<std::string> humanReaderNames;
-  SearcherInternal* internal;
-  std::string searchPattern;
-  std::string protocolPrefix;
-  std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int resultEnd;
-  std::string contentHumanReadableId;
-
- private:
-  void reset();
-
-};
-
-
-}
-
-#endif

include/server.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_H
+#define KIWIX_SERVER_H
+
+#include <string>
+#include <memory>
+
+namespace kiwix
+{
+  class Library;
+  class NameMapper;
+  class InternalServer;
+
+  class Server {
+     public:
+       /**
+        * The default constructor.
+        *
+        * @param library The library to serve.
+        */
+       Server(Library* library, NameMapper* nameMapper=nullptr);
+
+       virtual ~Server();
+
+       /**
+        * Serve the content.
+        */
+       bool start();
+
+       /**
+        * Stop the daemon.
+        */
+       void stop();
+
+       void setRoot(const std::string& root);
+       void setAddress(const std::string& addr) { m_addr = addr; }
+       void setPort(int port) { m_port = port; }
+       void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
+       void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
+       void setVerbose(bool verbose) { m_verbose = verbose; }
+       void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
+       void setTaskbar(bool withTaskbar, bool withLibraryButton)
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
+       int getPort();
+       std::string getAddress();
+
+     protected:
+       Library* mp_library;
+       NameMapper* mp_nameMapper;
+       std::string m_root = "";
+       std::string m_addr = "";
+       std::string m_indexTemplateString = "";
+       int m_port = 80;
+       int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
+       bool m_verbose = false;
+       bool m_withTaskbar = true;
+       bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
+       int m_ipConnectionLimit = 0;
+       std::unique_ptr<InternalServer> mp_server;
+  };
+}
+
+#endif

include/tools.h

@@ -0,0 +1,220 @@
+/*
+ * Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_TOOLS_H
+#define KIWIX_TOOLS_H
+
+#include <string>
+#include <vector>
+#include <map>
+
+namespace kiwix {
+
+/**
+ * Return the current directory.
+ *
+ * @return the current directory (utf8 encoded)
+ */
+std::string getCurrentDirectory();
+
+/**
+ * Return the data directory.
+ *
+ * The data directory is a directory where to put data (zim files, ...)
+ * It depends of the platform and it may be changed by user using environment variable.
+ *
+ * The resolution order is :
+ * - `KIWIX_DATA_DIR` env variable (if set).
+ * - On Windows :
+ *    . `$APPDATA/kiwix` if $APPDATA is set
+ *    . `$USERPROFILE/kiwix` if $USERPROFILE is set
+ * - Else :
+ *    . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
+ *    . `$HOME/.local/share/kiwx` if $HOWE is set
+ * - current directory
+ *
+ * @return the path of the data directory (utf8 encoded)
+ */
+std::string getDataDirectory();
+
+/** Return the path of the executable
+ *
+ * Some application may be packaged in auto extractible archive (Appimage) and the
+ * real executable is different of the path of the archive.
+ * If `realPathOnly` is true, return the path of the real executable instead of the
+ * archive launched by the user.
+ *
+ * @param realPathOnly If we must return the real path of the executable.
+ * @return the path of the executable (utf8 encoded)
+ */
+std::string getExecutablePath(bool realPathOnly = false);
+
+/** Tell if the path is a relative path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path A utf8 encoded path.
+ * @return true if the path is relative.
+ */
+bool isRelativePath(const std::string& path);
+
+/** Append a path to another one.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param basePath the base path.
+ * @param relativePath a path to add to the base path, must be a relative path.
+ * @return The concatenation of the paths, using the right separator.
+ */
+std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
+
+/** Remove the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The parent directory (or empty string if none).
+ */
+std::string removeLastPathElement(const std::string& path);
+
+/** Get the last element of a path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path a path.
+ * @return The base name of the path or empty string if none (ending with a separator).
+ */
+std::string getLastPathElement(const std::string& path);
+
+/** Compute the absolute path of a relative path based on another one
+ *
+ * Equivalent to appendToDirectory followed by a normalization of the path.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path (if empty, current directory is taken).
+ * @param relativePath the relative path.
+ * @return a absolute path.
+ */
+std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
+
+/** Compute the relative path of a path relative to another one
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate paths.
+ *
+ * @param path the base path.
+ * @param absolutePath the absolute path to find the relative path for.
+ * @return a relative path (pointing to absolutePath, relative to path).
+ */
+std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
+
+/** Sleep the current thread.
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * @param milliseconds The number of milliseconds to wait for.
+ */
+void sleep(unsigned int milliseconds);
+
+/** Split a string
+ *
+ * This function is provided as a small helper. It is probably better to use native tools.
+ *
+ * Assuming text = "foo:;bar;baz,oups;"
+ *
+ * split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
+ * split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
+ * split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
+ * split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
+ *
+ * @param str The string to split.
+ * @param delims A string of potential delimiters.
+ *               Each charater in the string can be a individual delimiters.
+ * @param dropEmpty true if empty part must be dropped from the result.
+ * @param keepDelim true if delimiter must be included from the result.
+ * @return a list of part (potentially containing delimiters)
+ */
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
+
+/** Convert language code from iso2 code to iso3
+ *
+ * This function is provided as a small helper. It is probably better to use native tools
+ * to manipulate locales.
+ *
+ * @param a2code a iso2 code string.
+ * @return the corresponding iso3 code.
+ * @throw std::out_of_range if iso2 code is not known.
+ */
+std::string converta2toa3(const std::string& a2code);
+
+/** Extracts content from given file.
+ *
+ * This function provides content of a file provided it's path.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Content of corresponding file in string format.
+ */
+std::string getFileContent(const std::string& path);
+
+/** Checks if file exists.
+ *
+ * This function returns boolean stating if file exists.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file exists or not.
+ */
+bool fileExists(const std::string& path);
+
+/** Checks if file is readable.
+ *
+ * This function returns boolean stating if file is readable.
+ *
+ * @param path The absolute path provided in string format.
+ * @return Boolean representing if file is readale or not.
+ */
+bool fileReadable(const std::string& path);
+
+/** Provides mimetype from filename.
+ *
+ * This function provides mimetype from file-name.
+ *
+ * @param filename string containing filename.
+ * @return mimetype from filename in string format.
+ */
+std::string getMimeTypeForFile(const std::string& filename);
+
+/** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 network interfaces
+ */
+std::map<std::string, std::string> getNetworkInterfaces();
+
+/** Provides the best IP address
+ * This function provides the best IP address from the list given by getNetworkInterfaces
+ */
+std::string getBestPublicIp();
+
+}
+#endif // KIWIX_TOOLS_H

include/version.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,22 +17,18 @@
  * MA 02110-1301, USA.
  */
 
-#ifndef KIWIX_OTHERTOOLS_H
+#ifndef KIWIX_VERSION_H
-#define KIWIX_OTHERTOOLS_H
+#define KIWIX_VERSION_H
 
-#ifdef _WIN32
+#include <string>
-#include <windows.h>
+#include <vector>
-#else
+#include <iostream>
-#include <unistd.h>
-#endif
-
-#include <pugixml.hpp>
 
 namespace kiwix
 {
-void sleep(unsigned int milliseconds);
+  typedef std::vector<std::pair<std::string, std::string>> LibVersions;
-std::string nodeToString(pugi::xml_node node);
+  LibVersions getVersions();
-std::string converta2toa3(const std::string& a2code);
+  void printVersions(std::ostream& out = std::cout);
 }
 
-#endif
+#endif // KIWIX_VERSION_H

include/xapianSearcher.h

@@ -1,98 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_XAPIAN_SEARCHER_H
-#define KIWIX_XAPIAN_SEARCHER_H
-
-#include <xapian.h>
-#include "reader.h"
-#include "searcher.h"
-
-#include <map>
-#include <string>
-
-using namespace std;
-
-namespace kiwix
-{
-class XapianSearcher;
-
-class XapianResult : public Result
-{
- public:
-  XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
-  virtual ~XapianResult(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual int get_readerIndex() { return 0; };
-
- private:
-  XapianSearcher* searcher;
-  Xapian::MSetIterator iterator;
-  Xapian::Document document;
-};
-
-class NoXapianIndexInZim : public exception
-{
-  virtual const char* what() const throw()
-  {
-    return "There is no fulltext index in the zim file";
-  }
-};
-
-class XapianSearcher
-{
-  friend class XapianResult;
-
- public:
-  XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
-  virtual ~XapianSearcher(){};
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int resultEnd,
-                     const bool verbose = false);
-  virtual Result* getNextResult();
-  void restart_search();
-
-  Xapian::MSet results;
-
- protected:
-  void closeIndex();
-  void openIndex(const string& xapianDirectoryPath);
-  void setup_queryParser();
-
-  Reader* reader;
-  Xapian::Database readableDatabase;
-  std::string language;
-  std::string stopwords;
-  Xapian::QueryParser queryParser;
-  Xapian::Stem stemmer;
-  Xapian::SimpleStopper stopper;
-  Xapian::MSetIterator current_result;
-  std::map<std::string, int> valuesmap;
-};
-}
-
-#endif

meson.build

@@ -1,101 +1,62 @@
-project('kiwix-lib', 'cpp',
+project('libkiwix', 'cpp',
-  version : '3.0.0',
+  version : '12.0.0',
-  license : 'GPL',
+  license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
-find_library_in_compiler = meson.version().version_compare('>=0.31.0')
 
-static_deps = get_option('android') or get_option('default_library') == 'static'
+static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
-if get_option('android')
+
-  extra_libs = ['-llog']
+# See https://github.com/kiwix/libkiwix/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
+  extra_libs = ['-latomic']
 else
   extra_libs = []
 endif
 
-thread_dep = dependency('threads')
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
+  # C++ std::thread is implemented using pthread on linux by gcc
+  thread_dep = dependency('threads')
+else
+  thread_dep = dependency('', required:false)
+endif
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
+microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+zlib_dep = dependency('zlib', static:static_deps)
+xapian_dep = dependency('xapian-core', static:static_deps)
+
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
+  error('Cannot found header mustache.hpp')
+endif
+
+libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
+if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
+  error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
+endif
+
 
 extra_cflags = ''
-if target_machine.system() == 'windows' and static_deps
+if host_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-
+if host_machine.system() == 'windows'
-
+  add_project_arguments('-DNOMINMAX', language: 'cpp')
-ctpp2_include_path = ''
-has_ctpp2_dep = false
-ctpp2_prefix_install = get_option('ctpp2-install-prefix')
-ctpp2_link_args = []
-if ctpp2_prefix_install == ''
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp')
-    if find_library_in_compiler
-      ctpp2_lib = compiler.find_library('ctpp2')
-    else
-      ctpp2_lib = find_library('ctpp2')
-    endif
-    ctpp2_link_args = ['-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      if find_library_in_compiler
-        iconv_lib = compiler.find_library('iconv', required:false)
-      else
-        iconv_lib = find_library('iconv', required:false)
-      endif
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
-else
-  if not find_library_in_compiler
-    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
-  endif
-  ctpp2_include_path = ctpp2_prefix_install + '/include'
-  ctpp2_include_args = ['-I'+ctpp2_include_path]
-  if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
-    ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
-    ctpp2_lib_path = join_paths(ctpp2_prefix_install, get_option('libdir'))
-    message(ctpp2_lib_path)
-    ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path, required:false)
-    if not ctpp2_lib.found()
-      ctpp2_lib_path = join_paths(ctpp2_prefix_install, 'lib')
-      message(ctpp2_lib_path)
-      ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
-    endif
-    ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
-    if meson.is_cross_build() and host_machine.system() == 'windows'
-      iconv_lib = compiler.find_library('iconv', required:false)
-      if iconv_lib.found()
-        ctpp2_link_args += ['-liconv']
-      endif
-    endif
-    has_ctpp2_dep = true
-    ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
-  else
-    message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
-  endif
 endif
 
-xapian_dep = dependency('xapian-core', required:false, static:static_deps)
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libcurl_dep]
+inc = include_directories('include', extra_include)
-if has_ctpp2_dep
-  all_deps += [ctpp2_dep]
-endif
-
-inc = include_directories('include')
 
 conf = configuration_data()
-conf.set('VERSION', '"@0@"'.format(meson.project_version()))
+conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
-conf.set('ENABLE_CTPP2', has_ctpp2_dep)
 
 if build_machine.system() == 'windows'
      extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -108,18 +69,11 @@ subdir('scripts')
 subdir('static')
 subdir('src')
 subdir('test')
-
+if get_option('doc')
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
+  subdir('docs')
-if xapian_dep.found()
-    pkg_requires += ['xapian-core']
 endif
 
-if has_ctpp2_dep
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
-  extra_libs += ctpp2_link_args
-  if ctpp2_include_path != ''
-    extra_cflags = ' -I'+ctpp2_include_path
-  endif
-endif
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

meson_options.txt

@@ -1,4 +1,4 @@
-option('ctpp2-install-prefix', type : 'string', value : '',
+option('static-linkage', type : 'boolean', value : false,
-  description : 'Prefix where ctpp libs has been installed')
+  description : 'Link statically with the dependencies.')
-option('android', type : 'boolean', value : false,
+option('doc', type : 'boolean', value : false,
-  description : 'Do we make a kiwix-lib for android')
+  description : 'Build the documentations.')

scripts/ctpp2c.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-
-ctpp2c=$1
-SOURCE=$(pwd)/$2
-DEST=$3
-
-$ctpp2c $SOURCE $DEST

scripts/format_code.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+
+# Compute 'src' path
+SCRIPT_DIR=$(dirname "$0")
+REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
+DIRS="src include"
+
+# Apply formating to all *.cpp and *.h files
+cd "$REPO_DIR"
+for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
+do
+  echo $FILE
+  clang-format -i -style=file "$FILE"
+done

scripts/kiwix-compile-i18n

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.data = f.read()
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_files', nargs='+',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    resources = [Resource(filename) for filename in args.i18n_resource_files]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_files ...
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -52,15 +52,21 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
 class Resource:
-    def __init__(self, base_dirs, filename):
+    def __init__(self, base_dirs, filename, cacheid=None):
-        filename = filename.strip()
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
@@ -71,7 +77,7 @@ class Resource:
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -81,28 +87,34 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>
@@ -113,7 +125,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,10 +151,11 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.
@@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -182,15 +202,17 @@ if __name__ == "__main__":
     parser.add_argument('--source_dir',
                         help="Additional directory where to look for resources.",
                         action='append')
-    parser.add_argument('resource_file',
+    parser.add_argument('resource_files', nargs='+',
                         help='The list of resources to compile.')
     args = parser.parse_args()
 
-    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
-    with open(args.resource_file, 'r') as f:
+    resources = []
-        resources = [Resource([base_dir]+source_dir, filename)
+    for resfile in args.resource_files:
-                        for filename in f.readlines()]
+        base_dir = os.path.dirname(os.path.realpath(resfile))
+        with open(resfile, 'r') as f:
+            resources += [Resource([base_dir]+source_dir, *line.strip().split())
+                            for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/kiwix-resources

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_resource_list_file(src_path, dst_path):
+    with open(src_path, 'r') as src:
+        with open(dst_path, 'w') as dst:
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        if resource.startswith('skin/'):
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,5 +1,13 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
-intermediate_ctpp2c = find_program('ctpp2c.sh')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/android/AndroidManifest.xml

@@ -1,13 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-
-  package="kiwix.org.kiwixlib"
->
-
-  <application android:allowBackup="true"
-    android:label="@string/app_name"
-    android:supportsRtl="true"
-  >
-
-  </application>
-
-</manifest>

src/android/kiwix.cpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwix.h"
-
-#include <iostream>
-#include <string>
-
-#include "unicode/putil.h"
-
-#include "utils.h"
-
-pthread_mutex_t globalLock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER;
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
-    JNIEnv* env, jobject obj, jstring dirStr)
-{
-  std::string cPath = jni2c(dirStr, env);
-
-  Lock l;
-  try {
-    u_setDataDirectory(cPath.c_str());
-  } catch (...) {
-    std::cerr << "Unable to set data directory " << cPath << std::endl;
-  }
-}

src/android/kiwixreader.cpp

@@ -1,424 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <zim/file.h>
-#include <android/log.h>
-#include "org_kiwix_kiwixlib_JNIKiwixReader.h"
-
-#include "common/base64.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
-    JNIEnv* env, jobject obj, jstring filename)
-{
-  std::string cPath = jni2c(filename, env);
-
-  __android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create reader with: %s", cPath.c_str());
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(cPath);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Error opening ZIM file");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Reader>::dispose(env, obj);
-}
-
-#define READER (Handle<kiwix::Reader>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
-{
-  jstring url;
-
-  try {
-    std::string cUrl = READER->getMainPage().getPath();
-    url = c2jni(cUrl, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM main page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    url = NULL;
-  }
-  return url;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
-{
-  jstring id;
-
-  try {
-    std::string cId = READER->getId();
-    id = c2jni(cId, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM id");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    id = NULL;
-  }
-
-  return id;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
-{
-  jint size;
-
-  try {
-    int cSize = READER->getFileSize();
-    size = c2jni(cSize);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM file size");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-  }
-
-  return size;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
-{
-  jstring creator;
-
-  try {
-    std::string cCreator = READER->getCreator();
-    creator = c2jni(cCreator, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM creator");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    creator = NULL;
-  }
-
-  return creator;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
-{
-  jstring publisher;
-
-  try {
-    std::string cPublisher = READER->getPublisher();
-    publisher = c2jni(cPublisher, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM publish");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    publisher = NULL;
-  }
-  return publisher;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
-{
-  jstring name;
-
-  try {
-    std::string cName = READER->getName();
-    name = c2jni(cName, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM name");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    name = NULL;
-  }
-  return name;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
-{
-  jstring favicon;
-
-  try {
-    std::string cContent;
-    std::string cMime;
-    READER->getFavicon(cContent, cMime);
-    favicon = c2jni(
-        base64_encode(cContent),
-        env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM favicon");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    favicon = NULL;
-  }
-  return favicon;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
-{
-  jstring date;
-
-  try {
-    std::string cDate = READER->getDate();
-    date = c2jni(cDate, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM date");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    date = NULL;
-  }
-  return date;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
-{
-  jstring language;
-
-  try {
-    std::string cLanguage = READER->getLanguage();
-    language = c2jni(cLanguage, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM language");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    language = NULL;
-  }
-
-  return language;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  jstring mimeType;
-
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    auto cMimeType = entry.getMimetype();
-    mimeType = c2jni(cMimeType, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get mime-type for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    mimeType = NULL;
-  }
-  return mimeType;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
-    JNIEnv* env, jobject obj, jstring url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
-{
-  /* Default values */
-  setStringObjValue("", titleObj, env);
-  setStringObjValue("", mimeTypeObj, env);
-  setIntObjValue(0, sizeObj, env);
-  jbyteArray data = env->NewByteArray(0);
-
-  /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
-  unsigned int cSize = 0;
-
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    cSize = entry.getSize();
-    setIntObjValue(cSize, sizeObj, env);
-
-    data = env->NewByteArray(cSize);
-    env->SetByteArrayRegion(
-        data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
-
-    setStringObjValue(entry.getMimetype(), mimeTypeObj, env);
-    setStringObjValue(entry.getTitle(), titleObj, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get content for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-  }
-
-  return data;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPart(
-    JNIEnv* env, jobject obj, jstring url, jint offset, jint len, jobject sizeObj)
-{
-  jbyteArray data = env->NewByteArray(0);
-  setIntObjValue(0, sizeObj, env);
-
-  /* Default values */
-  /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset);
-  unsigned int cLen = jni2c(len);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-
-    if (cLen == 0) {
-      setIntObjValue(entry.getSize(), sizeObj, env);
-    } else if (cOffset+cLen < entry.getSize()) {
-      auto blob = entry.getBlob(cOffset, cLen);
-      data = env->NewByteArray(cLen);
-      env->SetByteArrayRegion(
-          data, 0, cLen, reinterpret_cast<const jbyte*>(blob.data()));
-      setIntObjValue(cLen, sizeObj, env);
-    }
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-  }
-  return data;
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
-    JNIEnv* env, jobject obj, jstring url)
-{
-   jclass classPair = env->FindClass("org/kiwix/kiwixlib/Pair");
-   jmethodID midPairinit = env->GetMethodID(classPair, "<init>", "()V");
-   jobject pair = env->NewObject(classPair, midPairinit);
-   setPairObjValue("", 0, pair, env);
-
-   std::string cUrl = jni2c(url, env);
-   try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    auto part_info = entry.getDirectAccessInfo();
-    setPairObjValue(part_info.first, part_info.second, pair, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get direct access info for url: %s", cUrl.c_str());
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-  }
-  return pair;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
-                                                         jobject obj,
-                                                         jstring prefix,
-                                                         jint count)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count);
-
-  try {
-    if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
-      retVal = JNI_TRUE;
-    }
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get search results for pattern: %s", cPrefix.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
-                                                         jobject obj,
-                                                         jobject titleObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-
-  try {
-    if (READER->getNextSuggestion(cTitle)) {
-      setStringObjValue(cTitle, titleObj, env);
-      retVal = JNI_TRUE;
-    }
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get next suggestion");
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
-                                                           jobject obj,
-                                                           jstring title,
-                                                           jobject urlObj)
-{
-  std::string cTitle = jni2c(title, env);
-
-  try {
-    auto entry = READER->getEntryFromTitle(cTitle);
-    entry = entry.getFinalEntry();
-    setStringObjValue(entry.getPath(), urlObj, env);
-    return JNI_TRUE;
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", "Unable to get url for title %s: ", cTitle.c_str());
-    __android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
-  }
-
-  return JNI_FALSE;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
-    JNIEnv* env, jobject obj)
-{
-  jstring title;
-
-  try {
-    std::string cTitle = READER->getTitle();
-    title = c2jni(cTitle, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim title");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    title = NULL;
-  }
-  return title;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
-{
-  jstring description;
-
-  try {
-    std::string cDescription = READER->getDescription();
-    description = c2jni(cDescription, env);
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get zim description");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-    description = NULL;
-  }
-  return description;
-}
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
-    JNIEnv* env, jobject obj, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cUrl;
-
-  try {
-    std::string cUrl = READER->getRandomPage().getPath();
-    setStringObjValue(cUrl, urlObj, env);
-    retVal = JNI_TRUE;
-  } catch (std::exception& e) {
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get random page");
-    __android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
-  }
-  return retVal;
-}

src/android/kiwixsearcher.cpp

@@ -1,124 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <zim/file.h>
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"
-
-#include "reader.h"
-#include "searcher.h"
-#include "utils.h"
-
-#define SEARCHER (Handle<kiwix::Searcher>::getHandle(env, obj))
-#define RESULT (Handle<kiwix::Result>::getHandle(env, obj))
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Searcher>::dispose(env, obj);
-}
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNativeHandle(JNIEnv* env,
-                                                          jobject obj)
-{
-  kiwix::Searcher* searcher = new kiwix::Searcher();
-  return reinterpret_cast<jlong>(new Handle<kiwix::Searcher>(searcher));
-}
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
-    JNIEnv* env, jobject obj, jobject reader)
-{
-  auto searcher = SEARCHER;
-
-  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)), "");
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
-    JNIEnv* env, jobject obj, jstring query, jint count)
-{
-  std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count);
-
-  SEARCHER->search(cquery, 0, ccount);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNextResult(JNIEnv* env,
-                                                         jobject obj)
-{
-  jobject result = nullptr;
-
-  kiwix::Result* cresult = SEARCHER->getNextResult();
-  if (cresult != nullptr) {
-    jclass resultclass
-        = env->FindClass("org/kiwix/kiwixlib/JNIKiwixSearcher$Result");
-    jmethodID ctor = env->GetMethodID(
-        resultclass, "<init>", "(Lorg/kiwix/kiwixlib/JNIKiwixSearcher;JLorg/kiwix/kiwixlib/JNIKiwixSearcher;)V");
-    result = env->NewObject(resultclass, ctor, obj, reinterpret_cast<jlong>(new Handle<kiwix::Result>(cresult)), obj);
-  }
-  return result;
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_dispose(
-    JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Result>::dispose(env, obj);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getUrl(JNIEnv* env,
-                                                        jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_url(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getTitle(JNIEnv* env,
-                                                          jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_title(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getSnippet(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_snippet(), env);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getContent(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_content(), env);
-}

src/android/meson.build

@@ -1,26 +0,0 @@
-
-kiwix_jni = custom_target('jni',
-  input: ['org/kiwix/kiwixlib/JNIKiwix.java',
-          'org/kiwix/kiwixlib/JNIKiwixReader.java',
-          'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-          'org/kiwix/kiwixlib/JNIKiwixInt.java',
-          'org/kiwix/kiwixlib/JNIKiwixString.java',
-          'org/kiwix/kiwixlib/JNIKiwixBool.java',
-          'org/kiwix/kiwixlib/JNIKiwixException.java',
-          'org/kiwix/kiwixlib/Pair.java'],
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-kiwix_sources += [
-    'android/kiwix.cpp',
-    'android/kiwixreader.cpp',
-    'android/kiwixsearcher.cpp',
-    kiwix_jni]
-
-install_subdir('org', install_dir: 'kiwix-lib/java')
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/android/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -1,127 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.JNIKiwixString;
-import org.kiwix.kiwixlib.JNIKiwixInt;
-import org.kiwix.kiwixlib.JNIKiwixSearcher;
-import org.kiwix.kiwixlib.Pair;
-
-public class JNIKiwixReader
-{
-  public native String getMainPage();
-
-  public native String getTitle();
-
-  public native String getId();
-
-  public native String getLanguage();
-
-  public native String getMimeType(String url);
-
-  public native byte[] getContent(String url,
-                                  JNIKiwixString title,
-                                  JNIKiwixString mimeType,
-                                  JNIKiwixInt size);
-
-  /**
-   * getContentPart.
-   *
-   * Get only a part of the content of the article.
-   * Return a byte array of `len` size starting from offset `offset`.
-   * Set `size` to the number of bytes read
-   * (`len` if everything is ok, 0 in case of error).
-   * If `len` == 0, no bytes are read but `size` is set to the total size of the
-   * article.
-   */
-  public native byte[] getContentPart(String url,
-                                      int offest,
-                                      int len,
-                                      JNIKiwixInt size);
-
-  /**
-   * getDirectAccessInformation.
-   *
-   * Return information giving where the content is located in the zim file.
-   *
-   * Some contents (binary content) are stored uncompressed in the zim file.
-   * Knowing this information, it could be interesting to directly open
-   * the zim file (or zim part) and directly read the content from it (and so
-   * bypassing the libzim).
-   *
-   * Return a `Pair` (filename, offset) where the content is located.
-   *
-   * If the content cannot be directly accessed (content is compressed or zim
-   * file is cut in the middle of the content), the filename is an empty string
-   * and offset is zero.
-   */
-  public native Pair getDirectAccessInformation(String url);
-
-  public native boolean searchSuggestions(String prefix, int count);
-
-  public native boolean getNextSuggestion(JNIKiwixString title);
-
-  public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
-
-  public native String getDescription();
-
-  public native String getDate();
-
-  public native String getFavicon();
-
-  public native String getCreator();
-
-  public native String getPublisher();
-
-  public native String getName();
-
-  public native int getFileSize();
-
-  public native int getArticleCount();
-
-  public native int getMediaCount();
-
-  public native boolean getRandomPage(JNIKiwixString url);
-
-  public JNIKiwixSearcher search(String query, int count)
-  {
-    JNIKiwixSearcher searcher = new JNIKiwixSearcher();
-    searcher.addKiwixReader(this);
-    searcher.search(query, count);
-    return searcher;
-  }
-
-  public JNIKiwixReader(String filename) throws JNIKiwixException
-  {
-    nativeHandle = getNativeReader(filename);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException("Cannot open zimfile "+filename);
-    }
-  }
-  public JNIKiwixReader() {
-
-  }
-  public native void dispose();
-
-  private native long getNativeReader(String filename);
-  private long nativeHandle;
-}

src/android/org/kiwix/kiwixlib/JNIKiwixSearcher.java

@@ -1,67 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixReader;
-import java.util.Vector;
-
-public class JNIKiwixSearcher
-{
-  public class Result
-  {
-    private long nativeHandle;
-    private JNIKiwixSearcher searcher;
-    public Result(long handle, JNIKiwixSearcher _searcher)
-    {
-      nativeHandle = handle;
-      searcher = _searcher;
-    }
-    public native String getUrl();
-    public native String getTitle();
-    public native String getContent();
-    public native String getSnippet();
-    public native void dispose();
-  }
-
-  public JNIKiwixSearcher()
-  {
-    nativeHandle = getNativeHandle();
-    usedReaders = new Vector();
-  }
-  public native void dispose();
-
-  private native long getNativeHandle();
-  private long nativeHandle;
-  private Vector usedReaders;
-
-  public native void addReader(JNIKiwixReader reader);
-  public void addKiwixReader(JNIKiwixReader reader)
-  {
-    addReader(reader);
-    usedReaders.addElement(reader);
-  };
-
-  public native void search(String query, int count);
-
-  public native Result getNextResult();
-  public native boolean hasMoreResult();
-}

src/android/org/kiwix/kiwixlib/JNIKiwixString.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixString
-{
-  public String value;
-}

src/android/org/kiwix/kiwixlib/Pair.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class Pair
-{
-  public String filename;
-  public int offset;
-}

src/android/res/values/strings.xml

@@ -1,3 +0,0 @@
-<resources>
-  <string name="app_name">Kiwix Lib</string>
-</resources>

src/android/utils.h

@@ -1,150 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef _ANDROID_JNI_UTILS_H
-#define _ANDROID_JNI_UTILS_H
-
-#include <jni.h>
-
-#include <pthread.h>
-#include <string>
-
-extern pthread_mutex_t globalLock;
-
-inline jfieldID getHandleField(JNIEnv* env, jobject obj)
-{
-  jclass c = env->GetObjectClass(obj);
-  // J is the type signature for long:
-  return env->GetFieldID(c, "nativeHandle", "J");
-}
-
-class Lock
-{
- protected:
-  pthread_mutex_t* lock;
-
- public:
-  Lock() : lock(&globalLock) { pthread_mutex_lock(lock); }
-  Lock(const Lock&) = delete;
-  Lock& operator=(const Lock&) = delete;
-  Lock(Lock&& other) : lock(&globalLock) { other.lock = nullptr; }
-  virtual ~Lock()
-  {
-    if (lock) {
-      pthread_mutex_unlock(lock);
-    }
-  }
-};
-
-template <class T>
-class LockedHandle;
-
-template <class T>
-class Handle
-{
- protected:
-  T* h;
-
- public:
-  Handle(T* h) : h(h){};
-
-  // No destructor. This must and will be handled by dispose method.
-
-  static LockedHandle<T> getHandle(JNIEnv* env, jobject obj)
-  {
-    jlong handle = env->GetLongField(obj, getHandleField(env, obj));
-    return LockedHandle<T>(reinterpret_cast<Handle<T>*>(handle));
-  }
-
-  static void dispose(JNIEnv* env, jobject obj)
-  {
-    auto lHandle = getHandle(env, obj);
-    auto handle = lHandle.h;
-    delete handle->h;
-    delete handle;
-  }
-  friend class LockedHandle<T>;
-};
-
-template <class T>
-struct LockedHandle : public Lock {
-  Handle<T>* h;
-  LockedHandle(Handle<T>* h) : h(h) {}
-  T* operator->() { return h->h; }
-  T* operator*() { return h->h; }
-  operator bool() const { return (h->h != nullptr); }
-};
-
-/* c2jni type conversion functions */
-inline jboolean c2jni(const bool& val) { return val ? JNI_TRUE : JNI_FALSE; }
-inline jstring c2jni(const std::string& val, JNIEnv* env)
-{
-  return env->NewStringUTF(val.c_str());
-}
-
-inline jint c2jni(const int val) { return (jint)val; }
-inline jint c2jni(const unsigned val) { return (unsigned)val; }
-/* jni2c type conversion functions */
-inline bool jni2c(const jboolean& val) { return val == JNI_TRUE; }
-inline std::string jni2c(const jstring& val, JNIEnv* env)
-{
-  const char* chars = env->GetStringUTFChars(val, 0);
-  std::string ret(chars);
-  env->ReleaseStringUTFChars(val, chars);
-  return ret;
-}
-
-inline int jni2c(const jint val) { return (int)val; }
-/* Method to deal with variable passed by reference */
-inline void setStringObjValue(const std::string& value,
-                              const jobject obj,
-                              JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
-  env->SetObjectField(obj, objFid, c2jni(value, env));
-}
-
-inline void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "I");
-  env->SetIntField(obj, objFid, value);
-}
-
-inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
-  env->SetIntField(obj, objFid, c2jni(value));
-}
-
-inline void setPairObjValue(const std::string& filename, const int offset,
-                            const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID filenameFid = env->GetFieldID(objClass, "filename", "Ljava/lang/String;");
-  env->SetObjectField(obj, filenameFid, c2jni(filename, env));
-  jfieldID offsetFid = env->GetFieldID(objClass, "offset", "I");
-  env->SetIntField(obj, offsetFid, offset);
-}
-
-#endif // _ANDROID_JNI_UTILS_H

src/aria2.cpp

@@ -2,23 +2,40 @@
 
 #include "aria2.h"
 #include "xmlrpc.h"
+#include <iostream>
+#include <algorithm>
 #include <sstream>
 #include <thread>
 #include <chrono>
-#include <common/otherTools.h>
+#include "tools.h"
-#include <common/pathTools.h>
+#include "tools/pathTools.h"
-#include <downloader.h> // For AriaError
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "downloader.h" // For AriaError
+
+#ifdef _WIN32
+# define ARIA2_CMD "aria2c.exe"
+#else
+# define ARIA2_CMD "aria2c"
+# include <unistd.h>
+#endif
+
+
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (curlErrorBuffer[0] ? curlErrorBuffer : curl_easy_strerror(res)) << std::endl; \
+  }
 
 namespace kiwix {
 
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret("kiwixariarpc"),
+  m_secret(getNewRpcSecret())
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
 {
   m_downloadDir = getDataDirectory();
+  makeDirectory(m_downloadDir);
   std::vector<const char*> callCmd;
 
   std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
@@ -36,16 +53,22 @@ Aria2::Aria2():
   std::string rpc_secret = "--rpc-secret=" + m_secret;
   m_secret = "token:"+m_secret;
 
-#ifdef _WIN32
+  std::string aria2cmd = appendToDirectory(
-  callCmd.push_back("aria2c.exe");
+    removeLastPathElement(getExecutablePath(true)),
-#else
+    ARIA2_CMD);
-  callCmd.push_back("aria2c");
+  if (fileExists(aria2cmd)) {
-#endif
+    // A local aria2c exe exists (packaged with kiwix-desktop), use it.
+    callCmd.push_back(aria2cmd.c_str());
+  } else {
+    // Try to use a potential installed aria2c.
+    callCmd.push_back(ARIA2_CMD);
+  }
+  callCmd.push_back("--follow-metalink=mem");
   callCmd.push_back("--enable-rpc");
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
   callCmd.push_back(download_dir.c_str());
-  if (fileExists(session_file)) {
+  if (fileReadable(session_file)) {
     callCmd.push_back(inputFile.c_str());
   }
   callCmd.push_back(session.c_str());
@@ -61,31 +84,37 @@ Aria2::Aria2():
   callCmd.push_back("--max-concurrent-downloads=42");
   callCmd.push_back("--rpc-max-request-size=6M");
   callCmd.push_back("--file-allocation=none");
+  std::string launchCmd;
+  for (auto &cmd : callCmd) {
+      launchCmd.append(cmd).append(" ");
+  }
   mp_aria = Subprocess::run(callCmd);
-  mp_curl = curl_easy_init();
+
-  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
+  CURL* p_curl = curl_easy_init();
-  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
+  char curlErrorBuffer[CURL_ERROR_SIZE];
-  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
 
   int watchdog = 50;
   while(--watchdog) {
-    std::this_thread::sleep_for(std::chrono::microseconds(100));
+    sleep(10);
-    auto res = curl_easy_perform(mp_curl);
+    curlErrorBuffer[0] = 0;
+    auto res = curl_easy_perform(p_curl);
     if (res == CURLE_OK) {
       break;
+    } else if (watchdog == 1) {
+      LOG_ARIA_ERROR();
     }
   }
+  curl_easy_cleanup(p_curl);
   if (!watchdog) {
-    curl_easy_cleanup(mp_curl);
+    throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
-    throw std::runtime_error("Cannot connect to aria2c rpc");
   }
 }
 
-Aria2::~Aria2()
-{
-  curl_easy_cleanup(mp_curl);
-}
-
 void Aria2::close()
 {
   saveSession();
@@ -94,47 +123,61 @@ void Aria2::close()
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
-  auto str = static_cast<std::stringstream*>(userdata);
+  auto outStream = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
+  outStream->write(ptr, nmemb);
   return nmemb;
 }
 
 std::string Aria2::doRequest(const MethodCall& methodCall)
 {
-  pthread_mutex_lock(&m_lock);
   auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
+  std::stringstream outStream;
   CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+  long response_code;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+  char curlErrorBuffer[CURL_ERROR_SIZE];
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  CURL* p_curl = curl_easy_init();
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
-  res = curl_easy_perform(mp_curl);
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
-  if (res == CURLE_OK) {
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
-    long response_code;
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
-    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-    pthread_mutex_unlock(&m_lock);
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-    if (response_code != 200) {
+  curl_easy_setopt(p_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-      throw std::runtime_error("Invalid return code from aria");
+  curl_easy_setopt(p_curl, CURLOPT_WRITEDATA, &outStream);
-    }
+  curlErrorBuffer[0] = 0;
-    auto responseContent = stringstream.str();
+  res = curl_easy_perform(p_curl);
-    MethodResponse response(responseContent);
+  if (res != CURLE_OK) {
-    if (response.isFault()) {
+    LOG_ARIA_ERROR();
-      throw AriaError(response.getFault().getFaultString());
+    curl_easy_cleanup(p_curl);
-    }
+    throw std::runtime_error("Cannot perform request");
-    return responseContent;
   }
-  pthread_mutex_unlock(&m_lock);
+  curl_easy_getinfo(p_curl, CURLINFO_RESPONSE_CODE, &response_code);
-  throw std::runtime_error("Cannot perform request");
+  curl_easy_cleanup(p_curl);
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
 }
 
-std::string Aria2::addUri(const std::vector<std::string>& uris)
+std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
 {
   MethodCall methodCall("aria2.addUri", m_secret);
   auto uriParams = methodCall.newParamValue().getArray();
   for (auto& uri : uris) {
     uriParams.addValue().set(uri);
   }
+  for (auto& option : options) {
+    methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
+  }
   auto ret = doRequest(methodCall);
   MethodResponse response(ret);
   return response.getParamValue(0).getAsS();
@@ -153,12 +196,18 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
   return doRequest(methodCall);
 }
 
+std::string Aria2::getNewRpcSecret()
+{
+  std::string uuid = gen_uuid("");
+  uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
+  return uuid.substr(0, 9);
+}
+
 std::vector<std::string> Aria2::tellActive()
 {
   MethodCall methodCall("aria2.tellActive", m_secret);
   auto statusArray = methodCall.newParamValue().getArray();
   statusArray.addValue().set(std::string("gid"));
-  statusArray.addValue().set(std::string("following"));
   auto responseContent = doRequest(methodCall);
   MethodResponse response(responseContent);
   std::vector<std::string> activeGID;
@@ -173,6 +222,27 @@ std::vector<std::string> Aria2::tellActive()
   return activeGID;
 }
 
+std::vector<std::string> Aria2::tellWaiting()
+{
+  MethodCall methodCall("aria2.tellWaiting", m_secret);
+  methodCall.newParamValue().set(0);
+  methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> waitingGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      waitingGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return waitingGID;
+}
+
 void Aria2::saveSession()
 {
   MethodCall methodCall("aria2.saveSession", m_secret);
@@ -186,5 +256,25 @@ void Aria2::shutdown()
   doRequest(methodCall);
 }
 
+void Aria2::pause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.pause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::unpause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.unpause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::remove(const std::string& gid)
+{
+    MethodCall methodCall("aria2.remove", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
 
 } // end namespace kiwix

src/aria2.h

@@ -13,7 +13,6 @@
 
 #include <memory>
 #include <curl/curl.h>
-#include <pthread.h>
 
 namespace kiwix {
 
@@ -24,21 +23,23 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
-    CURL* mp_curl;
-    pthread_mutex_t m_lock;
-
     std::string doRequest(const MethodCall& methodCall);
 
   public:
     Aria2();
-    virtual ~Aria2();
+    virtual ~Aria2() = default;
     void close();
 
-    std::string addUri(const std::vector<std::string>& uri);
+    std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
+    static std::string getNewRpcSecret();
     std::vector<std::string> tellActive();
+    std::vector<std::string> tellWaiting();
     void saveSession();
     void shutdown();
+    void pause(const std::string& gid);
+    void unpause(const std::string& gid);
+    void remove(const std::string& gid);
 };
 
 }; //end namespace kiwix

src/book.cpp

@@ -18,78 +18,79 @@
  */
 
 #include "book.h"
-#include "reader.h"
 
-#include "common/base64.h"
+#include "tools.h"
-#include "common/regexTools.h"
+#include "tools/base64.h"
-#include "common/networkTools.h"
+#include "tools/regexTools.h"
+#include "tools/networkTools.h"
+#include "tools/otherTools.h"
+#include "tools/stringTools.h"
+#include "tools/pathTools.h"
+#include "tools/archiveTools.h"
 
+#include <zim/archive.h>
+#include <zim/item.h>
 #include <pugixml.hpp>
 
 namespace kiwix
 {
 /* Constructor */
-Book::Book() : m_readOnly(false)
+Book::Book() :
+  m_pathValid(false),
+  m_readOnly(false)
 {
 }
+
 /* Destructor */
 Book::~Book()
 {
 }
 
+Book::Illustrations Book::getIllustrations() const
+{
+  return m_illustrations;
+}
+
 bool Book::update(const kiwix::Book& other)
 {
   if (m_readOnly)
     return false;
 
-  m_readOnly = other.m_readOnly;
+  if (m_id != other.m_id)
+    return false;
 
-  if (m_path.empty()) {
+  *this = other;
-    m_path = other.m_path;
-  }
-
-  if (m_url.empty()) {
-    m_url = other.m_url;
-  }
-
-  if (m_tags.empty()) {
-    m_tags = other.m_tags;
-  }
-
-  if (m_name.empty()) {
-    m_name = other.m_name;
-  }
-
-  if (m_indexPath.empty()) {
-    m_indexPath = other.m_indexPath;
-    m_indexType = other.m_indexType;
-  }
-
-  if (m_faviconMimeType.empty()) {
-    m_favicon = other.m_favicon;
-    m_faviconMimeType = other.m_faviconMimeType;
-  }
   return true;
 }
 
-void Book::update(const kiwix::Reader& reader)
+void Book::update(const zim::Archive& archive) {
-{
+  m_path = archive.getFilename();
-  m_path = reader.getZimFilePath();
+  m_pathValid = true;
-  m_id = reader.getId();
+  m_id = std::string(archive.getUuid());
-  m_description = reader.getDescription();
+  m_title = getArchiveTitle(archive);
-  m_language = reader.getLanguage();
+  m_description = getMetaDescription(archive);
-  m_date = reader.getDate();
+  m_language = getMetaLanguage(archive);
-  m_creator = reader.getCreator();
+  m_creator = getMetaCreator(archive);
-  m_publisher = reader.getPublisher();
+  m_publisher = getMetaPublisher(archive);
-  m_title = reader.getTitle();
+  m_date = getMetaDate(archive);
-  m_name = reader.getName();
+  m_name = getMetaName(archive);
-  m_tags = reader.getTags();
+  m_flavour = getMetaFlavour(archive);
-  m_origId = reader.getOrigId();
+  m_tags = getMetaTags(archive);
-  m_articleCount = reader.getArticleCount();
+  m_category = getCategoryFromTags();
-  m_mediaCount = reader.getMediaCount();
+  m_articleCount = archive.getArticleCount();
-  m_size = reader.getFileSize() << 10;
+  m_mediaCount = archive.getMediaCount();
+  m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
-  reader.getFavicon(m_favicon, m_faviconMimeType);
+  m_illustrations.clear();
+  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+    const auto illustration = std::make_shared<Illustration>();
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
+    illustration->width = illustration->height = illustrationSize;
+    illustration->mimeType = illustrationItem.getMimetype();
+    illustration->data = illustrationItem.getData();
+    // NOTE: illustration->url is left uninitialized
+    m_illustrations.push_back(illustration);
+  }
 }
 
 #define ATTR(name) node.attribute(name).value()
@@ -101,36 +102,46 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
     path = computeAbsolutePath(baseDir, path);
   }
   m_path = path;
-  path = ATTR("indexPath");
+  m_pathValid = fileReadable(path);
-  if (!path.empty()) {
-    if (isRelativePath(path)) {
-      path = computeAbsolutePath(baseDir, path);
-    }
-    m_indexPath = path;
-    m_indexType = XAPIAN;
-  }
   m_title = ATTR("title");
-  m_name = ATTR("name");
-  m_tags = ATTR("tags");
   m_description = ATTR("description");
   m_language = ATTR("language");
-  m_date = ATTR("date");
   m_creator = ATTR("creator");
   m_publisher = ATTR("publisher");
+  m_date = ATTR("date");
   m_url = ATTR("url");
+  m_name = ATTR("name");
+  m_flavour = ATTR("flavour");
+  m_tags = ATTR("tags");
   m_origId = ATTR("origId");
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  m_favicon = base64_decode(ATTR("favicon"));
+  std::string favicon_mimetype = ATTR("faviconMimeType");
-  m_faviconMimeType = ATTR("faviconMimeType");
+  if (! favicon_mimetype.empty()) {
+    const auto favicon = std::make_shared<Illustration>();
+    favicon->data = base64_decode(ATTR("favicon"));
+    favicon->mimeType = favicon_mimetype;
+    favicon->url = ATTR("faviconUrl");
+    m_illustrations.assign(1, favicon);
+  }
   try {
     m_downloadId = ATTR("downloadId");
   } catch(...) {}
+  const auto catattr = node.attribute("category");
+  m_category = catattr.empty() ? getCategoryFromTags() : catattr.value();
 }
 #undef ATTR
 
 
+static std::string fromOpdsDate(const std::string& date)
+{
+  //The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
+  //and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
+  return date.substr(0, 10);
+}
+
+
 #define VALUE(name) node.child(name).child_value()
 void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
 {
@@ -138,11 +149,22 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   if (!m_id.compare(0, 9, "urn:uuid:")) {
     m_id.erase(0, 9);
   }
+  // No path on opds.
   m_title = VALUE("title");
-  m_description = VALUE("description");
+  m_description = VALUE("summary");
   m_language = VALUE("language");
-  m_date = VALUE("updated");
   m_creator = node.child("author").child("name").child_value();
+  m_publisher = node.child("publisher").child("name").child_value();
+  const std::string dcIssuedDate = VALUE("dc:issued");
+  m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
+  m_date = fromOpdsDate(m_date);
+  m_name = VALUE("name");
+  m_flavour = VALUE("flavour");
+  m_tags = VALUE("tags");
+  const auto catnode = node.child("category");
+  m_category = catnode.empty() ? getCategoryFromTags() : catnode.child_value();
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();
@@ -152,19 +174,22 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
       m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
     }
     if (rel == "http://opds-spec.org/image/thumbnail") {
-      m_faviconUrl = urlHost + linkNode.attribute("href").value();
+      const auto favicon = std::make_shared<Illustration>();
-      m_faviconMimeType = linkNode.attribute("type").value();
+      favicon->data.clear();
+      favicon->url = urlHost + linkNode.attribute("href").value();
+      favicon->mimeType = linkNode.attribute("type").value();
+      m_illustrations.assign(1, favicon);
     }
  }
 
 }
 #undef VALUE
 
-std::string Book::getHumanReadableIdFromPath()
+std::string Book::getHumanReadableIdFromPath() const
 {
   std::string id = m_path;
   if (!id.empty()) {
-    kiwix::removeAccents(id);
+    id = kiwix::removeAccents(id);
 
 #ifdef _WIN32
     id = replaceRegex(id, "", "^.*\\\\");
@@ -186,22 +211,84 @@ void Book::setPath(const std::string& path)
    : path;
 }
 
-void Book::setIndexPath(const std::string& indexPath)
+const Book::Illustration Book::missingDefaultIllustration;
+
+std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
 {
-  m_indexPath = isRelativePath(indexPath)
+  for ( const auto& ilPtr : m_illustrations ) {
-    ? computeAbsolutePath(getCurrentDirectory(), indexPath)
+    if (ilPtr->width == size && ilPtr->height == size) {
-    : indexPath;
+      return ilPtr;
+    }
+  }
+  throw std::runtime_error("Cannot find illustration");
+}
+
+const Book::Illustration& Book::getDefaultIllustration() const
+{
+  try {
+    return *getIllustration(48);
+  } catch (...) {
+    return missingDefaultIllustration;
+  }
+}
+
+const std::string& Book::Illustration::getData() const
+{
+  if (data.empty() && !url.empty()) {
+    const std::lock_guard<std::mutex> l(mutex);
+    if ( data.empty() ) {
+      try {
+        data = download(url);
+      } catch(...) {
+        std::cerr << "Cannot download favicon from " << url;
+      }
+    }
+  }
+  return data;
 }
 
 const std::string& Book::getFavicon() const {
-  if (m_favicon.empty() && !m_faviconUrl.empty()) {
+  return getDefaultIllustration().getData();
-    try {
+}
-      m_favicon = download(m_faviconUrl);
+
-    } catch(...) {
+const std::string& Book::getFaviconUrl() const
-      std::cerr << "Cannot download favicon from " << m_faviconUrl;
+{
-    }
+  return getDefaultIllustration().url;
+}
+
+const std::string& Book::getFaviconMimeType() const
+{
+  return getDefaultIllustration().mimeType;
+}
+
+std::string Book::getTagStr(const std::string& tagName) const {
+  return getTagValueFromTagList(convertTags(m_tags), tagName);
+}
+
+bool Book::getTagBool(const std::string& tagName) const {
+  return convertStrToBool(getTagStr(tagName));
+}
+
+std::string Book::getCategory() const
+{
+  return m_category;
+}
+
+std::string Book::getCategoryFromTags() const
+{
+  try
+  {
+    return getTagStr("category");
   }
-  return m_favicon;
+  catch ( const std::out_of_range& )
+  {
+    return "";
+  }
+}
+
+const std::vector<std::string> Book::getLanguages() const
+{
+  return kiwix::split(m_language, ",");
 }
 
 }

src/bookmark.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "bookmark.h"
+
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+/* Constructor */
+Bookmark::Bookmark()
+{
+}
+
+/* Destructor */
+Bookmark::~Bookmark()
+{
+}
+
+void Bookmark::updateFromXml(const pugi::xml_node& node)
+{
+  auto bookNode = node.child("book");
+  m_bookId = bookNode.child("id").child_value();
+  m_bookTitle = bookNode.child("title").child_value();
+  m_language = bookNode.child("language").child_value();
+  m_date = bookNode.child("date").child_value();
+  m_title = node.child("title").child_value();
+  m_url = node.child("url").child_value();
+}
+
+}

src/common/otherTools.cpp

@@ -1,300 +0,0 @@
-/*
- * Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/otherTools.h>
-#include <map>
-
-std::map<std::string, std::string> codeisomapping {
-  { "af", "afg" },
-  { "ax", "ala" },
-  { "al", "alb" },
-  { "dz", "dza" },
-  { "as", "asm" },
-  { "ad", "and" },
-  { "ao", "ago" },
-  { "ai", "aia" },
-  { "aq", "ata" },
-  { "ag", "atg" },
-  { "ar", "arg" },
-  { "am", "arm" },
-  { "aw", "abw" },
-  { "au", "aus" },
-  { "at", "aut" },
-  { "az", "aze" },
-  { "bs", "bhs" },
-  { "bh", "bhr" },
-  { "bd", "bgd" },
-  { "bb", "brb" },
-  { "by", "blr" },
-  { "be", "bel" },
-  { "bz", "blz" },
-  { "bj", "ben" },
-  { "bm", "bmu" },
-  { "bt", "btn" },
-  { "bo", "bol" },
-  { "ba", "bih" },
-  { "bw", "bwa" },
-  { "bv", "bvt" },
-  { "br", "bra" },
-  { "vg", "vgb" },
-  { "io", "iot" },
-  { "bn", "brn" },
-  { "bg", "bgr" },
-  { "bf", "bfa" },
-  { "bi", "bdi" },
-  { "kh", "khm" },
-  { "cm", "cmr" },
-  { "ca", "can" },
-  { "cv", "cpv" },
-  { "ky", "cym" },
-  { "cf", "caf" },
-  { "td", "tcd" },
-  { "cl", "chl" },
-  { "cn", "chn" },
-  { "hk", "hkg" },
-  { "mo", "mac" },
-  { "cx", "cxr" },
-  { "cc", "cck" },
-  { "co", "col" },
-  { "km", "com" },
-  { "cg", "cog" },
-  { "cd", "cod" },
-  { "ck", "cok" },
-  { "cr", "cri" },
-  { "ci", "civ" },
-  { "hr", "hrv" },
-  { "cu", "cub" },
-  { "cy", "cyp" },
-  { "cz", "cze" },
-  { "dk", "dnk" },
-  { "dj", "dji" },
-  { "dm", "dma" },
-  { "do", "dom" },
-  { "ec", "ecu" },
-  { "eg", "egy" },
-  { "sv", "slv" },
-  { "gq", "gnq" },
-  { "er", "eri" },
-  { "ee", "est" },
-  { "et", "eth" },
-  { "fk", "flk" },
-  { "fo", "fro" },
-  { "fj", "fji" },
-  { "fi", "fin" },
-  { "fr", "fra" },
-  { "gf", "guf" },
-  { "pf", "pyf" },
-  { "tf", "atf" },
-  { "ga", "gab" },
-  { "gm", "gmb" },
-  { "ge", "geo" },
-  { "de", "deu" },
-  { "gh", "gha" },
-  { "gi", "gib" },
-  { "gr", "grc" },
-  { "gl", "grl" },
-  { "gd", "grd" },
-  { "gp", "glp" },
-  { "gu", "gum" },
-  { "gt", "gtm" },
-  { "gg", "ggy" },
-  { "gn", "gin" },
-  { "gw", "gnb" },
-  { "gy", "guy" },
-  { "ht", "hti" },
-  { "hm", "hmd" },
-  { "va", "vat" },
-  { "hn", "hnd" },
-  { "hu", "hun" },
-  { "is", "isl" },
-  { "in", "ind" },
-  { "id", "idn" },
-  { "ir", "irn" },
-  { "iq", "irq" },
-  { "ie", "irl" },
-  { "im", "imn" },
-  { "il", "isr" },
-  { "it", "ita" },
-  { "jm", "jam" },
-  { "jp", "jpn" },
-  { "je", "jey" },
-  { "jo", "jor" },
-  { "kz", "kaz" },
-  { "ke", "ken" },
-  { "ki", "kir" },
-  { "kp", "prk" },
-  { "kr", "kor" },
-  { "kw", "kwt" },
-  { "kg", "kgz" },
-  { "la", "lao" },
-  { "lv", "lva" },
-  { "lb", "lbn" },
-  { "ls", "lso" },
-  { "lr", "lbr" },
-  { "ly", "lby" },
-  { "li", "lie" },
-  { "lt", "ltu" },
-  { "lu", "lux" },
-  { "mk", "mkd" },
-  { "mg", "mdg" },
-  { "mw", "mwi" },
-  { "my", "mys" },
-  { "mv", "mdv" },
-  { "ml", "mli" },
-  { "mt", "mlt" },
-  { "mh", "mhl" },
-  { "mq", "mtq" },
-  { "mr", "mrt" },
-  { "mu", "mus" },
-  { "yt", "myt" },
-  { "mx", "mex" },
-  { "fm", "fsm" },
-  { "md", "mda" },
-  { "mc", "mco" },
-  { "mn", "mng" },
-  { "me", "mne" },
-  { "ms", "msr" },
-  { "ma", "mar" },
-  { "mz", "moz" },
-  { "mm", "mmr" },
-  { "na", "nam" },
-  { "nr", "nru" },
-  { "np", "npl" },
-  { "nl", "nld" },
-  { "an", "ant" },
-  { "nc", "ncl" },
-  { "nz", "nzl" },
-  { "ni", "nic" },
-  { "ne", "ner" },
-  { "ng", "nga" },
-  { "nu", "niu" },
-  { "nf", "nfk" },
-  { "mp", "mnp" },
-  { "no", "nor" },
-  { "om", "omn" },
-  { "pk", "pak" },
-  { "pw", "plw" },
-  { "ps", "pse" },
-  { "pa", "pan" },
-  { "pg", "png" },
-  { "py", "pry" },
-  { "pe", "per" },
-  { "ph", "phl" },
-  { "pn", "pcn" },
-  { "pl", "pol" },
-  { "pt", "prt" },
-  { "pr", "pri" },
-  { "qa", "qat" },
-  { "re", "reu" },
-  { "ro", "rou" },
-  { "ru", "rus" },
-  { "rw", "rwa" },
-  { "bl", "blm" },
-  { "sh", "shn" },
-  { "kn", "kna" },
-  { "lc", "lca" },
-  { "mf", "maf" },
-  { "pm", "spm" },
-  { "vc", "vct" },
-  { "ws", "wsm" },
-  { "sm", "smr" },
-  { "st", "stp" },
-  { "sa", "sau" },
-  { "sn", "sen" },
-  { "rs", "srb" },
-  { "sc", "syc" },
-  { "sl", "sle" },
-  { "sg", "sgp" },
-  { "sk", "svk" },
-  { "si", "svn" },
-  { "sb", "slb" },
-  { "so", "som" },
-  { "za", "zaf" },
-  { "gs", "sgs" },
-  { "ss", "ssd" },
-  { "es", "esp" },
-  { "lk", "lka" },
-  { "sd", "sdn" },
-  { "sr", "sur" },
-  { "sj", "sjm" },
-  { "sz", "swz" },
-  { "se", "swe" },
-  { "ch", "che" },
-  { "sy", "syr" },
-  { "tw", "twn" },
-  { "tj", "tjk" },
-  { "tz", "tza" },
-  { "th", "tha" },
-  { "tl", "tls" },
-  { "tg", "tgo" },
-  { "tk", "tkl" },
-  { "to", "ton" },
-  { "tt", "tto" },
-  { "tn", "tun" },
-  { "tr", "tur" },
-  { "tm", "tkm" },
-  { "tc", "tca" },
-  { "tv", "tuv" },
-  { "ug", "uga" },
-  { "ua", "ukr" },
-  { "ae", "are" },
-  { "gb", "gbr" },
-  { "us", "usa" },
-  { "um", "umi" },
-  { "uy", "ury" },
-  { "uz", "uzb" },
-  { "vu", "vut" },
-  { "ve", "ven" },
-  { "vn", "vnm" },
-  { "vi", "vir" },
-  { "wf", "wlf" },
-  { "eh", "esh" },
-  { "ye", "yem" },
-  { "zm", "zmb" },
-  { "zw", "zwe" }
-};
-
-void kiwix::sleep(unsigned int milliseconds)
-{
-#ifdef _WIN32
-  Sleep(milliseconds);
-#else
-  usleep(1000 * milliseconds);
-#endif
-}
-
-
-struct XmlStringWriter: pugi::xml_writer
-{
-  std::string result;
-  virtual void write(const void* data, size_t size){
-    result.append(static_cast<const char*>(data), size);
-  }
-};
-
-std::string kiwix::nodeToString(pugi::xml_node node)
-{
-  XmlStringWriter writer;
-  node.print(writer, "  ");
-  return writer.result;
-}
-
-std::string kiwix::converta2toa3(const std::string& a2code){
-  return codeisomapping.at(a2code);
-}

src/common/pathTools.cpp

@@ -1,338 +0,0 @@
-/*
- * Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <common/pathTools.h>
-
-#ifdef __APPLE__
-#include <limits.h>
-#include <mach-o/dyld.h>
-#elif _WIN32
-#include <direct.h>
-#include <windows.h>
-#include "shlwapi.h"
-#define getcwd _getcwd  // stupid MSFT "deprecation" warning
-#endif
-
-#ifdef _WIN32
-#else
-#include <unistd.h>
-#endif
-
-#ifdef _WIN32
-#define SEPARATOR "\\"
-#else
-#define SEPARATOR "/"
-#include <unistd.h>
-#endif
-
-#include <stdlib.h>
-
-#ifndef PATH_MAX
-#define PATH_MAX 1024
-#endif
-
-bool isRelativePath(const string& path)
-{
-#ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
-#else
-  return path.empty() || path.substr(0, 1) == "/" ? false : true;
-#endif
-}
-
-string computeRelativePath(const string path, const string absolutePath)
-{
-  std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
-  std::vector<std::string> absolutePathParts
-      = kiwix::split(absolutePath, SEPARATOR);
-
-  unsigned int commonCount = 0;
-  while (commonCount < pathParts.size()
-         && commonCount < absolutePathParts.size()
-         && pathParts[commonCount] == absolutePathParts[commonCount]) {
-    if (!pathParts[commonCount].empty()) {
-      commonCount++;
-    }
-  }
-
-  string relativePath;
-#ifdef _WIN32
-  /* On Windows you have a token more because the root is represented
-     by a letter */
-  if (commonCount == 0) {
-    relativePath = "../";
-  }
-#endif
-
-  for (unsigned int i = commonCount; i < pathParts.size(); i++) {
-    relativePath += "../";
-  }
-  for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
-    relativePath += absolutePathParts[i];
-    relativePath += i + 1 < absolutePathParts.size() ? "/" : "";
-  }
-
-  return relativePath;
-}
-
-/* Warning: the relative path must be with slashes */
-string computeAbsolutePath(const string path, const string relativePath)
-{
-  string absolutePath;
-
-  if (path.empty()) {
-    char* path = NULL;
-    size_t size = 0;
-
-#ifdef _WIN32
-    path = _getcwd(path, size);
-#else
-    path = getcwd(path, size);
-#endif
-
-    absolutePath = string(path) + SEPARATOR;
-  } else {
-    absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
-                       ? path
-                       : path + SEPARATOR;
-  }
-
-#if _WIN32
-  char* cRelativePath = _strdup(relativePath.c_str());
-#else
-  char* cRelativePath = strdup(relativePath.c_str());
-#endif
-  char* token = strtok(cRelativePath, "/");
-
-  while (token != NULL) {
-    if (string(token) == "..") {
-      absolutePath = removeLastPathElement(absolutePath, true, false);
-      token = strtok(NULL, "/");
-    } else if (strcmp(token, ".") && strcmp(token, "")) {
-      absolutePath += string(token);
-      token = strtok(NULL, "/");
-      if (token != NULL) {
-        absolutePath += SEPARATOR;
-      }
-    } else {
-      token = strtok(NULL, "/");
-    }
-  }
-
-  return absolutePath;
-}
-
-string removeLastPathElement(const string path,
-                             const bool removePreSeparator,
-                             const bool removePostSeparator)
-{
-  string newPath = path;
-  size_t offset = newPath.find_last_of(SEPARATOR);
-  if (removePreSeparator &&
-#ifndef _WIN32
-      offset != newPath.find_first_of(SEPARATOR) &&
-#endif
-      offset == newPath.length() - 1) {
-    newPath = newPath.substr(0, offset);
-    offset = newPath.find_last_of(SEPARATOR);
-  }
-  newPath = removePostSeparator ? newPath.substr(0, offset)
-                                : newPath.substr(0, offset + 1);
-  return newPath;
-}
-
-string appendToDirectory(const string& directoryPath, const string& filename)
-{
-  string newPath = directoryPath + SEPARATOR + filename;
-  return newPath;
-}
-
-string getLastPathElement(const string& path)
-{
-  return path.substr(path.find_last_of(SEPARATOR) + 1);
-}
-
-unsigned int getFileSize(const string& path)
-{
-#ifdef _WIN32
-  struct _stat filestatus;
-  _stat(path.c_str(), &filestatus);
-#else
-  struct stat filestatus;
-  stat(path.c_str(), &filestatus);
-#endif
-
-  return filestatus.st_size / 1024;
-}
-
-string getFileSizeAsString(const string& path)
-{
-  ostringstream convert;
-  convert << getFileSize(path);
-  return convert.str();
-}
-
-string getFileContent(const string& path)
-{
-  std::ifstream f(path, std::ios::in|std::ios::ate);
-  std::string content;
-  if (f.is_open()) {
-    auto size = f.tellg();
-    content.reserve(size);
-    f.seekg(0, std::ios::beg);
-    content.assign((std::istreambuf_iterator<char>(f)),
-                    std::istreambuf_iterator<char>());
-  }
-  return content;
-}
-
-bool fileExists(const string& path)
-{
-#ifdef _WIN32
-  return PathFileExists(path.c_str());
-#else
-  bool flag = false;
-  fstream fin;
-  fin.open(path.c_str(), ios::in);
-  if (fin.is_open()) {
-    flag = true;
-  }
-  fin.close();
-  return flag;
-#endif
-}
-
-bool makeDirectory(const string& path)
-{
-#ifdef _WIN32
-  int status = _mkdir(path.c_str());
-#else
-  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-#endif
-  return status == 0;
-}
-
-string makeTmpDirectory()
-{
-#ifdef _WIN32
-  char cbase[MAX_PATH+1];
-  int base_len = GetTempPath(MAX_PATH+1, cbase);
-  UUID uuid;
-  UuidCreate(&uuid);
-  char* dir_name;
-  UuidToString(&uuid, reinterpret_cast<unsigned char**>(&dir_name));
-  string dir(cbase, base_len);
-  dir += dir_name;
-  _mkdir(dir.c_str());
-  RpcStringFree(reinterpret_cast<unsigned char**>(&dir_name));
-#else
-  string base = "/tmp";
-  auto _template = base + "/kiwix-lib_XXXXXX";
-  char* _template_array = new char[_template.size()+1];
-  memcpy(_template_array, _template.c_str(), _template.size());
-  string dir = mkdtemp(_template_array);
-  delete[] _template_array;
-#endif
-  return dir;
-}
-
-/* Try to create a link and if does not work then make a copy */
-bool copyFile(const string& sourcePath, const string& destPath)
-{
-  try {
-#ifndef _WIN32
-    if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
-#endif
-      std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
-      std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
-      outfile << infile.rdbuf();
-#ifndef _WIN32
-    }
-#endif
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return false;
-  }
-
-  return true;
-}
-
-string getExecutablePath()
-{
-  char binRootPath[PATH_MAX];
-
-#ifdef _WIN32
-  GetModuleFileName(NULL, binRootPath, PATH_MAX);
-  return std::string(binRootPath);
-#elif __APPLE__
-  uint32_t max = (uint32_t)PATH_MAX;
-  _NSGetExecutablePath(binRootPath, &max);
-  return std::string(binRootPath);
-#else
-  ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
-  if (size != -1) {
-    return std::string(binRootPath, size);
-  }
-#endif
-
-  return "";
-}
-
-bool writeTextFile(const string& path, const string& content)
-{
-  std::ofstream file;
-  file.open(path.c_str());
-  file << content;
-  file.close();
-  return true;
-}
-
-string getCurrentDirectory()
-{
-  char* a_cwd = getcwd(NULL, 0);
-  string s_cwd(a_cwd);
-  free(a_cwd);
-  return s_cwd;
-}
-
-string getDataDirectory()
-{
-#ifdef _WIN32
-  char* cDataDir = ::getenv("APPDATA");
-#else
-  char* cDataDir = ::getenv("KIWIX_DATA_DIR");
-#endif
-  std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (!dataDir.empty())
-    return dataDir;
-#ifdef _WIN32
-  cDataDir = ::getenv("USERPROFILE");
-  dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-#else
-  cDataDir = ::getenv("XDG_DATA_HOME");
-  dataDir = cDataDir==nullptr ? "" : cDataDir;
-  if (dataDir.empty()) {
-    cDataDir = ::getenv("HOME");
-    dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
-    dataDir = appendToDirectory(dataDir, ".local");
-    dataDir = appendToDirectory(dataDir, "share");
-  }
-#endif
-  return appendToDirectory(dataDir, "kiwix");
-}

src/config.h.in

@@ -1,4 +1,3 @@
 
-#mesondefine VERSION
+#mesondefine LIBKIWIX_VERSION
 
-#mesondefine ENABLE_CTPP2

src/ctpp2/CTPP2VMStringLoader.cpp

@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Renaud Gaudin <reg@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <ctpp2/CTPP2VMStringLoader.hpp>
-
-namespace CTPP // C++ Template Engine
-{
-
-//
-// Convert byte order
-//
-static void ConvertExecutable(VMExecutable * oCore)
-{
-	// Code entry point
-	oCore -> entry_point = Swap32(oCore -> entry_point);
-	// Offset of code segment
-	oCore -> code_offset = Swap32(oCore -> code_offset);
-
-	// Code segment size
-	oCore -> code_size   = Swap32(oCore -> code_size);
-
-	// Offset of static text segment
-	oCore -> syscalls_offset    = Swap32(oCore -> syscalls_offset);
-	// Static text segment size
-	oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
-
-	// Offset of static text index segment
-	oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
-	// Static text index segment size
-	oCore -> syscalls_index_size   = Swap32(oCore -> syscalls_index_size);
-
-	// Offset of static data segment
-	oCore -> static_data_offset    = Swap32(oCore -> static_data_offset);
-
-	// Static data segment size
-	oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
-
-	// Offset of static text segment
-	oCore -> static_text_offset    = Swap32(oCore -> static_text_offset);
-	// Static text segment size
-	oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
-
-	// Offset of static text index segment
-	oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
-	// Static text index segment size
-	oCore -> static_text_index_size   = Swap32(oCore -> static_text_index_size);
-
-	// Version 2.2+
-	// Offset of static data bit index
-	oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
-	/// Offset of static data bit index
-	oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
-
-	// Platform
-	oCore -> platform      = Swap64(oCore -> platform);
-
-	// Ugly-jolly hack!
-	// ... dereferencing type-punned pointer will break strict-aliasing rules ...
-	UINT_64 iTMP;
-	memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
-	iTMP = Swap64(iTMP);
-	memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
-
-	// Cyclic Redundancy Check
-	oCore -> crc = 0;
-
-	// Convert data structures
-
-	// Convert code segment
-	VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
-	UINT_32 iI = 0;
-	UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pInstructions -> instruction = Swap32(pInstructions -> instruction);
-		pInstructions -> argument    = Swap32(pInstructions -> argument);
-		pInstructions -> reserved    = Swap64(pInstructions -> reserved);
-		++pInstructions;
-	}
-
-	// Convert syscalls index
-	TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
-	iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static text index
-	pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
-	iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		pTextIndex -> offset = Swap32(pTextIndex -> offset);
-		pTextIndex -> length = Swap32(pTextIndex -> length);
-		++pTextIndex;
-	}
-
-	// Convert static data
-	StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
-	iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
-	for(iI = 0; iI < iSteps; ++iI)
-	{
-		(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
-		++pStaticDataVar;
-	}
-}
-
-//
-// Constructor
-//
-VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
-{
-    oCore = (VMExecutable *)malloc(rawContentSize + 1);
-    memcpy(oCore, rawContent, rawContentSize);
-
-	if (oCore -> magic[0] == 'C' &&
-	    oCore -> magic[1] == 'T' &&
-	    oCore -> magic[2] == 'P' &&
-	    oCore -> magic[3] == 'P')
-	{
-		// Check version
-		if (oCore -> version[0] >= 1)
-		{
-			// Platform-dependent data (byte order)
-			if (oCore -> platform == 0x4142434445464748ull)
-			{
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
-#endif
-
-				// Nothing to do, only check crc
-				UINT_32 iCRC = oCore -> crc;
-				oCore -> crc = 0;
-
-				// Calculate CRC of file
-                // KELSON: next line used to refer to oStat.st_size
-                // changed it to rawContentSize
-				if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
-				{
-					free(oCore);
-					throw CTPPLogicError("CRC checksum invalid");
-				}
-			}
-			// Platform-dependent data (byte order)
-			else if (oCore -> platform == 0x4847464544434241ull)
-			{
-				// Need to reconvert data
-#ifdef _DEBUG
-				fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
-#endif
-				ConvertExecutable(oCore);
-			}
-			else
-			{
-				free(oCore);
-				throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
-			}
-
-			// Check IEEE 754 format
-			if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
-			{
-				free(oCore);
-				throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
-			}
-		}
-
-		pVMMemoryCore = new VMMemoryCore(oCore);
-	}
-	else
-	{
-		free(oCore);
-		throw CTPPLogicError("Not an CTPP bytecode file.");
-	}
-}
-
-//
-// Get ready-to-run program
-//
-const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
-
-//
-// A destructor
-//
-VMStringLoader::~VMStringLoader() throw()
-{
-	delete pVMMemoryCore;
-	free(oCore);
-}
-
-} // namespace CTPP
-// End.

src/downloader.cpp

@@ -18,7 +18,8 @@
  */
 
 #include "downloader.h"
-#include "common/pathTools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
 
 #include <algorithm>
 #include <thread>
@@ -28,7 +29,7 @@
 
 #include "aria2.h"
 #include "xmlrpc.h"
-#include "common/otherTools.h"
+#include "tools/otherTools.h"
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -36,6 +37,8 @@ namespace kiwix
 
 void Download::updateStatus(bool follow)
 {
+  if (m_status == Download::K_REMOVED)
+    return;
   static std::vector<std::string> statusKey = {"status", "files", "totalLength",
                                                "completedLength", "followedBy",
                                                "downloadSpeed", "verifiedLength"};
@@ -72,12 +75,12 @@ void Download::updateStatus(bool follow)
     } catch (InvalidRPCNode& e) { }
   }
   m_status = status;
-  m_totalLength = structNode.getMember("totalLength").getValue().getAsI();
+  m_totalLength = extractFromString<uint64_t>(structNode.getMember("totalLength").getValue().getAsS());
-  m_completedLength = structNode.getMember("completedLength").getValue().getAsI();
+  m_completedLength = extractFromString<uint64_t>(structNode.getMember("completedLength").getValue().getAsS());
-  m_downloadSpeed = structNode.getMember("downloadSpeed").getValue().getAsI();
+  m_downloadSpeed = extractFromString<uint64_t>(structNode.getMember("downloadSpeed").getValue().getAsS());
   try {
     auto verifiedLengthValue = structNode.getMember("verifiedLength").getValue();
-    m_verifiedLength = verifiedLengthValue.getAsI();
+    m_verifiedLength = extractFromString<uint64_t>(verifiedLengthValue.getAsS());
   } catch (InvalidRPCNode& e) { m_verifiedLength = 0; }
   auto filesMember = structNode.getMember("files");
   auto fileStruct = filesMember.getValue().getArray().getValue(0).getStruct();
@@ -93,17 +96,57 @@ void Download::updateStatus(bool follow)
   }
 }
 
+void Download::resumeDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->unpause(m_followedBy);
+    else
+        mp_aria->unpause(m_did);
+    updateStatus(true);
+}
+
+void Download::pauseDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->pause(m_followedBy);
+    else
+        mp_aria->pause(m_did);
+    updateStatus(true);
+}
+
+void Download::cancelDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->remove(m_followedBy);
+    else
+        mp_aria->remove(m_did);
+    m_status = Download::K_REMOVED;
+}
+
 /* Constructor */
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  for (auto gid : mp_aria->tellActive()) {
+  try {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+    for (auto gid : mp_aria->tellWaiting()) {
-    m_knownDownloads[gid]->updateStatus();
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus(false);
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
+  }
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      if( m_knownDownloads.find(gid) == m_knownDownloads.end()) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads[gid]->updateStatus(false);
+      }
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
   }
 }
 
-
 /* Destructor */
 Downloader::~Downloader()
 {
@@ -114,7 +157,8 @@ void Downloader::close()
   mp_aria->close();
 }
 
-std::vector<std::string> Downloader::getDownloadIds() {
+std::vector<std::string> Downloader::getDownloadIds() const {
+  std::unique_lock<std::mutex> lock(m_lock);
   std::vector<std::string> ret;
   for(auto& p:m_knownDownloads) {
     ret.push_back(p.first);
@@ -122,33 +166,46 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri)
+std::shared_ptr<Download> Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
     auto& uris = d->getUris();
     if (std::find(uris.begin(), uris.end(), uri) != uris.end())
-      return d.get();
+      return d;
   }
   std::vector<std::string> uris = {uri};
-  auto gid = mp_aria->addUri(uris);
+  auto gid = mp_aria->addUri(uris, options);
-  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+  m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
-  return m_knownDownloads[gid].get();
+  return m_knownDownloads[gid];
 }
 
-Download* Downloader::getDownload(const std::string& did)
+std::shared_ptr<Download> Downloader::getDownload(const std::string& did)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   try {
-    return m_knownDownloads.at(did).get();
+    return m_knownDownloads.at(did);
-  } catch(exception& e) {
+  } catch(std::exception& e) {
+    for (auto gid : mp_aria->tellWaiting()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
+      }
+    }
     for (auto gid : mp_aria->tellActive()) {
       if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
-        return m_knownDownloads[gid].get();
+        return m_knownDownloads[gid];
       }
     }
     throw e;
   }
 }
 
+size_t Downloader::getNbDownload() const {
+  std::unique_lock<std::mutex> lock(m_lock);
+  return m_knownDownloads.size();
+}
+
 }

src/entry.cpp

@@ -1,138 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-
-namespace kiwix
-{
-
-Entry::Entry(zim::Article article)
-  : article(article)
-{
-}
-
-#define RETURN_IF_INVALID(WHAT) if(!good()) { return (WHAT); }
-
-std::string Entry::getPath() const
-{
-  RETURN_IF_INVALID("");
-  return article.getLongUrl();
-}
-
-std::string Entry::getTitle() const
-{
-  RETURN_IF_INVALID("");
-  return article.getTitle();
-}
-
-std::string Entry::getContent() const
-{
-  RETURN_IF_INVALID("");
-  return article.getData();
-}
-
-zim::Blob Entry::getBlob(offset_type offset) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset);
-}
-
-zim::Blob Entry::getBlob(offset_type offset, size_type size) const
-{
-  RETURN_IF_INVALID(zim::Blob());
-  return article.getData(offset, size);
-}
-
-std::pair<std::string, offset_type> Entry::getDirectAccessInfo() const
-{
-  RETURN_IF_INVALID(std::make_pair("", 0));
-  return article.getDirectAccessInformation();
-}
-
-size_type Entry::getSize() const
-{
-  RETURN_IF_INVALID(0);
-  return article.getArticleSize();
-}
-
-std::string Entry::getMimetype() const
-{
-  RETURN_IF_INVALID("");
-  try {
-    return article.getMimeType();
-  } catch (exception& e) {
-    return "application/octet-stream";
-  }
-}
-
-bool Entry::isRedirect() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isRedirect();
-}
-
-bool Entry::isLinkTarget() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isLinktarget();
-}
-
-bool Entry::isDeleted() const
-{
-  RETURN_IF_INVALID(false);
-  return article.isDeleted();
-}
-
-Entry Entry::getRedirectEntry() const
-{
-  RETURN_IF_INVALID(Entry());
-  if ( !article.isRedirect() ) {
-    throw NoEntry();
-  }
-
-  auto targeted_article = article.getRedirectArticle();
-  if ( !targeted_article.good()) {
-    throw NoEntry();
-  }
-  return targeted_article;
-}
-
-Entry Entry::getFinalEntry() const
-{
-  RETURN_IF_INVALID(Entry());
-  if (final_article.good()) {
-    return final_article;
-  }
-
-  int loopCounter = 42;
-  final_article = article;
-  while (final_article.isRedirect() && loopCounter--) {
-    final_article = final_article.getRedirectArticle();
-    if ( !final_article.good()) {
-      throw NoEntry();
-    }
-  }
-
-  return final_article;
-}
-
-}

src/html_dumper.cpp

@@ -0,0 +1,120 @@
+#include "html_dumper.h"
+#include "libkiwix-resources.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+#include "tools/regexTools.h"
+#include "server/i18n.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+HTMLDumper::HTMLDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
+{
+}
+/* Destructor */
+HTMLDumper::~HTMLDumper()
+{
+}
+
+namespace {
+
+std::string humanFriendlyTitle(std::string title)
+{
+  std::string humanFriendlyString = replaceRegex(title, "_", " ");
+  humanFriendlyString[0] = toupper(humanFriendlyString[0]);
+  return humanFriendlyString;
+}
+
+kainjow::mustache::list getTagList(std::string tags)
+{
+  const auto tagsList = kiwix::split(tags, ";", true, false);
+  kainjow::mustache::list finalTagList;
+  for (auto tag : tagsList) {
+  if (tag[0] != '_')
+    finalTagList.push_back(kainjow::mustache::object{
+      {"tag", tag}
+    });
+  }
+  return finalTagList;
+}
+
+} // unnamed namespace
+
+std::string HTMLDumper::dumpPlainHTML(kiwix::Filter filter) const
+{
+  kainjow::mustache::list booksData;
+  const auto filteredBooks = library->filter(filter);
+  const auto searchQuery = filter.getQuery();
+  auto languages = getLanguageData();
+  auto categories = getCategoryData();
+
+  for (auto &category : categories) {
+    const auto categoryName = category.get("name")->string_value();
+    if (categoryName == filter.getCategory()) {
+      category["selected"] = true;
+    }
+    category["hf_name"] = humanFriendlyTitle(categoryName);
+  }
+
+  for (auto &language : languages) {
+    if (language.get("lang_code")->string_value() == filter.getLang()) {
+      language["selected"] = true;
+    }
+  }
+
+  for ( const auto& bookId : filteredBooks ) {
+    const auto bookObj = library->getBookById(bookId);
+    const auto bookTitle = bookObj.getTitle();
+    std::string contentId = "";
+    try {
+      contentId = urlEncode(nameMapper->getNameForId(bookId));
+    } catch (...) {}
+    const auto bookDescription = bookObj.getDescription();
+    const auto langCode = bookObj.getCommaSeparatedLanguages();
+    const auto bookIconUrl = rootLocation + "/catalog/v2/illustration/" + bookId +  "/?size=48";
+    const auto tags = bookObj.getTags();
+    const auto downloadAvailable = (bookObj.getUrl() != "");
+    std::string faviconAttr = "style=background-image:url(" + bookIconUrl + ")";
+    
+    booksData.push_back(kainjow::mustache::object{
+      {"id", contentId},
+      {"title", bookTitle},
+      {"description", bookDescription},
+      {"langCode", langCode},
+      {"faviconAttr", faviconAttr},
+      {"tagList", getTagList(tags)},
+      {"downloadAvailable", downloadAvailable}
+    });
+  }
+
+  auto getTranslation = i18n::GetTranslatedStringWithMsgId(m_userLang);
+
+  const auto translations = kainjow::mustache::object{
+                    getTranslation("search"),
+                    getTranslation("download"),
+                    getTranslation("count-of-matching-books", {{"COUNT", to_string(filteredBooks.size())}}),
+                    getTranslation("book-filtering-all-categories"),
+                    getTranslation("book-filtering-all-languages"),
+                    getTranslation("powered-by-kiwix-html"),
+                    getTranslation("welcome-to-kiwix-server"),
+                    getTranslation("preview-book"),
+                    getTranslation("welcome-page-overzealous-filter", {{"URL", "?lang="}})
+  };
+
+  return render_template(
+             RESOURCE::templates::no_js_library_page_html,
+             kainjow::mustache::object{
+               {"root", rootLocation},
+               {"books", booksData },
+               {"searchQuery", searchQuery},
+               {"languages", languages},
+               {"categories", categories},
+               {"noResults", filteredBooks.size() == 0},
+               {"translations", translations}
+             }
+  );
+}
+
+} // namespace kiwix

src/kiwixserve.cpp

@@ -0,0 +1,82 @@
+#include "kiwixserve.h"
+#include "subprocess.h"
+
+#ifdef _WIN32
+    # define KIWIXSERVE_CMD "kiwix-serve.exe"
+    # include <windows.h>
+#else
+    # define KIWIXSERVE_CMD "kiwix-serve"
+    # include <unistd.h>
+#endif
+
+#include "tools.h"
+#include "tools/pathTools.h"
+#include "tools/stringTools.h"
+
+namespace kiwix {
+
+KiwixServe::KiwixServe(const std::string& libraryPath, int port)
+  : m_port(port),
+    m_libraryPath(libraryPath)
+{
+}
+
+KiwixServe::~KiwixServe()
+{
+    shutDown();
+}
+
+void KiwixServe::run()
+{
+    #ifdef _WIN32
+        int pid = GetCurrentProcessId();
+    #else
+        pid_t pid = getpid();
+
+    #endif
+
+    std::vector<const char*> callCmd;
+    std::string kiwixServeCmd = appendToDirectory(
+        removeLastPathElement(getExecutablePath(true)),
+        KIWIXSERVE_CMD);
+    if (fileExists(kiwixServeCmd)) {
+        // A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
+        callCmd.push_back(kiwixServeCmd.c_str());
+    } else {
+        // Try to use a potential installed kiwix-serve.
+        callCmd.push_back(KIWIXSERVE_CMD);
+    }
+    std::string attachProcessOpt = "-a" + to_string(pid);
+    std::string portOpt = "-p" + to_string(m_port);
+    callCmd.push_back(attachProcessOpt.c_str());
+    callCmd.push_back(portOpt.c_str());
+    callCmd.push_back("-l");
+    callCmd.push_back(m_libraryPath.c_str());
+    mp_kiwixServe = Subprocess::run(callCmd);
+}
+
+void KiwixServe::shutDown()
+{
+    if (mp_kiwixServe)
+        mp_kiwixServe->kill();
+}
+
+bool KiwixServe::isRunning()
+{
+    if (mp_kiwixServe) {
+        return (mp_kiwixServe->isRunning());
+    }
+    return false;
+}
+
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
+}

src/library_dumper.cpp

@@ -0,0 +1,120 @@
+#include "library_dumper.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibraryDumper::LibraryDumper(const Library* library, const NameMapper* nameMapper)
+  : library(library),
+    nameMapper(nameMapper)
+{
+}
+/* Destructor */
+LibraryDumper::~LibraryDumper()
+{
+}
+
+void LibraryDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+}
+
+namespace {
+
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"hbs", "srpskohrvatski"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"},
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
+  }
+}
+
+std::string getLanguageSelfName(const std::string& lang) {
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
+};
+
+} // unnamed namespace
+
+kainjow::mustache::list LibraryDumper::getCategoryData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : library->getBooksCategories() ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
+  }
+  return categoryData;
+}
+
+kainjow::mustache::list LibraryDumper::getLanguageData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+  return languageData;
+}
+
+} // namespace kiwix

src/libxml_dumper.cpp

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "libxml_dumper.h"
+#include "book.h"
+
+#include "tools.h"
+#include "tools/base64.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibXMLDumper::LibXMLDumper(const Library* library)
+  : library(library)
+{
+}
+/* Destructor */
+LibXMLDumper::~LibXMLDumper()
+{
+}
+
+#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
+#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
+
+void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
+  if (book.readOnly())
+    return;
+
+  auto entry_node = root_node.append_child("book");
+  ADD_ATTRIBUTE(entry_node, "id", book.getId());
+
+  if (!book.getPath().empty()) {
+    ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
+  }
+
+  if (book.getOrigId().empty()) {
+    ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
+    ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getCommaSeparatedLanguages());
+    ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
+    ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
+    ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
+    ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
+    ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
+    try {
+      auto defaultIllustration = book.getIllustration(48);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
+      ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
+      if (!defaultIllustration->getData().empty())
+        ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
+    } catch(...) {}
+  } else {
+    ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
+  }
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
+  ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
+
+  if (book.getArticleCount())
+    ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
+
+  if (book.getMediaCount())
+    ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
+
+  if (book.getSize())
+    ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
+
+  ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
+}
+
+#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
+
+void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
+
+  auto entry_node = root_node.append_child("bookmark");
+  auto book_node = entry_node.append_child("book");
+
+  try {
+    auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "id", book.getId());
+    ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
+    ADD_TEXT_ENTRY(book_node, "language", book.getCommaSeparatedLanguages());
+    ADD_TEXT_ENTRY(book_node, "date", book.getDate());
+  } catch (...) {
+    ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
+    ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
+    ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
+  }
+  ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
+  ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
+}
+
+
+std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node libraryNode = doc.append_child("library");
+
+  libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
+
+  if (library) {
+    for (auto& bookId: bookIds) {
+      handleBook(library->getBookById(bookId), libraryNode);
+    }
+  }
+  return nodeToString(libraryNode);
+}
+
+std::string LibXMLDumper::dumpLibXMLBookmark()
+{
+  pugi::xml_document doc;
+
+  /* Add the library node */
+  pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
+
+  if (library) {
+    for (auto& bookmark: library->getBookmarks()) {
+      handleBookmark(bookmark, bookmarksNode);
+    }
+  }
+  return nodeToString(bookmarksNode);
+}
+
+}

src/manager.cpp

@@ -19,35 +19,92 @@
 
 #include "manager.h"
 
+#include "tools.h"
+#include "tools/pathTools.h"
+
 #include <pugixml.hpp>
 
 namespace kiwix
 {
+
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+////////////////////////////////////////////////////////////////////////////////
+// LibraryManipulator
+////////////////////////////////////////////////////////////////////////////////
+
+LibraryManipulator::LibraryManipulator(Library* library)
+  : library(*library)
+{}
+
+LibraryManipulator::~LibraryManipulator()
+{}
+
+bool LibraryManipulator::addBookToLibrary(const Book& book)
+{
+  const auto ret = library.addBook(book);
+  if ( ret ) {
+    bookWasAddedToLibrary(book);
+  }
+  return ret;
+}
+
+void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
+{
+  library.addBookmark(bookmark);
+  bookmarkWasAddedToLibrary(bookmark);
+}
+
+uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
+{
+  const auto n = library.removeBooksNotUpdatedSince(rev);
+  if ( n != 0 ) {
+    booksWereRemovedFromLibrary();
+  }
+  return n;
+}
+
+void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
+{
+}
+
+void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
+{
+}
+
+void LibraryManipulator::booksWereRemovedFromLibrary()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Manager
+////////////////////////////////////////////////////////////////////////////////
+
 /* Constructor */
 Manager::Manager(LibraryManipulator* manipulator):
   writableLibraryPath(""),
-  manipulator(manipulator),
+  manipulator(manipulator, NoDelete())
-  mustDeleteManipulator(false)
 {
 }
 
 Manager::Manager(Library* library) :
   writableLibraryPath(""),
-  manipulator(new DefaultLibraryManipulator(library)),
+  manipulator(new LibraryManipulator(library))
-  mustDeleteManipulator(true)
 {
 }
 
-/* Destructor */
-Manager::~Manager()
-{
-  if (mustDeleteManipulator) {
-    delete manipulator;
-  }
-}
 bool Manager::parseXmlDom(const pugi::xml_document& doc,
-                          const bool readOnly,
+                          bool readOnly,
-                          const std::string& libraryPath)
+                          const std::string& libraryPath,
+                          bool trustLibrary)
 {
   pugi::xml_node libraryNode = doc.child("library");
 
@@ -59,14 +116,10 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 
     book.setReadOnly(readOnly);
     book.updateFromXml(bookNode,
-                       removeLastPathElement(libraryPath, true, false));
+                       removeLastPathElement(libraryPath));
 
-    /* Update the book properties with the new importer */
+    if (!trustLibrary && !book.getPath().empty()) {
-    if (libraryVersion.empty()
+      this->readBookFromPath(book.getPath(), &book);
-        || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
-      if (!book.getPath().empty()) {
-        this->readBookFromPath(book.getPath(), &book);
-      }
     }
     manipulator->addBookToLibrary(book);
   }
@@ -75,15 +128,16 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
 }
 
 bool Manager::readXml(const std::string& xml,
-                      const bool readOnly,
+                      bool readOnly,
-                      const std::string& libraryPath)
+                      const std::string& libraryPath,
+                      bool trustLibrary)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)xml.data(), xml.size());
+      = doc.load_buffer((void*)xml.data(), xml.size());
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, libraryPath);
+    this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
   }
 
   return true;
@@ -95,6 +149,15 @@ bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& url
 {
   pugi::xml_node libraryNode = doc.child("feed");
 
+  try {
+    m_totalBooks = strtoull(libraryNode.child("totalResults").child_value(), 0, 0);
+    m_startIndex = strtoull(libraryNode.child("startIndex").child_value(), 0, 0);
+    m_itemsPerPage = strtoull(libraryNode.child("itemsPerPage").child_value(), 0, 0);
+    m_hasSearchResult = true;
+  } catch(...) {
+    m_hasSearchResult = false;
+  }
+
   for (pugi::xml_node entryNode = libraryNode.child("entry"); entryNode;
        entryNode = entryNode.next_sibling("entry")) {
     kiwix::Book book;
@@ -115,7 +178,7 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
 {
   pugi::xml_document doc;
   pugi::xml_parse_result result
-      = doc.load_buffer_inplace((void*)content.data(), content.size());
+      = doc.load_buffer((void*)content.data(), content.size());
 
   if (result) {
     this->parseOpdsDom(doc, urlHost);
@@ -125,21 +188,22 @@ bool Manager::readOpds(const std::string& content, const std::string& urlHost)
   return false;
 }
 
-bool Manager::readFile(const std::string& path, const bool readOnly)
+bool Manager::readFile(
-{
+  const std::string& path,
-  return this->readFile(path, path, readOnly);
+  bool readOnly,
-}
+  bool trustLibrary)
-
-bool Manager::readFile(const std::string& nativePath,
-                       const std::string& UTF8Path,
-                       const bool readOnly)
 {
   bool retVal = true;
   pugi::xml_document doc;
-  pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
+
+#ifdef _WIN32
+  pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
+#else
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+#endif
 
   if (result) {
-    this->parseXmlDom(doc, readOnly, UTF8Path);
+    this->parseXmlDom(doc, readOnly, path, trustLibrary);
   } else {
     retVal = false;
   }
@@ -148,7 +212,7 @@ bool Manager::readFile(const std::string& nativePath,
    * able to know where to save the library if new content are
    * available */
   if (!readOnly) {
-    this->writableLibraryPath = UTF8Path;
+    this->writableLibraryPath = path;
   }
 
   return retVal;
@@ -165,16 +229,16 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
   kiwix::Book book;
 
   if (this->readBookFromPath(pathToOpen, &book)) {
-    if (pathToSave != pathToOpen) {
+    if (!pathToSave.empty() && pathToSave != pathToOpen) {
       book.setPath(isRelativePath(pathToSave)
                 ? computeAbsolutePath(
-                      removeLastPathElement(writableLibraryPath, true, false),
+                      removeLastPathElement(writableLibraryPath),
                       pathToSave)
                 : pathToSave);
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
+        || (!book.getTitle().empty() && !book.getLanguages().empty()
             && !book.getDate().empty())) {
       book.setUrl(url);
       manipulator->addBookToLibrary(book);
@@ -199,12 +263,15 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
 
 bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
+  std::string tmp_path = path;
+  if (isRelativePath(path)) {
+    tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
+  }
   try {
-    kiwix::Reader reader(path);
+    zim::Archive archive(tmp_path);
-    book->update(reader);
+    book->update(archive);
     book->setPathValid(true);
   } catch (const std::exception& e) {
-    std::cerr << "Invalid " << path << " : " << e.what() << std::endl;
     book->setPathValid(false);
     return false;
   }
@@ -212,4 +279,44 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
   return true;
 }
 
+bool Manager::readBookmarkFile(const std::string& path)
+{
+  pugi::xml_document doc;
+  pugi::xml_parse_result result = doc.load_file(path.c_str());
+
+  if (!result) {
+    return false;
+  }
+
+  pugi::xml_node libraryNode = doc.child("bookmarks");
+
+  for (pugi::xml_node node = libraryNode.child("bookmark"); node;
+       node = node.next_sibling("bookmark")) {
+    kiwix::Bookmark bookmark;
+
+    bookmark.updateFromXml(node);
+
+    manipulator->addBookmarkToLibrary(bookmark);
+  }
+
+  return true;
+}
+
+void Manager::reload(const Paths& paths)
+{
+  const auto libRevision = manipulator->getLibrary().getRevision();
+  for (std::string path : paths) {
+    if (!path.empty()) {
+      if ( kiwix::isRelativePath(path) )
+        path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
+
+      if (!readFile(path, false, true)) {
+        throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
+      }
+    }
+  }
+
+  manipulator->removeBooksNotUpdatedSince(libRevision);
+}
+
 }

src/meson.build

@@ -1,24 +1,38 @@
 kiwix_sources = [
   'book.cpp',
+  'bookmark.cpp',
   'library.cpp',
   'manager.cpp',
+  'libxml_dumper.cpp',
   'opds_dumper.cpp',
+  'html_dumper.cpp',
+  'library_dumper.cpp',
   'downloader.cpp',
-  'reader.cpp',
+  'server.cpp',
-  'entry.cpp',
+  'search_renderer.cpp',
-  'searcher.cpp',
   'subprocess.cpp',
   'aria2.cpp',
-  'common/base64.cpp',
+  'tools/base64.cpp',
-  'common/pathTools.cpp',
+  'tools/pathTools.cpp',
-  'common/regexTools.cpp',
+  'tools/regexTools.cpp',
-  'common/stringTools.cpp',
+  'tools/stringTools.cpp',
-  'common/networkTools.cpp',
+  'tools/networkTools.cpp',
-  'common/otherTools.cpp',
+  'tools/otherTools.cpp',
-  'xapian/htmlparse.cc',
+  'tools/archiveTools.cpp',
-  'xapian/myhtmlparse.cc'
+  'kiwixserve.cpp',
+  'name_mapper.cpp',
+  'server/byte_range.cpp',
+  'server/etag.cpp',
+  'server/request_context.cpp',
+  'server/response.cpp',
+  'server/internalServer.cpp',
+  'server/internalServer_catalog_v2.cpp',
+  'server/i18n.cpp',
+  'opds_catalog.cpp',
+  'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
 if host_machine.system() == 'windows'
   kiwix_sources += 'subprocess_windows.cpp'
@@ -26,32 +40,18 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if xapian_dep.found()
+install_dir = get_option('libdir')
-  kiwix_sources += ['xapianSearcher.cpp']
-endif
-
-if get_option('android')
-  subdir('android')
-  install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
-else
-  install_dir = get_option('libdir')
-endif
-
-
-if has_ctpp2_dep
-  kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
-endif
 
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,
                input : 'config.h.in')
 install_headers(config_h, subdir:'kiwix')
 
-kiwixlib = library('kiwix',
+libkiwix = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
-                   install_dir: install_dir,
+                   install_dir: install_dir)
-                   install_rpath: '$ORIGIN')

src/microhttpd_wrapper.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,9 +17,8 @@
  * MA 02110-1301, USA.
  */
 
-package org.kiwix.kiwixlib;
+#include <microhttpd.h>
 
-public class JNIKiwixInt
+#if MHD_VERSION < 0x00097002
-{
+typedef int MHD_Result;
-  public int value;
+#endif
-}

src/name_mapper.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "name_mapper.h"
+#include "library.h"
+#include "tools/regexTools.h"
+#include <iostream>
+
+namespace kiwix {
+
+HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
+  for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = library.getBookById(bookId);
+    auto bookName = currentBook.getHumanReadableIdFromPath();
+    m_idToName[bookId] = bookName;
+    m_nameToId[bookName] = bookId;
+
+    if (!withAlias)
+      continue;
+
+    auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
+    if (aliasName == bookName) {
+      continue;
+    }
+    if (m_nameToId.find(aliasName) == m_nameToId.end()) {
+      m_nameToId[aliasName] = bookId;
+    } else {
+      auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
+      std::cerr << "Path collision: " << alreadyPresentPath
+                << " and " << currentBook.getPath()
+                << " can't share the same URL path '" << aliasName << "'."
+                << " Therefore, only " << alreadyPresentPath
+                << " will be served." << std::endl;
+    }
+  }
+}
+
+std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
+  return m_idToName.at(id);
+}
+
+std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
+  return m_nameToId.at(name);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// UpdatableNameMapper
+////////////////////////////////////////////////////////////////////////////////
+
+UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+  : library(lib)
+  , withAlias(withAlias)
+{
+  update();
+}
+
+void UpdatableNameMapper::update()
+{
+  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  std::lock_guard<std::mutex> lock(mutex);
+  nameMapper.reset(newNameMapper);
+}
+
+UpdatableNameMapper::NameMapperHandle
+UpdatableNameMapper::currentNameMapper() const
+{
+  // Return a copy of the handle to the current NameMapper object. It will
+  // ensure that the object survives any call to UpdatableNameMapper::update()
+  // made before the completion of any pending operation on that object.
+  std::lock_guard<std::mutex> lock(mutex);
+  return nameMapper;
+}
+
+std::string UpdatableNameMapper::getNameForId(const std::string& id) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getNameForId(id);
+}
+
+std::string UpdatableNameMapper::getIdForName(const std::string& name) const
+{
+  // Ensure that the current nameMapper object survives a concurrent call
+  // to UpdatableNameMapper::update()
+  return currentNameMapper()->getIdForName(name);
+}
+
+}

src/opds_catalog.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "opds_catalog.h"
+#include "tools/stringTools.h"
+
+#include <sstream>
+
+namespace kiwix
+{
+
+namespace
+{
+const char opdsSearchEndpoint[] = "/catalog/v2/entries";
+
+enum Separator { AMP };
+
+std::ostringstream& operator<<(std::ostringstream& oss, Separator sep)
+{
+  if ( oss.tellp() > 0 )
+    oss << "&";
+  return oss;
+}
+
+std::string buildSearchString(const Filter& f)
+{
+  std::ostringstream oss;
+  if ( f.hasQuery() )
+    oss << AMP << "q=" << urlEncode(f.getQuery());
+
+  if ( f.hasCategory() )
+    oss << AMP << "category=" << urlEncode(f.getCategory());
+
+  if ( f.hasLang() )
+    oss << AMP << "lang=" << urlEncode(f.getLang());
+
+  if ( f.hasName() )
+    oss << AMP << "name=" << urlEncode(f.getName());
+
+  if ( !f.getAcceptTags().empty() )
+    oss << AMP << "tag=" << urlEncode(join(f.getAcceptTags(), ";"));
+
+  return oss.str();
+}
+
+} // unnamed namespace
+
+std::string getSearchUrl(const Filter& f)
+{
+  const std::string searchString = buildSearchString(f);
+
+  if ( searchString.empty() )
+    return opdsSearchEndpoint;
+  else
+    return opdsSearchEndpoint + ("?" + searchString);
+}
+
+} // namespace kiwix

src/opds_dumper.cpp

@@ -20,13 +20,18 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include <common/otherTools.h>
+#include "libkiwix-resources.h"
+#include <mustache.hpp>
+
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
 
 namespace kiwix
 {
+
 /* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
+OPDSDumper::OPDSDumper(const Library* library, const NameMapper* nameMapper)
-  : library(library)
+  : LibraryDumper(library, nameMapper)
 {
 }
 /* Destructor */
@@ -34,104 +39,170 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-std::string gen_date_str()
+namespace
 {
-  auto now = time(0);
-  auto tm = localtime(&now);
 
-  std::stringstream is;
+const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
-  is << std::setw(2) << std::setfill('0')
+
-     << 1900+tm->tm_year << "-"
+typedef kainjow::mustache::data MustacheData;
-     << std::setw(2) << std::setfill('0') << tm->tm_mon << "-"
+typedef kainjow::mustache::list BooksData;
-     << std::setw(2) << std::setfill('0') << tm->tm_mday << "T"
+typedef kainjow::mustache::list IllustrationInfo;
-     << std::setw(2) << std::setfill('0') << tm->tm_hour << ":"
+
-     << std::setw(2) << std::setfill('0') << tm->tm_min << ":"
+IllustrationInfo getBookIllustrationInfo(const Book& book)
-     << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z";
+{
-  return is.str();
+    kainjow::mustache::list illustrations;
+    if ( book.isPathValid() ) {
+      for ( const auto& illustration : book.getIllustrations() ) {
+        // For now, we are handling only sizexsize@1 illustration.
+        // So we can simply pass one size to mustache.
+        illustrations.push_back(kainjow::mustache::object{
+          {"icon_size", to_string(illustration->width)},
+          {"icon_mimetype", illustration->mimeType}
+        });
+      }
+    }
+    return illustrations;
 }
 
-void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+std::string fullEntryXML(const Book& book, const std::string& rootLocation, const std::string& contentId)
 {
-  m_totalResults = totalResults;
+    const auto bookDate = book.getDate() + "T00:00:00Z";
-  m_startIndex = startIndex,
+    const kainjow::mustache::object data{
-  m_count = count;
+      {"root",  rootLocation},
-  m_isSearchResult = true;
+      {"id", book.getId()},
+      {"name", book.getName()},
+      {"title", book.getTitle()},
+      {"description", book.getDescription()},
+      {"language", book.getCommaSeparatedLanguages()},
+      {"content_id",  urlEncode(contentId)},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+      {"book_date", bookDate},
+      {"category", book.getCategory()},
+      {"flavour", book.getFlavour()},
+      {"tags", book.getTags()},
+      {"article_count", to_string(book.getArticleCount())},
+      {"media_count", to_string(book.getMediaCount())},
+      {"author_name", book.getCreator()},
+      {"publisher_name", book.getPublisher()},
+      {"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
+      {"size", to_string(book.getSize())},
+      {"icons", getBookIllustrationInfo(book)},
+    };
+    return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
+std::string partialEntryXML(const Book& book, const std::string& rootLocation)
-
+{
-pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
+    const auto bookDate = book.getDate() + "T00:00:00Z";
-  auto entry_node = root_node.append_child("entry");
+    const kainjow::mustache::object data{
-  ADD_TEXT_ENTRY(entry_node, "title", book.getTitle());
+      {"root",  rootLocation},
-  ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId());
+      {"endpoint_root", rootLocation + "/catalog/v2"},
-  ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
+      {"id", book.getId()},
-  ADD_TEXT_ENTRY(entry_node, "updated", date);
+      {"title", book.getTitle()},
-  ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
-
+    };
-  auto content_node = entry_node.append_child("link");
+    const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
-  content_node.append_attribute("type") = "text/html";
+    return render_template(xmlTemplate, data);
-  content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str();
-
-  auto author_node = entry_node.append_child("author");
-  ADD_TEXT_ENTRY(author_node, "name", book.getCreator());
-
-  if (! book.getUrl().empty()) {
-    auto acquisition_link = entry_node.append_child("link");
-    acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access";
-    acquisition_link.append_attribute("type") = "application/x-zim";
-    acquisition_link.append_attribute("href") = book.getUrl().c_str();
-    acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str();
-  }
-
-  if (! book.getFaviconMimeType().empty() ) {
-    auto image_link = entry_node.append_child("link");
-    image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail";
-    image_link.append_attribute("type") = book.getFaviconMimeType().c_str();
-    image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str();
-  }
-  return entry_node;
 }
 
-string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds)
+BooksData getBooksData(const Library* library, const NameMapper* nameMapper, const std::vector<std::string>& bookIds, const std::string& rootLocation, bool partial)
 {
-  date = gen_date_str();
+  BooksData booksData;
-  pugi::xml_document doc;
+  for ( const auto& bookId : bookIds ) {
-
+    try {
-  auto root_node = doc.append_child("feed");
+      const Book book = library->getBookByIdThreadSafe(bookId);
-  root_node.append_attribute("xmlns") = "http://www.w3.org/2005/Atom";
+      const std::string contentId = nameMapper->getNameForId(bookId);
-  root_node.append_attribute("xmlns:opds") = "http://opds-spec.org/2010/catalog";
+      const auto entryXML = partial
-
+                          ? partialEntryXML(book, rootLocation)
-  ADD_TEXT_ENTRY(root_node, "id", id);
+                          : fullEntryXML(book, rootLocation, contentId);
-
+      booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
-  ADD_TEXT_ENTRY(root_node, "title", title);
+    } catch ( const std::out_of_range& ) {
-  ADD_TEXT_ENTRY(root_node, "updated", date);
+      // the book was removed from the library since its id was obtained
-
+      // ignore it
-  if (m_isSearchResult) {
-    ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults));
-    ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex));
-    ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count));
-  }
-
-  auto self_link_node = root_node.append_child("link");
-  self_link_node.append_attribute("rel") = "self";
-  self_link_node.append_attribute("href") = "";
-  self_link_node.append_attribute("type") = "application/atom+xml";
-
-
-  if (!searchDescriptionUrl.empty() ) {
-    auto search_link = root_node.append_child("link");
-    search_link.append_attribute("rel") = "search";
-    search_link.append_attribute("type") = "application/opensearchdescription+xml";
-    search_link.append_attribute("href") = searchDescriptionUrl.c_str();
-  }
-
-  if (library) {
-    for (auto& bookId: bookIds) {
-      handleBook(library->getBookById(bookId), root_node);
     }
   }
 
-  return nodeToString(root_node);
+  return booksData;
+}
+
+} // unnamed namespace
+
+string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
+{
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, false);
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"root", rootLocation},
+     {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", booksData }
+  };
+
+  return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
+}
+
+string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
+{
+  const auto endpointRoot = rootLocation + "/catalog/v2";
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, partial);
+
+  const char* const endpoint = partial ? "/partial_entries" : "/entries";
+  const kainjow::mustache::object template_data{
+     {"date", gen_date_str()},
+     {"endpoint_root", endpointRoot},
+     {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
+     {"query", query.empty() ? "" : "?" + query},
+     {"totalResults", to_string(m_totalResults)},
+     {"startIndex", to_string(m_startIndex)},
+     {"itemsPerPage", to_string(m_count)},
+     {"books", booksData },
+     {"dump_partial_entries", MustacheData(partial)}
+  };
+
+  return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
+}
+
+std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
+{
+  const auto book = library->getBookById(bookId);
+  const std::string contentId = nameMapper->getNameForId(bookId);
+  return XML_HEADER
+         + "\n"
+         + fullEntryXML(book, rootLocation, contentId);
+}
+
+std::string OPDSDumper::categoriesOPDSFeed() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData = getCategoryData();
+  return render_template(
+             RESOURCE::templates::catalog_v2_categories_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/categories")},
+               {"categories", categoryData }
+             }
+  );
+}
+
+std::string OPDSDumper::languagesOPDSFeed() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData = getLanguageData();
+  return render_template(
+             RESOURCE::templates::catalog_v2_languages_xml,
+             kainjow::mustache::object{
+               {"date", now},
+               {"endpoint_root", rootLocation + "/catalog/v2"},
+               {"feed_id", gen_uuid(libraryId + "/languages")},
+               {"languages", languageData }
+             }
+  );
 }
 
 }

src/reader.cpp

@@ -1,862 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-
-inline char hi(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[(v >> 4) & 0xf];
-}
-
-inline char lo(char v)
-{
-  char hex[] = "0123456789abcdef";
-  return hex[v & 0xf];
-}
-
-std::string hexUUID(std::string in)
-{
-  std::ostringstream out;
-  for (unsigned n = 0; n < 4; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 4; n < 6; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 6; n < 8; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 8; n < 10; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  out << '-';
-  for (unsigned n = 10; n < 16; ++n) {
-    out << hi(in[n]) << lo(in[n]);
-  }
-  std::string op = out.str();
-  return op;
-}
-
-namespace kiwix
-{
-/* Constructor */
-Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
-{
-  string tmpZimFilePath = zimFilePath;
-
-  /* Remove potential trailing zimaa */
-  size_t found = tmpZimFilePath.rfind("zimaa");
-  if (found != string::npos && tmpZimFilePath.size() > 5
-      && found == tmpZimFilePath.size() - 5) {
-    tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
-  }
-
-  this->zimFileHandler = new zim::File(tmpZimFilePath);
-
-  if (this->zimFileHandler != NULL) {
-    this->firstArticleOffset
-        = this->zimFileHandler->getNamespaceBeginOffset('A');
-    this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
-    this->nsACount = this->zimFileHandler->getNamespaceCount('A');
-    this->nsICount = this->zimFileHandler->getNamespaceCount('I');
-    this->zimFilePath = zimFilePath;
-  }
-
-  /* initialize random seed: */
-  srand(time(NULL));
-}
-
-/* Destructor */
-Reader::~Reader()
-{
-  if (this->zimFileHandler != NULL) {
-    delete this->zimFileHandler;
-  }
-}
-
-zim::File* Reader::getZimFileHandler() const
-{
-  return this->zimFileHandler;
-}
-std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
-{
-  std::map<const std::string, unsigned int> counters;
-  string mimeType, item, counterString;
-  unsigned int counter;
-
-  zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
-
-  if (article.good()) {
-    stringstream ssContent(article.getData());
-
-    while (getline(ssContent, item, ';')) {
-      stringstream ssItem(item);
-      getline(ssItem, mimeType, '=');
-      getline(ssItem, counterString, '=');
-      if (!counterString.empty() && !mimeType.empty()) {
-        sscanf(counterString.c_str(), "%u", &counter);
-        counters.insert(pair<string, int>(mimeType, counter));
-      }
-    }
-  }
-
-  return counters;
-}
-
-/* Get the count of articles which can be indexed/displayed */
-unsigned int Reader::getArticleCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  if (counterMap.empty()) {
-    counter = this->nsACount;
-  } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
-    }
-  }
-
-  return counter;
-}
-
-/* Get the count of medias content in the ZIM file */
-unsigned int Reader::getMediaCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  if (counterMap.empty()) {
-    counter = this->nsICount;
-  } else {
-    auto it = counterMap.find("image/jpeg");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/gif");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-
-    it = counterMap.find("image/png");
-    if (it != counterMap.end()) {
-      counter += it->second;
-    }
-  }
-  return counter;
-}
-
-/* Get the total of all items of a ZIM file, redirects included */
-unsigned int Reader::getGlobalCount() const
-{
-  return this->zimFileHandler->getCountArticles();
-}
-
-/* Return the UID of the ZIM file */
-string Reader::getId() const
-{
-  std::ostringstream s;
-  s << this->zimFileHandler->getFileheader().getUuid();
-  return s.str();
-}
-
-/* Return a page url from a title */
-bool Reader::getPageUrlFromTitle(const string& title, string& url) const
-{
-  try {
-    auto entry = getEntryFromTitle(title);
-    entry = entry.getFinalEntry();
-    url = entry.getPath();
-    return true;
-  } catch (NoEntry& e) {
-    return false;
-  }
-}
-
-/* Return an URL from a title */
-string Reader::getRandomPageUrl() const
-{
-  return getRandomPage().getPath();
-}
-
-Entry Reader::getRandomPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  zim::Article article;
-  std::string mainPagePath = this->getMainPage().getPath();
-  int watchdog = 42;
-
-  do {
-    auto idx = this->firstArticleOffset
-          + (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
-                             * this->nsACount);
-    article = zimFileHandler->getArticle(idx);
-    if (!watchdog--) {
-      throw NoEntry();
-    }
-  } while (!article.good() && article.getLongUrl() == mainPagePath);
-
-  return article;
-}
-
-/* Return the welcome page URL */
-string Reader::getMainPageUrl() const
-{
-  return getMainPage().getPath();
-}
-
-Entry Reader::getMainPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  string url = "";
-
-  zim::Article article;
-  if (this->zimFileHandler->getFileheader().hasMainPage())
-  {
-    article = zimFileHandler->getArticle(
-        this->zimFileHandler->getFileheader().getMainPage());
-  }
-
-  if (!article.good())
-  {
-    return getFirstPage();
-  }
-
-  return article;
-}
-
-bool Reader::getFavicon(string& content, string& mimeType) const
-{
-  static const char* const paths[] = {"-/favicon.png", "I/favicon.png", "I/favicon", "-/favicon"};
-
-  for (auto &path: paths) {
-    try {
-      auto entry = getEntryFromPath(path);
-      entry = entry.getFinalEntry();
-      content = entry.getContent();
-      mimeType = entry.getMimetype();
-      return true;
-    } catch(NoEntry& e) {};
-  }
-
-  return false;
-}
-
-string Reader::getZimFilePath() const
-{
-  return this->zimFilePath;
-}
-/* Return a metatag value */
-bool Reader::getMetatag(const string& name, string& value) const
-{
-  try {
-    auto entry = getEntryFromPath("M/"+name);
-    value = entry.getContent();
-    return true;
-  } catch(NoEntry& e) {
-    return false;
-  }
-}
-
-string Reader::getTitle() const
-{
-  string value;
-  this->getMetatag("Title", value);
-  if (value.empty()) {
-    value = getLastPathElement(zimFileHandler->getFilename());
-    std::replace(value.begin(), value.end(), '_', ' ');
-    size_t pos = value.find(".zim");
-    value = value.substr(0, pos);
-  }
-  return value;
-}
-
-string Reader::getName() const
-{
-  string value;
-  this->getMetatag("Name", value);
-  return value;
-}
-
-string Reader::getTags() const
-{
-  string value;
-  this->getMetatag("Tags", value);
-  return value;
-}
-
-string Reader::getDescription() const
-{
-  string value;
-  this->getMetatag("Description", value);
-
-  /* Mediawiki Collection tends to use the "Subtitle" name */
-  if (value.empty()) {
-    this->getMetatag("Subtitle", value);
-  }
-
-  return value;
-}
-
-string Reader::getLanguage() const
-{
-  string value;
-  this->getMetatag("Language", value);
-  return value;
-}
-
-string Reader::getDate() const
-{
-  string value;
-  this->getMetatag("Date", value);
-  return value;
-}
-
-string Reader::getCreator() const
-{
-  string value;
-  this->getMetatag("Creator", value);
-  return value;
-}
-
-string Reader::getPublisher() const
-{
-  string value;
-  this->getMetatag("Publisher", value);
-  return value;
-}
-
-string Reader::getOrigId() const
-{
-  string value;
-  this->getMetatag("startfileuid", value);
-  if (value.empty()) {
-    return "";
-  }
-  std::string id = value;
-  std::string origID;
-  std::string temp = "";
-  unsigned int k = 0;
-  char tempArray[16] = "";
-  for (unsigned int i = 0; i < id.size(); i++) {
-    if (id[i] == '\n') {
-      tempArray[k] = atoi(temp.c_str());
-      temp = "";
-      k++;
-    } else {
-      temp += id[i];
-    }
-  }
-  origID = hexUUID(tempArray);
-  return origID;
-}
-
-/* Return the first page URL */
-string Reader::getFirstPageUrl() const
-{
-  return getFirstPage().getPath();
-}
-
-Entry Reader::getFirstPage() const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  auto firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
-  auto article = zimFileHandler->getArticle(firstPageOffset);
-
-  if (! article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-bool _parseUrl(const string& url, char* ns, string& title)
-{
-  /* Offset to visit the url */
-  unsigned int urlLength = url.size();
-  unsigned int offset = 0;
-
-  /* Ignore the first '/' */
-  if (url[offset] == '/')
-    offset++;
-
-  if (url[offset] == '/' || offset >= urlLength)
-    return false;
-
-  /* Get namespace */
-  *ns = url[offset++];
-
-  if (url[offset] != '/' || offset >= urlLength)
-    return false;
-
-  offset++;
-
-  if ( offset >= urlLength)
-    return false;
-
-  /* Get content title */
-  title = url.substr(offset, urlLength - offset);
-
-  return true;
-}
-
-bool Reader::parseUrl(const string& url, char* ns, string& title) const
-{
-  return _parseUrl(url, ns, title);
-}
-
-Entry Reader::getEntryFromPath(const std::string& path) const
-{
-  char ns = 0;
-  std::string short_url;
-
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-  _parseUrl(path, &ns, short_url);
-
-  if (short_url.empty() && ns == 0) {
-    return getMainPage();
-  }
-
-  auto article = zimFileHandler->getArticle(ns, short_url);
-  if (!article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-Entry Reader::getEntryFromEncodedPath(const std::string& path) const
-{
-  return getEntryFromPath(urlDecode(path, true));
-}
-
-Entry Reader::getEntryFromTitle(const std::string& title) const
-{
-  if (!this->zimFileHandler) {
-    throw NoEntry();
-  }
-
-  auto article = this->zimFileHandler->getArticleByTitle('A', title);
-  if (!article.good()) {
-    throw NoEntry();
-  }
-
-  return article;
-}
-
-/* Return article by url */
-bool Reader::getArticleObjectByDecodedUrl(const string& url,
-                                          zim::Article& article) const
-{
-  if (this->zimFileHandler == NULL) {
-    return false;
-  }
-
-  /* Parse the url */
-  char ns = 0;
-  string urlStr;
-  _parseUrl(url, &ns, urlStr);
-
-  /* Main page */
-  if (urlStr.empty() && ns == 0) {
-    _parseUrl(this->getMainPage().getPath(), &ns, urlStr);
-  }
-
-  /* Extract the content from the zim file */
-  article = zimFileHandler->getArticle(ns, urlStr);
-  return article.good();
-}
-
-/* Return the mimeType without the content */
-bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
-{
-  try {
-    auto entry = getEntryFromPath(url);
-    mimeType = entry.getMimetype();
-    return true;
-  } catch (NoEntry& e) {
-    mimeType = "";
-    return false;
-  }
-}
-
-bool get_content_by_decoded_url(const Reader& reader,
-                                const string& url,
-                                string& content,
-                                string& title,
-                                unsigned int& contentLength,
-                                string& contentType,
-                                string& baseUrl)
-{
-  content = "";
-  contentType = "";
-  contentLength = 0;
-
-  try {
-    auto entry = reader.getEntryFromPath(url);
-    entry = entry.getFinalEntry();
-    baseUrl = entry.getPath();
-    contentType = entry.getMimetype();
-    content = entry.getContent();
-    contentLength = entry.getSize();
-    title = entry.getTitle();
-
-    /* Try to set a stub HTML header/footer if necesssary */
-    if (contentType.find("text/html") != string::npos
-      && content.find("<body") == std::string::npos
-      && content.find("<BODY") == std::string::npos) {
-      content = "<html><head><title>" + title +
-              "</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
-              "charset=utf-8\" /></head><body>" +
-              content + "</body></html>";
-    }
-    return true;
-  } catch (NoEntry& e) {
-    return false;
-  }
-}
-
-
-/* Get a content from a zim file */
-bool Reader::getContentByUrl(const string& url,
-                             string& content,
-                             string& title,
-                             unsigned int& contentLength,
-                             string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
-}
-
-bool Reader::getContentByEncodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                kiwix::urlDecode(url),
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType) const
-{
-  std::string stubRedirectUrl;
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                stubRedirectUrl);
-}
-
-bool Reader::getContentByDecodedUrl(const string& url,
-                                    string& content,
-                                    string& title,
-                                    unsigned int& contentLength,
-                                    string& contentType,
-                                    string& baseUrl) const
-{
-  return get_content_by_decoded_url(*this,
-                                url,
-                                content,
-                                title,
-                                contentLength,
-                                contentType,
-                                baseUrl);
-}
-
-/* Check if an article exists */
-bool Reader::urlExists(const string& url) const
-{
-  return pathExists(url);
-}
-
-bool Reader::pathExists(const string& path) const
-{
-  if (!zimFileHandler)
-  {
-    return false;
-  }
-
-  char ns = 0;
-  string titleStr;
-  _parseUrl(path, &ns, titleStr);
-  zim::File::const_iterator findItr = zimFileHandler->find(ns, titleStr);
-  return findItr != zimFileHandler->end() && findItr->getUrl() == titleStr;
-}
-
-/* Does the ZIM file has a fulltext index */
-bool Reader::hasFulltextIndex() const
-{
-  if (!zimFileHandler || zimFileHandler->is_multiPart() )
-  {
-    return false;
-  }
-
-  return ( pathExists("Z//fulltextIndex/xapian")
-        || pathExists("X/fulltext/xapian"));
-}
-
-/* Search titles by prefix */
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               const bool reset)
-{
-  bool retVal = false;
-  zim::File::const_iterator articleItr;
-
-  /* Reset the suggestions otherwise check if the suggestions number is less
-   * than the suggestionsCount */
-  if (reset) {
-    this->suggestions.clear();
-    this->suggestionsOffset = this->suggestions.begin();
-  } else {
-    if (this->suggestions.size() > suggestionsCount) {
-      return false;
-    }
-  }
-
-  /* Return if no prefix */
-  if (prefix.size() == 0) {
-    return false;
-  }
-
-  for (articleItr = zimFileHandler->findByTitle('A', prefix);
-       articleItr != zimFileHandler->end()
-       && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
-       ++articleItr) {
-    /* Extract the interesting part of article title & url */
-    std::string normalizedArticleTitle
-        = kiwix::normalize(articleItr->getTitle());
-    std::string articleFinalUrl = "/A/" + articleItr->getUrl();
-    if (articleItr->isRedirect()) {
-      zim::Article article = *articleItr;
-      unsigned int loopCounter = 0;
-      while (article.isRedirect() && loopCounter++ < 42) {
-        article = article.getRedirectArticle();
-      }
-      articleFinalUrl = "/A/" + article.getUrl();
-    }
-
-    /* Go through all already found suggestions and skip if this
-       article is already in the suggestions list (with an other
-       title) */
-    bool insert = true;
-    std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
-         suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
-        insert = false;
-        break;
-      } else if (result < 0) {
-        break;
-      }
-    }
-
-    /* Insert if possible */
-    if (insert) {
-      std::vector<std::string> suggestion;
-      suggestion.push_back(articleItr->getTitle());
-      suggestion.push_back(articleFinalUrl);
-      suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
-    }
-
-    /* Suggestions where found */
-    retVal = true;
-  }
-
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return retVal;
-}
-
-std::vector<std::string> Reader::getTitleVariants(
-    const std::string& title) const
-{
-  std::vector<std::string> variants;
-  variants.push_back(title);
-  variants.push_back(kiwix::ucFirst(title));
-  variants.push_back(kiwix::lcFirst(title));
-  variants.push_back(kiwix::toTitle(title));
-  return variants;
-}
-
-/* Try also a few variations of the prefix to have better results */
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount)
-{
-  std::vector<std::string> variants = this->getTitleVariants(prefix);
-  bool retVal;
-
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
-  /* Try to search in the title using fulltext search database */
-  const zim::Search* suggestionSearch
-      = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
-
-  if (suggestionSearch->get_matches_estimated()) {
-    for (auto current = suggestionSearch->begin();
-         current != suggestionSearch->end();
-         current++) {
-      std::vector<std::string> suggestion;
-      suggestion.push_back(current->getTitle());
-      suggestion.push_back("/A/" + current->getUrl());
-      suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
-    }
-    this->suggestionsOffset = this->suggestions.begin();
-    retVal = true;
-  } else {
-    for (std::vector<std::string>::iterator variantsItr = variants.begin();
-         variantsItr != variants.end();
-         variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
-               || retVal;
-    }
-  }
-
-  return retVal;
-}
-
-/* Get next suggestion */
-bool Reader::getNextSuggestion(string& title)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset))[0];
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-bool Reader::getNextSuggestion(string& title, string& url)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset))[0];
-    url = (*(this->suggestionsOffset))[1];
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-/* Check if the file has as checksum */
-bool Reader::canCheckIntegrity() const
-{
-  return this->zimFileHandler->getChecksum() != "";
-}
-
-/* Return true if corrupted, false otherwise */
-bool Reader::isCorrupted() const
-{
-  try {
-    if (this->zimFileHandler->verify() == true) {
-      return false;
-    }
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return true;
-  }
-
-  return true;
-}
-
-/* Return the file size, works also for splitted files */
-unsigned int Reader::getFileSize() const
-{
-  zim::File* file = this->getZimFileHandler();
-  zim::offset_type size = 0;
-
-  if (file != NULL) {
-    size = file->getFilesize();
-  }
-
-  return (size / 1024);
-}
-}

src/search_renderer.cpp

@@ -0,0 +1,236 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include <cmath>
+
+#include "search_renderer.h"
+#include "library.h"
+#include "name_mapper.h"
+
+#include "tools/archiveTools.h"
+
+#include <zim/search.h>
+
+#include <mustache.hpp>
+#include "libkiwix-resources.h"
+#include "tools/stringTools.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
+{}
+
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                      unsigned int start, unsigned int estimatedResultCount)
+    : m_srs(srs),
+      mp_nameMapper(mapper),
+      mp_library(library),
+      protocolPrefix("zim://"),
+      searchProtocolPrefix("search://"),
+      estimatedResultCount(estimatedResultCount),
+      resultStart(start)
+{}
+
+/* Destructor */
+SearchRenderer::~SearchRenderer() = default;
+
+void SearchRenderer::setSearchPattern(const std::string& pattern)
+{
+  searchPattern = pattern;
+}
+
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
+{
+  searchBookQuery = bookQuery;
+}
+
+void SearchRenderer::setProtocolPrefix(const std::string& prefix)
+{
+  this->protocolPrefix = prefix;
+}
+
+void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
+{
+  this->searchProtocolPrefix = prefix;
+}
+
+std::string extractValueFromQuery(const std::string& query, const std::string& key) {
+  const std::string p = key + "=";
+  const size_t i = query.find(p);
+  if (i == std::string::npos) {
+    return "";
+  }
+  std::string r = query.substr(i + p.size());
+  return r.substr(0, r.find("&"));
+}
+
+kainjow::mustache::data buildQueryData
+(
+  const std::string& searchProtocolPrefix,
+  const std::string& pattern,
+  const std::string& bookQuery
+) {
+  kainjow::mustache::data query;
+  query.set("pattern", kiwix::encodeDiples(pattern));
+  std::ostringstream ss;
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
+  ss << "&" << bookQuery;
+  query.set("unpaginatedQuery", ss.str());
+  auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
+  if(!lang.empty()) {
+    query.set("lang", lang);
+  }
+  return query;
+}
+
+kainjow::mustache::data buildPagination(
+  unsigned int pageLength,
+  unsigned int resultsCount,
+  unsigned int resultsStart
+)
+{
+  assert(pageLength!=0);
+  kainjow::mustache::data pagination;
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  if (resultsCount == 0) {
+    // Easy case
+    pagination.set("itemsPerPage", to_string(pageLength));
+    pagination.set("hasPages", false);
+    pagination.set("pages", pages);
+    return pagination;
+  }
+
+  // First we want to display pages starting at a multiple of `pageLength`
+  // so, let's calculate the start index of the current page.
+  auto currentPage = resultsStart/pageLength;
+  auto lastPage = ((resultsCount-1)/pageLength);
+  auto lastPageStart = lastPage*pageLength;
+  auto nbPages = lastPage + 1;
+
+  auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
+  auto lastPageGenerated = std::min(currentPage+4, lastPage);
+
+  if (nbPages != 1) {
+    if (firstPageGenerated!=0) {
+      kainjow::mustache::data page;
+      page.set("label", "◀");
+      page.set("start", to_string(0));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+
+    for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
+      kainjow::mustache::data page;
+      page.set("label", to_string(i+1));
+      page.set("start", to_string(i*pageLength));
+      page.set("current", bool(i == currentPage));
+      pages.push_back(page);
+    }
+
+    if (lastPageGenerated!=lastPage) {
+      kainjow::mustache::data page;
+      page.set("label", "▶");
+      page.set("start", to_string(lastPageStart));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+  }
+
+  pagination.set("itemsPerPage", to_string(pageLength));
+  pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
+  pagination.set("pages", pages);
+  return pagination;
+}
+
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+{
+  const std::string absPathPrefix = protocolPrefix;
+  // Build the results list
+  kainjow::mustache::data items{kainjow::mustache::data::type::list};
+  for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
+    kainjow::mustache::data result;
+    const std::string zim_id(it.getZimId());
+    const auto path = mp_nameMapper->getNameForId(zim_id) + "/" + it.getPath();
+    result.set("title", it.getTitle());
+    result.set("absolutePath", absPathPrefix + urlEncode(path));
+    result.set("snippet", it.getSnippet());
+    if (mp_library) {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
+    if (it.getWordCount() >= 0) {
+      result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
+    }
+
+    items.push_back(result);
+  }
+  kainjow::mustache::data results;
+  results.set("items", items);
+  results.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  results.set("hasResults", estimatedResultCount != 0);
+  results.set("start", kiwix::beautifyInteger(resultStart));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
+
+  // pagination
+  auto pagination = buildPagination(
+    pageLength,
+    estimatedResultCount,
+    resultStart
+  );
+
+  kainjow::mustache::data query = buildQueryData(
+    searchProtocolPrefix,
+    searchPattern,
+    searchBookQuery
+  );
+
+
+  kainjow::mustache::data allData;
+  allData.set("searchProtocolPrefix", searchProtocolPrefix);
+  allData.set("results", results);
+  allData.set("pagination", pagination);
+  allData.set("query", query);
+
+  kainjow::mustache::mustache tmpl(tmpl_str);
+
+  std::stringstream ss;
+  tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  if (!tmpl.is_valid()) {
+    throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
+  }
+  return ss.str();
+}
+
+std::string SearchRenderer::getHtml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_html);
+}
+
+std::string SearchRenderer::getXml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_xml);
+}
+
+
+}

src/searcher.cpp

@@ -1,465 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <cmath>
-
-#include "searcher.h"
-#include "reader.h"
-#include "xapianSearcher.h"
-
-#include <zim/search.h>
-
-#ifdef ENABLE_CTPP2
-#include <ctpp2/CDT.hpp>
-#include <ctpp2/CTPP2FileLogger.hpp>
-#include <ctpp2/CTPP2SimpleVM.hpp>
-#include "ctpp2/CTPP2VMStringLoader.hpp"
-#include "kiwixlib-resources.h"
-
-using namespace CTPP;
-#endif
-
-#define MAX_SEARCH_LEN 140
-
-namespace kiwix
-{
-class _Result : public Result
-{
- public:
-  _Result(zim::Search::iterator& iterator);
-  virtual ~_Result(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual int get_readerIndex();
-
- private:
-  zim::Search::iterator iterator;
-};
-
-struct SearcherInternal {
-  const zim::Search* _search;
-  XapianSearcher* _xapianSearcher;
-  zim::Search::iterator current_iterator;
-
-  SearcherInternal() : _search(NULL), _xapianSearcher(NULL) {}
-  ~SearcherInternal()
-  {
-    if (_search != NULL) {
-      delete _search;
-    }
-    if (_xapianSearcher != NULL) {
-      delete _xapianSearcher;
-    }
-  }
-};
-
-/* Constructor */
-Searcher::Searcher(const string& xapianDirectoryPath,
-                   Reader* reader,
-                   const string& humanReadableName)
-    : internal(new SearcherInternal()),
-      searchPattern(""),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      resultCountPerPage(0),
-      estimatedResultCount(0),
-      resultStart(0),
-      resultEnd(0),
-      contentHumanReadableId(humanReadableName)
-{
-  loadICUExternalTables();
-  if (!reader || !reader->hasFulltextIndex()) {
-    internal->_xapianSearcher = new XapianSearcher(xapianDirectoryPath, reader);
-  }
-  this->humanReaderNames.push_back(humanReadableName);
-}
-
-Searcher::Searcher(const std::string& humanReadableName)
-    : internal(new SearcherInternal()),
-      searchPattern(""),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      resultCountPerPage(0),
-      estimatedResultCount(0),
-      resultStart(0),
-      resultEnd(0),
-      contentHumanReadableId(humanReadableName)
-{
-  loadICUExternalTables();
-}
-
-/* Destructor */
-Searcher::~Searcher()
-{
-  delete internal;
-}
-
-bool Searcher::add_reader(Reader* reader, const std::string& humanReadableName)
-{
-  if (!reader->hasFulltextIndex()) {
-      return false;
-  }
-  this->readers.push_back(reader);
-  this->humanReaderNames.push_back(humanReadableName);
-  return true;
-}
-
-/* Search strings in the database */
-void Searcher::search(std::string& search,
-                      unsigned int resultStart,
-                      unsigned int resultEnd,
-                      const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing query `" << search << "'" << endl;
-  }
-
-  /* If resultEnd & resultStart inverted */
-  if (resultStart > resultEnd) {
-    resultEnd += resultStart;
-    resultStart = resultEnd - resultStart;
-    resultEnd -= resultStart;
-  }
-
-  /* Try to find results */
-  if (resultStart != resultEnd) {
-    /* Avoid big researches */
-    this->resultCountPerPage = resultEnd - resultStart;
-    if (this->resultCountPerPage > MAX_SEARCH_LEN) {
-      resultEnd = resultStart + MAX_SEARCH_LEN;
-      this->resultCountPerPage = MAX_SEARCH_LEN;
-    }
-
-    /* Perform the search */
-    this->searchPattern = search;
-    this->resultStart = resultStart;
-    this->resultEnd = resultEnd;
-    string unaccentedSearch = removeAccents(search);
-    if (internal->_xapianSearcher) {
-      internal->_xapianSearcher->searchInIndex(
-          unaccentedSearch, resultStart, resultEnd, verbose);
-      this->estimatedResultCount
-          = internal->_xapianSearcher->results.get_matches_estimated();
-    } else {
-      std::vector<const zim::File*> zims;
-      for (auto current = this->readers.begin(); current != this->readers.end();
-           current++) {
-        if ( (*current)->hasFulltextIndex() ) {
-            zims.push_back((*current)->getZimFileHandler());
-        }
-      }
-      zim::Search* search = new zim::Search(zims);
-      search->set_query(unaccentedSearch);
-      search->set_range(resultStart, resultEnd);
-      internal->_search = search;
-      internal->current_iterator = internal->_search->begin();
-      this->estimatedResultCount = internal->_search->get_matches_estimated();
-    }
-  }
-
-  return;
-}
-
-
-void Searcher::geo_search(float latitude, float longitude, float distance,
-                          unsigned int resultStart,
-                          unsigned int resultEnd,
-                          const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-  }
-
-  /* If resultEnd & resultStart inverted */
-  if (resultStart > resultEnd) {
-    resultEnd += resultStart;
-    resultStart = resultEnd - resultStart;
-    resultEnd -= resultStart;
-  }
-
-  /* Try to find results */
-  if (resultStart == resultEnd) {
-    return;
-  }
-
-  if (internal->_xapianSearcher) {
-    return;
-  }
-
-  /* Avoid big researches */
-  this->resultCountPerPage = resultEnd - resultStart;
-  if (this->resultCountPerPage > MAX_SEARCH_LEN) {
-    resultEnd = resultStart + MAX_SEARCH_LEN;
-    this->resultCountPerPage = MAX_SEARCH_LEN;
-  }
-
-  /* Perform the search */
-  std::ostringstream oss;
-  oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
-  this->searchPattern = oss.str();
-  this->resultStart = resultStart;
-  this->resultEnd = resultEnd;
-
-  std::vector<const zim::File*> zims;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    zims.push_back((*current)->getZimFileHandler());
-  }
-  zim::Search* search = new zim::Search(zims);
-  search->set_query("");
-  search->set_georange(latitude, longitude, distance);
-  search->set_range(resultStart, resultEnd);
-  internal->_search = search;
-  internal->current_iterator = internal->_search->begin();
-  this->estimatedResultCount = internal->_search->get_matches_estimated();
-}
-
-
-void Searcher::restart_search()
-{
-  if (internal->_xapianSearcher) {
-    internal->_xapianSearcher->restart_search();
-  } else if (internal->_search) {
-    internal->current_iterator = internal->_search->begin();
-  }
-}
-
-Result* Searcher::getNextResult()
-{
-  if (internal->_xapianSearcher) {
-    return internal->_xapianSearcher->getNextResult();
-  } else if (internal->_search &&
-             internal->current_iterator != internal->_search->end()) {
-    Result* result = new _Result(internal->current_iterator);
-    internal->current_iterator++;
-    return result;
-  }
-  return NULL;
-}
-
-/* Reset the results */
-void Searcher::reset()
-{
-  this->estimatedResultCount = 0;
-  this->searchPattern = "";
-  return;
-}
-
-void Searcher::suggestions(std::string& search, const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing suggestion query `" << search << "`" << endl;
-  }
-
-  this->searchPattern = search;
-  this->resultStart = 0;
-  this->resultEnd = 10;
-  string unaccentedSearch = removeAccents(search);
-
-  if (internal->_xapianSearcher) {
-    /* [TODO] Suggestion on a external database ?
-     * We do not support that. */
-    this->estimatedResultCount = 0;
-  } else {
-    std::vector<const zim::File*> zims;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      zims.push_back((*current)->getZimFileHandler());
-    }
-    zim::Search* search = new zim::Search(zims);
-    search->set_query(unaccentedSearch);
-    search->set_range(resultStart, resultEnd);
-    search->set_suggestion_mode(true);
-    internal->_search = search;
-    internal->current_iterator = internal->_search->begin();
-    this->estimatedResultCount = internal->_search->get_matches_estimated();
-  }
-}
-
-/* Return the result count estimation */
-unsigned int Searcher::getEstimatedResultCount()
-{
-  return this->estimatedResultCount;
-}
-
-bool Searcher::setProtocolPrefix(const std::string prefix)
-{
-  this->protocolPrefix = prefix;
-  return true;
-}
-
-bool Searcher::setSearchProtocolPrefix(const std::string prefix)
-{
-  this->searchProtocolPrefix = prefix;
-  return true;
-}
-
-_Result::_Result(zim::Search::iterator& iterator)
-    : iterator(iterator)
-{
-}
-
-std::string _Result::get_url()
-{
-  return iterator.get_url();
-}
-std::string _Result::get_title()
-{
-  return iterator.get_title();
-}
-int _Result::get_score()
-{
-  return iterator.get_score();
-}
-std::string _Result::get_snippet()
-{
-  return iterator.get_snippet();
-}
-std::string _Result::get_content()
-{
-  if (iterator->good()) {
-    return iterator->getData();
-  }
-  return "";
-}
-int _Result::get_size()
-{
-  return iterator.get_size();
-}
-int _Result::get_wordCount()
-{
-  return iterator.get_wordCount();
-}
-int _Result::get_readerIndex()
-{
-  return iterator.get_fileIndex();
-}
-#ifdef ENABLE_CTPP2
-
-string Searcher::getHtml()
-{
-  SimpleVM oSimpleVM(
-      1024, //iIMaxFunctions (default value)
-      4096, //iIMaxArgStackSize (default value)
-      4096, //iIMaxCodeStackSize (default value)
-      10240 * 2 //iIMaxSteps (default*2)
-  );
-
-  // Fill data
-  CDT oData;
-  CDT resultsCDT(CDT::ARRAY_VAL);
-
-  this->restart_search();
-  Result* p_result = NULL;
-  while ((p_result = this->getNextResult())) {
-    CDT result;
-    result["title"] = p_result->get_title();
-    result["url"] = p_result->get_url();
-    result["snippet"] = p_result->get_snippet();
-    result["contentId"] = humanReaderNames[p_result->get_readerIndex()];
-
-    if (p_result->get_size() >= 0) {
-      result["size"] = kiwix::beautifyInteger(p_result->get_size());
-    }
-
-    if (p_result->get_wordCount() >= 0) {
-      result["wordCount"] = kiwix::beautifyInteger(p_result->get_wordCount());
-    }
-
-    resultsCDT.PushBack(result);
-    delete p_result;
-  }
-  this->restart_search();
-  oData["results"] = resultsCDT;
-
-  // pages
-  CDT pagesCDT(CDT::ARRAY_VAL);
-
-  unsigned int pageStart
-      = this->resultStart / this->resultCountPerPage >= 5
-            ? this->resultStart / this->resultCountPerPage - 4
-            : 0;
-  unsigned int pageCount
-      = this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
-
-  if (pageCount > 10) {
-    pageCount = 10;
-  } else if (pageCount == 1) {
-    pageCount = 0;
-  }
-
-  for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
-    CDT page;
-    page["label"] = i + 1;
-    page["start"] = i * this->resultCountPerPage;
-    page["end"] = (i + 1) * this->resultCountPerPage;
-
-    if (i * this->resultCountPerPage == this->resultStart) {
-      page["selected"] = true;
-    }
-
-    pagesCDT.PushBack(page);
-  }
-  oData["pages"] = pagesCDT;
-
-  oData["count"] = kiwix::beautifyInteger(this->estimatedResultCount);
-  oData["searchPattern"] = kiwix::encodeDiples(this->searchPattern);
-  oData["searchPatternEncoded"] = urlEncode(this->searchPattern);
-  oData["resultStart"] = this->resultStart + 1;
-  oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount
-                            ? this->estimatedResultCount
-                            : this->resultEnd);
-  oData["resultRange"] = this->resultCountPerPage;
-  oData["resultLastPageStart"]
-      = this->estimatedResultCount > this->resultCountPerPage
-            ? std::round(this->estimatedResultCount / this->resultCountPerPage) * this->resultCountPerPage
-            : 0;
-  oData["protocolPrefix"] = this->protocolPrefix;
-  oData["searchProtocolPrefix"] = this->searchProtocolPrefix;
-  oData["contentId"] = this->contentHumanReadableId;
-
-  std::string template_ct2 = RESOURCE::results_ct2;
-  VMStringLoader oLoader(template_ct2.c_str(), template_ct2.size());
-
-  FileLogger oLogger(stderr);
-
-  // DEBUG only (write output to stdout)
-  // oSimpleVM.Run(oData, oLoader, stdout, oLogger);
-
-  std::string sResult;
-  oSimpleVM.Run(oData, oLoader, sResult, oLogger);
-
-  return sResult;
-}
-
-#endif
-}

src/server.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "server.h"
+
+#include "library.h"
+#include "name_mapper.h"
+
+#include <string>
+
+#include <zim/item.h>
+#include "server/internalServer.h"
+
+namespace kiwix {
+
+Server::Server(Library* library, NameMapper* nameMapper) :
+  mp_library(library),
+  mp_nameMapper(nameMapper),
+  mp_server(nullptr)
+{
+}
+
+Server::~Server() = default;
+
+bool Server::start() {
+  mp_server.reset(new InternalServer(
+    mp_library,
+    mp_nameMapper,
+    m_addr,
+    m_port,
+    m_root,
+    m_nbThreads,
+    m_multizimSearchLimit,
+    m_verbose,
+    m_withTaskbar,
+    m_withLibraryButton,
+    m_blockExternalLinks,
+    m_indexTemplateString,
+    m_ipConnectionLimit));
+  return mp_server->start();
+}
+
+void Server::stop() {
+  if (mp_server) {
+    mp_server->stop();
+    mp_server.reset(nullptr);
+  }
+}
+
+void Server::setRoot(const std::string& root)
+{
+  m_root = root;
+  if (m_root[0] != '/') {
+    m_root = "/" + m_root;
+  }
+  if (m_root.back() == '/') {
+    m_root.erase(m_root.size() - 1);
+  }
+}
+
+int Server::getPort()
+{
+  return mp_server->getPort();
+}
+
+std::string Server::getAddress()
+{
+  return mp_server->getAddress();
+}
+
+}

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/etag.cpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "etag.h"
+
+#include "tools/stringTools.h"
+
+#include <algorithm>
+#include <sstream>
+
+namespace kiwix {
+
+namespace {
+
+// Characters in the options part of the ETag could in principle be picked up
+// from the latin alphabet in natural order (the character corresponding to
+// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
+// this file). However it is better to have some mnemonics in the option names,
+// hence below variable: all_options[opt] corresponds to the character going
+// into the ETag for ETag::Option opt.
+// IMPORTANT: The characters in all_options must come in sorted order (so that
+// IMPORTANT: isValidOptionsString() works correctly).
+const char all_options[] = "Zz";
+
+static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
+
+bool isValidETagBody(const std::string& s)
+{
+  return !s.empty() && s.find_first_of("\"/") == std::string::npos;
+}
+
+bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
+{
+  std::string::size_type i = 0;
+  for ( const char c : s )
+  {
+    const std::string::size_type j = sortedString.find(c, i);
+    if ( j == std::string::npos )
+      return false;
+    i = j+1;
+  }
+  return true;
+}
+
+bool isValidOptionsString(const std::string& s)
+{
+  return isSubsequenceOf(s, all_options);
+}
+
+} // namespace
+
+
+void ETag::set_option(Option opt)
+{
+  if ( ! get_option(opt) )
+  {
+    m_options.push_back(all_options[opt]);
+    std::sort(m_options.begin(), m_options.end());
+  }
+}
+
+bool ETag::get_option(Option opt) const
+{
+  return m_options.find(all_options[opt]) != std::string::npos;
+}
+
+std::string ETag::get_etag() const
+{
+  if ( m_body.empty() )
+    return std::string();
+
+  return "\"" + m_body + "/" + m_options + "\"";
+}
+
+ETag::ETag(const std::string& body, const std::string& options)
+{
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
+  {
+    m_body = body;
+    m_options = options;
+  }
+}
+
+ETag ETag::parse(std::string s)
+{
+  if ( kiwix::startsWith("W/", s) )
+    s = s.substr(2);
+
+  if ( s.front() != '"' || s.back() != '"' )
+    return ETag();
+
+  s = s.substr(1, s.size()-2);
+
+  const std::string::size_type i = s.find('/');
+  if ( i == std::string::npos )
+    return ETag();
+
+  return ETag(s.substr(0, i), s.substr(i+1));
+}
+
+ETag ETag::match(const std::string& etags, const std::string& body)
+{
+  std::istringstream ss(etags);
+  std::string etag_str;
+  while ( ss >> etag_str )
+  {
+    if ( etag_str.back() == ',' )
+      etag_str.pop_back();
+
+    const ETag etag = parse(etag_str);
+    if ( etag && etag.m_body == body )
+      return etag;
+  }
+
+  return ETag();
+}
+
+} // namespace kiwix

src/server/etag.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_ETAG_H
+#define KIWIXLIB_SERVER_ETAG_H
+
+#include <string>
+
+namespace kiwix {
+
+// The ETag string used by Kiwix server (more precisely, its value inside the
+// double quotes) consists of two parts:
+//
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
+//
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
+//               the values of some of the headers of the response
+//
+// The two parts are separated with a slash (/) symbol (which is always present,
+// even when the the options part is empty). Neither portion of a Kiwix ETag
+// may contain the slash symbol.
+// Examples of valid Kiwix server ETags (including the double quotes):
+//
+//   "abcdefghijklmn/"
+//   "1234567890/z"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
+//
+// The options part of the Kiwix ETag allows to correctly set the required
+// headers when responding to a conditional If-None-Match request with a 304
+// (Not Modified) response without following the full code path that would
+// discover the necessary options.
+
+class ETag
+{
+  public: // types
+    enum Option {
+      ZIM_CONTENT,
+      COMPRESSED_CONTENT,
+      OPTION_COUNT
+    };
+
+  public: // functions
+    ETag() {}
+
+    void set_body(const std::string& s) { m_body = s; }
+    void set_option(Option opt);
+
+    explicit operator bool() const { return !m_body.empty(); }
+
+    bool get_option(Option opt) const;
+    std::string get_etag() const;
+
+
+    static ETag match(const std::string& etags, const std::string& server_id);
+
+  private: // functions
+    ETag(const std::string& serverId, const std::string& options);
+
+    static ETag parse(std::string s);
+
+  private: // data
+    std::string m_body;
+    std::string m_options;
+};
+
+} // namespace kiwix
+
+#endif // KIWIXLIB_SERVER_ETAG_H

src/server/i18n.cpp

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "i18n.h"
+
+#include "tools/otherTools.h"
+
+#include <algorithm>
+#include <map>
+
+namespace kiwix
+{
+
+const char* I18nStringTable::get(const std::string& key) const
+{
+  const I18nString* const begin = entries;
+  const I18nString* const end = begin + entryCount;
+  const I18nString* found = std::lower_bound(begin, end, key,
+      [](const I18nString& a, const std::string& k) {
+        return a.key < k;
+  });
+  return (found == end || found->key != key) ? nullptr : found->value;
+}
+
+namespace i18n
+{
+// this data is generated by the i18n resource compiler
+extern const I18nStringTable stringTables[];
+extern const size_t langCount;
+}
+
+namespace
+{
+
+class I18nStringDB
+{
+public: // functions
+  I18nStringDB() {
+    for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
+      const auto& t = kiwix::i18n::stringTables[i];
+      lang2TableMap[t.lang] = &t;
+    }
+    enStrings = lang2TableMap.at("en");
+  };
+
+  std::string get(const std::string& lang, const std::string& key) const {
+    const char* s = getStringsFor(lang)->get(key);
+    if ( s == nullptr ) {
+      s = enStrings->get(key);
+      if ( s == nullptr ) {
+        throw std::runtime_error("Invalid message id");
+      }
+    }
+    return s;
+  }
+
+  size_t getStringCount(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang)->entryCount;
+    } catch(const std::out_of_range&) {
+      return 0;
+    }
+  }
+
+private: // functions
+  const I18nStringTable* getStringsFor(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang);
+    } catch(const std::out_of_range&) {
+      return enStrings;
+    }
+  }
+
+private: // data
+  std::map<std::string, const I18nStringTable*> lang2TableMap;
+  const I18nStringTable* enStrings;
+};
+
+const I18nStringDB& getStringDb()
+{
+  static const I18nStringDB stringDb;
+  return stringDb;
+}
+
+} // unnamed namespace
+
+std::string getTranslatedString(const std::string& lang, const std::string& key)
+{
+  return getStringDb().get(lang, key);
+}
+
+namespace i18n
+{
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params)
+{
+  const std::string tmpl = getTranslatedString(lang, key);
+  return render_template(tmpl, params);
+}
+
+} // namespace i18n
+
+std::string ParameterizedMessage::getText(const std::string& lang) const
+{
+  return i18n::expandParameterizedString(lang, msgId, params);
+}
+
+namespace
+{
+
+LangPreference parseSingleLanguagePreference(const std::string& s)
+{
+  const size_t langStart = s.find_first_not_of(" \t\n");
+  if ( langStart == std::string::npos ) {
+    return {"", 0};
+  }
+
+  const size_t langEnd = s.find(';', langStart);
+  if ( langEnd == std::string::npos ) {
+    return {s.substr(langStart), 1};
+  }
+
+  const std::string lang = s.substr(langStart, langEnd - langStart);
+  // We don't care about langEnd == langStart which will result in an empty
+  // language name - it will be dismissed by parseUserLanguagePreferences()
+
+  float q = 1.0;
+  int nCharsScanned;
+  if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
+       && langEnd + 1 + nCharsScanned == s.size() ) {
+    return {lang, q};
+  }
+
+  return {"", 0};
+}
+
+} // unnamed namespace
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s)
+{
+  UserLangPreferences result;
+  std::istringstream iss(s);
+  std::string singleLangPrefStr;
+  while ( std::getline(iss, singleLangPrefStr, ',') )
+  {
+    const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
+    if ( !langPref.lang.empty() && langPref.preference > 0 ) {
+      result.push_back(langPref);
+    }
+  }
+
+  return result;
+}
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
+{
+  if ( prefs.empty() ) {
+    return "en";
+  }
+
+  std::string bestLangSoFar("en");
+  float bestScoreSoFar = 0;
+  const auto& stringDb = getStringDb();
+  for ( const auto& entry : prefs ) {
+    const float score = entry.preference * stringDb.getStringCount(entry.lang);
+    if ( score > bestScoreSoFar ) {
+      bestScoreSoFar = score;
+      bestLangSoFar = entry.lang;
+    }
+  }
+  return bestLangSoFar;
+}
+
+} // namespace kiwix