fixup! fixup! fixup! fixup! Introduce Manager::addBooksFromDirectory()

skipInvalid=false makes little sense, if an invalid book is found, we can simply choose to ignore it rather stopping the whole operation midway.

.github/move.yml

@@ -1,27 +0,0 @@
-# Configuration for Move Issues - https://github.com/dessant/move-issues
-
-# Delete the command comment when it contains no other content
-deleteCommand: true
-
-# Close the source issue after moving
-closeSourceIssue: true
-
-# Lock the source issue after moving
-lockSourceIssue: false
-
-# Mention issue and comment authors
-mentionAuthors: true
-
-# Preserve mentions in the issue content
-keepContentMentions: true
-
-# Move labels that also exist on the target repository
-moveLabels: true
-
-# Set custom aliases for targets
-# aliases:
-#   r: repo
-#   or: owner/repo
-
-# Repository to extend settings from
-# _extends: repo

.github/workflows/ci.yml

@@ -3,147 +3,198 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
 
 jobs:
-  Macos:
-    runs-on: macos-latest
+  macOS:
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - macos-aarch64-dyn
+          - macos-x86_64-dyn
+          - ios-arm64-dyn
+          - ios-x86_64-dyn
+        include:
+          - target: macos-aarch64-dyn
+            arch_name: arm64-apple-macos
+            run_test: true
+          - target: macos-x86_64-dyn
+            arch_name: x86_64-apple-darwin
+            run_test: true
+          - target: ios-arm64-dyn
+            arch_name: aarch64-apple-ios
+            run_test: false
+          - target: ios-x86_64-dyn
+            arch_name: x86-apple-ios-simulator
+            run_test: false
+    runs-on: macos-15
+
+    env:
+      HOME: /Users/runner
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Setup python 3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
+      - name: Retrieve source code
+        uses: actions/checkout@v4
+
       - name: Install packages
         run: |
           brew update
-          brew install gcovr pkg-config ninja
-      - name: Install python modules
-        run: pip3 install meson==0.49.2 pytest
-      - name: Install deps
-        shell: bash
-        run: |
-          ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
-          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
+          brew install ninja meson
+
+      - name: Install dependencies
+        uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+        with:
+          target_platform: ${{ matrix.target }}
+
       - name: Compile
+        env:
+          PKG_CONFIG_PATH: ${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib/pkgconfig
+          CPPFLAGS: -I${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/include
+          MESON_OPTION: --default-library=shared -Db_coverage=true
+          MESON_CROSSFILE: ${{env.HOME}}/BUILD_${{matrix.arch_name}}/meson_cross_file.txt
         shell: bash
         run: |
-          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
-          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
-          meson . build --default-library=shared -Db_coverage=true
-          cd build
-          ninja
-      - name: Test
-        shell: bash
-        run: |
-          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
-          cd build
-          meson test --verbose
+          if [ -e $MESON_CROSSFILE ]; then
+            MESON_OPTION="$MESON_OPTION --cross-file $MESON_CROSSFILE -Dstatic-linkage=true"
+          fi
+          meson . build ${MESON_OPTION}
+          ninja -C build
+
+      - name: Test libkiwix
+        if: matrix.run_test
         env:
           SKIP_BIG_MEMORY_TEST: 1
+          LD_LIBRARY_PATH: ${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib:${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib64
+        run: meson test -C build --verbose
+
+  Windows:
+    runs-on: windows-2022
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install packages
+        run:
+          choco install pkgconfiglite ninja
+
+      - name: Install python modules
+        run: pip3 install meson
+
+      - name: Setup MSVC compiler
+        uses: bus1/cabuild/action/msdevshell@v1
+        with:
+          architecture: x64
+
+      - name: Install dependencies
+        uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+        with:
+          target_platform: win-x86_64-static
+
+      - name: Compile
+        shell: cmd
+        run: |
+          set PKG_CONFIG_PATH=%cd%\BUILD_win-amd64\INSTALL\lib\pkgconfig
+          set CPPFLAGS=-I%cd%\BUILD_win-amd64\INSTALL\include
+          meson.exe setup . build -Dwerror=false --default-library=static --buildtype=release
+          cd build
+          ninja.exe
+
+      - name: Test
+        shell: cmd
+        run: |
+          cd build
+          meson.exe test --verbose
+        env:
+          WAIT_TIME_FACTOR_TEST: 10
 
   Linux:
     strategy:
       fail-fast: false
       matrix:
-        name:
-          - native_static
-          - native_dyn
-          - android_arm
-          - android_arm64
-          - win32_static
-          - win32_dyn
+        target:
+          - linux-x86_64-static
+          - linux-x86_64-dyn
+          - android-arm
+          - android-arm64
+        image_variant: ['jammy']
         include:
-          - name: native_static
-            target: native_static
-            image_variant: bionic
+          - target: linux-x86_64-static
             lib_postfix: '/x86_64-linux-gnu'
-          - name: native_dyn
-            target: native_dyn
-            image_variant: bionic
+            arch_name: linux-x86_64
+            run_test: true
+            coverage: true
+          - target: linux-x86_64-dyn
             lib_postfix: '/x86_64-linux-gnu'
-          - name: android_arm
-            target: android_arm
-            image_variant: bionic
+            arch_name: linux-x86_64
+            run_test: true
+            coverage: true
+          - target: android-arm
             lib_postfix: '/arm-linux-androideabi'
-          - name: android_arm64
-            target: android_arm64
-            image_variant: bionic
+            arch_name: arm-linux-androideabi
+            run_test: false
+            coverage: false
+          - target: android-arm64
             lib_postfix: '/aarch64-linux-android'
-          - name: win32_static
-            target: win32_static
-            image_variant: f35
-            lib_postfix: '64'
-          - name: win32_dyn
-            target: win32_dyn
-            image_variant: f35
-            lib_postfix: '64'
+            arch_name: aarch64-linux-android
+            run_test: false
+            coverage: false
     env:
       HOME: /home/runner
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2025-06-07"
     steps:
     - name: Checkout code
-      shell: python
-      run: |
-        from subprocess import check_call
-        from os import environ
-        command = [
-          'git', 'clone',
-          'https://github.com/${{github.repository}}',
-          '--depth=1',
-          '--branch', '${{ github.head_ref || github.ref_name }}'
-        ]
-        check_call(command, cwd=environ['HOME'])
-    - name: Install deps
-      shell: bash
-      run: |
-        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_libkiwix.tar.xz
-        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
+      uses: actions/checkout@v4
+    - name: Install dependencies
+      uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+      with:
+        target_platform: ${{ matrix.target }}
     - name: Compile
       shell: bash
       run: |
         meson --version
-        if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
+        if [[ "${{matrix.target}}" =~ .*-dyn ]]; then
           MESON_OPTION="--default-library=shared"
         else
           MESON_OPTION="--default-library=static"
         fi
-        if [[ "${{matrix.target}}" =~ native_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
+        if [ -e "${{env.HOME}}/BUILD_${{matrix.arch_name}}/meson_cross_file.txt" ]; then
+          MESON_OPTION="$MESON_OPTION --cross-file ${{env.HOME}}/BUILD_${{matrix.arch_name}}/meson_cross_file.txt"
         else
-          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
+          MESON_OPTION="$MESON_OPTION -Db_coverage=true"
         fi
-        if [[ "${{matrix.target}}" =~ android_.* ]]; then
+        if [[ "${{matrix.target}}" =~ android-.* ]]; then
           MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
         fi
-        cd $HOME/libkiwix
         meson . build ${MESON_OPTION}
         cd build
         ninja
       env:
-        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
-        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
+        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.arch_name}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.arch_name}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.arch_name}}/INSTALL/include"
     - name: Test
-      if: startsWith(matrix.target, 'native_')
+      if: matrix.run_test
       shell: bash
       run: |
-        cd $HOME/libkiwix/build
+        cd build
         meson test --verbose
-        ninja coverage
+        if [[ "${{matrix.coverage}}" = "true" ]]; then
+          ninja coverage
+        fi
       env:
-        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
+        LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.arch_name}}/INSTALL/lib:/home/runner/BUILD_${{matrix.arch_name}}/INSTALL/lib${{matrix.lib_postfix}}"
         SKIP_BIG_MEMORY_TEST: 1
+
     - name: Publish coverage
-      shell: bash
-      run: |
-        cd $HOME/libkiwix
-        curl https://codecov.io/bash -o codecov.sh
-        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
-        rm codecov.sh
-      if: startsWith(matrix.target, 'native_')
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      if: matrix.coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -1,19 +1,29 @@
 name: Packages
-on: [push, pull_request]
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
 
 jobs:
   build-deb:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
         distro:
+#          - debian-unstable
+#          - debian-trixie
+#          - debian-bookworm
+#          - debian-bullseye
+          - ubuntu-noble
           - ubuntu-jammy
-          - ubuntu-impish
-          - ubuntu-focal
-          - ubuntu-bionic
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # Determine which PPA we should upload to
       - name: PPA
@@ -21,19 +31,55 @@ jobs:
         run: |
           if [[ $REF == refs/tags* ]]
           then
-            echo "::set-output name=ppa::kiwixteam/release"
+            echo "ppa=kiwixteam/release" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=ppa::kiwixteam/dev"
+            echo "ppa=kiwixteam/dev" >> $GITHUB_OUTPUT
           fi
         env:
           REF: ${{ github.ref }}
 
-      - uses: legoktm/gh-action-auto-dch@master
+      - uses: legoktm/gh-action-auto-dch@main
         with:
           fullname: Kiwix builder
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
+#      - uses: legoktm/gh-action-build-deb@debian-unstable
+#        if: matrix.distro == 'debian-unstable'
+#        name: Build package for debian-unstable
+#        id: build-debian-unstable
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@b47978ba8498dc8b8153cc3b5f99a5fc1afa5de1 # pin@debian-trixie
+#        if: matrix.distro == 'debian-trixie'
+#        name: Build package for debian-trixie
+#        id: build-debian-trixie
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@1f4e86a6bb34aaad388167eaf5eb85d553935336 # pin@debian-bookworm
+#        if: matrix.distro == 'debian-bookworm'
+#        name: Build package for debian-bookworm
+#        id: build-debian-bookworm
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@084b4263209252ec80a75d2c78a586192c17f18d # pin@debian-bullseye
+#        if: matrix.distro == 'debian-bullseye'
+#        name: Build package for debian-bullseye
+#        id: build-debian-bullseye
+#        with:
+#          args: --no-sign
+
+      - uses: legoktm/gh-action-build-deb@9114a536498b65c40b932209b9833aa942bf108d # pin@ubuntu-noble
+        if: matrix.distro == 'ubuntu-noble'
+        name: Build package for ubuntu-noble
+        id: build-ubuntu-noble
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
       - uses: legoktm/gh-action-build-deb@ubuntu-jammy
         if: matrix.distro == 'ubuntu-jammy'
         name: Build package for ubuntu-jammy
@@ -42,50 +88,24 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
-      - uses: legoktm/gh-action-build-deb@ubuntu-focal
-        if: matrix.distro == 'ubuntu-focal'
-        name: Build package for ubuntu-focal
-        id: build-ubuntu-focal
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
-      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
-        if: matrix.distro == 'ubuntu-bionic'
-        name: Build package for ubuntu-bionic
-        id: build-ubuntu-bionic
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: Packages for ${{ matrix.distro }}
           path: output
 
-      - uses: legoktm/gh-action-dput@master
+      - uses: legoktm/gh-action-dput@main
         name: Upload dev package
-        # Only upload on pushes to master
-        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        # Only upload on pushes to main
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/dev
           packages: output/*_source.changes
 
-      - uses: legoktm/gh-action-dput@master
+      - uses: legoktm/gh-action-dput@main
         name: Upload release package
-        # Only upload on pushes to master or tag
-        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        if: github.event_name == 'release' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/release
           packages: output/*_source.changes
-

.readthedocs.yaml

@@ -0,0 +1,21 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: docs/requirements.txt

ChangeLog

@@ -1,3 +1,164 @@
+libkiwix 14.1.1
+===============
+
+* Server:
+  - Fix regression for kiwix-serve --nosearchbar (@veloman-yunkan #1250)
+  - Avoid results content interpretation... crash in fulltext search (@vighnesh-sawant #1241)
+  - Fix for intermittent /content/blank.html errors (@veloman-yunkan #1249)
+
+libkiwix 14.1.0
+===============
+
+ * Server:
+  - Viewer detects & tracks intrapage navigation anchors too (@veloman-yunkan #1213)
+  - Add support for catalog only mode (@veloman-yunkan #1219)
+  - Add API which returns server access url (@vighnesh-sawant #1234)
+  - Fix chrome searchbar placeholder text overflow (@aditii2712 #1185)
+  - Fix magnet link queryStyring (@rgaudin #1160)
+  - Improve chrome printing stylesheet (@kelson42 #1202)
+  - Default white background (@kelson42 #1205)
+
+ * Other:
+  - Switched to the new libzim illustrations API (@veloman-yunkan #1226)
+  - Stop building Windows with DEBUG symbols in CI (@kelson42 #1165)
+  - Update many things in the CI/CD (@kelson42 #1203 #1194 #1209 #1207 #1235)
+  - Requires now libzim 9.4.0 (@kelson42 #1231)
+  - Fix compilation for FreeBSD (@OICe2 #1173 #1174)
+  - Wait up to 1s to let aria2c to start before complaining (@kelson42 #1169)
+
+libkiwix 14.0.0
+===============
+
+ * Server:
+  - Support of IPv6 (@veloman-yunkan @aryanA101a #1074 #1093)
+  - Better public IP configuration/detection (@sgourdas #1132)
+  - Fix API errors in catalog searches if Xapian keyword in used (@veloman-yunkan #1137)
+  - Clearly define which Web browsers are supported (@kelson42 @rgaudin @jaifroid @benoit74 #1132)
+  - Improve welcome page download buttons (@veloman-yunkan #1094)
+  - Better handling of external (non-HTTP) links (@veloman-yunkan #1123)
+  - Fix book illustration size on welcome page to 48x48 pixels (@veloman-yunkan #1127)
+  - Remove "Multiple Languages" in language filter (@veloman-yunkan #1098)
+  - Stop transforming tags casing (@kelson42 @veloman-yunkan #1079 #1121)
+  - ZIM file size consistently advertised in MiB (@harsha-mangena #1132)
+  - Few new supported languages in the filter (@kelson42 #1080)
+  - Improve accesskeys (@kelson42 #1075)
+  - Add OpenSearch <link> to head of pages (@kelson42 #1070)
+ * Compilation/Packaging:
+  - Multiple fixes around deb packaging (@kelson42 #1108 #1114 #1135)
+  - Generating of libkiwix.pc via Meson (@veloman-yunkan #1133)
+  - Native Windows CI/CD (@mgautierfr @kelson42 #1113 #1125)
+  - Better check (maximum) libzim version (@kelson42 #1124)
+  - Multiple automated tests improvements (@veloman-yunkan #1068 #1067)
+ * Other:
+  - Deleted supported env. variable `$KIWIX_DATA_DIR` and `kiwix::getDataDirectory()` (@sgourdas #1107)
+  - New string slugification for filenames (@shaopenglin #1105)
+  - Multiple improvements around aria2c download mgmt. (@veloman-yunkan #1097)
+
+libkiwix 13.1.0
+===============
+
+ * Server:
+  - Properly translated error pages (@veloman-yunkan #1032)
+  - Properly translated search result page (@veloman-yunkan #1046)
+  - Default UI language is resolved in frontend (@veloman-yunkan #1044)
+  - Better support of older Web browsers by polyfilling replaceAll() (@veloman-yunkan #1054)
+ * New API to migrate bookmarks between books (@mgautierfr #1043)
+ * Fixed compilation on Haiku OS (@Begasus #1048)
+
+libkiwix 13.0.0
+===============
+
+ * Server:
+  - Improved look & feel of kiwix-serve UI (@veloman-yunkan #917 #1021)
+  - Increase tolerance to malformed (control characters) ZIM entry titles (@veloman-yunkan #1023)
+  - API allowing to filter many categories at once (@juuz0 #974)
+  - Cookie-less user language control (@veloman-yumkan #997)
+  - Hack to fix Mirrorbrain based broken magnet URLs (@rgaudin #1001)
+ * Fix handling of books with 'Name' metadata with dots (@mgautier #1016)
+ * New method beautifyFileSize() to provide nice-looking book sizes (@vuuz0 #971)
+ * Fix a few missing includes (@mgautierfr #978)
+ * New functions to read - kiwix-serve - languages and categories streams (@juuz0 #967)
+ * Add support of Fon language (@kelson42 #1013)
+ * C++17 code base compliancy (@mgautierfr #996)
+ * Use everywhere std::shared_ptr in place of raw pointer (@mgautierfr #991)
+ * Do not use [[nodiscard]] attribute on compiler not supporting it (@mgautierfr #1003)
+ * Add a non minified version of autoComplete.js (@mgautierfr #1008)
+ * Multiple CI/CD improvements (@kelson42 #982)
+
+libkiwix 12.1.0
+===============
+
+ * Server:
+  - Introduce a `/nojs` endpoint to browse catalog and zim files with a browser without js (@juuz0 #897)
+  - Translate the viewer (@veloman-yunkan #871 #846)
+  - Display `mul` on tile when zim is multi-languages (@juuz0 #934)
+  - Suggestion links point to the `/content` endpoint (@veloman-yunkan #862)
+  - Correctly compress web fonts in http answers (@kelson42 #856)
+  - Correctly encode link in suggestions (@veloman-yunkan #859 #860 #963)
+  - Correctly encode url redirection (@veloman-yunkan #866 #890)
+  - Properly handle user language, through cookies and http headers (@veloman-yunkan #849 #869)
+  - Fix url encoding (@veloman-yunkan #870)
+  - Fix viewer for viewer for SeaMonkey (@veloman-yunkan #887)
+  - Make the downloader threadsafe (@mgautierfr #886)
+  - Add RSS feed in the main page (pointing to the catalog) (@juuz0 #882 #920)
+  - Correctly set the mimetype for json and ico (@veloman-yunkan #892)
+  - `count=-1` correspond to unlimited count (instead of 0) (@veloman-yunkan #894)
+  - Keep the navigation bar on top (@juuz0 #896)
+  - Make the viewer's iframe "safe" (@veloman-yunkan #906 #930)
+  - Correctly escape search link in XML Opds output (@veloman-yunkan #936)
+  - Store values needed for the viewer js in the url fragment instead of the query string (@juuz0 #907)
+  - Get rid of legacy OPDS API usage in the viewer (@veloman-yunkan #939)
+  - Fix charset encoding declaration in OPDS response MIME types (@veloman-yunkan #942)
+  - Fix PDF in the viewer (@veloman-yunkan #940)
+  - Fix external links handling in the viewer (@veloman-yunkan #959)
+  - Add tests of searching with accents (@mgautierfs #954)
+ * Fix handling of missing illustration in the book (@veloman-yunkan #961)
+ * Add support for multi languages zim files (@veloman-yunkan #904)
+ * Fix includes for openbsd (@bentley #949)
+ * Fix pathes in git to allow git clone on Windows (@adamlamar #868)
+ * Switch to `main` as principal branch (instead of `master`) (@kelson42)
+ * Remove libkiwix android publisher from the repository (@kelson42 #884)
+ * Various fixes of meson and CI. (@mgautierfr @kelson42)
+
+libkiwix 12.0.0
+===============
+
+ * [API Break] Remove wrapper around libzim (@mgautierfr #789)
+ * Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
+ * Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
+ * Prevent search on multi language content (@veloman-yunkan #838)
+ * Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
+ * Catalog:
+  - Include tags in free text catalog search (@veloman-yunkan #802)
+  - Illustration's url is based on book's uuid (@veloman-yunkan #804)
+  - Cleanup of the opds-dumper (@veloman-yunkan #829)
+  - Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
+  - Make opds-dumper respect the namemapper (@mgautierfr #837)
+ * Server:
+  - Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
+  - Better http caching (@veloman-yunkan #833)
+  - Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
+  - Better redirection of main page (@veloman-yunkan #827)
+  - Remove jquery (@mgautierfr @juuz0 #796)
+  - Better Viewer of zim content :
+    . Introduce `/content` endpoints (@veloman-yunkan #806)
+    . Switch to iframe based content viewer (@veloman-yunkan #716)
+  - Optimised design of the welcome page:
+    . Alignement (@juuz0 @kelson42 #786)
+    . Exit download modal on pressing escape key (@juzz0 #800)
+    . Add favicon for different devices (@juzz0 #805)
+    . Fix auto hidding of the toolbar (@veloman-yunkan #821)
+    . Allow user to filter books by tags in the front page (@juuz0 #711)
+ * CI :
+   - Trigger CI on pull_request (@kelson42 #791)
+   - Drop Ubuntu Impish packaging (@legoktm #825)
+   - Add Ubuntu Kinetic packaging (@legoktm #801)
+ * Testing:
+   - Test ICULanguageInfo (@veloman-yunkan #795)
+   - Introduce fake `test` language to test i18n (@veloman-yunkan #848)
+ * Fix documentation (@kelson42 #816)
+ * Udpate translation (#787 #839 #847)
+
 libkiwix 11.0.0
 ===============
 
@@ -5,7 +166,7 @@ libkiwix 11.0.0
  * [server] Use gzip compression instead of deflat (mgautierfr #757)
  * [server] Version the static resources. This allow better invalidating
    browser cache when resources are changed (@veloman-yunkan #712)
- * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server|front] Use integer to query the host for page length (@juuz0 #772)
  * [server] Improve multizim search API:
    - Improvement of the cache system
    - Better API to select on which books to search in.

README.md

@@ -7,10 +7,10 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
-[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/kiwix/libkiwix/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
 [![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
-[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -24,9 +24,9 @@ with the Libkiwix compilation itself, we recommend to have a look to
 Preamble
 --------
 
-Although the Libkiwix can be (cross-)compiled on/for many sytems, the
+Although the Libkiwix can be (cross-)compiled on/for many systems, the
 following documentation explains how to do it on POSIX ones. It is
-primarly thought for GNU/Linux systems and has been tested on recent
+primarily thought for GNU/Linux systems and has been tested on recent
 releases of Ubuntu and Fedora.
 
 Dependencies
@@ -54,7 +54,7 @@ The following dependency needs to be available at runtime:
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
 compilation script will tell you if one of them is missing or too old.
-In the worse case, you will have to download and compile bleeding edge
+In the worst case, you will have to download and compile bleeding edge
 version by hand.
 
 If you want to install these dependencies locally, then use the
@@ -101,6 +101,33 @@ meson . build -Dwrapper=android -Dwerror=false
 ninja -C build
 ```
 
+Static files compilation
+------------------------
+
+Libkiwix has a few static files 'compiled' within the binary
+code. This is mostly Javascript/HTML/pictures necessary for the HTTP
+daemon.
+
+These static files are available in the `static` directory and are
+compiled by custom Python code available in this repository `scripts`
+directory. This happens automatically at compilation time without any
+additional command to run.
+
+To avoid HTTP caching issues, the URLs (to the static content) are
+appended with a `cacheid` parameter (this is called "cache
+busting"). This `cacheid` value derived from the
+[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
+static file. As a consequence, each time you change a static file, the
+corresponding `cacheid` value will change.
+
+To properly test this feature, this `cacheid` needs to be added
+manually to the automated tests and has to be commited. After
+modifying the needed static file, [run the automated
+tests](#Testing). They will fail, but the inspection of the testing
+log will give you the new `cacheid` value(s). Finally update
+`test/server.cpp` with the appropriate `cacheid` value(s) which have
+changed.
+
 Testing
 -------
 
@@ -124,7 +151,7 @@ where you want to install the libraries. After the installation
 succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
-------------
+--------------
 
 If you want to uninstall the Kiwix library:
 ```bash
@@ -134,28 +161,6 @@ ninja -C build uninstall
 Like for the installation, you might need to run the command as `root`
 (or using `sudo`).
 
-Troubleshooting
----------------
-
-If you need to install Meson "manually":
-```bash
-virtualenv -p python3 ./ # Create virtualenv
-source bin/activate      # Activate the virtualenv
-pip3 install meson       # Install Meson
-hash -r                  # Refresh bash paths
-```
-
-If you need to install Ninja "manually":
-```bash
-git clone git://github.com/ninja-build/ninja.git
-cd ninja
-git checkout release
-./configure.py --bootstrap
-mkdir ../bin
-cp ninja ../bin
-cd ..
-```
-
 Custom Index Page
 -----------------
 
@@ -185,7 +190,7 @@ To use JS provided by kiwix-serve you can use the following template to start wi
 - To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
 - To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
 - To add language select box add - `<select id="languageFilter"></select>` under body tag.
-- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add category select box add - `<select id="categoryFilter"></select>` under body tag.
 - To add search box for books use following form -
     ```
         <form id='kiwixSearchForm'>
@@ -196,7 +201,7 @@ To use JS provided by kiwix-serve you can use the following template to start wi
 
 
 If you compile manually Libmicrohttpd, you might need to compile it
-without GNU TLS, a bug here will empeach further compilation
+without GNU TLS, a bug here will impeach further compilation
 otherwise.
 
 If the compilation still fails, you might need to get a more recent
@@ -205,6 +210,28 @@ distribution. Try then with a source tarball distributed by the
 problematic upstream project or even directly from the source code
 repository.
 
+Troubleshooting
+---------------
+
+If you need to install Meson "manually":
+```bash
+virtualenv -p python3 ./ # Create virtualenv
+source bin/activate      # Activate the virtualenv
+pip3 install meson       # Install Meson
+hash -r                  # Refresh bash paths
+```
+
+If you need to install Ninja "manually":
+```bash
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+mkdir ../bin
+cp ninja ../bin
+cd ..
+```
+
 License
 -------
 

android-kiwix-lib-publisher/.gitignore

@@ -1,13 +0,0 @@
-*.iml
-.gradle
-/local.properties
-/.idea/caches
-/.idea/libraries
-/.idea/modules.xml
-/.idea/workspace.xml
-/.idea/navEditor.xml
-/.idea/assetWizardSettings.xml
-.DS_Store
-/build
-/captures
-.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -1,25 +0,0 @@
-// Top-level build file where you can add configuration options common to all sub-projects/modules.
-
-buildscript {
-    repositories {
-        google()
-        jcenter()
-
-    }
-    dependencies {
-        classpath 'com.android.tools.build:gradle:3.4.1'
-        // NOTE: Do not place your application dependencies here; they belong
-        // in the individual module build.gradle files
-    }
-}
-
-allprojects {
-    repositories {
-        google()
-        jcenter()
-    }
-}
-
-task clean(type: Delete) {
-    delete rootProject.buildDir
-}

android-kiwix-lib-publisher/gradle.properties

@@ -1,15 +0,0 @@
-# Project-wide Gradle settings.
-# IDE (e.g. Android Studio) users:
-# Gradle settings configured through the IDE *will override*
-# any settings specified in this file.
-# For more details on how to configure your build environment visit
-# http://www.gradle.org/docs/current/userguide/build_environment.html
-# Specifies the JVM arguments used for the daemon process.
-# The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx1536m
-# When configured, Gradle will run in incubating parallel mode.
-# This option should only be used with decoupled projects. More details, visit
-# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
-# org.gradle.parallel=true
-# Kotlin code style for this project: "official" or "obsolete":
-kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +0,0 @@
-#Wed Jun 19 15:28:39 BST 2019
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -1,172 +0,0 @@
-#!/usr/bin/env sh
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
-exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -1,84 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -1 +0,0 @@
-/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -1,64 +0,0 @@
-apply plugin: 'com.android.library'
-apply plugin: 'maven'
-
-android {
-    compileSdkVersion 28
-    defaultConfig {
-        minSdkVersion 15
-        targetSdkVersion 28
-        versionCode 1
-        versionName "1.0"
-    }
-    buildTypes {
-        release {
-            minifyEnabled false
-            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
-        }
-    }
-}
-
-dependencies {
-    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
-}
-
-task writePom {
-  pom {
-    project {
-        groupId 'org.kiwix.kiwixlib'
-        artifactId 'kiwixlib'
-        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
-        packaging 'aar'
-        name 'kiwixlib'
-        url 'https://github.com/kiwix/libkiwix'
-        licenses {
-            license {
-                name 'GPLv3'
-                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
-            }
-        }
-      developers {
-          developer {
-              id 'kiwix'
-              name 'kiwix'
-              email 'contact@kiwix.org'
-          }
-      }
-      scm {
-          connection 'https://github.com/kiwix/libkiwix.git'
-          developerConnection 'https://github.com/kiwix/libkiwix.git'
-          url 'https://github.com/kiwix/libkiwix'
-      }
-    }
-  }.withXml {
-    def dependenciesNode = asNode().appendNode('dependencies')
-
-    //Iterate over the implementation dependencies, adding a <dependency> node for each
-    configurations.implementation.allDependencies.each {
-      def dependencyNode = dependenciesNode.appendNode('dependency')
-      dependencyNode.appendNode('groupId', it.group)
-      dependencyNode.appendNode('artifactId', it.name)
-      dependencyNode.appendNode('version', it.version)
-    }
-  }.writeTo("$buildDir/pom.xml")
-}
-

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -1,21 +0,0 @@
-# Add project specific ProGuard rules here.
-# You can control the set of applied configuration files using the
-# proguardFiles setting in build.gradle.
-#
-# For more details, see
-#   http://developer.android.com/guide/developing/tools/proguard.html
-
-# If your project uses WebView with JS, uncomment the following
-# and specify the fully qualified class name to the JavaScript interface
-# class:
-#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
-#   public *;
-#}
-
-# Uncomment this to preserve the line number information for
-# debugging stack traces.
-#-keepattributes SourceFile,LineNumberTable
-
-# If you keep the line number information, uncomment this to
-# hide the original source file name.
-#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,10 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="org.kiwix.kiwixlib">
-
-    <application
-        android:allowBackup="true"
-        android:supportsRtl="true">
-
-    </application>
-
-</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -1 +0,0 @@
-include ':kiwixLibAndroid'

debian/control

@@ -3,13 +3,12 @@ Priority: optional
 Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
  meson,
- pkg-config,
- libzim-dev (>= 7.2.0~),
+ pkgconf,
+ libzim-dev (>= 9.0), libzim-dev (<< 10.0),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
  libkainjow-mustache-dev,
- liblzma-dev,
  libmicrohttpd-dev,
  libpugixml-dev,
  zlib1g-dev
@@ -22,12 +21,13 @@ Package: libkiwix-dev
 Section: libdevel
 Architecture: any
 Multi-Arch: same
-Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
- libzim-dev (>= 7.2.0~),
+Depends: libkiwix14 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 9.0), libzim-dev (<< 10.0),
  libicu-dev,
  libpugixml-dev,
  libcurl4-gnutls-dev,
- libmicrohttpd-dev
+ libmicrohttpd-dev,
+ zlib1g-dev
 Description: library of common code for Kiwix (development)
  Kiwix is an offline Wikipedia reader. libkiwix provides the
  software core for Kiwix, and contains the code shared by all
@@ -35,11 +35,12 @@ Description: library of common code for Kiwix (development)
  .
  This package contains development files.
 
-Package: libkiwix10
+Package: libkiwix14
 Architecture: any
 Multi-Arch: same
 Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
-Conflicts: libkiwix0, libkiwix3, libkiwix9
+Conflicts: libkiwix0, libkiwix3, libkiwix9, libkiwix10, libkiwix11, libkiwix12, libkiwix13
+Replaces: libkiwix0, libkiwix3, libkiwix9, libkiwix10, libkiwix11, libkiwix12, libkiwix13
 Description: library of common code for Kiwix
  Kiwix is an offline Wikipedia reader. libkiwix provides the
  software core for Kiwix, and contains the code shared by all

docs/conf.py

@@ -24,8 +24,6 @@ author = 'libkiwix-team'
 
 # -- General configuration ---------------------------------------------------
 
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -42,9 +40,7 @@ templates_path = ['_templates']
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
-
-if not on_rtd:
-    html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinx_rtd_theme'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

docs/index.rst

@@ -12,4 +12,3 @@ Welcome to libkiwix's documentation!
 
    usage
    api/ref_api
-   widget

docs/requirements.txt

@@ -1,2 +1,3 @@
 breathe
 exhale
+sphinx_rtd_theme

docs/widget.rst

@@ -1,82 +0,0 @@
-Kiwix serve widget
-====================
-
-Introduction
-------------
-
-The kiwix-serve widget provides an easy to embed way to show the `kiwix-serve` homepage.
-
-Usage
------
-
-To use the widget, simply add an iframe with its `src` attribute set to the `widget` endpoint.
-Example HTML Page ::
-
-    <!DOCTYPE html>
-    <html lang="en">
-    <head>
-    <title>Widget Test</title>
-    </head>
-    <body>
-    <iframe src="http://192.168.18.8:8080/widget?disabledesc&disablefilter&disabledownload" width=1000 height=1000></iframe>
-    </body>
-    </html>
-
-This creates an iframe with the kiwix-serve homepage contents.
-
-Arguments are explained below.
-
-Possible Arguments
--------------------
-
-Currently, the following arguments are supported.
-
-disabledesc (value = N/A)
-    Disables the description part of a tile.
-
-disablefilter (value = N/A)
-    Disables the search filters: language, category, tag and search function.
-
-disableclick (value = N/A)
-    Disables clicking the book to open it for reading.
-
-disabledownload (value = N/A)
-    Disables the download button (if avaialable at all) on the tile.
-
-
-Custom CSS and JS
------------------
-
-You can add your custom CSS rules and Javascript code to the widget.
-
-To do that, use the following code as template::
-
-    <iframe id="receiver" src="http://192.168.18.8:8080/widget?disabledesc=&disablefilter=&disabledownload=" width="1000" height="1000">
-    <p>Your browser does not support iframes.</p>
-    </iframe>
-
-    <script>
-    window.onload = function() {
-    var receiver = document.getElementById('receiver').contentWindow;
-    function sendMessage() {
-        let msg = {
-        css: `
-        .book__header {
-            color:red;
-        }`,
-        js: `
-        function widgetTest() {
-        console.log("Testing widget");
-        }
-        widgetTest();
-        `
-        }
-        receiver.postMessage(msg, 'http://192.168.18.8:8080/widget');
-    }
-    sendMessage();
-    }
-    </script>
-
-
-The CSS/JS fields are optional, you may send both or only one.
-

format_code.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/bash
-
-files=(
-"include/library.h"
-"include/common/stringTools.h"
-"include/common/pathTools.h"
-"include/common/otherTools.h"
-"include/common/regexTools.h"
-"include/common/networkTools.h"
-"include/common/archiveTools.h"
-"include/manager.h"
-"include/reader.h"
-"include/kiwix.h"
-"include/xapianSearcher.h"
-"include/searcher.h"
-"src/library.cpp"
-"src/android/kiwix.cpp"
-"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
-"src/searcher.cpp"
-"src/common/pathTools.cpp"
-"src/common/regexTools.cpp"
-"src/common/otherTools.cpp"
-"src/common/archiveTools.cpp"
-"src/common/networkTools.cpp"
-"src/common/stringTools.cpp"
-"src/xapianSearcher.cpp"
-"src/manager.cpp"
-"src/reader.cpp"
-)
-
-for i in "${files[@]}"
-do
-  echo $i
-  clang-format -i -style=file $i
-done

include/book.h

@@ -79,7 +79,9 @@ class Book
   bool isPathValid() const { return m_pathValid; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
-  const std::string& getLanguage() const { return m_language; }
+  DEPRECATED const std::string& getLanguage() const { return m_language; }
+  const std::string& getCommaSeparatedLanguages() const { return m_language; }
+  const std::vector<std::string> getLanguages() const;
   const std::string& getCreator() const { return m_creator; }
   const std::string& getPublisher() const { return m_publisher; }
   const std::string& getDate() const { return m_date; }

include/bookmark.h

@@ -29,19 +29,33 @@ class xml_node;
 namespace kiwix
 {
 
+class Book;
 /**
  * A class to store information about a bookmark (an article in a book)
  */
 class Bookmark
 {
  public:
+  /**
+   *  Create an empty bookmark.
+   *
+   * Bookmark must be populated with `set*` methods
+   */
   Bookmark();
+
+  /**
+   * Create a bookmark given a Book, a path and a title.
+   */
+  Bookmark(const Book& book, const std::string& path, const std::string& title);
+
   ~Bookmark();
 
   void updateFromXml(const pugi::xml_node& node);
 
   const std::string& getBookId() const { return m_bookId; }
   const std::string& getBookTitle()  const { return m_bookTitle; }
+  const std::string& getBookName()  const { return m_bookName; }
+  const std::string& getBookFlavour()  const { return m_bookFlavour; }
   const std::string& getUrl() const { return m_url; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getLanguage() const { return m_language; }
@@ -49,6 +63,8 @@ class Bookmark
 
   void setBookId(const std::string& bookId) { m_bookId = bookId; }
   void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
+  void setBookName(const std::string& bookName) { m_bookName = bookName; }
+  void setBookFlavour(const std::string& bookFlavour) { m_bookFlavour = bookFlavour; }
   void setUrl(const std::string& url) { m_url = url; }
   void setTitle(const std::string& title) { m_title = title; }
   void setLanguage(const std::string& language) { m_language = language; }
@@ -57,6 +73,8 @@ class Bookmark
  protected:
   std::string m_bookId;
   std::string m_bookTitle;
+  std::string m_bookName;
+  std::string m_bookFlavour;
   std::string m_url;
   std::string m_title;
   std::string m_language;

include/common.h

@@ -16,6 +16,7 @@
 
 namespace kiwix {
 
+enum class IpMode { IPV4, IPV6, ALL, AUTO }; // AUTO: Server decides the protocol
 typedef zim::size_type size_type;
 typedef zim::offset_type offset_type;
 

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <memory>
 #include <stdexcept>
+#include <mutex>
 
 namespace kiwix
 {
@@ -43,6 +44,14 @@ class AriaError : public std::runtime_error {
 };
 
 
+/**
+ * A representation of a current download.
+ *
+ * `Download` is not thread safe. User must care to not call method on a
+ * same download from different threads.
+ * However, it is safe to use different `Download`s from different threads.
+ */
+
 class Download {
  public:
   typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
@@ -53,19 +62,89 @@ class Download {
     : mp_aria(p_aria),
       m_status(K_UNKNOWN),
       m_did(did) {};
-  void updateStatus(bool follow=false);
+
+  /**
+   * Update the status of the download.
+   *
+   * This call make an aria rpc call and is blocking.
+   * Some download (started with a metalink) are in fact several downloads.
+   * - A first one to download the metadlink.
+   * - A second one to download the real file.
+   *
+   * If `follow` is true, updateStatus tries to detect that and tracks
+   * the second download when the first one is finished.
+   * By passing false to `follow`, `Download` will only track the first download.
+   *
+   * `getFoo` methods are based on the last statusUpdate.
+   *
+   * @param follow: Do we have to follow following downloads.
+   */
+  void updateStatus(bool follow);
+
+  /**
+   * Pause the download (and call updateStatus)
+   */
   void pauseDownload();
+
+  /**
+   * Resume the download (and call updateStatus)
+   */
   void resumeDownload();
+
+  /**
+   * Cancel the download.
+   *
+   * A canceled downlod cannot be resume and updateStatus does nothing.
+   * However, you can still get information based on the last known information.
+   */
   void cancelDownload();
-  StatusResult getStatus()          { return m_status; }
-  std::string  getDid()             { return m_did; }
-  std::string  getFollowedBy()      { return m_followedBy; }
-  uint64_t     getTotalLength()     { return m_totalLength; }
-  uint64_t     getCompletedLength() { return m_completedLength; }
-  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
-  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
-  std::string  getPath()            { return m_path; }
-  std::vector<std::string>&  getUris()             { return m_uris; }
+
+  /*
+   * Get the status of the download.
+   */
+  StatusResult getStatus() const             { return m_status; }
+
+  /*
+   * Get the id of the download.
+   */
+  const std::string&  getDid() const         { return m_did; }
+
+  /*
+   * Get the id of the "second" download.
+   *
+   * Set only if the "first" download is a metalink and is complete.
+   */
+  const std::string&  getFollowedBy() const  { return m_followedBy; }
+
+  /*
+   * Get the total length of the download.
+   */
+  uint64_t     getTotalLength() const        { return m_totalLength; }
+
+  /*
+   * Get the completed length of the download.
+   */
+  uint64_t     getCompletedLength() const    { return m_completedLength; }
+
+  /*
+   * Get the download speed of the download.
+   */
+  uint64_t     getDownloadSpeed() const      { return m_downloadSpeed; }
+
+  /*
+   * Get the verified length of the download.
+   */
+  uint64_t     getVerifiedLength() const     { return m_verifiedLength; }
+
+  /*
+   * Get the path (local file) of the download.
+   */
+  const std::string&  getPath() const        { return m_path; }
+
+  /*
+   * Get the download uris of the download.
+   */
+  const std::vector<std::string>&  getUris() const { return m_uris; }
 
  protected:
   std::shared_ptr<Aria2> mp_aria;
@@ -83,23 +162,69 @@ class Download {
 /**
  * A tool to download things.
  *
+ * A Downloader manages `Download` using aria2 in the background.
+ * `Downloader` is threadsafe.
+ * However, the returned `Download`s are NOT threadsafe.
  */
 class Downloader
 {
- public:
-  Downloader();
+ public: // types
+  typedef std::vector<std::pair<std::string, std::string>> Options;
+
+ public: // functions
+  /*
+  * Create a new Downloader object.
+  *
+  * @param sessionFileDir: The directory where aria2 will store its session file.
+  */
+  explicit Downloader(std::string sessionFileDir);
   virtual ~Downloader();
 
   void close();
 
-  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
-  Download* getDownload(const std::string& did);
+  /**
+   * Start a new download.
+   *
+   * This method is thread safe and returns a pointer to a newly created
+   * `Download` or an existing one with a matching URI. In the latter case
+   * the options parameter is ignored, which can lead to surprising results.
+   * For example, if the old and new download requests (sharing the same URI)
+   * have different values for the download directory or output file name
+   * options, after the download is reported to be complete the downloaded file
+   * will be present only at the location specified for the first request.
+   *
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param uri: The uri of the thing to download.
+   * @param downloadDir: The download directory where the thing should be stored (takes precedence over any "dir" in `options`).
+   * @param options: A series of pair <option_name, option_value> to pass to aria.
+   * @return: The newly created Download.
+   */
+  std::shared_ptr<Download> startDownload(const std::string& uri, const std::string& downloadDir, Options options = {});
 
-  size_t getNbDownload() { return m_knownDownloads.size(); }
-  std::vector<std::string> getDownloadIds();
+  /**
+   * Get a download corrsponding to a download id (did)
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param did: The download id to search for.
+   * @return: The Download corresponding to did.
+   * @throw: Throw std::out_of_range if did is not found.
+   */
+  std::shared_ptr<Download> getDownload(const std::string& did);
 
- private:
-  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
+  /**
+   * Get the number of downloads currently managed.
+   */
+  size_t getNbDownload() const;
+
+  /**
+   * Get the ids of the managed downloads.
+   */
+  std::vector<std::string> getDownloadIds() const;
+
+ private: // data
+  mutable std::mutex m_lock;
+  std::map<std::string, std::shared_ptr<Download>> m_knownDownloads;
   std::shared_ptr<Aria2> mp_aria;
 };
 }

include/i18n.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ * Copyright 2024 Veloman Yunkan <veloman.yunkan@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU  General Public License as published by
@@ -17,34 +17,21 @@
  * MA 02110-1301, USA.
  */
 
-#ifndef KIWIX_SERVER_I18N
-#define KIWIX_SERVER_I18N
+#ifndef KIWIX_I18N
+#define KIWIX_I18N
 
+#include <map>
 #include <string>
-#include <mustache.hpp>
 
 namespace kiwix
 {
 
-struct I18nString {
-  const char* const key;
-  const char* const value;
-};
-
-struct I18nStringTable {
-  const char* const lang;
-  const size_t entryCount;
-  const I18nString* const entries;
-
-  const char* get(const std::string& key) const;
-};
-
 std::string getTranslatedString(const std::string& lang, const std::string& key);
 
 namespace i18n
 {
 
-typedef kainjow::mustache::object Parameters;
+typedef std::map<std::string, std::string> Parameters;
 
 std::string expandParameterizedString(const std::string& lang,
                                       const std::string& key,
@@ -71,10 +58,10 @@ private:
 
 } // namespace i18n
 
-struct ParameterizedMessage
+class ParameterizedMessage
 {
 public: // types
-  typedef kainjow::mustache::object Parameters;
+  typedef i18n::Parameters Parameters;
 
 public: // functions
   ParameterizedMessage(const std::string& msgId, const Parameters& params)
@@ -84,11 +71,22 @@ public: // functions
 
   std::string getText(const std::string& lang) const;
 
+  const std::string& getMsgId()  const { return msgId; }
+  const Parameters&  getParams() const { return params; }
+
 private: // data
   const std::string msgId;
   const Parameters  params;
 };
 
+inline ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
+{
+  const ParameterizedMessage::Parameters noParams;
+  return ParameterizedMessage(msgId, noParams);
+}
+
+std::string translateBookCategory(const std::string& lang, const std::string& category);
+
 } // namespace kiwix
 
-#endif // KIWIX_SERVER_I18N
+#endif // KIWIX_I18N

include/library.h

@@ -34,6 +34,10 @@
 
 #define KIWIX_LIBRARY_VERSION "20110515"
 
+namespace Xapian {
+class WritableDatabase;
+};
+
 namespace kiwix
 {
 
@@ -51,6 +55,22 @@ enum supportedListMode {
   NOVALID = 1 << 5
 };
 
+enum MigrationMode {
+  /** When migrating bookmarks, do not allow to migrate to an older book than the currently pointed one
+   * (or date stored in the bookmark if book is invalid)
+   *
+   * If no newer books are found, no upgrade is made.
+   */
+  UPGRADE_ONLY = 0,
+
+  /** Try hard to do a migration. This mostly does:
+   * - Try to find a newer book.
+   * - If book is invalid: find a best book, potentially older.
+   * Older book will never be returned if current book is a valid one.
+   */
+  ALLOW_DOWNGRADE = 1,
+};
+
 class Filter {
   public: // types
     using Tags = std::vector<std::string>;
@@ -67,6 +87,7 @@ class Filter {
     std::string _query;
     bool _queryIsPartial;
     std::string _name;
+    std::string _flavour;
 
   public: // functions
     Filter();
@@ -105,13 +126,30 @@ class Filter {
     Filter& acceptTags(const Tags& tags);
     Filter& rejectTags(const Tags& tags);
 
+    /**
+     *  Set the filter to only accept books in the specified category.
+     *
+     *  Multiple categories can be specified as a comma-separated list (in
+     *  which case a book in any of those categories will match).
+     */
     Filter& category(std::string category);
+
+    /**
+     *  Set the filter to only accept books in the specified language.
+     *
+     *  Multiple languages can be specified as a comma-separated list (in
+     *  which case a book in any of those languages will match).
+     */
     Filter& lang(std::string lang);
+
     Filter& publisher(std::string publisher);
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
     Filter& query(std::string query, bool partial=true);
     Filter& name(std::string name);
+    Filter& flavour(std::string flavour);
+    Filter& clearLang();
+    Filter& clearCategory();
 
     bool hasQuery() const;
     const std::string& getQuery() const { return _query; }
@@ -132,6 +170,9 @@ class Filter {
     bool hasCreator() const;
     const std::string& getCreator() const { return _creator; }
 
+    bool hasFlavour() const;
+    const std::string& getFlavour() const { return _flavour; }
+
     const Tags& getAcceptTags() const { return _acceptTags; }
     const Tags& getRejectTags() const { return _rejectTags; }
 
@@ -157,31 +198,53 @@ class ZimSearcher : public zim::Searcher
     std::mutex m_mutex;
 };
 
+template<typename, typename>
+class ConcurrentCache;
+
+template<typename, typename>
+class MultiKeyCache;
+
+using LibraryPtr = std::shared_ptr<Library>;
+using ConstLibraryPtr = std::shared_ptr<const Library>;
+
+
+// Some compiler we use don't have [[nodiscard]] attribute.
+// We don't want to declare `create` with it in this case.
+#define LIBKIWIX_NODISCARD
+#if defined __has_cpp_attribute
+  #if __has_cpp_attribute (nodiscard)
+    #undef LIBKIWIX_NODISCARD
+    #define LIBKIWIX_NODISCARD [[nodiscard]]
+  #endif
+#endif
+
 /**
  * A Library store several books.
  */
-class Library
+class Library: public std::enable_shared_from_this<Library>
 {
-  // all data fields must be added in LibraryBase
-  mutable std::mutex m_mutex;
-
  public:
   typedef uint64_t Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
   typedef std::set<std::string> BookIdSet;
 
- public:
+ private:
   Library();
+
+ public:
+  LIBKIWIX_NODISCARD static LibraryPtr create() {
+    return LibraryPtr(new Library());
+  }
   ~Library();
 
   /**
    * Library is not a copiable object. However it can be moved.
    */
   Library(const Library& ) = delete;
-  Library(Library&& );
+  Library(Library&& ) = delete;
   void operator=(const Library& ) = delete;
-  Library& operator=(Library&& );
+  Library& operator=(Library&& ) = delete;
 
   /**
    * Add a book to the library.
@@ -208,7 +271,7 @@ class Library
   void addBookmark(const Bookmark& bookmark);
 
   /**
-   * Remove a bookmarkk
+   * Remove a bookmark
    *
    * @param zimId The zimId of the bookmark.
    * @param url The url of the bookmark.
@@ -216,6 +279,66 @@ class Library
    */
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
+  /**
+   * Migrate all invalid bookmarks.
+   *
+   * All invalid bookmarks (ie pointing to unknown books, no check is made on bookmark pointing to
+   * invalid articles of valid book) will be migrated (if possible) to a better book.
+   * "Better book", will be determined using method `getBestTargetBookId`.
+   *
+   * @return A tuple<int, int>: <The number of bookmarks updated>, <Number of invalid bookmarks before migration was performed>.
+   */
+  std::tuple<int, int> migrateBookmarks(MigrationMode migrationMode = ALLOW_DOWNGRADE);
+
+  /**
+   * Migrate all bookmarks associated to a specific book.
+   *
+   * All bookmarks associated to `sourceBookId` book will be migrated to a better book.
+   * "Better book", will be determined using method `getBestTargetBookId`.
+   *
+   * @param sourceBookId the source bookId of the bookmarks to migrate.
+   * @param migrationMode how we will find the best book.
+   * @return The number of bookmarks updated.
+   */
+  int migrateBookmarks(const std::string& sourceBookId, MigrationMode migrationMode = UPGRADE_ONLY);
+
+  /**
+   * Migrate bookmarks
+   *
+   * Migrate all bookmarks pointing to `source` to `destination`.
+   *
+   * @param sourceBookId the source bookId of the bookmarks to migrate.
+   * @param targetBookId the destination bookId to migrate the bookmarks to.
+   * @return The number of bookmarks updated.
+   */
+  int migrateBookmarks(const std::string& sourceBookId, const std::string& targetBookId);
+ 
+  /**
+   * Get the best available bookId for a bookmark.
+   *
+   * Given a bookmark, return the best available bookId.
+   * "best available bookId" is determined using heuristitcs based on book name, flavour and date.
+   *
+   * @param bookmark The bookmark to search the bookId for.
+   * @param migrationMode The migration mode to use.
+   * @return A bookId. Potentially empty string if no suitable book found.
+   */
+  std::string getBestTargetBookId(const Bookmark& bookmark, MigrationMode migrationMode) const;
+
+  /**
+   * Get the best bookId for a combination of book's name, flavour and date.
+   *
+   * Given a bookName (mandatory), try to find the best book.
+   * If preferedFlavour is given, will try to find a book with the same flavour. If not found, return a book with a different flavour.
+   * If minDate is given, return a book newer than minDate. If not found, return a empty bookId.
+   *
+   * @param bookName The name of the book
+   * @param preferedFlavour The prefered flavour.
+   * @param minDate the minimal book date acceptable. Must be a string in the format "YYYY-MM-DD".
+   * @return A bookId corresponding to the query, or empty string if not found.
+   */
+  std::string getBestTargetBookId(const std::string& bookName, const std::string& preferedFlavour="", const std::string& minDate="") const;
+
   // XXX: This is a non-thread-safe operation
   const Book& getBookById(const std::string& id) const;
   // XXX: This is a non-thread-safe operation
@@ -332,8 +455,8 @@ class Library
   /**
    * Return the current revision of the library.
    *
-   * The revision of the library is updated (incremented by one) only by
-   * the addBook() operation.
+   * The revision of the library is updated (incremented by one) by
+   * the addBook() and removeBookById() operations.
    *
    * @return Current revision of the library.
    */
@@ -352,19 +475,37 @@ class Library
 
 private: // types
   typedef const std::string& (Book::*BookStrPropMemFn)() const;
-  struct Impl;
+  struct Entry : Book
+  {
+    Library::Revision lastUpdatedRevision = 0;
+  };
 
 private: // functions
   AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
+  std::string getBestFromBookCollection(BookIdCollection books, const Bookmark& bookmark, MigrationMode migrationMode) const;
+  unsigned int getBookCount_not_protected(const bool localBooks, const bool remoteBooks) const;
   void updateBookDB(const Book& book);
   void dropCache(const std::string& bookId);
 
 private: //data
-  std::unique_ptr<Impl> mp_impl;
+  mutable std::recursive_mutex m_mutex;
+  Library::Revision m_revision;
+  std::map<std::string, Entry> m_books;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+  std::unique_ptr<Xapian::WritableDatabase> m_bookDB;
 };
 
+// We don't need it anymore and we don't want to polute any other potential usage
+// of `LIBKIWIX_NODISCARD` token.
+#undef LIBKIWIX_NODISCARD
+
+
 }
 
 #endif

include/manager.h

@@ -37,10 +37,10 @@ namespace kiwix
 class LibraryManipulator
 {
  public: // functions
-  explicit LibraryManipulator(Library* library);
+  explicit LibraryManipulator(LibraryPtr library);
   virtual ~LibraryManipulator();
 
-  Library& getLibrary() const { return library; }
+  LibraryPtr getLibrary() const { return library; }
 
   bool addBookToLibrary(const Book& book);
   void addBookmarkToLibrary(const Bookmark& bookmark);
@@ -52,7 +52,7 @@ class LibraryManipulator
   virtual void booksWereRemovedFromLibrary();
 
  private: // data
-  kiwix::Library& library;
+  LibraryPtr library;
 };
 
 /**
@@ -64,8 +64,8 @@ class Manager
   typedef std::vector<std::string> Paths;
 
  public: // functions
-  explicit Manager(LibraryManipulator* manipulator);
-  explicit Manager(Library* library);
+  explicit Manager(LibraryManipulator manipulator);
+  explicit Manager(LibraryPtr library);
 
   /**
    * Read a `library.xml` and add book in the file to the library.
@@ -155,6 +155,15 @@ class Manager
                        const std::string& url = "",
                        const bool checkMetaData = false);
 
+  /**
+   * Add all books from the directory tree into the library.
+   * 
+   * @param path          The path of the directory to scan.
+   * @param verboseFlag   Verbose logs flag.
+   */
+  void addBooksFromDirectory(const std::string& path,
+                             const bool verboseFlag = false);
+
   std::string writableLibraryPath;
 
   bool m_hasSearchResult = false;
@@ -163,7 +172,7 @@ class Manager
   uint64_t m_itemsPerPage = 0;
 
  protected:
-  std::shared_ptr<kiwix::LibraryManipulator> manipulator;
+  kiwix::LibraryManipulator manipulator;
 
   bool readBookFromPath(const std::string& path, Book* book);
   bool parseXmlDom(const pugi::xml_document& doc,

include/meson.build

@@ -4,15 +4,15 @@ headers = [
   'common.h',
   'library.h',
   'manager.h',
-  'libxml_dumper.h',
-  'opds_dumper.h',
   'downloader.h',
   'search_renderer.h',
   'server.h',
+  'spelling_correction.h',
   'kiwixserve.h',
   'name_mapper.h',
   'tools.h',
-  'version.h'
+  'version.h',
+  'i18n.h'
 ]
 
 install_headers(headers, subdir:'kiwix')

include/name_mapper.h

@@ -50,16 +50,19 @@ class HumanReadableNameMapper : public NameMapper {
     std::map<std::string, std::string> m_nameToId;
 
   public:
-    HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
+    HumanReadableNameMapper(const kiwix::Library& library, bool withAlias);
     virtual ~HumanReadableNameMapper() = default;
     virtual std::string getNameForId(const std::string& id) const;
     virtual std::string getIdForName(const std::string& name) const;
+
+  private:
+    void mapName(const kiwix::Library& lib, std::string name, std::string id);
 };
 
 class UpdatableNameMapper : public NameMapper {
     typedef std::shared_ptr<NameMapper> NameMapperHandle;
   public:
-    UpdatableNameMapper(Library& library, bool withAlias);
+    UpdatableNameMapper(std::shared_ptr<Library> library, bool withAlias);
 
     virtual std::string getNameForId(const std::string& id) const;
     virtual std::string getIdForName(const std::string& name) const;
@@ -71,7 +74,7 @@ class UpdatableNameMapper : public NameMapper {
 
   private:
     mutable std::mutex mutex;
-    Library& library;
+    std::shared_ptr<Library> library;
     NameMapperHandle nameMapper;
     const bool withAlias;
 };

include/search_renderer.h

@@ -37,29 +37,11 @@ class SearchRenderer
   /**
    * Construct a SearchRenderer from a SearchResultSet.
    *
-   * The constructed version of the SearchRenderer will not introduce
-   * the book name for each result. It is better to use the other constructor
-   * with a Library pointer to have a better html page.
-   *
    * @param srs The `SearchResultSet` to render.
-   * @param mapper The `NameMapper` to use to do the rendering.
    * @param start The start offset used for the srs.
    * @param estimatedResultCount The estimatedResultCount of the whole search
    */
-  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
-                 unsigned int start, unsigned int estimatedResultCount);
-
-  /**
-   * Construct a SearchRenderer from a SearchResultSet.
-   *
-   * @param srs The `SearchResultSet` to render.
-   * @param mapper The `NameMapper` to use to do the rendering.
-   * @param library The `Library` to use to look up book details for search results.
-   * @param start The start offset used for the srs.
-   * @param estimatedResultCount The estimatedResultCount of the whole search
-   */
-  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
-                 unsigned int start, unsigned int estimatedResultCount);
+  SearchRenderer(zim::SearchResultSet srs, unsigned int start, unsigned int estimatedResultCount);
 
   ~SearchRenderer();
 
@@ -90,24 +72,39 @@ class SearchRenderer
     this->pageLength  = pageLength;
   }
 
-  std::string renderTemplate(const std::string& tmpl_str);
+  /**
+   * set user language
+   */
+  void setUserLang(const std::string& lang){
+    this->userlang  = lang;
+  }
 
   /**
    * Generate the html page with the resutls of the search.
+   *
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+                    May be nullptr. In this case, bookName is not set in the rendered string.
+   * @return The html string
    */
-  std::string getHtml();
+  std::string getHtml(const NameMapper& mapper, const Library* library);
 
-    /**
+  /**
    * Generate the xml page with the resutls of the search.
+   *
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+                    May be nullptr. In this case, bookName is not set in the rendered string.
+   * @return The xml string
    */
-  std::string getXml();
+  std::string getXml(const NameMapper& mapper, const Library* library);
 
+ protected: // function
+  std::string renderTemplate(const std::string& tmpl_str, const NameMapper& mapper, const Library *library);
 
  protected:
   std::string beautifyInteger(const unsigned int number);
   zim::SearchResultSet m_srs;
-  NameMapper* mp_nameMapper;
-  Library* mp_library;
   std::string searchBookQuery;
   std::string searchPattern;
   std::string protocolPrefix;
@@ -115,6 +112,7 @@ class SearchRenderer
   unsigned int pageLength;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
+  std::string userlang = "en";
 };
 
 

include/server.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <memory>
+#include "tools.h"
 
 namespace kiwix
 {
@@ -36,7 +37,7 @@ namespace kiwix
         *
         * @param library The library to serve.
         */
-       Server(Library* library, NameMapper* nameMapper=nullptr);
+       Server(std::shared_ptr<Library> library, std::shared_ptr<NameMapper> nameMapper=nullptr);
 
        virtual ~Server();
 
@@ -51,7 +52,7 @@ namespace kiwix
        void stop();
 
        void setRoot(const std::string& root);
-       void setAddress(const std::string& addr) { m_addr = addr; }
+       void setAddress(const std::string& addr);
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
        void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
@@ -62,14 +63,19 @@ namespace kiwix
         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
        void setBlockExternalLinks(bool blockExternalLinks)
         { m_blockExternalLinks = blockExternalLinks; }
-       int getPort();
-       std::string getAddress();
+       void setCatalogOnlyMode(bool enable) { m_catalogOnlyMode = enable; }
+       void setContentServerUrl(std::string url) { m_contentServerUrl = url; }
+       void setIpMode(IpMode mode) { m_ipMode = mode; }
+       int getPort() const;
+       IpAddress getAddress() const;
+       IpMode getIpMode() const;
+       std::vector<std::string> getServerAccessUrls() const;
 
      protected:
-       Library* mp_library;
-       NameMapper* mp_nameMapper;
+       std::shared_ptr<Library> mp_library;
+       std::shared_ptr<NameMapper> mp_nameMapper;
        std::string m_root = "";
-       std::string m_addr = "";
+       IpAddress m_addr;
        std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
@@ -78,7 +84,10 @@ namespace kiwix
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
        bool m_blockExternalLinks = false;
+       IpMode m_ipMode = IpMode::AUTO;
        int m_ipConnectionLimit = 0;
+       bool m_catalogOnlyMode = false;
+       std::string m_contentServerUrl;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

include/spelling_correction.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2025 Veloman Yunkan
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SPELLING_CORRECTION_H
+#define KIWIX_SPELLING_CORRECTION_H
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace zim
+{
+class Archive;
+}
+
+namespace Xapian
+{
+class Database;
+}
+
+namespace kiwix
+{
+
+class SpellingsDB
+{
+public: // functions
+  SpellingsDB(const zim::Archive& archive, std::filesystem::path cacheDirPath);
+  ~SpellingsDB();
+
+  SpellingsDB(const SpellingsDB& ) = delete;
+  void operator=(const SpellingsDB& ) = delete;
+
+  std::vector<std::string> getSpellingCorrections(const std::string& word, uint32_t maxCount) const;
+
+private: // data
+  std::unique_ptr<Xapian::Database> impl_;
+};
+
+} // namespace kiwix
+
+#endif // KIWIX_SPELLING_CORRECTION_H

include/tools.h

@@ -23,8 +23,21 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <cstdint>
+#include "common.h"
 
-namespace kiwix {
+namespace kiwix
+{
+
+struct IpAddress
+{
+    std::string addr;  // IPv4 address
+    std::string addr6; // IPv6 address
+};
+
+typedef std::pair<std::string, std::string> LangNameCodePair;
+typedef std::vector<LangNameCodePair> FeedLanguages;
+typedef std::vector<std::string> FeedCategories;
 
 /**
  * Return the current directory.
@@ -33,26 +46,6 @@ namespace kiwix {
  */
 std::string getCurrentDirectory();
 
-/**
- * Return the data directory.
- *
- * The data directory is a directory where to put data (zim files, ...)
- * It depends of the platform and it may be changed by user using environment variable.
- *
- * The resolution order is :
- * - `KIWIX_DATA_DIR` env variable (if set).
- * - On Windows :
- *    . `$APPDATA/kiwix` if $APPDATA is set
- *    . `$USERPROFILE/kiwix` if $USERPROFILE is set
- * - Else :
- *    . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
- *    . `$HOME/.local/share/kiwx` if $HOWE is set
- * - current directory
- *
- * @return the path of the data directory (utf8 encoded)
- */
-std::string getDataDirectory();
-
 /** Return the path of the executable
  *
  * Some application may be packaged in auto extractible archive (Appimage) and the
@@ -206,15 +199,73 @@ bool fileReadable(const std::string& path);
 std::string getMimeTypeForFile(const std::string& filename);
 
 /** Provides all available network interfaces
+ *
+ * This function provides the available IPv4 and IPv6 network interfaces
+ * as a map from the interface name to its IPv4 and/or IPv6 address(es).
+ */
+std::map<std::string, IpAddress> getNetworkInterfacesIPv4Or6();
+
+/** Provides all available IPv4 network interfaces
  *
  * This function provides the available IPv4 network interfaces
+ * as a map from the interface name to its IPv4 address.
+ *
+ * Provided for backward compatibility with libkiwix v13.1.0.
  */
 std::map<std::string, std::string> getNetworkInterfaces();
 
 /** Provides the best IP address
- * This function provides the best IP address from the list given by getNetworkInterfaces
+ * This function provides the best IP addresses for both ipv4 and ipv6 protocols,
+ * in an IpAddress struct, based on the list given by getNetworkInterfacesIPv4Or6()
+ */
+IpAddress getBestPublicIps();
+
+/** Provides the best IPv4 adddress
+ * Equivalent to getBestPublicIp(false). Provided for backward compatibility
+ * with libkiwix v13.1.0.
  */
 std::string getBestPublicIp();
 
+/** Converts file size to human readable format.
+ *
+ * This function will convert a number to its equivalent size using units.
+ *
+ * @param number file size in bytes.
+ * @return a human-readable string representation of the size, e.g., "2.3 KB", "1.8 MB", "5.2 GB".
+ */
+std::string beautifyFileSize(uint64_t number);
+
+/**
+ * Load languages stored in an OPDS stream.
+ *
+ * @param content the OPDS stream.
+ * @return vector containing pairs of language code and their corresponding full language name.
+ */
+FeedLanguages readLanguagesFromFeed(const std::string& content);
+
+/**
+ * Load categories stored in an OPDS stream .
+ *
+ * @param content the OPDS stream.
+ * @return vector containing category strings.
+ */
+FeedCategories readCategoriesFromFeed(const std::string& content);
+
+/**
+ * Retrieve the full language name associated with a given ISO 639-3 language code.
+ *
+ * @param lang ISO 639-3 language code.
+ * @return full language name.
+ */
+std::string getLanguageSelfName(const std::string& lang);
+
+/**
+ * Slugifies the filename by converting any characters reserved by the operating
+ * system to '_'. Note filename is only the file name and not a path.
+ *
+ * @param filename Valid UTF-8 encoded file name string. 
+ * @return slugified string.
+ */
+std::string getSlugifiedFileName(const std::string& filename);
 }
 #endif // KIWIX_TOOLS_H

kiwix.pc.in

@@ -1,10 +0,0 @@
-prefix=@prefix@
-libdir=${prefix}/lib64
-includedir=${prefix}/include
-
-Name: libkiwix
-Description: A library that contains a lot of things used by used by other kiwix programs
-Version: @version@
-Requires: @requires@
-Libs: -L${libdir} -lkiwix @extra_libs@
-Cflags: -I${includedir}/ @extra_cflags@

meson.build

@@ -1,26 +1,61 @@
 project('libkiwix', 'cpp',
-  version : '11.0.0',
+  version : '14.1.1',
   license : 'GPLv3+',
-  default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
+  default_options : ['c_std=c11', 'cpp_std=c++17', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
-
 static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
+extra_libs = []
 
-# See https://github.com/kiwix/libkiwix/issues/371
-if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
-  extra_libs = ['-latomic']
-else
-  extra_libs = []
+# Atomics as compiled by GCC or clang can lead to external references to
+# functions depending on the type size and the platform. LLVM provides them in
+# 'libcompiler_rt', which clang normally automatically links in, while GNU
+# provides them in 'libatomic', which GCC *does not* link in automatically (but
+# this is probably going to change, see
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81358). Regardless of the setup
+# of the compiler driver itself (GCC or clang), we can thus assume that if some
+# atomic references can't be resolved, then 'libatomic' is missing.
+atomics_program = '''
+#include <atomic>
+#include <cstdint>
+
+using namespace std;
+
+int main() {
+  volatile atomic_bool a_b(true);
+  volatile atomic_ullong a_ull(-1);
+  // Next two lines are to cover atomic<socket_t> from 'httplib.h'.
+  volatile atomic<uint32_t> a_u32(-1);
+  volatile atomic<uint64_t> a_u64(-1);
+
+  return atomic_load(&a_b) == false && atomic_load(&a_ull) == 0 &&
+    atomic_load(&a_u32) == 0 && atomic_load(&a_u64) == 0;
+}
+'''
+if not compiler.links(atomics_program,
+                      name: 'compiler driver readily supports atomics')
+  libatomic = compiler.find_library('atomic')
+  compiler.links(atomics_program, name: 'atomics work with libatomic',
+                 dependencies: libatomic, required: true)
+  extra_libs += ['-latomic']
 endif
 
-if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
-  # C++ std::thread is implemented using pthread on linux by gcc
+# C++ std::thread is implemented using pthread on Linux by GCC, and on FreeBSD
+# for both GCC and LLVM.
+if (host_machine.system() == 'linux' and compiler.get_id() == 'gcc') or \
+   host_machine.system() == 'freebsd'
   thread_dep = dependency('threads')
 else
   thread_dep = dependency('', required:false)
 endif
+
 libicu_dep = dependency('icu-i18n', static:static_deps)
+if libicu_dep.version().version_compare('>= 76')
+  libicu_deps = [libicu_dep, dependency('icu-uc', static:static_deps)]
+else
+  libicu_deps = [libicu_dep]
+endif
+
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
@@ -35,9 +70,10 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
-if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
-  error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
+libzim_dep = dependency('libzim', version:['>=9.4.0', '<10.0.0'], static:static_deps)
+
+if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN', dependencies: libzim_dep)
+  error('Libzim seems to be compiled without Xapian. Xapian support is mandatory.')
 endif
 
 
@@ -49,21 +85,25 @@ endif
 
 if host_machine.system() == 'windows'
   add_project_arguments('-DNOMINMAX', language: 'cpp')
+  extra_libs += ['-liphlpapi']
 endif
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
+if build_machine.system() == 'windows'
+  extra_libs += ['-lshlwapi', '-lwinmm']
+endif
+
+
+# Dependencies as string
+all_deps = [thread_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
+
+# Dependencies as array
+all_deps += libicu_deps
 
 inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
 
-if build_machine.system() == 'windows'
-     extra_link_args = ['-lshlwapi', '-lwinmm']
-else
-     extra_link_args = []
-endif
-
 subdir('include')
 subdir('scripts')
 subdir('static')
@@ -73,17 +113,10 @@ if get_option('doc')
   subdir('docs')
 endif
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
-
-pkg_conf = configuration_data()
-pkg_conf.set('prefix', get_option('prefix'))
-pkg_conf.set('requires', ' '.join(pkg_requires))
-pkg_conf.set('extra_libs', ' '.join(extra_libs))
-pkg_conf.set('extra_cflags', extra_cflags)
-pkg_conf.set('version', meson.project_version())
-configure_file(output : 'kiwix.pc',
-               configuration : pkg_conf,
-               input : 'kiwix.pc.in',
-               install_dir: get_option('libdir')+'/pkgconfig'
-              )
-
+pkg_mod = import('pkgconfig')
+pkg_mod.generate(libraries : [libkiwix] + extra_libs,
+                 version : meson.project_version(),
+                 name : 'libkiwix',
+                 filebase : 'libkiwix',
+                 description : 'A library that contains useful primitives that Kiwix readers have in common',
+                 extra_cflags: extra_cflags)

scripts/format_code.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+
+# Compute 'src' path
+SCRIPT_DIR=$(dirname "$0")
+REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
+DIRS="src include"
+
+# Apply formating to all *.cpp and *.h files
+cd "$REPO_DIR"
+for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
+do
+  echo $FILE
+  clang-format -i -style=file "$FILE"
+done

scripts/kiwix-compile-i18n

@@ -61,7 +61,7 @@ lang_table_entry_cxx_template = '''
 
 cxxfile_template = '''// This file is automatically generated. Do not modify it.
 
-#include "server/i18n.h"
+#include "server/i18n_utils.h"
 
 namespace kiwix {
 namespace i18n {

scripts/kiwix-compile-resources

@@ -52,26 +52,60 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
+BINARY_RESOURCE_EXTENSIONS = {'.ico', '.png', '.ttf'}
+
+TEXT_RESOURCE_EXTENSIONS = {
+    '.css',
+    '.html',
+    '.js',
+    '.json',
+    '.svg',
+    '.tmpl',
+    '.webmanifest',
+    '.xml',
+}
+
+if not BINARY_RESOURCE_EXTENSIONS.isdisjoint(TEXT_RESOURCE_EXTENSIONS):
+    raise RuntimeError(f"The following file type extensions are declared to be both binary and text: {BINARY_RESOURCE_EXTENSIONS.intersection(TEXT_RESOURCE_EXTENSIONS)}")
+
+def is_binary_resource(filename):
+    _, extension = os.path.splitext(filename)
+    is_binary = extension in BINARY_RESOURCE_EXTENSIONS
+    is_text = extension in TEXT_RESOURCE_EXTENSIONS
+    if not is_binary and not is_text:
+        # all file type extensions of static resources must be listed
+        # in either BINARY_RESOURCE_EXTENSIONS or TEXT_RESOURCE_EXTENSIONS
+        raise RuntimeError(f"Unknown file type extension: {extension}")
+    return is_binary
+
 class Resource:
-    def __init__(self, base_dirs, filename):
-        filename = filename.strip()
+    def __init__(self, base_dirs, filename, cacheid=None):
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
                 with open(os.path.join(base_dir, filename), 'rb') as f:
                     self.data = f.read()
+                    if not is_binary_resource(filename):
+                        self.data = self.data.replace(b"\r\n", b"\n")
                 found = True
                 break
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -93,6 +127,12 @@ class Resource:
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
             namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@@ -123,7 +163,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,6 +179,7 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
@@ -159,8 +205,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -182,15 +230,17 @@ if __name__ == "__main__":
     parser.add_argument('--source_dir',
                         help="Additional directory where to look for resources.",
                         action='append')
-    parser.add_argument('resource_file',
+    parser.add_argument('resource_files', nargs='+',
                         help='The list of resources to compile.')
     args = parser.parse_args()
 
-    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
-    with open(args.resource_file, 'r') as f:
-        resources = [Resource([base_dir]+source_dir, filename)
-                        for filename in f.readlines()]
+    resources = []
+    for resfile in args.resource_files:
+        base_dir = os.path.dirname(os.path.realpath(resfile))
+        with open(resfile, 'r') as f:
+            resources += [Resource([base_dir]+source_dir, *line.strip().split())
+                            for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-compile-resources.1

@@ -2,7 +2,7 @@
 .SH NAME
 kiwix-compile-resources \- helper to compile and generate some Kiwix resources
 .SH SYNOPSIS
-\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
 .SH DESCRIPTION
 .TP
 resource_file

scripts/kiwix-resources

@@ -99,16 +99,24 @@ def preprocess_resource(resource_path):
             print(preprocessed_content, end='', file=target)
 
 
-def copy_file(src_path, dst_path):
-    with open(src_path, 'rb') as src:
-        with open(dst_path, 'wb') as dst:
-            dst.write(src.read())
+def copy_resource_list_file(src_path, dst_path):
+    with open(src_path, 'r') as src:
+        with open(dst_path, 'w') as dst:
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
 
 def preprocess_resources(resource_file_path):
     resource_filename = os.path.basename(resource_file_path)
     for resource in read_resource_file(resource_file_path):
-        preprocess_resource(resource)
-    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+        if resource.startswith('skin/'):
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

src/aria2.cpp

@@ -4,6 +4,7 @@
 #include "xmlrpc.h"
 #include <iostream>
 #include <algorithm>
+#include <fstream>
 #include <sstream>
 #include <thread>
 #include <chrono>
@@ -24,25 +25,46 @@
 #define LOG_ARIA_ERROR() \
   { \
     std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
-    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+    std::cerr << (curlErrorBuffer[0] ? curlErrorBuffer : curl_easy_strerror(res)) << std::endl; \
   }
 
 namespace kiwix {
 
-Aria2::Aria2():
+namespace {
+
+void pauseAnyActiveDownloads(const std::string& ariaSessionFilePath)
+{
+    std::ifstream inputFile(ariaSessionFilePath);
+    if ( !inputFile )
+        return;
+
+    std::ostringstream ss;
+    std::string line;
+    while ( std::getline(inputFile, line) ) {
+        if ( !startsWith(line, " pause=") ) {
+            ss << line << "\n";
+        }
+        if ( !line.empty() && line[0] != ' ' && line[0] != '#' ) {
+            ss << " pause=true\n";
+        }
+    }
+
+    std::ofstream outputFile(ariaSessionFilePath);
+    outputFile << ss.str();
+}
+
+} // unnamed namespace
+
+Aria2::Aria2(std::string sessionFileDir):
   mp_aria(nullptr),
   m_port(42042),
-  m_secret(getNewRpcSecret()),
-  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
-  mp_curl(nullptr)
+  m_secret(getNewRpcSecret())
 {
-  m_downloadDir = getDataDirectory();
-  makeDirectory(m_downloadDir);
   std::vector<const char*> callCmd;
 
   std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
-  std::string download_dir = "--dir=" + getDataDirectory();
-  std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
+  std::string session_file = appendToDirectory(sessionFileDir, "kiwix.session");
+  pauseAnyActiveDownloads(session_file);
   std::string session = "--save-session=" + session_file;
   std::string inputFile = "--input-file=" + session_file;
 //  std::string log_dir = "--log=\"" + logDir + "\"";
@@ -69,7 +91,6 @@ Aria2::Aria2():
   callCmd.push_back("--enable-rpc");
   callCmd.push_back(rpc_secret.c_str());
   callCmd.push_back(rpc_port.c_str());
-  callCmd.push_back(download_dir.c_str());
   if (fileReadable(session_file)) {
     callCmd.push_back(inputFile.c_str());
   }
@@ -91,36 +112,42 @@ Aria2::Aria2():
       launchCmd.append(cmd).append(" ");
   }
   mp_aria = Subprocess::run(callCmd);
-  mp_curl = curl_easy_init();
 
-  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
-  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
-  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
+  CURL* p_curl = curl_easy_init();
+  char curlErrorBuffer[CURL_ERROR_SIZE];
 
-  int watchdog = 50;
-  while(--watchdog) {
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
+  curl_easy_setopt(p_curl, CURLOPT_TIMEOUT_MS, 100);
+
+  typedef std::chrono::duration<double> Seconds;
+
+  const double MAX_WAITING_TIME_SECONDS = 1;
+  const auto t0 = std::chrono::steady_clock::now();
+  bool maxWaitingTimeWasExceeded = false;
+
+  CURLcode res = CURLE_OK;
+  while ( !maxWaitingTimeWasExceeded ) {
     sleep(10);
-    m_curlErrorBuffer[0] = 0;
-    auto res = curl_easy_perform(mp_curl);
+    curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(p_curl);
     if (res == CURLE_OK) {
       break;
-    } else if (watchdog == 1) {
-      LOG_ARIA_ERROR();
     }
+
+    const auto dt = std::chrono::steady_clock::now() - t0;
+    const double elapsedTime = std::chrono::duration_cast<Seconds>(dt).count();
+    maxWaitingTimeWasExceeded = elapsedTime > MAX_WAITING_TIME_SECONDS;
   }
-  if (!watchdog) {
-    curl_easy_cleanup(mp_curl);
+  curl_easy_cleanup(p_curl);
+  if ( maxWaitingTimeWasExceeded ) {
+    LOG_ARIA_ERROR();
     throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
-Aria2::~Aria2()
-{
-  std::unique_lock<std::mutex> lock(m_lock);
-  curl_easy_cleanup(mp_curl);
-}
-
 void Aria2::close()
 {
   saveSession();
@@ -140,20 +167,25 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
   std::stringstream outStream;
   CURLcode res;
   long response_code;
-  {
-    std::unique_lock<std::mutex> lock(m_lock);
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
-    m_curlErrorBuffer[0] = 0;
-    res = curl_easy_perform(mp_curl);
-    if (res != CURLE_OK) {
-      LOG_ARIA_ERROR();
-      throw std::runtime_error("Cannot perform request");
-    }
-    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  char curlErrorBuffer[CURL_ERROR_SIZE];
+  CURL* p_curl = curl_easy_init();
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+  curl_easy_setopt(p_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(p_curl, CURLOPT_WRITEDATA, &outStream);
+  curlErrorBuffer[0] = 0;
+  res = curl_easy_perform(p_curl);
+  if (res != CURLE_OK) {
+    LOG_ARIA_ERROR();
+    curl_easy_cleanup(p_curl);
+    throw std::runtime_error("Cannot perform request");
   }
+  curl_easy_getinfo(p_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(p_curl);
 
   auto responseContent = outStream.str();
   if (response_code != 200) {

src/aria2.h

@@ -12,7 +12,6 @@
 #include "xmlrpc.h"
 
 #include <memory>
-#include <mutex>
 #include <curl/curl.h>
 
 namespace kiwix {
@@ -23,16 +22,11 @@ class Aria2
     std::unique_ptr<Subprocess> mp_aria;
     int m_port;
     std::string m_secret;
-    std::string m_downloadDir;
-    std::unique_ptr<char[]> m_curlErrorBuffer;
-    CURL* mp_curl;
-    std::mutex m_lock;
-
     std::string doRequest(const MethodCall& methodCall);
 
   public:
-    Aria2();
-    virtual ~Aria2();
+    explicit Aria2(std::string sessionFileDir);
+    virtual ~Aria2() = default;
     void close();
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});

src/book.cpp

@@ -66,7 +66,7 @@ bool Book::update(const kiwix::Book& other)
 void Book::update(const zim::Archive& archive) {
   m_path = archive.getFilename();
   m_pathValid = true;
-  m_id = getArchiveId(archive);
+  m_id = std::string(archive.getUuid());
   m_title = getArchiveTitle(archive);
   m_description = getMetaDescription(archive);
   m_language = getMetaLanguage(archive);
@@ -77,15 +77,16 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = getArchiveArticleCount(archive);
-  m_mediaCount = getArchiveMediaCount(archive);
+  m_articleCount = archive.getArticleCount();
+  m_mediaCount = archive.getMediaCount();
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
   m_illustrations.clear();
-  for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
+  for ( const auto& illustrationInfo : archive.getIllustrationInfos() ) {
     const auto illustration = std::make_shared<Illustration>();
-    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
-    illustration->width = illustration->height = illustrationSize;
+    const zim::Item illustrationItem = archive.getIllustrationItem(illustrationInfo);
+    illustration->width = illustrationInfo.width;
+    illustration->height = illustrationInfo.height;
     illustration->mimeType = illustrationItem.getMimetype();
     illustration->data = illustrationItem.getData();
     // NOTE: illustration->url is left uninitialized
@@ -117,11 +118,12 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
   m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
   m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
   m_size = strtoull(ATTR("size"), 0, 0) << 10;
-  std::string favicon_mimetype = ATTR("faviconMimeType");
-  if (! favicon_mimetype.empty()) {
+  const std::string faviconMimeType = ATTR("faviconMimeType");
+  const std::string faviconBase64EncodedData = ATTR("favicon");
+  if ( !faviconMimeType.empty() && !faviconBase64EncodedData.empty() ) {
     const auto favicon = std::make_shared<Illustration>();
-    favicon->data = base64_decode(ATTR("favicon"));
-    favicon->mimeType = favicon_mimetype;
+    favicon->data = base64_decode(faviconBase64EncodedData);
+    favicon->mimeType = faviconMimeType;
     favicon->url = ATTR("faviconUrl");
     m_illustrations.assign(1, favicon);
   }
@@ -286,4 +288,9 @@ std::string Book::getCategoryFromTags() const
   }
 }
 
+const std::vector<std::string> Book::getLanguages() const
+{
+  return kiwix::split(m_language, ",");
+}
+
 }

src/bookmark.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "bookmark.h"
+#include "book.h"
 
 #include <pugixml.hpp>
 
@@ -28,6 +29,17 @@ Bookmark::Bookmark()
 {
 }
 
+Bookmark::Bookmark(const Book& book, const std::string& path, const std::string& title):
+  m_bookId(book.getId()),
+  m_bookTitle(book.getTitle()),
+  m_bookName(book.getName()),
+  m_bookFlavour(book.getFlavour()),
+  m_url(path),
+  m_title(title),
+  m_language(book.getCommaSeparatedLanguages()),
+  m_date(book.getDate())
+{}
+
 /* Destructor */
 Bookmark::~Bookmark()
 {
@@ -38,6 +50,8 @@ void Bookmark::updateFromXml(const pugi::xml_node& node)
   auto bookNode = node.child("book");
   m_bookId = bookNode.child("id").child_value();
   m_bookTitle = bookNode.child("title").child_value();
+  m_bookName = bookNode.child("name").child_value();
+  m_bookFlavour = bookNode.child("flavour").child_value();
   m_language = bookNode.child("language").child_value();
   m_date = bookNode.child("date").child_value();
   m_title = node.child("title").child_value();

src/downloader.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "downloader.h"
+#include "tools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
 
@@ -124,38 +125,50 @@ void Download::cancelDownload()
 }
 
 /* Constructor */
-Downloader::Downloader() :
-  mp_aria(new Aria2())
+Downloader::Downloader(std::string sessionFileDir) :
+  mp_aria(new Aria2(sessionFileDir))
 {
-  try {
-    for (auto gid : mp_aria->tellActive()) {
-      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
-    }
-  } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
-  }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
       m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
+      m_knownDownloads[gid]->updateStatus(false);
     }
   } catch (std::exception& e) {
     std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      if( m_knownDownloads.find(gid) == m_knownDownloads.end()) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads[gid]->updateStatus(false);
+      }
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
+  }
 }
 
 /* Destructor */
 Downloader::~Downloader()
 {
+  close();
 }
 
 void Downloader::close()
 {
-  mp_aria->close();
+  if ( mp_aria ) {
+    try {
+      mp_aria->close();
+    } catch (const std::exception& err) {
+      std::cerr << "ERROR: Failed to save the downloader state: "
+                << err.what() << std::endl;
+    }
+    mp_aria.reset();
+  }
 }
 
-std::vector<std::string> Downloader::getDownloadIds() {
+std::vector<std::string> Downloader::getDownloadIds() const {
+  std::unique_lock<std::mutex> lock(m_lock);
   std::vector<std::string> ret;
   for(auto& p:m_knownDownloads) {
     ret.push_back(p.first);
@@ -163,42 +176,82 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
+namespace
 {
+
+bool downloadCanBeReused(const Download& d,
+                         const std::string& uri,
+                         const Downloader::Options& /*options*/)
+{
+  const auto& uris = d.getUris();
+  const bool sameURI = std::find(uris.begin(), uris.end(), uri) != uris.end();
+
+  if ( !sameURI )
+    return false;
+
+  switch ( d.getStatus() ) {
+  case Download::K_ERROR:
+  case Download::K_UNKNOWN:
+  case Download::K_REMOVED:
+    return false;
+
+  case Download::K_ACTIVE:
+  case Download::K_WAITING:
+  case Download::K_PAUSED:
+    return true; // XXX: what if options are different?
+
+  case Download::K_COMPLETE:
+    return fileExists(d.getPath()); // XXX: what if options are different?
+  }
+
+  return false;
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Download> Downloader::startDownload(const std::string& uri, const std::string& downloadDir, Options options)
+{
+  std::unique_lock<std::mutex> lock(m_lock);
+  options.erase(std::remove_if(options.begin(), options.end(), [](const auto& option) {
+    return option.first == "dir";
+  }), options.end());
+  options.push_back({"dir", downloadDir});
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
-    auto& uris = d->getUris();
-    if (std::find(uris.begin(), uris.end(), uri) != uris.end())
-      return d.get();
+    if ( downloadCanBeReused(*d, uri, options) )
+      return d;
   }
   std::vector<std::string> uris = {uri};
   auto gid = mp_aria->addUri(uris, options);
-  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-  return m_knownDownloads[gid].get();
+  m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+  return m_knownDownloads[gid];
 }
 
-Download* Downloader::getDownload(const std::string& did)
+std::shared_ptr<Download> Downloader::getDownload(const std::string& did)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   try {
-    m_knownDownloads.at(did).get()->updateStatus(true);
-    return m_knownDownloads.at(did).get();
+    return m_knownDownloads.at(did);
   } catch(std::exception& e) {
-    for (auto gid : mp_aria->tellActive()) {
-      if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
-      }
-    }
     for (auto gid : mp_aria->tellWaiting()) {
       if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
       }
-  }
+    }
+    for (auto gid : mp_aria->tellActive()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
+      }
+    }
     throw e;
   }
 }
 
+size_t Downloader::getNbDownload() const {
+  std::unique_lock<std::mutex> lock(m_lock);
+  return m_knownDownloads.size();
+}
+
 }

src/html_dumper.cpp

@@ -0,0 +1,144 @@
+#include "html_dumper.h"
+#include "libkiwix-resources.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+#include "tools/regexTools.h"
+#include "server/i18n_utils.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+HTMLDumper::HTMLDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
+{
+}
+/* Destructor */
+HTMLDumper::~HTMLDumper()
+{
+}
+
+namespace {
+
+std::string humanFriendlyTitle(std::string title)
+{
+  std::string humanFriendlyString = replaceRegex(title, "_", " ");
+  humanFriendlyString[0] = toupper(humanFriendlyString[0]);
+  return humanFriendlyString;
+}
+
+kainjow::mustache::object getLangTag(const std::vector<std::string>& bookLanguages) {
+  std::string langShortString = "";
+  std::string langFullString = "???";
+
+  //if more than 1 languages then show "mul" else show the language
+  if(bookLanguages.size() > 1) {
+    std::vector<std::string> mulLanguages;
+    langShortString = "mul";
+    for (const auto& lang : bookLanguages) {
+      const std::string fullLang = getLanguageSelfName(lang);
+      mulLanguages.push_back(fullLang);
+    }
+    langFullString = kiwix::join(mulLanguages, ",");
+  } else if(bookLanguages.size() == 1) {
+    langShortString = bookLanguages[0];
+    langFullString = getLanguageSelfName(langShortString);
+  }
+
+  kainjow::mustache::object langTag;
+    langTag["langShortString"] = langShortString;
+    langTag["langFullString"] = langFullString;
+  return langTag;
+}
+
+kainjow::mustache::list getTagList(std::string tags)
+{
+  const auto tagsList = kiwix::split(tags, ";", true, false);
+  kainjow::mustache::list finalTagList;
+  for (auto tag : tagsList) {
+  if (tag[0] != '_')
+    finalTagList.push_back(kainjow::mustache::object{
+      {"tag", tag}
+    });
+  }
+  return finalTagList;
+}
+
+} // unnamed namespace
+
+std::string HTMLDumper::dumpPlainHTML(kiwix::Filter filter) const
+{
+  kainjow::mustache::list booksData;
+  const auto filteredBooks = library->filter(filter);
+  const auto searchQuery = filter.getQuery();
+  auto languages = getLanguageData();
+  auto categories = getCategoryData();
+
+  for (auto &category : categories) {
+    const auto categoryName = category.get("name")->string_value();
+    if (categoryName == filter.getCategory()) {
+      category["selected"] = true;
+    }
+    category["hf_name"] = humanFriendlyTitle(categoryName);
+  }
+
+  for (auto &language : languages) {
+    if (language.get("lang_code")->string_value() == filter.getLang()) {
+      language["selected"] = true;
+    }
+  }
+
+  for ( const auto& bookId : filteredBooks ) {
+    const auto bookObj = library->getBookById(bookId);
+    const auto bookTitle = bookObj.getTitle();
+    std::string contentId = "";
+    try {
+      contentId = urlEncode(nameMapper->getNameForId(bookId));
+    } catch (...) {}
+    const auto bookDescription = bookObj.getDescription();
+    const auto bookIconUrl = rootLocation + "/catalog/v2/illustration/" + bookId +  "/?size=48";
+    const auto tags = bookObj.getTags();
+    const auto downloadAvailable = (bookObj.getUrl() != "");
+    const auto langTagObj = getLangTag(bookObj.getLanguages());
+    std::string faviconAttr = "style=background-image:url(" + bookIconUrl + ")";
+    booksData.push_back(kainjow::mustache::object{
+      {"id", contentId},
+      {"title", bookTitle},
+      {"description", bookDescription},
+      {"langTag", langTagObj},
+      {"faviconAttr", faviconAttr},
+      {"tagList", getTagList(tags)},
+      {"downloadAvailable", downloadAvailable}
+    });
+  }
+
+  auto getTranslation = i18n::GetTranslatedStringWithMsgId(m_userLang);
+
+  const auto translations = kainjow::mustache::object{
+                    getTranslation("search"),
+                    getTranslation("download"),
+                    getTranslation("count-of-matching-books", {{"COUNT", to_string(filteredBooks.size())}}),
+                    getTranslation("book-filtering-all-categories"),
+                    getTranslation("book-filtering-all-languages"),
+                    getTranslation("powered-by-kiwix-html"),
+                    getTranslation("welcome-to-kiwix-server"),
+                    getTranslation("preview-book"),
+                    getTranslation("welcome-page-overzealous-filter", {{"URL", "?lang="}})
+  };
+
+  return render_template(
+             RESOURCE::templates::no_js_library_page_html,
+             kainjow::mustache::object{
+               {"root", rootLocation},
+               {"contentAccessUrl", onlyAsNonEmptyMustacheValue(contentAccessUrl)},
+               {"books", booksData },
+               {"searchQuery", searchQuery},
+               {"languages", languages},
+               {"categories", categories},
+               {"noResults", filteredBooks.size() == 0},
+               {"translations", translations}
+             }
+  );
+}
+
+} // namespace kiwix

src/html_dumper.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_HTML_DUMPER_H
+#define KIWIX_HTML_DUMPER_H
+
+#include <string>
+
+#include "library_dumper.h"
+
+namespace kiwix
+{
+
+/**
+ * A class to dump Library in HTML format.
+ */
+class HTMLDumper : public LibraryDumper
+{
+ public:
+  HTMLDumper(const Library* library, const NameMapper* NameMapper);
+  ~HTMLDumper();
+
+
+  /**
+   * Dump library in HTML
+   *
+   * @return HTML content
+   */
+  std::string dumpPlainHTML(kiwix::Filter filter) const;
+};
+
+}
+
+#endif // KIWIX_HTML_DUMPER_H

src/library.cpp

@@ -39,6 +39,8 @@
 namespace kiwix
 {
 
+
+
 namespace
 {
 
@@ -58,6 +60,8 @@ bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
       && book1.getPath() == book2.getPath();
 }
 
+} // unnamed namespace
+
 template<typename Key, typename Value>
 class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
 {
@@ -79,49 +83,8 @@ class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
     }
 };
 
-} // unnamed namespace
-
-struct Library::Impl
-{
-  struct Entry : Book
-  {
-    Library::Revision lastUpdatedRevision = 0;
-  };
-
-  Library::Revision m_revision;
-  std::map<std::string, Entry> m_books;
-  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
-  std::unique_ptr<ArchiveCache> mp_archiveCache;
-  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
-  std::unique_ptr<SearcherCache> mp_searcherCache;
-  std::vector<kiwix::Bookmark> m_bookmarks;
-  Xapian::WritableDatabase m_bookDB;
-
-  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
-
-  Impl();
-  ~Impl();
-
-  Impl(Impl&& );
-  Impl& operator=(Impl&& );
-};
-
-Library::Impl::Impl()
-  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
-    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
-    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
-{
-}
-
-Library::Impl::~Impl()
-{
-}
-
-Library::Impl::Impl(Library::Impl&& ) = default;
-Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
-
 unsigned int
-Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
+Library::getBookCount_not_protected(const bool localBooks, const bool remoteBooks) const
 {
   unsigned int result = 0;
   for (auto& pair: m_books) {
@@ -136,50 +99,41 @@ Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
 
 /* Constructor */
 Library::Library()
-  : mp_impl(new Library::Impl)
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB(new Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY))
 {
 }
 
-Library::Library(Library&& other)
-  : mp_impl(std::move(other.mp_impl))
-{
-}
-
-Library& Library::operator=(Library&& other)
-{
-  mp_impl = std::move(other.mp_impl);
-  return *this;
-}
-
 /* Destructor */
 Library::~Library() = default;
 
 bool Library::addBook(const Book& book)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  ++mp_impl->m_revision;
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  ++m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
-    auto& oldbook = mp_impl->m_books.at(book.getId());
+    auto& oldbook = m_books.at(book.getId());
     if ( ! booksReferToTheSameArchive(oldbook, book) ) {
       dropCache(book.getId());
     }
     oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
                           // XXX: Then m_bookDB will become out-of-sync with
                           // XXX: the real contents of the library.
-    oldbook.lastUpdatedRevision = mp_impl->m_revision;
+    oldbook.lastUpdatedRevision = m_revision;
     return false;
   } catch (std::out_of_range&) {
-    auto& newEntry = mp_impl->m_books[book.getId()];
+    auto& newEntry = m_books[book.getId()];
     static_cast<Book&>(newEntry) = book;
-    newEntry.lastUpdatedRevision = mp_impl->m_revision;
-    size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
+    newEntry.lastUpdatedRevision = m_revision;
+    size_t new_cache_size = static_cast<size_t>(std::ceil(getBookCount_not_protected(true, true)*0.1));
     if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
-      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+      mp_archiveCache->setMaxSize(new_cache_size);
     }
     if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
-      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+      mp_searcherCache->setMaxSize(new_cache_size);
     }
     return true;
   }
@@ -187,33 +141,186 @@ bool Library::addBook(const Book& book)
 
 void Library::addBookmark(const Bookmark& bookmark)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  mp_impl->m_bookmarks.push_back(bookmark);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
-      mp_impl->m_bookmarks.erase(it);
+      m_bookmarks.erase(it);
       return true;
     }
   }
   return false;
 }
 
+std::tuple<int, int> Library::migrateBookmarks(MigrationMode migrationMode) {
+  std::set<std::string> sourceBooks;
+  int invalidBookmarks = 0;
+  {
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
+    for(auto& bookmark:m_bookmarks) {
+      if (m_books.find(bookmark.getBookId()) == m_books.end()) {
+        invalidBookmarks += 1;
+        sourceBooks.insert(bookmark.getBookId());
+      }
+    }
+  }
+  int changed = 0;
+  for(auto& sourceBook:sourceBooks) {
+    changed += migrateBookmarks(sourceBook, migrationMode);
+  }
+  return std::make_tuple(changed, invalidBookmarks);
+}
+
+std::string Library::getBestFromBookCollection(BookIdCollection books, const Bookmark& bookmark, MigrationMode migrationMode) const {
+  // This function try to get the best book for a bookmark from a book collection.
+  // It assumes that all books in the collection are "acceptable".
+  // (this definiton is not clear but for now it is book's name is equal to bookmark's bookName)
+  //
+  // The algorithm first sort the colletion by "flavour equality" and date.
+  // "flavour equality" is if book's flavour is same that bookmark's flavour (let's say "flavourA" here)
+  // So we have the sorted collection:
+  // - flavourA, date 5
+  // - flavourA, date 4
+  // - flavourB, date 6
+  // - flavourC, date 5
+  // - flavourB, date 3
+  //
+  // Then, depending of migrationMode:
+  // - If ALLOW_DOWNGRADE => take the first one
+  // - If UPGRADE_ONLY => loop on books until we find a book newer than bookmark.
+  //   So if bookmark date is 5 => flavourB, date 6
+  //      if bookmark date is 4 => flavourA, date 5
+  //      if bookmark date is 7 => No book
+
+  if (books.empty()) {
+    return "";
+  }
+
+  sort(books, DATE, false);
+  stable_sort(books.begin(), books.end(), [&](const std::string& bookId1, const std::string& bookId2) {
+    const auto& book1 = getBookById(bookId1);
+    const auto& book2 = getBookById(bookId2);
+    bool same_flavour1 = book1.getFlavour() == bookmark.getBookFlavour();
+    bool same_flavour2 = book2.getFlavour() == bookmark.getBookFlavour();
+    // return True if bookId1 is before bookId2, ie if same_flavour1 and not same_flavour2
+    return same_flavour1 > same_flavour2;
+  });
+
+  if (migrationMode == ALLOW_DOWNGRADE) {
+    return books[0];
+  } else {
+    for (const auto& bookId: books) {
+      const auto& book = getBookById(bookId);
+      if (book.getDate() >= bookmark.getDate()) {
+        return bookId;
+      }
+    }
+  }
+
+  return "";
+}
+
+std::string remove_quote(std::string input) {
+  std::replace(input.begin(), input.end(), '"', ' ');
+  return input;
+}
+
+std::string Library::getBestTargetBookId(const std::string& bookName, const std::string& preferedFlavour, const std::string& minDate) const {
+  // Let's reuse our algorithm based on bookmark.
+  MigrationMode migrationMode = UPGRADE_ONLY;
+  auto bookmark = Bookmark();
+  bookmark.setBookName(bookName);
+  bookmark.setBookFlavour(preferedFlavour);
+
+  if (minDate.empty()) {
+    migrationMode = ALLOW_DOWNGRADE;
+  } else {
+    bookmark.setDate(minDate);
+  }
+
+  return getBestTargetBookId(bookmark, migrationMode);
+}
+
+std::string Library::getBestTargetBookId(const Bookmark& bookmark, MigrationMode migrationMode) const {
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  // Search for a existing book with the same name
+  auto book_filter = Filter();
+  if (!bookmark.getBookName().empty()) {
+    book_filter.name(bookmark.getBookName());
+  } else {
+    // We don't have a name stored (older bookmarks)
+    // Fallback on title (All bookmarks should have one, but let's be safe against wrongly filled bookmark)
+    if (bookmark.getBookTitle().empty()) {
+        // No bookName nor bookTitle, no way to find target book.
+        return "";
+    }
+    book_filter.query("title:\"" + remove_quote(bookmark.getBookTitle()) + "\"");
+  }
+  auto targetBooks = filter(book_filter);
+  auto bestBook = getBestFromBookCollection(targetBooks, bookmark, migrationMode);
+  if (bestBook.empty()) {
+    try {
+      getBookById(bookmark.getBookId());
+      return bookmark.getBookId();
+    } catch (std::out_of_range&) {}
+  }
+  return bestBook;
+}
+
+int Library::migrateBookmarks(const std::string& sourceBookId, MigrationMode migrationMode) {
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+
+  Bookmark firstBookmarkToChange;
+  for(auto& bookmark:m_bookmarks) {
+    if (bookmark.getBookId() == sourceBookId) {
+      firstBookmarkToChange = bookmark;
+      break;
+    }
+  }
+
+  if (firstBookmarkToChange.getBookId().empty()) {
+    return 0;
+  }
+
+  std::string betterBook = getBestTargetBookId(firstBookmarkToChange, migrationMode);
+
+  if (betterBook.empty()) {
+    return 0;
+  }
+
+  return migrateBookmarks(sourceBookId, betterBook);
+}
+
+int Library::migrateBookmarks(const std::string& sourceBookId, const std::string& targetBookId) {
+  if (sourceBookId == targetBookId) {
+    return 0;
+  }
+  int changed = 0;
+  for (auto& bookmark:m_bookmarks) {
+    if (bookmark.getBookId() == sourceBookId) {
+      bookmark.setBookId(targetBookId);
+      changed +=1;
+    }
+  }
+  return changed;
+}
+
 
 void Library::dropCache(const std::string& id)
 {
-  mp_impl->mp_archiveCache->drop(id);
-  mp_impl->mp_searcherCache->drop(id);
+  mp_archiveCache->drop(id);
+  mp_searcherCache->drop(id);
 }
 
 bool Library::removeBookById(const std::string& id)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  mp_impl->m_bookDB.delete_document("Q" + id);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  m_bookDB->delete_document("Q" + id);
   dropCache(id);
   // We do not change the cache size here
   // Most of the time, the book is remove in case of library refresh, it is
@@ -221,21 +328,25 @@ bool Library::removeBookById(const std::string& id)
   // Having a too big cache is not a problem here (or it would have been before)
   // (And setMaxSize doesn't actually reduce the cache size, extra cached items
   //  will be removed in put or getOrPut).
-  return mp_impl->m_books.erase(id) == 1;
+  const bool bookWasRemoved = m_books.erase(id) == 1;
+  if ( bookWasRemoved ) {
+    ++m_revision;
+  }
+  return bookWasRemoved;
 }
 
 Library::Revision Library::getRevision() const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  return mp_impl->m_revision;
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  return m_revision;
 }
 
 uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
 {
   BookIdCollection booksToRemove;
   {
-    std::lock_guard<std::mutex> lock(m_mutex);
-    for ( const auto& entry : mp_impl->m_books) {
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
+    for ( const auto& entry : m_books) {
       if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
         booksToRemove.push_back(entry.first);
       }
@@ -254,12 +365,12 @@ const Book& Library::getBookById(const std::string& id) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  return mp_impl->m_books.at(id);
+  return m_books.at(id);
 }
 
 Book Library::getBookByIdThreadSafe(const std::string& id) const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   return getBookById(id);
 }
 
@@ -267,7 +378,7 @@ const Book& Library::getBookByPath(const std::string& path) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  for(auto& it: mp_impl->m_books) {
+  for(auto& it: m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
       return book;
@@ -280,7 +391,7 @@ const Book& Library::getBookByPath(const std::string& path) const
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
   try {
-    return mp_impl->mp_archiveCache->getOrPut(id,
+    return mp_archiveCache->getOrPut(id,
     [&](){
       auto book = getBookById(id);
       if (!book.isPathValid()) {
@@ -297,7 +408,7 @@ std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
 {
   assert(!ids.empty());
   try {
-    return mp_impl->mp_searcherCache->getOrPut(ids,
+    return mp_searcherCache->getOrPut(ids,
     [&](){
       std::vector<zim::Archive> archives;
       for(auto& id:ids) {
@@ -317,8 +428,8 @@ std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
-  return mp_impl->getBookCount(localBooks, remoteBooks);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  return getBookCount_not_protected(localBooks, remoteBooks);
 }
 
 bool Library::writeToFile(const std::string& path) const
@@ -330,7 +441,7 @@ bool Library::writeToFile(const std::string& path) const
   dumper.setBaseDir(baseDir);
   std::string xml;
   {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     xml = dumper.dumpLibXMLContent(allBookIds);
   };
   return writeTextFile(path, xml);
@@ -346,10 +457,10 @@ bool Library::writeBookmarksToFile(const std::string& path) const
 
 Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   AttributeCounts propValueCounts;
 
-  for (const auto& pair: mp_impl->m_books) {
+  for (const auto& pair: m_books) {
     const auto& book = pair.second;
     if (book.getOrigId().empty()) {
       propValueCounts[(book.*p)()] += 1;
@@ -369,20 +480,35 @@ std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
 
 std::vector<std::string> Library::getBooksLanguages() const
 {
-  return getBookPropValueSet(&Book::getLanguage);
+  std::vector<std::string> langs;
+  for ( const auto& langAndCount : getBooksLanguagesWithCounts() ) {
+    langs.push_back(langAndCount.first);
+  }
+  return langs;
 }
 
 Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
 {
-  return getBookAttributeCounts(&Book::getLanguage);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  AttributeCounts langsWithCounts;
+
+  for (const auto& pair: m_books) {
+    const auto& book = pair.second;
+    if (book.getOrigId().empty()) {
+      for ( const auto& lang : book.getLanguages() ) {
+        ++langsWithCounts[lang];
+      }
+    }
+  }
+  return langsWithCounts;
 }
 
 std::vector<std::string> Library::getBooksCategories() const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   std::set<std::string> categories;
 
-  for (const auto& pair: mp_impl->m_books) {
+  for (const auto& pair: m_books) {
     const auto& book = pair.second;
     const auto& c = book.getCategory();
     if ( !c.empty() ) {
@@ -406,12 +532,12 @@ std::vector<std::string> Library::getBooksPublishers() const
 const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
 {
   if (!onlyValidBookmarks) {
-    return mp_impl->m_bookmarks;
+    return m_bookmarks;
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
-  std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto& bookmark:mp_impl->m_bookmarks) {
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  for(auto& bookmark:m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
     }
@@ -421,10 +547,10 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
 
 Library::BookIdCollection Library::getBooksIds() const
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
-  for (auto& pair: mp_impl->m_books) {
+  for (auto& pair: m_books) {
     bookIds.push_back(pair.first);
   }
 
@@ -436,12 +562,14 @@ void Library::updateBookDB(const Book& book)
 {
   Xapian::Stem stemmer;
   Xapian::TermGenerator indexer;
-  const std::string lang = book.getLanguage();
-  try {
-    stemmer = Xapian::Stem(iso639_3ToXapian(lang));
-    indexer.set_stemmer(stemmer);
-    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
-  } catch (...) {}
+  const auto langs = book.getLanguages();
+  if ( langs.size() == 1 ) {
+    try {
+      stemmer = Xapian::Stem(iso639_3ToXapian(langs[0]));
+      indexer.set_stemmer(stemmer);
+      indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
+    } catch (...) {}
+  }
   Xapian::Document doc;
   indexer.set_document(doc);
 
@@ -456,10 +584,13 @@ void Library::updateBookDB(const Book& book)
   // Index all fields for field-based search
   indexer.index_text(title, 1, "S");
   indexer.index_text(desc,  1, "XD");
-  indexer.index_text(lang,  1, "L");
+  for ( const auto& lang : langs ) {
+    indexer.index_text(lang,  1, "L");
+  }
   indexer.index_text(normalizeText(book.getCreator()),   1, "A");
   indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
-  indexer.index_text(normalizeText(book.getName()),      1, "XN");
+  doc.add_term("XN"+normalizeText(book.getName()));
+  indexer.index_text(normalizeText(book.getFlavour()),  1, "XF");
   indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
 
   for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
@@ -475,7 +606,7 @@ void Library::updateBookDB(const Book& book)
 
   doc.set_data(book.getId());
 
-  mp_impl->m_bookDB.replace_document(idterm, doc);
+  m_bookDB->replace_document(idterm, doc);
 }
 
 namespace
@@ -500,6 +631,7 @@ Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
   queryParser.add_prefix("title", "S");
   queryParser.add_prefix("description", "XD");
   queryParser.add_prefix("name", "XN");
+  queryParser.add_prefix("flavour", "XF");
   queryParser.add_prefix("category", "XC");
   queryParser.add_prefix("lang", "L");
   queryParser.add_prefix("publisher", "XP");
@@ -513,8 +645,6 @@ Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
   //queryParser.set_stemmer(Xapian::Stem(iso639_3ToXapian(???)));
   //queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_SOME);
   const auto flags = Xapian::QueryParser::FLAG_PHRASE
-                   | Xapian::QueryParser::FLAG_BOOLEAN
-                   | Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE
                    | Xapian::QueryParser::FLAG_LOVEHATE
                    | Xapian::QueryParser::FLAG_WILDCARD
                    | partialQueryFlag;
@@ -526,14 +656,35 @@ Xapian::Query nameQuery(const std::string& name)
   return Xapian::Query("XN" + normalizeText(name));
 }
 
-Xapian::Query categoryQuery(const std::string& category)
+Xapian::Query flavourQuery(const std::string& name)
 {
-  return Xapian::Query("XC" + normalizeText(category));
+  return Xapian::Query("XF" + normalizeText(name));
 }
 
-Xapian::Query langQuery(const std::string& lang)
+Xapian::Query multipleParamQuery(const std::string& commaSeparatedList, const std::string& prefix)
 {
-  return Xapian::Query("L" + normalizeText(lang));
+  Xapian::Query q;
+  bool firstIteration = true;
+  for ( const auto& elem : kiwix::split(commaSeparatedList, ",") ) {
+    const Xapian::Query singleQuery(prefix + normalizeText(elem));
+    if ( firstIteration ) {
+      q = singleQuery;
+      firstIteration = false;
+    } else {
+      q = Xapian::Query(Xapian::Query::OP_OR, q, singleQuery);
+    }
+  }
+  return q;
+}
+
+Xapian::Query categoryQuery(const std::string& commaSeparatedCategoryList)
+{
+  return multipleParamQuery(commaSeparatedCategoryList, "XC");
+}
+
+Xapian::Query langQuery(const std::string& commaSeparatedLanguageList)
+{
+  return multipleParamQuery(commaSeparatedLanguageList, "L");
 }
 
 Xapian::Query publisherQuery(const std::string& publisher)
@@ -577,6 +728,9 @@ Xapian::Query buildXapianQuery(const Filter& filter)
   if ( filter.hasName() ) {
     q = Xapian::Query(Xapian::Query::OP_AND, q, nameQuery(filter.getName()));
   }
+  if ( filter.hasFlavour() )  {
+    q = Xapian::Query(Xapian::Query::OP_AND, q, flavourQuery(filter.getFlavour()));     
+  }
   if ( filter.hasCategory() ) {
     q = Xapian::Query(Xapian::Query::OP_AND, q, categoryQuery(filter.getCategory()));
   }
@@ -607,10 +761,10 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
 
   BookIdCollection bookIds;
 
-  std::lock_guard<std::mutex> lock(m_mutex);
-  Xapian::Enquire enquire(mp_impl->m_bookDB);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
+  Xapian::Enquire enquire(*m_bookDB);
   enquire.set_query(query);
-  const auto results = enquire.get_mset(0, mp_impl->m_books.size());
+  const auto results = enquire.get_mset(0, m_books.size());
   for ( auto it = results.begin(); it != results.end(); ++it  ) {
     bookIds.push_back(it.get_document().get_data());
   }
@@ -622,9 +776,9 @@ Library::BookIdCollection Library::filter(const Filter& filter) const
 {
   BookIdCollection result;
   const auto preliminaryResult = filterViaBookDB(filter);
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   for(auto id : preliminaryResult) {
-    if(filter.accept(mp_impl->m_books.at(id))) {
+    if(filter.accept(m_books.at(id))) {
       result.push_back(id);
     }
   }
@@ -696,7 +850,7 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
   // NOTE: for the entire duration of the sort. Will need to obtain (under a
   // NOTE: lock) the required atributes from the books once, and then the
   // NOTE: sorting will run on a copy of data without locking.
-  std::lock_guard<std::mutex> lock(m_mutex);
+  std::lock_guard<std::recursive_mutex> lock(m_mutex);
   switch(sort) {
     case TITLE:
       std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
@@ -742,6 +896,7 @@ enum filterTypes {
   QUERY = FLAG(12),
   NAME = FLAG(13),
   CATEGORY = FLAG(14),
+  FLAVOUR = FLAG(15),
 };
 
 Filter& Filter::local(bool accept)
@@ -843,6 +998,25 @@ Filter& Filter::name(std::string name)
   activeFilters |= NAME;
   return *this;
 }
+  
+Filter& Filter::flavour(std::string flavour)
+{
+  _flavour = flavour;
+  activeFilters |= FLAVOUR;
+  return *this;
+}
+
+Filter& Filter::clearLang()
+{
+  activeFilters &= ~LANG;
+  return *this;
+}
+
+Filter& Filter::clearCategory()
+{
+  activeFilters &= ~CATEGORY;
+  return *this;
+}
 
 #define ACTIVE(X) (activeFilters & (X))
 #define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
@@ -876,6 +1050,12 @@ bool Filter::hasCreator() const
   return ACTIVE(_CREATOR);
 }
 
+bool Filter::hasFlavour() const
+{
+  return ACTIVE(FLAVOUR);
+}
+
+
 bool Filter::accept(const Book& book) const
 {
   auto local = !book.getPath().empty();

src/library_dumper.cpp

@@ -0,0 +1,61 @@
+#include "library_dumper.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibraryDumper::LibraryDumper(const Library* library, const NameMapper* nameMapper)
+  : library(library),
+    nameMapper(nameMapper)
+{
+}
+/* Destructor */
+LibraryDumper::~LibraryDumper()
+{
+}
+
+void LibraryDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+}
+
+kainjow::mustache::list LibraryDumper::getCategoryData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : library->getBooksCategories() ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
+  }
+  return categoryData;
+}
+
+kainjow::mustache::list LibraryDumper::getLanguageData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+  return languageData;
+}
+
+} // namespace kiwix

src/library_dumper.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBRARY_DUMPER_H
+#define KIWIX_LIBRARY_DUMPER_H
+
+#include <string>
+
+
+#include "library.h"
+#include "name_mapper.h"
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+/**
+ * A base class to dump Library in various formats.
+ *
+ */
+class LibraryDumper
+{
+ public:
+  LibraryDumper(const Library* library, const NameMapper* NameMapper);
+  ~LibraryDumper();
+
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
+
+  /**
+   * Set the root location used when generating url.
+   *
+   * @param rootLocation the root location to use.
+   */
+  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+
+  /**
+   * Set the URL for accessing book content
+   *
+   * @param url the URL of the /content endpoint of the content server
+   */
+  void setContentAccessUrl(const std::string& url) { this->contentAccessUrl = url; }
+
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+
+  /**
+   * Sets user default language
+   *
+   * @param userLang the user language to be set
+   */
+  void setUserLanguage(std::string userLang) { this->m_userLang = userLang; }
+
+  /**
+   * Get the data of categories
+   */
+  kainjow::mustache::list getCategoryData() const;
+
+  /**
+   * Get the data of languages
+   */
+  kainjow::mustache::list getLanguageData() const;
+
+ protected:
+  const kiwix::Library* const library;
+  const kiwix::NameMapper* const nameMapper;
+  std::string libraryId;
+  std::string rootLocation;
+  std::string contentAccessUrl;
+  std::string m_userLang;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+};
+}
+
+#endif // KIWIX_LIBRARY_DUMPER_H

src/libxml_dumper.cpp

@@ -54,7 +54,7 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
   if (book.getOrigId().empty()) {
     ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
     ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
-    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getCommaSeparatedLanguages());
     ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
     ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
@@ -97,11 +97,15 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
     auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
-    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "name", book.getName());
+    ADD_TEXT_ENTRY(book_node, "flavour", book.getFlavour());
+    ADD_TEXT_ENTRY(book_node, "language", book.getCommaSeparatedLanguages());
     ADD_TEXT_ENTRY(book_node, "date", book.getDate());
   } catch (...) {
     ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
+    ADD_TEXT_ENTRY(book_node, "name", bookmark.getBookName());
+    ADD_TEXT_ENTRY(book_node, "flavour", bookmark.getBookFlavour());
     ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
     ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
   }
@@ -135,7 +139,7 @@ std::string LibXMLDumper::dumpLibXMLBookmark()
   pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
 
   if (library) {
-    for (auto& bookmark: library->getBookmarks()) {
+    for (auto& bookmark: library->getBookmarks(false)) {
       handleBookmark(bookmark, bookmarksNode);
     }
   }

src/manager.cpp

@@ -23,26 +23,24 @@
 #include "tools/pathTools.h"
 
 #include <pugixml.hpp>
+#include <filesystem>
+#include <iostream>
+#include <set>
+#include <queue>
+#include <cctype>
+#include <algorithm>
+
+namespace fs = std::filesystem;
 
 namespace kiwix
 {
 
-namespace
-{
-
-struct NoDelete
-{
-  template<class T> void operator()(T*) {}
-};
-
-} // unnamed namespace
-
 ////////////////////////////////////////////////////////////////////////////////
 // LibraryManipulator
 ////////////////////////////////////////////////////////////////////////////////
 
-LibraryManipulator::LibraryManipulator(Library* library)
-  : library(*library)
+LibraryManipulator::LibraryManipulator(LibraryPtr library)
+  : library(library)
 {}
 
 LibraryManipulator::~LibraryManipulator()
@@ -50,7 +48,7 @@ LibraryManipulator::~LibraryManipulator()
 
 bool LibraryManipulator::addBookToLibrary(const Book& book)
 {
-  const auto ret = library.addBook(book);
+  const auto ret = library->addBook(book);
   if ( ret ) {
     bookWasAddedToLibrary(book);
   }
@@ -59,13 +57,13 @@ bool LibraryManipulator::addBookToLibrary(const Book& book)
 
 void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
 {
-  library.addBookmark(bookmark);
+  library->addBookmark(bookmark);
   bookmarkWasAddedToLibrary(bookmark);
 }
 
 uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
 {
-  const auto n = library.removeBooksNotUpdatedSince(rev);
+  const auto n = library->removeBooksNotUpdatedSince(rev);
   if ( n != 0 ) {
     booksWereRemovedFromLibrary();
   }
@@ -89,15 +87,15 @@ void LibraryManipulator::booksWereRemovedFromLibrary()
 ////////////////////////////////////////////////////////////////////////////////
 
 /* Constructor */
-Manager::Manager(LibraryManipulator* manipulator):
+Manager::Manager(LibraryManipulator manipulator):
   writableLibraryPath(""),
-  manipulator(manipulator, NoDelete())
+  manipulator(manipulator)
 {
 }
 
-Manager::Manager(Library* library) :
+Manager::Manager(LibraryPtr library) :
   writableLibraryPath(""),
-  manipulator(new LibraryManipulator(library))
+  manipulator(LibraryManipulator(library))
 {
 }
 
@@ -121,7 +119,7 @@ bool Manager::parseXmlDom(const pugi::xml_document& doc,
     if (!trustLibrary && !book.getPath().empty()) {
       this->readBookFromPath(book.getPath(), &book);
     }
-    manipulator->addBookToLibrary(book);
+    manipulator.addBookToLibrary(book);
   }
 
   return true;
@@ -166,7 +164,7 @@ bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& url
     book.updateFromOpds(entryNode, urlHost);
 
     /* Update the book properties with the new importer */
-    manipulator->addBookToLibrary(book);
+    manipulator.addBookToLibrary(book);
   }
 
   return true;
@@ -238,10 +236,10 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
+        || (!book.getTitle().empty() && !book.getLanguages().empty()
             && !book.getDate().empty())) {
       book.setUrl(url);
-      manipulator->addBookToLibrary(book);
+      manipulator.addBookToLibrary(book);
       return book.getId();
     }
   }
@@ -261,6 +259,58 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
           .empty());
 }
 
+void Manager::addBooksFromDirectory(const std::string& path,
+                                    const bool verboseFlag)
+{
+  std::set<std::string> iteratedDirs;
+  std::queue<std::string> dirQueue;
+  dirQueue.push(fs::absolute(path).u8string());
+  int totalBooksAdded = 0;
+  if (verboseFlag)
+    std::cout << "Adding books from the directory tree: " << dirQueue.front() << std::endl;
+
+  while (!dirQueue.empty()) {
+    const auto currentPath = dirQueue.front();
+    dirQueue.pop();
+    if (verboseFlag)
+      std::cout << "Visiting directory: " << currentPath << std::endl;
+    for (const auto& dirEntry : fs::directory_iterator(currentPath)) {
+      auto resolvedPath = dirEntry.path();
+      if (fs::is_symlink(dirEntry)) {
+        try {
+          resolvedPath = fs::canonical(dirEntry.path());
+        } catch (const std::exception& e) {
+          std::cerr << "Could not resolve symlink " << resolvedPath.u8string() << " to a valid path. Skipping..." << std::endl;
+          continue;
+        }
+      }
+      const std::string pathString = resolvedPath.u8string();
+      std::string resolvedPathExtension = resolvedPath.extension();
+      std::transform(resolvedPathExtension.begin(), resolvedPathExtension.end(), resolvedPathExtension.begin(),
+          [](unsigned char c){ return std::tolower(c); });
+      if (fs::is_directory(resolvedPath)) {
+        if (iteratedDirs.find(pathString) == iteratedDirs.end())
+          dirQueue.push(pathString);
+        else if (verboseFlag)
+          std::cout << "Already iterated over " << pathString << ". Skipping..." << std::endl;
+      } else if (resolvedPathExtension == ".zim" || resolvedPathExtension == ".zimaa") {
+        if (!this->addBookFromPath(pathString, pathString, "", false)) {
+          std::cerr << "Could not add " << pathString << " into the library." << std::endl;
+        } else if (verboseFlag) {
+          std::cout << "Added " << pathString << " into the library." << std::endl;
+          totalBooksAdded++;
+        }
+      } else if (verboseFlag) {
+        std::cout << "Skipped " << pathString << " - unsupported file type or permission denied." << std::endl;
+      }
+    }
+    iteratedDirs.insert(currentPath);
+  }
+
+  if (verboseFlag)
+    std::cout << "Traversal completed. Total books added: " << totalBooksAdded << std::endl;
+}
+
 bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
 {
   std::string tmp_path = path;
@@ -296,7 +346,7 @@ bool Manager::readBookmarkFile(const std::string& path)
 
     bookmark.updateFromXml(node);
 
-    manipulator->addBookmarkToLibrary(bookmark);
+    manipulator.addBookmarkToLibrary(bookmark);
   }
 
   return true;
@@ -304,7 +354,7 @@ bool Manager::readBookmarkFile(const std::string& path)
 
 void Manager::reload(const Paths& paths)
 {
-  const auto libRevision = manipulator->getLibrary().getRevision();
+  const auto libRevision = manipulator.getLibrary()->getRevision();
   for (std::string path : paths) {
     if (!path.empty()) {
       if ( kiwix::isRelativePath(path) )
@@ -316,7 +366,7 @@ void Manager::reload(const Paths& paths)
     }
   }
 
-  manipulator->removeBooksNotUpdatedSince(libRevision);
+  manipulator.removeBooksNotUpdatedSince(libRevision);
 }
 
 }

src/meson.build

@@ -5,6 +5,8 @@ kiwix_sources = [
   'manager.cpp',
   'libxml_dumper.cpp',
   'opds_dumper.cpp',
+  'html_dumper.cpp',
+  'library_dumper.cpp',
   'downloader.cpp',
   'server.cpp',
   'search_renderer.cpp',
@@ -15,6 +17,8 @@ kiwix_sources = [
   'tools/regexTools.cpp',
   'tools/stringTools.cpp',
   'tools/networkTools.cpp',
+  'tools/opdsParsingTools.cpp',
+  'tools/languageTools.cpp',
   'tools/otherTools.cpp',
   'tools/archiveTools.cpp',
   'kiwixserve.cpp',
@@ -24,9 +28,10 @@ kiwix_sources = [
   'server/request_context.cpp',
   'server/response.cpp',
   'server/internalServer.cpp',
-  'server/internalServer_catalog_v2.cpp',
+  'server/internalServer_catalog.cpp',
   'server/i18n.cpp',
   'opds_catalog.cpp',
+  'spelling_correction.cpp',
   'version.cpp'
 ]
 kiwix_sources += lib_resources
@@ -45,7 +50,7 @@ config_h = configure_file(output : 'kiwix_config.h',
                input : 'config.h.in')
 install_headers(config_h, subdir:'kiwix')
 
-kiwixlib = library('kiwix',
+libkiwix = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,

src/name_mapper.cpp

@@ -24,33 +24,37 @@
 
 namespace kiwix {
 
-HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
-  for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
+HumanReadableNameMapper::HumanReadableNameMapper(const kiwix::Library& library, bool withAlias) {
+  for (auto& bookId: library.filter(kiwix::Filter())) {
     auto& currentBook = library.getBookById(bookId);
     auto bookName = currentBook.getHumanReadableIdFromPath();
     m_idToName[bookId] = bookName;
-    m_nameToId[bookName] = bookId;
+    mapName(library, bookName, bookId);
 
     if (!withAlias)
       continue;
 
     auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
-    if (aliasName == bookName) {
-      continue;
-    }
-    if (m_nameToId.find(aliasName) == m_nameToId.end()) {
-      m_nameToId[aliasName] = bookId;
-    } else {
-      auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
-      std::cerr << "Path collision: " << alreadyPresentPath
-                << " and " << currentBook.getPath()
-                << " can't share the same URL path '" << aliasName << "'."
-                << " Therefore, only " << alreadyPresentPath
-                << " will be served." << std::endl;
+    if (aliasName != bookName) {
+      mapName(library, aliasName, bookId);
     }
   }
 }
 
+void HumanReadableNameMapper::mapName(const Library& library, std::string name, std::string bookId) {
+  if (m_nameToId.find(name) == m_nameToId.end()) {
+    m_nameToId[name] = bookId;
+  } else {
+    const auto& currentBook = library.getBookById(bookId);
+    auto alreadyPresentPath = library.getBookById(m_nameToId[name]).getPath();
+    std::cerr << "Path collision: '" << alreadyPresentPath
+              << "' and '" << currentBook.getPath()
+              << "' can't share the same URL path '" << name << "'."
+              << " Therefore, only '" << alreadyPresentPath
+              << "' will be served." << std::endl;
+  }
+}
+
 std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
   return m_idToName.at(id);
 }
@@ -63,7 +67,7 @@ std::string HumanReadableNameMapper::getIdForName(const std::string& name) const
 // UpdatableNameMapper
 ////////////////////////////////////////////////////////////////////////////////
 
-UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
+UpdatableNameMapper::UpdatableNameMapper(LibraryPtr lib, bool withAlias)
   : library(lib)
   , withAlias(withAlias)
 {
@@ -72,7 +76,7 @@ UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
 
 void UpdatableNameMapper::update()
 {
-  const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
+  const auto newNameMapper = new HumanReadableNameMapper(*library, withAlias);
   std::lock_guard<std::mutex> lock(mutex);
   nameMapper.reset(newNameMapper);
 }

src/opds_dumper.cpp

@@ -20,7 +20,7 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include <mustache.hpp>
 
 #include "tools/stringTools.h"
@@ -30,8 +30,8 @@ namespace kiwix
 {
 
 /* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
-  : library(library)
+OPDSDumper::OPDSDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
 {
 }
 /* Destructor */
@@ -39,16 +39,11 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
-{
-  m_totalResults = totalResults;
-  m_startIndex = startIndex,
-  m_count = count;
-}
-
 namespace
 {
 
+const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
+
 typedef kainjow::mustache::data MustacheData;
 typedef kainjow::mustache::list BooksData;
 typedef kainjow::mustache::list IllustrationInfo;
@@ -56,29 +51,32 @@ typedef kainjow::mustache::list IllustrationInfo;
 IllustrationInfo getBookIllustrationInfo(const Book& book)
 {
     kainjow::mustache::list illustrations;
-    if ( book.isPathValid() ) {
-      for ( const auto& illustration : book.getIllustrations() ) {
-        // For now, we are handling only sizexsize@1 illustration.
-        // So we can simply pass one size to mustache.
-        illustrations.push_back(kainjow::mustache::object{
-          {"icon_size", to_string(illustration->width)},
-          {"icon_mimetype", illustration->mimeType}
-        });
-      }
+    for ( const auto& illustration : book.getIllustrations() ) {
+      // For now, we are handling only sizexsize@1 illustration.
+      // So we can simply pass one size to mustache.
+      illustrations.push_back(kainjow::mustache::object{
+        {"icon_size", to_string(illustration->width)},
+        {"icon_mimetype", illustration->mimeType}
+      });
     }
     return illustrations;
 }
 
-kainjow::mustache::object getSingleBookData(const Book& book)
+std::string fullEntryXML(const Book& book,
+                         const std::string& rootLocation,
+                         const std::string& contentAccessUrl,
+                         const std::string& contentId)
 {
     const auto bookDate = book.getDate() + "T00:00:00Z";
-    return kainjow::mustache::object{
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
+      {"contentAccessUrl",  onlyAsNonEmptyMustacheValue(contentAccessUrl)},
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
-      {"language", book.getLanguage()},
-      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"language", book.getCommaSeparatedLanguages()},
+      {"content_id",  urlEncode(contentId)},
       {"updated", bookDate}, // XXX: this should be the entry update datetime
       {"book_date", bookDate},
       {"category", book.getCategory()},
@@ -92,27 +90,39 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
+    return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+std::string partialEntryXML(const Book& book, const std::string& rootLocation)
 {
-  auto data = getSingleBookData(book);
-  data["with_xml_header"] = MustacheData(withXMLHeader);
-  data["dump_partial_entries"] = MustacheData(partial);
-  data["endpoint_root"] = endpointRoot;
-  data["root"] = rootLocation;
-  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+    const auto bookDate = book.getDate() + "T00:00:00Z";
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
+      {"endpoint_root", rootLocation + "/catalog/v2"},
+      {"id", book.getId()},
+      {"title", book.getTitle()},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+    };
+    const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
+    return render_template(xmlTemplate, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library,
+                       const NameMapper* nameMapper,
+                       const std::vector<std::string>& bookIds,
+                       const std::string& rootLocation,
+                       const std::string& contentAccessUrl,
+                       bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
     try {
       const Book book = library->getBookByIdThreadSafe(bookId);
-      booksData.push_back(kainjow::mustache::object{
-          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
-      });
+      const std::string contentId = nameMapper->getNameForId(bookId);
+      const auto entryXML = partial
+                          ? partialEntryXML(book, rootLocation)
+                          : fullEntryXML(book, rootLocation, contentAccessUrl, contentId);
+      booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
     } catch ( const std::out_of_range& ) {
       // the book was removed from the library since its id was obtained
       // ignore it
@@ -122,64 +132,11 @@ BooksData getBooksData(const Library* library, const std::vector<std::string>& b
   return booksData;
 }
 
-std::map<std::string, std::string> iso639_3 = {
-  {"atj", "atikamekw"},
-  {"azb", "آذربایجان دیلی"},
-  {"bcl", "central bikol"},
-  {"bgs", "tagabawa"},
-  {"bxr", "буряад хэлэн"},
-  {"cbk", "chavacano"},
-  {"cdo", "閩東語"},
-  {"dag", "Dagbani"},
-  {"diq", "dimli"},
-  {"dty", "डोटेली"},
-  {"eml", "emiliân-rumagnōl"},
-  {"fbs", "српскохрватски"},
-  {"ido", "ido"},
-  {"kbp", "kabɩyɛ"},
-  {"kld", "Gamilaraay"},
-  {"lbe", "лакку маз"},
-  {"lbj", "ལ་དྭགས་སྐད་"},
-  {"map", "Austronesian"},
-  {"mhr", "марий йылме"},
-  {"mnw", "ဘာသာမန်"},
-  {"myn", "mayan"},
-  {"nah", "nahuatl"},
-  {"nai", "north American Indian"},
-  {"nds", "plattdütsch"},
-  {"nrm", "bhasa narom"},
-  {"olo", "livvi"},
-  {"pih", "Pitcairn-Norfolk"},
-  {"pnb", "Western Panjabi"},
-  {"rmr", "Caló"},
-  {"rmy", "romani shib"},
-  {"roa", "romance languages"},
-  {"twi", "twi"}
-};
-
-std::once_flag fillLanguagesFlag;
-
-void fillLanguagesMap()
-{
-  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
-    const ICULanguageInfo lang(*icuLangPtr);
-    iso639_3.insert({lang.iso3Code(), lang.selfName()});
-  }
-}
-
-std::string getLanguageSelfName(const std::string& lang) {
-  const auto itr = iso639_3.find(lang);
-  if (itr != iso639_3.end()) {
-    return itr->second;
-  }
-  return lang;
-};
-
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, contentAccessUrl, false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
@@ -197,20 +154,20 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, contentAccessUrl, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
+  const std::string url = endpoint + (query.empty() ? "" : "?" + query);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
      {"filter", onlyAsNonEmptyMustacheValue(query)},
-     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"self_url", url},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
-     {"books", booksData },
-     {"dump_partial_entries", MustacheData(partial)}
+     {"books", booksData }
   };
 
   return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
@@ -218,23 +175,17 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+  const auto book = library->getBookById(bookId);
+  const std::string contentId = nameMapper->getNameForId(bookId);
+  return XML_HEADER
+         + "\n"
+         + fullEntryXML(book, rootLocation, contentAccessUrl, contentId);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list categoryData;
-  for ( const auto& category : library->getBooksCategories() ) {
-    const auto urlencodedCategoryName = urlEncode(category);
-    categoryData.push_back(kainjow::mustache::object{
-      {"name", category},
-      {"urlencoded_name",  urlencodedCategoryName},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
-    });
-  }
-
+  kainjow::mustache::list categoryData = getCategoryData();
   return render_template(
              RESOURCE::templates::catalog_v2_categories_xml,
              kainjow::mustache::object{
@@ -249,21 +200,7 @@ std::string OPDSDumper::categoriesOPDSFeed() const
 std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list languageData;
-  std::call_once(fillLanguagesFlag, fillLanguagesMap);
-  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
-    const std::string languageCode = langAndBookCount.first;
-    const int bookCount = langAndBookCount.second;
-    const auto languageSelfName = getLanguageSelfName(languageCode);
-    languageData.push_back(kainjow::mustache::object{
-      {"lang_code",  languageCode},
-      {"lang_self_name", languageSelfName},
-      {"book_count", to_string(bookCount)},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
-    });
-  }
-
+  kainjow::mustache::list languageData = getLanguageData();
   return render_template(
              RESOURCE::templates::catalog_v2_languages_xml,
              kainjow::mustache::object{

src/opds_dumper.h

@@ -27,6 +27,8 @@
 #include <pugixml.hpp>
 
 #include "library.h"
+#include "name_mapper.h"
+#include "library_dumper.h"
 
 using namespace std;
 
@@ -37,11 +39,10 @@ namespace kiwix
  * A tool to dump a `Library` into a opds stream.
  *
  */
-class OPDSDumper
+class OPDSDumper : public LibraryDumper
 {
  public:
-  OPDSDumper() = default;
-  OPDSDumper(Library* library);
+  OPDSDumper(const Library* library, const NameMapper* NameMapper);
   ~OPDSDumper();
 
   /**
@@ -84,37 +85,6 @@ class OPDSDumper
    * @return The OPDS feed.
    */
   std::string languagesOPDSFeed() const;
-
-  /**
-   * Set the id of the library.
-   *
-   * @param id the id to use.
-   */
-  void setLibraryId(const std::string& id) { this->libraryId = id;}
-
-  /**
-   * Set the root location used when generating url.
-   *
-   * @param rootLocation the root location to use.
-   */
-  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
-
-  /**
-   * Set some informations about the search results.
-   *
-   * @param totalResult the total number of results of the search.
-   * @param startIndex the start index of the result.
-   * @param count the number of result of the current set (or page).
-   */
-  void setOpenSearchInfo(int totalResult, int startIndex, int count);
-
- protected:
-  kiwix::Library* library;
-  std::string libraryId;
-  std::string rootLocation;
-  int m_totalResults;
-  int m_startIndex;
-  int m_count;
 };
 }
 

src/search_renderer.cpp

@@ -29,23 +29,49 @@
 #include <zim/search.h>
 
 #include <mustache.hpp>
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include "tools/stringTools.h"
 
+#include "server/i18n_utils.h"
+
 namespace kiwix
 {
 
-/* Constructor */
-SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
-                      unsigned int start, unsigned int estimatedResultCount)
-    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
-{}
+namespace
+{
 
-SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+ParameterizedMessage searchResultsPageTitleMsg(const std::string& searchPattern)
+{
+  return ParameterizedMessage("search-results-page-title",
+                              {{"SEARCH_PATTERN", searchPattern}}
+  );
+}
+
+ParameterizedMessage searchResultsPageHeaderMsg(const std::string& searchPattern,
+                                                const kainjow::mustache::data& r)
+{
+  if ( r.get("count")->string_value() == "0" ) {
+    return ParameterizedMessage("empty-search-results-page-header",
+                                {{"SEARCH_PATTERN", searchPattern}}
+    );
+  } else {
+    return ParameterizedMessage("search-results-page-header",
+                                {
+                                  {"SEARCH_PATTERN", searchPattern},
+                                  {"START", r.get("startLabel")->string_value()},
+                                  {"END",   r.get("end")  ->string_value()},
+                                  {"COUNT", r.get("count")->string_value()},
+                                }
+    );
+  }
+}
+
+} // unnamed namespace
+
+/* Constructor */
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs,
                       unsigned int start, unsigned int estimatedResultCount)
     : m_srs(srs),
-      mp_nameMapper(mapper),
-      mp_library(library),
       protocolPrefix("zim://"),
       searchProtocolPrefix("search://"),
       estimatedResultCount(estimatedResultCount),
@@ -94,7 +120,7 @@ kainjow::mustache::data buildQueryData
   kainjow::mustache::data query;
   query.set("pattern", kiwix::encodeDiples(pattern));
   std::ostringstream ss;
-  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
   ss << "&" << bookQuery;
   query.set("unpaginatedQuery", ss.str());
   auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
@@ -164,22 +190,33 @@ kainjow::mustache::data buildPagination(
   return pagination;
 }
 
-std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str, const NameMapper& nameMapper, const Library* library)
 {
-  const std::string absPathPrefix = protocolPrefix + "content/";
+  const std::string absPathPrefix = protocolPrefix;
   // Build the results list
   kainjow::mustache::data items{kainjow::mustache::data::type::list};
   for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
-    std::string zim_id(it.getZimId());
+    const std::string zim_id(it.getZimId());
+    const auto path = nameMapper.getNameForId(zim_id) + "/" + it.getPath();
     result.set("title", it.getTitle());
-    result.set("absolutePath", absPathPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
+    result.set("absolutePath", absPathPrefix + urlEncode(path));
     result.set("snippet", it.getSnippet());
-    if (mp_library) {
-      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    if (library) {
+      const std::string bookTitle = library->getBookById(zim_id).getTitle();
+      const ParameterizedMessage bookInfoMsg("search-result-book-info",
+          {{"BOOK_TITLE", bookTitle}}
+      );
+      result.set("bookInfo",  bookInfoMsg.getText(userlang)); // for HTML
+      result.set("bookTitle", bookTitle); // for XML
     }
     if (it.getWordCount() >= 0) {
-      result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
+      const auto wordCountStr = kiwix::beautifyInteger(it.getWordCount());
+      const ParameterizedMessage wordCountMsg("word-count",
+          {{"COUNT", wordCountStr}}
+      );
+      result.set("wordCountInfo", wordCountMsg.getText(userlang)); // for HTML
+      result.set("wordCount", wordCountStr); // for XML
     }
 
     items.push_back(result);
@@ -187,9 +224,9 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
   kainjow::mustache::data results;
   results.set("items", items);
   results.set("count", kiwix::beautifyInteger(estimatedResultCount));
-  results.set("hasResults", estimatedResultCount != 0);
   results.set("start", kiwix::beautifyInteger(resultStart));
-  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
+  results.set("startLabel", kiwix::beautifyInteger(resultStart+1));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength, estimatedResultCount)));
 
   // pagination
   auto pagination = buildPagination(
@@ -204,12 +241,15 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
     searchBookQuery
   );
 
-
-  kainjow::mustache::data allData;
-  allData.set("protocolPrefix", protocolPrefix);
-  allData.set("results", results);
-  allData.set("pagination", pagination);
-  allData.set("query", query);
+  const auto pageHeaderMsg = searchResultsPageHeaderMsg(searchPattern, results);
+  const kainjow::mustache::object allData{
+    {"PAGE_TITLE", searchResultsPageTitleMsg(searchPattern).getText(userlang)},
+    {"PAGE_HEADER", pageHeaderMsg.getText(userlang)},
+    {"searchProtocolPrefix", searchProtocolPrefix},
+    {"results", results},
+    {"pagination", pagination},
+    {"query", query},
+  };
 
   kainjow::mustache::mustache tmpl(tmpl_str);
 
@@ -221,14 +261,14 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
   return ss.str();
 }
 
-std::string SearchRenderer::getHtml()
+std::string SearchRenderer::getHtml(const NameMapper& mapper, const Library* library)
 {
-  return renderTemplate(RESOURCE::templates::search_result_html);
+  return renderTemplate(RESOURCE::templates::search_result_html, mapper, library);
 }
 
-std::string SearchRenderer::getXml()
+std::string SearchRenderer::getXml(const NameMapper& mapper, const Library* library)
 {
-  return renderTemplate(RESOURCE::templates::search_result_xml);
+  return renderTemplate(RESOURCE::templates::search_result_xml, mapper, library);
 }
 
 

src/server.cpp

@@ -29,7 +29,23 @@
 
 namespace kiwix {
 
-Server::Server(Library* library, NameMapper* nameMapper) :
+namespace
+{
+
+std::string makeServerUrl(std::string host, int port, std::string root)
+{
+  const int httpDefaultPort = 80;
+
+  if (port == httpDefaultPort) {
+    return "http://" + host + root;
+  } else {
+    return "http://" + host + ":" + std::to_string(port) + root;
+  }
+}
+
+}  // unnamed namespace
+
+Server::Server(LibraryPtr library, std::shared_ptr<NameMapper> nameMapper) :
   mp_library(library),
   mp_nameMapper(nameMapper),
   mp_server(nullptr)
@@ -51,9 +67,18 @@ bool Server::start() {
     m_withTaskbar,
     m_withLibraryButton,
     m_blockExternalLinks,
+    m_ipMode,
     m_indexTemplateString,
-    m_ipConnectionLimit));
-  return mp_server->start();
+    m_ipConnectionLimit,
+    m_catalogOnlyMode,
+    m_contentServerUrl));
+  if (mp_server->start()) {
+    // this syncs m_addr of InternalServer and Server as they may diverge
+    m_addr = mp_server->getAddress();
+    return true;
+  } else {
+    return false;
+  }
 }
 
 void Server::stop() {
@@ -66,22 +91,53 @@ void Server::stop() {
 void Server::setRoot(const std::string& root)
 {
   m_root = root;
-  if (m_root[0] != '/') {
-    m_root = "/" + m_root;
-  }
-  if (m_root.back() == '/') {
-    m_root.erase(m_root.size() - 1);
+  while (!m_root.empty() && m_root.back() == '/')
+    m_root.pop_back();
+
+  while (!m_root.empty() && m_root.front() == '/')
+    m_root = m_root.substr(1);
+  m_root = m_root.empty() ? m_root : "/" + m_root;
+}
+
+void Server::setAddress(const std::string& addr)
+{
+  m_addr.addr.clear();
+  m_addr.addr6.clear();
+
+  if (addr.empty()) return;
+
+  if (addr.find(':') != std::string::npos) { // IPv6
+    m_addr.addr6 = (addr[0] == '[') ? addr.substr(1, addr.length() - 2) : addr; // Remove brackets if any
+  } else {
+    m_addr.addr = addr;
   }
 }
 
-int Server::getPort()
+int Server::getPort() const
 {
-  return mp_server->getPort();
+  return m_port;
 }
 
-std::string Server::getAddress()
+IpAddress Server::getAddress() const
 {
-  return mp_server->getAddress();
+  return m_addr;
+}
+
+IpMode Server::getIpMode() const
+{
+  return mp_server->getIpMode();
+}
+
+std::vector<std::string> Server::getServerAccessUrls() const
+{
+  std::vector<std::string> result;
+  if (!m_addr.addr.empty()) {
+    result.push_back(makeServerUrl(m_addr.addr, m_port, m_root));
+  }
+  if (!m_addr.addr6.empty()) {
+    result.push_back(makeServerUrl("[" + m_addr.addr6 + "]", m_port, m_root));
+  }
+  return result;
 }
 
 }

src/server/etag.cpp

@@ -37,11 +37,11 @@ namespace {
 // into the ETag for ETag::Option opt.
 // IMPORTANT: The characters in all_options must come in sorted order (so that
 // IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
+const char all_options[] = "Zz";
 
 static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
 
-bool isValidServerId(const std::string& s)
+bool isValidETagBody(const std::string& s)
 {
   return !s.empty() && s.find_first_of("\"/") == std::string::npos;
 }
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
 
 std::string ETag::get_etag() const
 {
-  if ( m_serverId.empty() )
+  if ( m_body.empty() )
     return std::string();
 
-  return "\"" + m_serverId + "/" + m_options + "\"";
+  return "\"" + m_body + "/" + m_options + "\"";
 }
 
-ETag::ETag(const std::string& serverId, const std::string& options)
+ETag::ETag(const std::string& body, const std::string& options)
 {
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
   {
-    m_serverId = serverId;
+    m_body = body;
     m_options = options;
   }
 }
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
   return ETag(s.substr(0, i), s.substr(i+1));
 }
 
-ETag ETag::match(const std::string& etags, const std::string& server_id)
+ETag ETag::match(const std::string& etags, const std::string& body)
 {
   std::istringstream ss(etags);
   std::string etag_str;
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
       etag_str.pop_back();
 
     const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
+    if ( etag && etag.m_body == body )
       return etag;
   }
 

src/server/etag.h

@@ -28,10 +28,11 @@ namespace kiwix {
 // The ETag string used by Kiwix server (more precisely, its value inside the
 // double quotes) consists of two parts:
 //
-// 1. ServerId - The string obtained on server start up
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
 //
-// 2. Options -  Zero or more characters encoding the values of some of the
-//               headers of the response
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
+//               the values of some of the headers of the response
 //
 // The two parts are separated with a slash (/) symbol (which is always present,
 // even when the the options part is empty). Neither portion of a Kiwix ETag
@@ -40,7 +41,7 @@ namespace kiwix {
 //
 //   "abcdefghijklmn/"
 //   "1234567890/z"
-//   "1234567890/cz"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
 //
 // The options part of the Kiwix ETag allows to correctly set the required
 // headers when responding to a conditional If-None-Match request with a 304
@@ -51,7 +52,7 @@ class ETag
 {
   public: // types
     enum Option {
-      CACHEABLE_ENTITY,
+      ZIM_CONTENT,
       COMPRESSED_CONTENT,
       OPTION_COUNT
     };
@@ -59,10 +60,10 @@ class ETag
   public: // functions
     ETag() {}
 
-    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_body(const std::string& s) { m_body = s; }
     void set_option(Option opt);
 
-    explicit operator bool() const { return !m_serverId.empty(); }
+    explicit operator bool() const { return !m_body.empty(); }
 
     bool get_option(Option opt) const;
     std::string get_etag() const;
@@ -76,7 +77,7 @@ class ETag
     static ETag parse(std::string s);
 
   private: // data
-    std::string m_serverId;
+    std::string m_body;
     std::string m_options;
 };
 

src/server/i18n.cpp

@@ -17,7 +17,7 @@
  * MA 02110-1301, USA.
  */
 
-#include "i18n.h"
+#include "i18n_utils.h"
 
 #include "tools/otherTools.h"
 
@@ -70,6 +70,14 @@ public: // functions
     return s;
   }
 
+  size_t getStringCount(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang)->entryCount;
+    } catch(const std::out_of_range&) {
+      return 0;
+    }
+  }
+
 private: // functions
   const I18nStringTable* getStringsFor(const std::string& lang) const {
     try {
@@ -84,13 +92,17 @@ private: // data
   const I18nStringTable* enStrings;
 };
 
+const I18nStringDB& getStringDb()
+{
+  static const I18nStringDB stringDb;
+  return stringDb;
+}
+
 } // unnamed namespace
 
 std::string getTranslatedString(const std::string& lang, const std::string& key)
 {
-  static const I18nStringDB stringDb;
-
-  return stringDb.get(lang, key);
+  return getStringDb().get(lang, key);
 }
 
 namespace i18n
@@ -100,8 +112,12 @@ std::string expandParameterizedString(const std::string& lang,
                                       const std::string& key,
                                       const Parameters& params)
 {
+  kainjow::mustache::object mustacheParams;
+  for( const auto& kv : params ) {
+    mustacheParams[kv.first] = kv.second;
+  }
   const std::string tmpl = getTranslatedString(lang, key);
-  return render_template(tmpl, params);
+  return render_template(tmpl, mustacheParams);
 }
 
 } // namespace i18n
@@ -111,4 +127,79 @@ std::string ParameterizedMessage::getText(const std::string& lang) const
   return i18n::expandParameterizedString(lang, msgId, params);
 }
 
+namespace
+{
+
+LangPreference parseSingleLanguagePreference(const std::string& s)
+{
+  const size_t langStart = s.find_first_not_of(" \t\n");
+  if ( langStart == std::string::npos ) {
+    return {"", 0};
+  }
+
+  const size_t langEnd = s.find(';', langStart);
+  if ( langEnd == std::string::npos ) {
+    return {s.substr(langStart), 1};
+  }
+
+  const std::string lang = s.substr(langStart, langEnd - langStart);
+  // We don't care about langEnd == langStart which will result in an empty
+  // language name - it will be dismissed by parseUserLanguagePreferences()
+
+  float q = 1.0;
+  int nCharsScanned;
+  if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
+       && langEnd + 1 + nCharsScanned == s.size() ) {
+    return {lang, q};
+  }
+
+  return {"", 0};
+}
+
+} // unnamed namespace
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s)
+{
+  UserLangPreferences result;
+  std::istringstream iss(s);
+  std::string singleLangPrefStr;
+  while ( std::getline(iss, singleLangPrefStr, ',') )
+  {
+    const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
+    if ( !langPref.lang.empty() && langPref.preference > 0 ) {
+      result.push_back(langPref);
+    }
+  }
+
+  return result;
+}
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
+{
+  if ( prefs.empty() ) {
+    return "en";
+  }
+
+  std::string bestLangSoFar("en");
+  float bestScoreSoFar = 0;
+  const auto& stringDb = getStringDb();
+  for ( const auto& entry : prefs ) {
+    const float score = entry.preference * stringDb.getStringCount(entry.lang);
+    if ( score > bestScoreSoFar ) {
+      bestScoreSoFar = score;
+      bestLangSoFar = entry.lang;
+    }
+  }
+  return bestLangSoFar;
+}
+
+std::string translateBookCategory(const std::string& lang, const std::string& category)
+{
+    try {
+        return getTranslatedString(lang, "book-category." + category);
+    } catch (...) {
+        return category;
+    }
+}
+
 } // namespace kiwix

src/server/i18n_utils.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_I18N_UTILS
+#define KIWIX_SERVER_I18N_UTILS
+
+#include "i18n.h"
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+struct I18nString {
+  const char* const key;
+  const char* const value;
+};
+
+struct I18nStringTable {
+  const char* const lang;
+  const size_t entryCount;
+  const I18nString* const entries;
+
+  const char* get(const std::string& key) const;
+};
+
+namespace i18n
+{
+
+class GetTranslatedStringWithMsgId
+{
+  typedef kainjow::mustache::basic_data<std::string> MustacheString;
+  typedef std::pair<std::string, MustacheString> MsgIdAndTranslation;
+
+public:
+  explicit GetTranslatedStringWithMsgId(const std::string& lang) : m_lang(lang) {}
+
+  MsgIdAndTranslation operator()(const std::string& key) const
+  {
+    return {key, getTranslatedString(m_lang, key)};
+  }
+
+  MsgIdAndTranslation operator()(const std::string& key, const Parameters& params) const
+  {
+    return {key, expandParameterizedString(m_lang, key, params)};
+  }
+
+private:
+  const std::string m_lang;
+};
+
+} // namespace i18n
+
+struct LangPreference
+{
+  const std::string lang;
+  const float preference;
+};
+
+typedef std::vector<LangPreference> UserLangPreferences;
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s);
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs);
+
+} // namespace kiwix
+
+#endif // KIWIX_SERVER_I18N_UTILS

src/server/internalServer.h

@@ -27,6 +27,7 @@ extern "C" {
 
 #include "library.h"
 #include "name_mapper.h"
+#include "tools.h"
 
 #include <zim/search.h>
 #include <zim/suggestion.h>
@@ -88,16 +89,14 @@ class SearchInfo {
 
 
 typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
-typedef ConcurrentCache<std::string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
-
 class OPDSDumper;
+class LibraryDumper;
 
 class InternalServer {
   public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
+    InternalServer(LibraryPtr library,
+                   std::shared_ptr<NameMapper> nameMapper,
+                   IpAddress addr,
                    int port,
                    std::string root,
                    int nbThreads,
@@ -106,8 +105,11 @@ class InternalServer {
                    bool withTaskbar,
                    bool withLibraryButton,
                    bool blockExternalLinks,
+                   IpMode ipMode,
                    std::string indexTemplateString,
-                   int ipConnectionLimit);
+                   int ipConnectionLimit,
+                   bool catalogOnlyMode,
+                   std::string zimViewerURL);
     virtual ~InternalServer();
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
@@ -119,13 +121,15 @@ class InternalServer {
                                void** cont_cls);
     bool start();
     void stop();
-    std::string getAddress() { return m_addr; }
-    int getPort() { return m_port; }
+    IpAddress getAddress() const { return m_addr; }
+    int getPort() const { return m_port; }
+    IpMode getIpMode() const { return m_ipMode; }
 
   private: // functions
     std::unique_ptr<Response> handle_request(const RequestContext& request);
     std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
     std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_viewer_settings(const RequestContext& request);
     std::unique_ptr<Response> handle_skin(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
@@ -133,9 +137,11 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
     std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
+    std::unique_ptr<Response> handle_no_js(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_search_request(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
     std::unique_ptr<Response> handle_catch(const RequestContext& request);
@@ -143,49 +149,57 @@ class InternalServer {
     std::unique_ptr<Response> handle_content(const RequestContext& request);
     std::unique_ptr<Response> handle_raw(const RequestContext& request);
     std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
-    std::unique_ptr<Response> handle_widget(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
 
     MustacheData get_default_data() const;
 
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
     std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
     SearchInfo getSearchInfo(const RequestContext& r) const;
 
     bool isLocallyCustomizedResource(const std::string& url) const;
 
+    std::string getLibraryId() const;
+
+    std::string getNoJSDownloadPageHTML(const std::string& bookId, const std::string& userLang) const;
+    OPDSDumper getOPDSDumper() const;
+    void setContentAccessUrl(LibraryDumper& libDumper) const;
+
+  private: // types
+    class LockableSuggestionSearcher;
+    typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+    typedef ConcurrentCache<std::string, std::shared_ptr<LockableSuggestionSearcher>> SuggestionSearcherCache;
+
   private: // data
-    std::string m_addr;
+    IpAddress m_addr;
     int m_port;
-    std::string m_root;
+    std::string m_root;                   // URI-encoded
+    std::string m_rootPrefixOfDecodedURL; // URI-decoded
     int m_nbThreads;
     unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
+    IpMode m_ipMode;
     std::string m_indexTemplateString;
     int m_ipConnectionLimit;
     struct MHD_Daemon* mp_daemon;
 
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
+    LibraryPtr mp_library;
+    std::shared_ptr<NameMapper> mp_nameMapper;
 
     SearchCache searchCache;
     SuggestionSearcherCache suggestionSearcherCache;
 
     std::string m_server_id;
-    std::string m_library_id;
 
     class CustomizedResources;
     std::unique_ptr<CustomizedResources> m_customizedResources;
 
-    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
+    const bool m_catalogOnlyMode;
+    const std::string m_contentServerUrl;
 };
 
 }

src/server/internalServer_catalog.cpp

@@ -24,7 +24,7 @@
 #include "request_context.h"
 #include "response.h"
 #include "tools/otherTools.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include <mustache.hpp>
 
@@ -33,6 +33,78 @@
 
 namespace kiwix {
 
+namespace
+{
+
+enum OPDSResponseKind
+{
+  OPDS_ENTRY,
+  OPDS_NAVIGATION_FEED,
+  OPDS_ACQUISITION_FEED
+};
+
+const std::string opdsMimeType[] = {
+  "application/atom+xml;type=entry;profile=opds-catalog;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=navigation;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=acquisition;charset=utf-8"
+};
+
+} // unnamed namespace
+
+OPDSDumper InternalServer::getOPDSDumper() const
+{
+  kiwix::OPDSDumper opdsDumper(mp_library.get(), mp_nameMapper.get());
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(getLibraryId());
+  setContentAccessUrl(opdsDumper);
+  return opdsDumper;
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return UrlNotFoundResponse(request);
+  }
+
+  if (url == "v2") {
+    return handle_catalog_v2(request);
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return UrlNotFoundResponse(request);
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper = getOPDSDumper();
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    bookIdsToDump = search_catalog(request, opdsDumper);
+    uuid = zim::Uuid::generate();
+  }
+
+  auto response = ContentResponse::build(
+      opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()),
+      opdsMimeType[OPDS_ACQUISITION_FEED]);
+  return std::move(response);
+}
+
 std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -43,15 +115,14 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
+    return UrlNotFoundResponse(request);
   }
 
   if (url == "root.xml") {
     return handle_catalog_v2_root(request);
   } else if (url == "searchdescription.xml") {
     const std::string endpoint_root = m_root + "/catalog/v2";
-    return ContentResponse::build(*this,
+    return ContentResponse::build(
         RESOURCE::catalog_v2_searchdescription_xml,
         kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
         "application/opensearchdescription+xml"
@@ -70,40 +141,36 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   } else if (url == "illustration") {
     return handle_catalog_v2_illustration(request);
   } else {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
+    return UrlNotFoundResponse(request);
   }
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
 {
+  const std::string libraryId = getLibraryId();
   return ContentResponse::build(
-             *this,
              RESOURCE::templates::catalog_v2_root_xml,
              kainjow::mustache::object{
                {"date", gen_date_str()},
                {"endpoint_root", m_root + "/catalog/v2"},
-               {"feed_id", gen_uuid(m_library_id)},
-               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
-               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
-               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
+               {"feed_id", gen_uuid(libraryId)},
+               {"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
+               {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
+               {"category_list_feed_id", gen_uuid(libraryId + "/categories")},
+               {"language_list_feed_id", gen_uuid(libraryId + "/languages")}
              },
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
 {
-  OPDSDumper opdsDumper(mp_library);
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  kiwix::OPDSDumper opdsDumper = getOPDSDumper();
   const auto bookIds = search_catalog(request, opdsDumper);
   const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
-             *this,
              opdsFeed,
-             "application/atom+xml;profile=opds-catalog;kind=acquisition"
+             opdsMimeType[OPDS_ACQUISITION_FEED]
   );
 }
 
@@ -112,42 +179,32 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
   try {
     mp_library->getBookById(entryId);
   } catch (const std::out_of_range&) {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
+    return UrlNotFoundResponse(request);
   }
 
-  OPDSDumper opdsDumper(mp_library);
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  kiwix::OPDSDumper opdsDumper = getOPDSDumper();
   const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
   return ContentResponse::build(
-             *this,
              opdsFeed,
-             "application/atom+xml;type=entry;profile=opds-catalog"
+             opdsMimeType[OPDS_ENTRY]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  kiwix::OPDSDumper opdsDumper = getOPDSDumper();
   return ContentResponse::build(
-             *this,
              opdsDumper.categoriesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  kiwix::OPDSDumper opdsDumper = getOPDSDumper();
   return ContentResponse::build(
-             *this,
              opdsDumper.languagesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
@@ -158,10 +215,12 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const R
     auto book = mp_library->getBookByIdThreadSafe(bookId);
     auto size = request.get_argument<unsigned int>("size");
     auto illustration = book.getIllustration(size);
-    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+    return ContentResponse::build(
+               illustration->getData(),
+               illustration->mimeType
+    );
   } catch(...) {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
+    return UrlNotFoundResponse(request);
   }
 }
 

src/server/request_context.cpp

@@ -25,8 +25,10 @@
 #include <sstream>
 #include <cstdio>
 #include <atomic>
+#include <cctype>
 
 #include "tools/stringTools.h"
+#include "i18n_utils.h"
 
 namespace kiwix {
 
@@ -47,39 +49,29 @@ RequestMethod str2RequestMethod(const std::string& method) {
   else                          return RequestMethod::OTHER;
 }
 
-std::string
-fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
-{
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    return full_url;
-  } else if (full_url == rootLocation) {
-    return "/";
-  } else if (full_url.size() > rootLocation.size() &&
-             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-    return full_url.substr(rootLocation.size());
-  } else {
-    return "";
-  }
-}
-
 } // unnamed namespace
 
-RequestContext::RequestContext(struct MHD_Connection* connection,
-                               std::string rootLocation,
-                               const std::string& _url,
+RequestContext::RequestContext(const std::string& _rootLocation, // URI-encoded
+                               const std::string& unrootedUrl,   // URI-decoded
                                const std::string& _method,
-                               const std::string& version) :
-  full_url(_url),
-  url(fullURL2LocalURL(_url, rootLocation)),
+                               const std::string& version,
+                               const NameValuePairs& headers,
+                               const NameValuePairs& queryArgs) :
+  rootLocation(_rootLocation),
+  url(unrootedUrl),
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingGzip(false),
   byteRange_()
 {
-  MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
-  MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
+  for ( const auto& kv : headers ) {
+    add_header(kv.first, kv.second);
+  }
+
+  for ( const auto& kv : queryArgs ) {
+    add_argument(kv.first, kv.second);
+  }
 
   try {
     acceptEncodingGzip =
@@ -89,25 +81,30 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   try {
     byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
+
+  userlang = determine_user_language();
 }
 
 RequestContext::~RequestContext()
 {}
 
-MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                       const char *key, const char *value)
+void RequestContext::add_header(const char *key, const char *value)
 {
-  RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[lcAll(key)] = value;
-  return MHD_YES;
+  this->headers[lcAll(key)] = value;
 }
 
-MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                         const char *key, const char* value)
+void RequestContext::add_argument(const char *key, const char* value)
 {
-  RequestContext *_this = static_cast<RequestContext*>(__this);
+  RequestContext *_this = this;
   _this->arguments[key].push_back(value == nullptr ? "" : value);
-  return MHD_YES;
+  if ( ! _this->queryString.empty() ) {
+    _this->queryString += "&";
+  }
+  _this->queryString += urlEncode(key);
+  if ( value ) {
+    _this->queryString += "=";
+    _this->queryString += urlEncode(value);
+  }
 }
 
 void RequestContext::print_debug_info() const {
@@ -131,7 +128,6 @@ void RequestContext::print_debug_info() const {
     printf("\n");
   }
   printf("Parsed : \n");
-  printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
   printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
@@ -169,11 +165,15 @@ std::string RequestContext::get_url_part(int number) const {
 }
 
 std::string RequestContext::get_full_url() const {
-  return full_url;
+  return rootLocation + urlEncode(url);
+}
+
+std::string RequestContext::get_root_path() const {
+  return rootLocation.empty() ? "/" : rootLocation;
 }
 
 bool RequestContext::is_valid_url() const {
-  return !url.empty();
+  return url.empty() || url[0] == '/';
 }
 
 ByteRange RequestContext::get_range() const {
@@ -190,16 +190,24 @@ std::string RequestContext::get_header(const std::string& name) const {
 }
 
 std::string RequestContext::get_user_language() const
+{
+  return userlang.lang;
+}
+
+RequestContext::UserLanguage RequestContext::determine_user_language() const
 {
   try {
-    return get_argument("userlang");
+    return {UserLanguage::SelectorKind::QUERY_PARAM, get_argument("userlang")};
   } catch(const std::out_of_range&) {}
 
   try {
-    return get_header("Accept-Language");
+    const std::string acceptLanguage = get_header("Accept-Language");
+    const auto userLangPrefs = parseUserLanguagePreferences(acceptLanguage);
+    const auto lang = selectMostSuitableLanguage(userLangPrefs);
+    return {UserLanguage::SelectorKind::ACCEPT_LANGUAGE_HEADER, lang};
   } catch(const std::out_of_range&) {}
 
-  return "en";
+  return {UserLanguage::SelectorKind::DEFAULT, "en"};
 }
 
 std::string RequestContext::get_requested_format() const

src/server/request_context.h

@@ -29,7 +29,7 @@
 #include <stdexcept>
 
 #include "byte_range.h"
-#include "tools/stringTools.h"
+#include "../tools/stringTools.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -55,12 +55,17 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
+  public: // types
+    typedef std::vector<std::pair<const char*, const char*>> NameValuePairs;
+
   public: // functions
-    RequestContext(struct MHD_Connection* connection,
-                   std::string rootLocation,
-                   const std::string& url,
+    RequestContext(const std::string& rootLocation, // URI-encoded
+                   const std::string& unrootedUrl,  // URI-decoded
                    const std::string& method,
-                   const std::string& version);
+                   const std::string& version,
+                   const NameValuePairs& headers,
+                   const NameValuePairs& queryArgs);
+
     ~RequestContext();
 
     void print_debug_info() const;
@@ -91,16 +96,15 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
+    std::string get_root_path() const;
 
-    std::string get_query(bool mustEncode = false) const {
-      return get_query([](const std::string& key) {return true;}, mustEncode);
-    }
+    std::string get_query() const { return queryString; }
 
     template<class F>
     std::string get_query(F filter, bool mustEncode) const {
       std::string q;
       const char* sep = "";
-      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value):value; };
       for ( const auto& a : arguments ) {
         if (!filter(a.first)) {
           continue;
@@ -120,8 +124,22 @@ class RequestContext {
     std::string get_user_language() const;
     std::string get_requested_format() const;
 
+  private: // types
+    struct UserLanguage
+    {
+      enum SelectorKind
+      {
+        QUERY_PARAM,
+        ACCEPT_LANGUAGE_HEADER,
+        DEFAULT
+      };
+
+      SelectorKind selectedBy;
+      std::string  lang;
+    };
+
   private: // data
-    std::string full_url;
+    std::string rootLocation;
     std::string url;
     RequestMethod method;
     std::string version;
@@ -132,10 +150,14 @@ class RequestContext {
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::vector<std::string>> arguments;
+    std::string queryString;
+    UserLanguage userlang;
 
   private: // functions
-    static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+    UserLanguage determine_user_language() const;
+
+    void add_header(const char* name, const char* value);
+    void add_argument(const char* name, const char* value);
 };
 
 template<> std::string RequestContext::get_argument(const std::string& name) const;

src/server/response.cpp

@@ -20,7 +20,7 @@
 #include "response.h"
 #include "request_context.h"
 #include "internalServer.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
@@ -32,6 +32,9 @@
 #include <zlib.h>
 
 #include <array>
+#include <list>
+#include <map>
+#include <regex>
 
 // This is somehow a magic value.
 // If this value is too small, we will compress (and lost cpu time) too much
@@ -47,6 +50,8 @@ namespace kiwix {
 
 namespace
 {
+typedef kainjow::mustache::data MustacheData;
+
 // some utilities
 
 std::string get_mime_type(const zim::Item& item)
@@ -64,7 +69,13 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/javascript") != std::string::npos
       || mimeType.find("application/atom") != std::string::npos
       || mimeType.find("application/opensearchdescription") != std::string::npos
-      || mimeType.find("application/json") != std::string::npos;
+      || mimeType.find("application/json") != std::string::npos
+
+      // Web fonts
+      || mimeType.find("application/font-") != std::string::npos
+      || mimeType.find("application/x-font-") != std::string::npos
+      || mimeType.find("application/vnd.ms-fontobject") != std::string::npos
+      || mimeType.find("font/") != std::string::npos;
 }
 
 bool compress(std::string &content) {
@@ -102,96 +113,359 @@ bool compress(std::string &content) {
 }
 
 
+const char* getCacheControlHeader(Response::Kind k)
+{
+  switch(k) {
+    case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
+    case Response::ZIM_CONTENT:     return "max-age=3600, must-revalidate";
+    default:                        return "max-age=0, must-revalidate";
+  }
+}
 
 } // unnamed namespace
 
-Response::Response(bool verbose)
-  : m_verbose(verbose),
-    m_returnCode(MHD_HTTP_OK)
+Response::Response()
+  : m_returnCode(MHD_HTTP_OK)
 {
   add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
-std::unique_ptr<Response> Response::build(const InternalServer& server)
+void Response::set_kind(Kind k)
 {
-  return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
+  m_kind = k;
+  if ( k == ZIM_CONTENT )
+    m_etag.set_option(ETag::ZIM_CONTENT);
 }
 
-std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
+std::unique_ptr<Response> Response::build()
 {
-  auto response = Response::build(server);
+  return std::make_unique<Response>();
+}
+
+std::unique_ptr<Response> Response::build_304(const ETag& etag)
+{
+  auto response = Response::build();
   response->set_code(MHD_HTTP_NOT_MODIFIED);
   response->m_etag = etag;
+  if ( etag.get_option(ETag::ZIM_CONTENT) ) {
+    response->set_kind(Response::ZIM_CONTENT);
+  }
   if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
     response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
   }
   return response;
 }
 
-const UrlNotFoundMsg urlNotFoundMsg;
-const InvalidUrlMsg invalidUrlMsg;
 
-std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
+namespace
 {
-  return getTranslatedString(m_request.get_user_language(), msgId);
+
+// This class was introduced in order to work around the missing support
+// for std::variant (and std::optional) under some of the current build
+// platforms.
+template<class T>
+class Optional
+{
+public: // functions
+    Optional() {}
+    Optional(const T& t) : ptr(new T(t)) {}
+    Optional(const Optional& o) : ptr(o.has_value() ? new T(*o) : nullptr) {}
+    Optional(Optional&& o) : ptr(std::move(o.ptr)) {}
+
+    Optional& operator=(const Optional& o)
+    {
+      *this = Optional(o);
+      return *this;
+    }
+
+    Optional& operator=(Optional&& o)
+    {
+      ptr = std::move(o.ptr);
+      return *this;
+    }
+
+    bool has_value() const { return ptr.get() != nullptr; }
+    const T& operator*() const { return *ptr; }
+    T& operator*() { return *ptr; }
+
+private: // data
+    std::unique_ptr<T> ptr;
+};
+
+} // unnamed namespace
+
+class ContentResponseBlueprint::Data
+{
+public:
+  typedef std::list<Data>             List;
+  typedef std::map<std::string, Data> Object;
+
+private:
+  // std::variant<std::string, bool, List, Object> data;
+  // XXX: libkiwix is compiled on platforms where std::variant
+  // XXX: is not yet supported. Hence this hack. Only one
+  // XXX: of the below data members is expected to contain a value.
+  Optional<std::string> m_stringValue;
+  Optional<bool>        m_boolValue;
+  Optional<List>        m_listValue;
+  Optional<Object>      m_objectValue;
+
+public:
+  Data() {}
+  Data(const std::string& s) : m_stringValue(s) {}
+  Data(bool b)               : m_boolValue(b)   {}
+  Data(const List& l)        : m_listValue(l)   {}
+  Data(const Object& o)      : m_objectValue(o) {}
+
+  MustacheData toMustache(const std::string& lang) const;
+
+  Data& operator[](const std::string& key)
+  {
+    return (*m_objectValue)[key];
+  }
+
+  void push_back(const Data& d) { (*m_listValue).push_back(d); }
+
+  static Data onlyAsNonEmptyValue(const std::string& s)
+  {
+    return s.empty() ? Data(false) : Data(s);
+  }
+
+  static Data from(const ParameterizedMessage& pmsg)
+  {
+    Object obj;
+    for(const auto& kv : pmsg.getParams()) {
+      obj[kv.first] = kv.second;
+    }
+    return Object{
+              { "msgid", pmsg.getMsgId() },
+              { "params", Data(obj) }
+    };
+  }
+
+  static Data fromMsgId(const std::string& nonParameterizedMsgId)
+  {
+    return from(nonParameterizedMessage(nonParameterizedMsgId));
+  }
+
+  static Data staticMultiParagraphText(const std::string& msgIdPrefix, size_t n)
+  {
+    Object paragraphs;
+    for ( size_t i = 1; i <= n; ++i ) {
+      std::ostringstream oss;
+      oss << "p" << i;
+      const std::string pId = oss.str();
+      paragraphs[pId] = fromMsgId(msgIdPrefix + "." + pId);
+    }
+    return paragraphs;
+  }
+
+  std::string asJSON() const;
+  void dumpJSON(std::ostream& os) const;
+
+private:
+  bool isString() const { return m_stringValue.has_value(); }
+  bool isList()   const { return m_listValue.has_value();   }
+  bool isObject() const { return m_objectValue.has_value(); }
+
+  const std::string& stringValue() const { return *m_stringValue; }
+  bool               boolValue()   const { return *m_boolValue;   }
+  const List&        listValue()   const { return *m_listValue;   }
+  const Object&      objectValue() const { return *m_objectValue; }
+
+  const Data* get(const std::string& key) const
+  {
+    if ( !isObject() )
+      return nullptr;
+
+    const auto& obj = objectValue();
+    const auto it = obj.find(key);
+    return it != obj.end() ? &it->second : nullptr;
+  }
+};
+
+MustacheData ContentResponseBlueprint::Data::toMustache(const std::string& lang) const
+{
+  if ( this->isList() ) {
+    kainjow::mustache::list l;
+    for ( const auto& x : this->listValue() ) {
+      l.push_back(x.toMustache(lang));
+    }
+    return l;
+  } else if ( this->isObject() ) {
+    const Data* msgId = this->get("msgid");
+    const Data* msgParams = this->get("params");
+    if ( msgId && msgId->isString() && msgParams && msgParams->isObject() ) {
+      std::map<std::string, std::string> params;
+      for(const auto& kv : msgParams->objectValue()) {
+        params[kv.first] = kv.second.stringValue();
+      }
+      const ParameterizedMessage msg(msgId->stringValue(), ParameterizedMessage::Parameters(params));
+      return msg.getText(lang);
+    } else {
+      kainjow::mustache::object o;
+      for ( const auto& kv : this->objectValue() ) {
+        o[kv.first] = kv.second.toMustache(lang);
+      }
+      return o;
+    }
+  } else if ( this->isString() ) {
+    return this->stringValue();
+  } else {
+    return this->boolValue();
+  }
 }
 
+void ContentResponseBlueprint::Data::dumpJSON(std::ostream& os) const
+{
+  if ( this->isString() ) {
+    os << '"' << escapeForJSON(this->stringValue()) << '"';
+  } else if ( this->isList() ) {
+    const char * sep = " ";
+    os << "[";
+
+    for ( const auto& x :  this->listValue() ) {
+      os << sep;
+      x.dumpJSON(os);
+      sep = ", ";
+    }
+    os << " ]";
+  } else if ( this->isObject() ) {
+    const char * sep = " ";
+    os << "{";
+    for ( const auto& kv : this->objectValue() ) {
+      os << sep << '"' << kv.first << "\" : ";
+      kv.second.dumpJSON(os);
+      sep = ", ";
+    }
+    os << " }";
+  } else {
+    os << (this->boolValue() ? "true" : "false");
+  }
+}
+
+std::string ContentResponseBlueprint::Data::asJSON() const
+{
+  std::ostringstream oss;
+  this->dumpJSON(oss);
+
+  // This JSON is going to be used in HTML inside a <script></script> tag.
+  // If it contains "</script>" (or "</script >") as a substring, then the HTML
+  // parser will be confused. Since for a valid JSON that may happen only inside
+  // a JSON string, we can safely take advantage of the answers to
+  // https://stackoverflow.com/questions/28259389/how-to-put-script-in-a-javascript-string
+  // and work around the issue by inserting an otherwise harmless backslash.
+  return std::regex_replace(oss.str(), std::regex("</script"), "</scr\\ipt");
+}
+
+ContentResponseBlueprint::ContentResponseBlueprint(const RequestContext* request,
+                         int httpStatusCode,
+                         const std::string& mimeType,
+                         const std::string& templateStr,
+                         bool includeKiwixResponseData)
+  : m_request(*request)
+  , m_httpStatusCode(httpStatusCode)
+  , m_mimeType(mimeType)
+  , m_template(templateStr)
+  , m_includeKiwixResponseData(includeKiwixResponseData)
+  , m_data(new Data)
+{}
+
+ContentResponseBlueprint::~ContentResponseBlueprint() = default;
+
 std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
 {
-  auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
-  r->set_code(m_httpStatusCode);
-  if ( m_taskbarInfo ) {
-    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
+  kainjow::mustache::data d = m_data->toMustache(m_request.get_user_language());
+  if ( m_includeKiwixResponseData ) {
+    d.set("KIWIX_RESPONSE_TEMPLATE", escapeForJSON(m_template, false));
+    d.set("KIWIX_RESPONSE_DATA", m_data->asJSON());
   }
+  auto r = ContentResponse::build(m_template, d, m_mimeType);
+  r->set_code(m_httpStatusCode);
   return r;
 }
 
-HTTPErrorResponse::HTTPErrorResponse(const InternalServer& server,
-                                     const RequestContext& request,
+NewHTTP404Response::NewHTTP404Response(const RequestContext& request,
+                                       const std::string& root,
+                                       const std::string& urlPath)
+  : ContentResponseBlueprint(&request,
+                             MHD_HTTP_NOT_FOUND,
+                             "text/html; charset=utf-8",
+                             RESOURCE::templates::sexy404_html,
+                             /*includeKiwixResponseData=*/true)
+{
+  *this->m_data = Data(Data::Object{
+                    {"root", root },
+                    {"url_path", urlPath},
+                    {"PAGE_TITLE",   Data::fromMsgId("new-404-page-title")},
+                    {"PAGE_HEADING", Data::fromMsgId("new-404-page-heading")},
+                    {"404_img_text", Data::fromMsgId("404-img-text")},
+                    {"path_was_not_found_msg", Data::fromMsgId("path-was-not-found")},
+                    {"advice", Data::staticMultiParagraphText("404-advice", 5)},
+  });
+}
+
+BlockExternalLinkResponse::BlockExternalLinkResponse(const RequestContext& request,
+                                       const std::string& root,
+                                       const std::string& externalUrl)
+  : ContentResponseBlueprint(&request,
+                             MHD_HTTP_OK,
+                             "text/html; charset=utf-8",
+                             RESOURCE::templates::captured_external_html,
+                             /*includeKiwixResponseData=*/true)
+{
+  *this->m_data = Data(Data::Object{
+                    {"root", root },
+                    {"external_link_detected", Data::fromMsgId("external-link-detected") },
+                    {"url", externalUrl },
+                    {"caution_warning", Data::fromMsgId("caution-warning") },
+                    {"external_link_intro", Data::fromMsgId("external-link-intro") },
+                    {"advice", Data::staticMultiParagraphText("external-link-advice", 3)},
+  });
+}
+
+HTTPErrorResponse::HTTPErrorResponse(const RequestContext& request,
                                      int httpStatusCode,
                                      const std::string& pageTitleMsgId,
                                      const std::string& headingMsgId,
-                                     const std::string& cssUrl)
-  : ContentResponseBlueprint(&server,
-                             &request,
+                                     const std::string& cssUrl,
+                                     bool includeKiwixResponseData)
+  : ContentResponseBlueprint(&request,
                              httpStatusCode,
                              request.get_requested_format() == "html" ? "text/html; charset=utf-8" : "application/xml; charset=utf-8",
-                             request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml)
+                             request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml,
+                             includeKiwixResponseData)
 {
-  kainjow::mustache::list emptyList;
-  this->m_data = kainjow::mustache::object{
-                    {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
-                    {"PAGE_TITLE",   getMessage(pageTitleMsgId)},
-                    {"PAGE_HEADING", getMessage(headingMsgId)},
+  Data::List emptyList;
+  *this->m_data = Data(Data::Object{
+                    {"CSS_URL", Data::onlyAsNonEmptyValue(cssUrl) },
+                    {"PAGE_TITLE",   Data::fromMsgId(pageTitleMsgId)},
+                    {"PAGE_HEADING", Data::fromMsgId(headingMsgId)},
                     {"details", emptyList}
-  };
+  });
 }
 
-HTTP404Response::HTTP404Response(const InternalServer& server,
-                                 const RequestContext& request)
-  : HTTPErrorResponse(server,
-                      request,
+HTTP404Response::HTTP404Response(const RequestContext& request)
+  : HTTPErrorResponse(request,
                       MHD_HTTP_NOT_FOUND,
                       "404-page-title",
-                      "404-page-heading")
+                      "404-page-heading",
+                      std::string(),
+                      /*includeKiwixResponseData=*/true)
 {
 }
 
-HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
+UrlNotFoundResponse::UrlNotFoundResponse(const RequestContext& request)
+    : HTTP404Response(request)
 {
-  const std::string requestUrl = m_request.get_full_url();
-  return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
-}
-
-HTTPErrorResponse& HTTPErrorResponse::operator+(const std::string& msg)
-{
-  m_data["details"].push_back({"p", msg});
-  return *this;
+  const std::string requestUrl = urlDecode(m_request.get_full_url(), false);
+  *this += ParameterizedMessage("url-not-found", {{"url", requestUrl}});
 }
 
 HTTPErrorResponse& HTTPErrorResponse::operator+(const ParameterizedMessage& details)
 {
-  return *this + details.getText(m_request.get_user_language());
+  (*m_data)["details"].push_back(Data::Object{{"p", Data::from(details)}});
+  return *this;
 }
 
 HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& details)
@@ -201,67 +475,51 @@ HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& det
 }
 
 
-HTTP400Response::HTTP400Response(const InternalServer& server,
-                                 const RequestContext& request)
-  : HTTPErrorResponse(server,
-                      request,
+HTTP400Response::HTTP400Response(const RequestContext& request)
+  : HTTPErrorResponse(request,
                       MHD_HTTP_BAD_REQUEST,
                       "400-page-title",
-                      "400-page-heading")
+                      "400-page-heading",
+                      std::string(),
+                      /*includeKiwixResponseData=*/true)
 {
-}
-
-HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
-{
-  std::string requestUrl = m_request.get_full_url();
+  std::string requestUrl = urlDecode(m_request.get_full_url(), false);
   const auto query = m_request.get_query();
   if (!query.empty()) {
     requestUrl += "?" + encodeDiples(query);
   }
-  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
-  return *this + msgTmpl.render({"url", requestUrl});
+  *this += ParameterizedMessage("invalid-request", {{"url", requestUrl}});
 }
 
-HTTP500Response::HTTP500Response(const InternalServer& server,
-                                 const RequestContext& request)
-  : HTTPErrorResponse(server,
-                      request,
-                      MHD_HTTP_INTERNAL_SERVER_ERROR,
-                      "500-page-title",
-                      "500-page-heading")
+HTTP500Response::HTTP500Response(const RequestContext& request,
+                                 const std::string& root,
+                                 const std::string& urlPath,
+                                 const std::string& errorText)
+  : ContentResponseBlueprint(&request,
+                             MHD_HTTP_INTERNAL_SERVER_ERROR,
+                             "text/html; charset=utf-8",
+                             RESOURCE::templates::sexy500_html,
+                             /*includeKiwixResponseData=*/true)
 {
-  // operator+() is a state-modifying operator (akin to operator+=)
-  *this + "An internal server error occured. We are sorry about that :/";
+  auto pageParams = Data::Object{
+    {"root", root },
+    {"url_path", urlPath},
+    {"PAGE_TITLE",   Data::fromMsgId("500-page-title")},
+    {"PAGE_HEADING", Data::fromMsgId("500-page-heading")},
+    {"PAGE_TEXT",    Data::fromMsgId("500-page-text")},
+    {"500_img_text", Data::fromMsgId("500-img-text")},
+  };
+
+  if ( !errorText.empty() ) {
+    pageParams["error"] = errorText;
+  }
+
+  *this->m_data = Data(pageParams);
 }
 
-std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
+std::unique_ptr<Response> Response::build_416(size_t resourceLength)
 {
-  // We want a 500 response to be a minimalistic one (so that the server doesn't
-  // have to provide additional resources required for its proper rendering)
-  // ";raw=true" in the MIME-type below disables response decoration
-  // (see ContentResponse::contentDecorationAllowed())
-  const std::string mimeType = "text/html;charset=utf-8;raw=true";
-  auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
-  r->set_code(m_httpStatusCode);
-  return r;
-}
-
-ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
-{
-  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
-  return *this;
-}
-
-ContentResponseBlueprint& ContentResponseBlueprint::operator+=(const TaskbarInfo& taskbarInfo)
-{
-  // operator+() is already a state-modifying operator (akin to operator+=)
-  return *this + taskbarInfo;
-}
-
-
-std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
-{
-  auto response = Response::build(server);
+  auto response = Response::build();
 // [FIXME] (compile with recent enough version of libmicrohttpd)
 //  response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
   response->set_code(416);
@@ -273,9 +531,9 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
 }
 
 
-std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
+std::unique_ptr<Response> Response::build_redirect(const std::string& redirectUrl)
 {
-  auto response = Response::build(server);
+  auto response = Response::build();
   response->m_returnCode = MHD_HTTP_FOUND;
   response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
   return response;
@@ -337,52 +595,6 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar(const std::string& lang)
-{
-  i18n::GetTranslatedString t(lang);
-  kainjow::mustache::object data{
-    {"root", m_root},
-    {"content", m_bookName},
-    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
-    {"title", m_bookTitle},
-    {"withlibrarybutton", m_withLibraryButton},
-    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
-    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
-    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
-    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
-  };
-  auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    head_content);
-
-  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<body[^>]*>",
-    taskbar_part);
-}
-
-
-void ContentResponse::inject_externallinks_blocker()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    script_tag);
-}
-
-void ContentResponse::inject_root_link(){
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    "<link type=\"root\" href=\"" + m_root + "\">");
-}
-
 bool
 ContentResponse::can_compress(const RequestContext& request) const
 {
@@ -391,16 +603,6 @@ ContentResponse::can_compress(const RequestContext& request) const
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
-bool
-ContentResponse::contentDecorationAllowed() const
-{
-  if (m_raw) {
-    return false;
-  }
-  return (startsWith(m_mimeType, "text/html")
-    && m_mimeType.find(";raw=true") == std::string::npos);
-}
-
 MHD_Response*
 Response::create_mhd_response(const RequestContext& request)
 {
@@ -411,17 +613,6 @@ Response::create_mhd_response(const RequestContext& request)
 MHD_Response*
 ContentResponse::create_mhd_response(const RequestContext& request)
 {
-  if (contentDecorationAllowed()) {
-    inject_root_link();
-
-    if (m_withTaskbar) {
-      introduce_taskbar(request.get_user_language());
-    }
-    if (m_blockExternalLinks) {
-      inject_externallinks_blocker();
-    }
-  }
-
   const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
@@ -437,12 +628,12 @@ ContentResponse::create_mhd_response(const RequestContext& request)
   return response;
 }
 
-MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
+MHD_Result Response::send(const RequestContext& request, bool verbose, MHD_Connection* connection)
 {
   MHD_Response* response = create_mhd_response(request);
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+                          getCacheControlHeader(m_kind));
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
     MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
@@ -453,7 +644,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
-  if (m_verbose)
+  if (verbose)
     print_response_info(m_returnCode, response);
 
   auto ret = MHD_queue_response(connection, m_returnCode, response);
@@ -461,91 +652,60 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
-{
-  m_bookName = bookName;
-  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
-}
-
-
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
-  Response(verbose),
-  m_root(root),
+ContentResponse::ContentResponse(const std::string& content, const std::string& mimetype) :
+  Response(),
   m_content(content),
-  m_mimeType(mimetype),
-  m_raw(raw),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  m_bookName(""),
-  m_bookTitle("")
+  m_mimeType(mimetype)
 {
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
 std::unique_ptr<ContentResponse> ContentResponse::build(
-  const InternalServer& server,
   const std::string& content,
-  const std::string& mimetype,
-  bool isHomePage,
-  bool raw)
+  const std::string& mimetype)
 {
-   return std::unique_ptr<ContentResponse>(new ContentResponse(
-        server.m_root,
-        server.m_verbose.load(),
-        raw,
-        server.m_withTaskbar && !isHomePage,
-        server.m_withLibraryButton,
-        server.m_blockExternalLinks,
-        content,
-        mimetype));
+   return std::make_unique<ContentResponse>(content, mimetype);
 }
 
 std::unique_ptr<ContentResponse> ContentResponse::build(
-  const InternalServer& server,
   const std::string& template_str,
   kainjow::mustache::data data,
-  const std::string& mimetype,
-  bool isHomePage)
+  const std::string& mimetype)
 {
   auto content = render_template(template_str, data);
-  return ContentResponse::build(server, content, mimetype, isHomePage);
+  return ContentResponse::build(content, mimetype);
 }
 
-ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
-  Response(verbose),
+ItemResponse::ItemResponse(const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
+  Response(),
   m_item(item),
   m_mimeType(mimetype)
 {
   m_byteRange = byterange;
-  set_cacheable();
+  set_kind(Response::ZIM_CONTENT);
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
+std::unique_ptr<Response> ItemResponse::build(const RequestContext& request, const zim::Item& item)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
-    response->set_cacheable();
+    auto response = ContentResponse::build(item.getData(), mimetype);
+    response->set_kind(Response::ZIM_CONTENT);
     response->m_byteRange = byteRange;
     return std::move(response);
   }
 
   if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
-    auto response = Response::build_416(server, item.getSize());
-    response->set_cacheable();
+    auto response = Response::build_416(item.getSize());
+    response->set_kind(Response::ZIM_CONTENT);
     return response;
   }
 
-  return std::unique_ptr<Response>(new ItemResponse(
-        server.m_verbose.load(),
-        item,
-        mimetype,
-        byteRange));
+  return std::make_unique<ItemResponse>(item, mimetype, byteRange);
 }
 
 MHD_Response*

src/server/response.h

@@ -27,7 +27,7 @@
 #include <mustache.hpp>
 #include "byte_range.h"
 #include "etag.h"
-#include "i18n.h"
+#include "i18n_utils.h"
 
 #include <zim/item.h>
 
@@ -41,24 +41,32 @@ class Archive;
 
 namespace kiwix {
 
-class InternalServer;
 class RequestContext;
 
 class Response {
   public:
-    Response(bool verbose);
+    enum Kind
+    {
+      STATIC_RESOURCE,
+      ZIM_CONTENT,
+      DYNAMIC_CONTENT
+    };
+
+  public:
+    Response();
     virtual ~Response() = default;
 
-    static std::unique_ptr<Response> build(const InternalServer& server);
-    static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
-    static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
-    static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
+    static std::unique_ptr<Response> build();
+    static std::unique_ptr<Response> build_304(const ETag& etag);
+    static std::unique_ptr<Response> build_416(size_t resourceLength);
+    static std::unique_ptr<Response> build_redirect(const std::string& redirectUrl);
 
-    MHD_Result send(const RequestContext& request, MHD_Connection* connection);
+    MHD_Result send(const RequestContext& request, bool verbose, MHD_Connection* connection);
 
     void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_kind(Kind k);
+    Kind get_kind() const { return m_kind; }
+    void set_etag_body(const std::string& id) { m_etag.set_body(id); }
     void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
@@ -68,7 +76,7 @@ class Response {
     MHD_Response* create_error_response(const RequestContext& request) const;
 
   protected: // data
-    bool m_verbose;
+    Kind m_kind = DYNAMIC_CONTENT;
     int m_returnCode;
     ByteRange m_byteRange;
     ETag m_etag;
@@ -81,164 +89,109 @@ class Response {
 class ContentResponse : public Response {
   public:
     ContentResponse(
-      const std::string& root,
-      bool verbose,
-      bool raw,
-      bool withTaskbar,
-      bool withLibraryButton,
-      bool blockExternalLinks,
       const std::string& content,
       const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
-      const InternalServer& server,
       const std::string& content,
-      const std::string& mimetype,
-      bool isHomePage = false,
-      bool raw = false);
+      const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
-      const InternalServer& server,
       const std::string& template_str,
       kainjow::mustache::data data,
-      const std::string& mimetype,
-      bool isHomePage = false);
+      const std::string& mimetype);
 
-    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
+    const std::string& getContent() const { return m_content; }
+    const std::string& getMimeType() const { return m_mimeType; }
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar(const std::string& lang);
-    void inject_externallinks_blocker();
-    void inject_root_link();
     bool can_compress(const RequestContext& request) const;
-    bool contentDecorationAllowed() const;
 
 
   private:
-    std::string m_root;
     std::string m_content;
     std::string m_mimeType;
-    bool m_raw;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    std::string m_bookName;
-    std::string m_bookTitle;
  };
 
-struct TaskbarInfo
-{
-  const std::string bookName;
-  const zim::Archive* const archive;
-
-  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
-    : bookName(bookName)
-    , archive(a)
-  {}
-};
-
 class ContentResponseBlueprint
 {
 public: // functions
-  ContentResponseBlueprint(const InternalServer* server,
-                           const RequestContext* request,
+  ContentResponseBlueprint(const RequestContext* request,
                            int httpStatusCode,
                            const std::string& mimeType,
-                           const std::string& templateStr)
-    : m_server(*server)
-    , m_request(*request)
-    , m_httpStatusCode(httpStatusCode)
-    , m_mimeType(mimeType)
-    , m_template(templateStr)
-  {}
+                           const std::string& templateStr,
+                           bool includeKiwixResponseData = false);
 
-  virtual ~ContentResponseBlueprint() = default;
+  ~ContentResponseBlueprint();
 
-  operator std::unique_ptr<ContentResponse>() const
+  operator std::unique_ptr<Response>() const
   {
     return generateResponseObject();
   }
 
-  operator std::unique_ptr<Response>() const
-  {
-    return operator std::unique_ptr<ContentResponse>();
-  }
+  std::unique_ptr<ContentResponse> generateResponseObject() const;
 
+protected: // types
+  class Data;
 
-  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
-  ContentResponseBlueprint& operator+=(const TaskbarInfo& taskbarInfo);
-
-protected: // functions
-  std::string getMessage(const std::string& msgId) const;
-  virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
-
-public: //data
-  const InternalServer& m_server;
+protected: //data
   const RequestContext& m_request;
   const int m_httpStatusCode;
   const std::string m_mimeType;
   const std::string m_template;
-  kainjow::mustache::data m_data;
-  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
+  const bool m_includeKiwixResponseData;
+  std::unique_ptr<Data> m_data;
+};
+
+struct NewHTTP404Response : ContentResponseBlueprint
+{
+  NewHTTP404Response(const RequestContext& request,
+                     const std::string& root,
+                     const std::string& urlPath);
 };
 
 struct HTTPErrorResponse : ContentResponseBlueprint
 {
-  HTTPErrorResponse(const InternalServer& server,
-                    const RequestContext& request,
+  HTTPErrorResponse(const RequestContext& request,
                     int httpStatusCode,
                     const std::string& pageTitleMsgId,
                     const std::string& headingMsgId,
-                    const std::string& cssUrl = "");
+                    const std::string& cssUrl = "",
+                    bool includeKiwixResponseData = false);
 
-  using ContentResponseBlueprint::operator+;
-  using ContentResponseBlueprint::operator+=;
-  HTTPErrorResponse& operator+(const std::string& msg);
   HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
   HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
 };
 
-class UrlNotFoundMsg {};
-
-extern const UrlNotFoundMsg urlNotFoundMsg;
-
 struct HTTP404Response : HTTPErrorResponse
 {
-  HTTP404Response(const InternalServer& server,
-                  const RequestContext& request);
-
-  using HTTPErrorResponse::operator+;
-  HTTPErrorResponse& operator+(UrlNotFoundMsg /*unused*/);
+  explicit HTTP404Response(const RequestContext& request);
 };
 
-class InvalidUrlMsg {};
-
-extern const InvalidUrlMsg invalidUrlMsg;
+struct UrlNotFoundResponse : HTTP404Response
+{
+  explicit UrlNotFoundResponse(const RequestContext& request);
+};
 
 struct HTTP400Response : HTTPErrorResponse
 {
-  HTTP400Response(const InternalServer& server,
-                  const RequestContext& request);
-
-  using HTTPErrorResponse::operator+;
-  HTTPErrorResponse& operator+(InvalidUrlMsg /*unused*/);
+  explicit HTTP400Response(const RequestContext& request);
 };
 
-struct HTTP500Response : HTTPErrorResponse
+struct HTTP500Response : ContentResponseBlueprint
 {
-  HTTP500Response(const InternalServer& server,
-                  const RequestContext& request);
-
-private: // overrides
-  // generateResponseObject() is overriden in order to produce a minimal
-  // response without any need for additional resources from the server
-  std::unique_ptr<ContentResponse> generateResponseObject() const override;
+  HTTP500Response(const RequestContext& request,
+                  const std::string& root,
+                  const std::string& urlPath,
+                  const std::string& error = "");
 };
 
 class ItemResponse : public Response {
   public:
-    ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
+    ItemResponse(const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
+    static std::unique_ptr<Response> build(const RequestContext& request, const zim::Item& item);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
@@ -247,6 +200,13 @@ class ItemResponse : public Response {
     std::string m_mimeType;
 };
 
+struct BlockExternalLinkResponse : ContentResponseBlueprint
+{
+  BlockExternalLinkResponse(const RequestContext& request,
+                            const std::string& root,
+                            const std::string& externalUrl);
+};
+
 }
 
 #endif //KIWIXLIB_SERVER_RESPONSE_H

src/spelling_correction.cpp

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2025 Veloman Yunkan
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "spelling_correction.h"
+#include "zim/archive.h"
+
+#include <sstream>
+#include <stdexcept>
+
+#include <xapian.h>
+
+namespace kiwix
+{
+
+namespace
+{
+
+std::vector<std::string> getAllTitles(const zim::Archive& a)
+{
+  std::vector<std::string> result;
+  for (const auto& entry : a.iterByPath() ) {
+     result.push_back(entry.getTitle());
+  }
+  return result;
+}
+
+void createXapianDB(std::string path, const zim::Archive& archive)
+{
+  const int flags = Xapian::DB_BACKEND_GLASS|Xapian::DB_CREATE;
+  const auto tmpDbPath = path + ".tmp";
+  Xapian::WritableDatabase db(tmpDbPath, flags);
+  for (const auto& t : getAllTitles(archive)) {
+    db.add_spelling(t);
+  }
+  db.commit();
+  db.compact(path, Xapian::DBCOMPACT_SINGLE_FILE);
+  db.close();
+  std::filesystem::remove_all(tmpDbPath);
+}
+
+std::string spellingsDBPathForZIMArchive(std::filesystem::path cacheDirPath, const zim::Archive& a)
+{
+  // The version of spellings DB must be updated each time an important change
+  // to the implementation is made that renders using the previous version
+  // impossible or undesirable.
+  const char SPELLINGS_DB_VERSION[] = "0.1";
+
+  std::ostringstream filename;
+  filename << a.getUuid() << ".spellingsdb.v" << SPELLINGS_DB_VERSION;
+  return (cacheDirPath / filename.str()).string();
+}
+
+std::unique_ptr<Xapian::Database> openOrCreateXapianDB(std::filesystem::path cacheDirPath, const zim::Archive& archive)
+{
+  const auto path = spellingsDBPathForZIMArchive(cacheDirPath, archive);
+  try
+  {
+    return std::make_unique<Xapian::Database>(path);
+  }
+  catch (const Xapian::DatabaseOpeningError& )
+  {
+    createXapianDB(path, archive);
+    return std::make_unique<Xapian::Database>(path);
+  }
+}
+
+} // unnamed namespace
+
+SpellingsDB::SpellingsDB(const zim::Archive& archive, std::filesystem::path cacheDirPath)
+  : impl_(openOrCreateXapianDB(cacheDirPath, archive))
+{
+}
+
+SpellingsDB::~SpellingsDB()
+{
+}
+
+std::vector<std::string> SpellingsDB::getSpellingCorrections(const std::string& word, uint32_t maxCount) const
+{
+  if ( maxCount > 1 ) {
+    throw std::runtime_error("More than one spelling correction was requested");
+  }
+
+  std::vector<std::string> result;
+  const auto term = impl_->get_spelling_suggestion(word, 3);
+  if ( !term.empty() ) {
+    result.push_back(term);
+  }
+  return result;
+}
+
+} // namespace kiwix

src/tools/archiveTools.cpp

@@ -93,10 +93,6 @@ std::string getMetaFlavour(const zim::Archive& archive) {
   return getMetadata(archive, "Flavour");
 }
 
-std::string getArchiveId(const zim::Archive& archive) {
-  return (std::string) archive.getUuid();
-}
-
 bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType){
   try {
@@ -109,46 +105,6 @@ bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
   return false;
 }
 
-// should this be in libzim
-unsigned int getArchiveMediaCount(const zim::Archive& archive) {
-  std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
-  unsigned int counter = 0;
-
-  for (auto &pair:counterMap) {
-    if (startsWith(pair.first, "image/") ||
-        startsWith(pair.first, "video/") ||
-        startsWith(pair.first, "audio/")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-unsigned int getArchiveArticleCount(const zim::Archive& archive) {
-  // [HACK]
-  // getArticleCount() returns different things depending of the "version" of the zim.
-  // On old zim (<=6), it returns the number of entry in `A` namespace
-  // On recent zim (>=7), it returns:
-  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
-  //  - the number of front article if a frontArticleIndex is present
-  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
-  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
-  if (archive.hasNewNamespaceScheme()) {
-    //The articleCount is "good"
-    return archive.getArticleCount();
-  } else {
-    // We have to parse the `M/Counter` metadata
-    unsigned int counter = 0;
-    for(const auto& pair:parseArchiveCounter(archive)) {
-      if (startsWith(pair.first, "text/html")) {
-        counter += pair.second;
-      }
-    }
-    return counter;
-  }
-}
-
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }
@@ -169,14 +125,4 @@ zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path
   }
   throw zim::EntryNotFound("Cannot find entry for non empty path");
 }
-
-MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
-  try {
-    auto counterContent = archive.getMetadata("Counter");
-    return parseMimetypeCounter(counterContent);
-  } catch (zim::EntryNotFound& e) {
-    return {};
-  }
-}
-
 } // kiwix

src/tools/archiveTools.h

@@ -40,7 +40,6 @@ namespace kiwix
     std::string getMetaCreator(const zim::Archive& archive);
     std::string getMetaPublisher(const zim::Archive& archive);
     std::string getMetaFlavour(const zim::Archive& archive);
-    std::string getArchiveId(const zim::Archive& archive);
 
     bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType);
@@ -52,9 +51,6 @@ namespace kiwix
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
 
     zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
-
-    MimeCounterType parseArchiveCounter(const zim::Archive& archive);
-
 }
 
 #endif

src/tools/languageTools.cpp

@@ -0,0 +1,86 @@
+#include "tools.h"
+#include "stringTools.h"
+#include <mutex>
+
+namespace kiwix
+{
+
+namespace
+{
+
+// These mappings are not provided by the ICU library, any such mappings can be manually added here
+std::map<std::string, std::string> iso639_3 = {
+    {"ami", "Amis"},
+    {"atj", "atikamekw"},
+    {"azb", "آذربایجان دیلی"},
+    {"bcl", "central bikol"},
+    {"bgs", "tagabawa"},
+    {"blk", "ပအိုဝ်ႏ"},
+    {"bxr", "буряад хэлэн"},
+    {"cbk", "chavacano"},
+    {"cdo", "閩東語"},
+    {"dag", "Dagbani"},
+    {"diq", "dimli"},
+    {"dty", "डोटेली"},
+    {"eml", "emiliân-rumagnōl"},
+    {"fbs", "српскохрватски"},
+    {"fon", "fɔ̀ngbè"},
+    {"gcr", "Kriyòl gwiyannen"},
+    {"guw", "Gungbe"},
+    {"hbs", "srpskohrvatski"},
+    {"hyw", "հայերէն/հայերեն"},
+    {"ido", "ido"},
+    {"kbp", "kabɩyɛ"},
+    {"kld", "Gamilaraay"},
+    {"lbe", "лакку маз"},
+    {"lbj", "ལ་དྭགས་སྐད་"},
+    {"lld", "ladin"},
+    {"map", "Austronesian"},
+    {"mhr", "марий йылме"},
+    {"mnw", "ဘာသာမန်"},
+    {"myn", "mayan"},
+    {"nah", "nahuatl"},
+    {"nai", "north American Indian"},
+    {"nds", "plattdütsch"},
+    {"nrm", "bhasa narom"},
+    {"olo", "livvi"},
+    {"pih", "Pitcairn-Norfolk"},
+    {"pnb", "Western Panjabi"},
+    {"pwn", "Pinayuanan"},
+    {"rmr", "Caló"},
+    {"rmy", "romani shib"},
+    {"roa", "romance languages"},
+    {"skr", "سرائیکی"},
+    {"szy", "Sakizaya"},
+    {"tay", "Tayal"},
+    {"tgl", "Wikang Tagalog"},
+    {"twi", "Akwapem Twi"},
+// ICU for Ubuntu versions <= focal (20.04) returns "" for the language code ""
+// unlike the later versions - which returns "und". We map this value to "Undetermined" for a common ground.
+    {"", "Undetermined"},
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+    for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+        const kiwix::ICULanguageInfo lang(*icuLangPtr);
+        iso639_3.insert({lang.iso3Code(), lang.selfName()});
+    }
+    iso639_3.erase("mul");
+}
+
+} // unnamed namespace
+
+std::string getLanguageSelfName(const std::string& lang)
+{
+    std::call_once(fillLanguagesFlag, fillLanguagesMap);
+    const auto itr = iso639_3.find(lang);
+    if (itr != iso639_3.end()) {
+        return itr->second;
+    }
+    return lang;
+};
+
+} // namespace kiwix

src/tools/networkTools.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "tools.h"
+#include "stringTools.h"
 #include <tools/networkTools.h>
 
 #include <stdio.h>
@@ -32,17 +33,30 @@
 #include <stdexcept>
 
 #ifdef _WIN32
+#include <iphlpapi.h>
 #include <winsock2.h>
 #include <ws2tcpip.h>
 #include <iostream>
 #else
 #include <unistd.h>
-#include <sys/ioctl.h>
+#include <arpa/inet.h>
+#include <ifaddrs.h>
 #include <sys/socket.h>
+#include <netinet/in.h>
 #include <net/if.h>
 #include <netdb.h>
 #endif
 
+#ifdef __HAIKU__
+#include <sys/sockio.h>
+#endif
+
+namespace kiwix
+{
+
+namespace
+{
+
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
   auto str = static_cast<std::stringstream*>(userdata);
@@ -50,7 +64,15 @@ size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdat
   return nmemb;
 }
 
-std::string kiwix::download(const std::string& url) {
+void updatePublicIpAddress(IpAddress& publicIpAddr, const IpAddress& interfaceIpAddr)
+{
+  if (publicIpAddr.addr.empty())  publicIpAddr.addr  = interfaceIpAddr.addr;
+  if (publicIpAddr.addr6.empty()) publicIpAddr.addr6 = interfaceIpAddr.addr6;
+}
+
+} // unnamed namespace
+
+std::string download(const std::string& url) {
   auto curl = curl_easy_init();
   std::stringstream ss;
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
@@ -71,103 +93,161 @@ std::string kiwix::download(const std::string& url) {
   return ss.str();
 }
 
-std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
-  std::map<std::string, std::string> interfaces;
+namespace
+{
 
 #ifdef _WIN32
-  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
-  if (sd == INVALID_SOCKET) {
-    std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
-    return interfaces;
-  }
 
-  INTERFACE_INFO InterfaceList[20];
-  unsigned long nBytesReturned;
-  if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
-      sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
-    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
-    return interfaces;
-  }
+std::map<std::string, IpAddress> getNetworkInterfacesWin() {
+  std::map<std::string, IpAddress> interfaces;
 
-  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
-  for (int i = 0; i < nNumInterfaces; ++i) {
-    sockaddr_in *pAddress;
-    pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
-    if(pAddress->sin_family == AF_INET) {
-    /* Add to the map */
-    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
-    interfaces[interfaceName] = interfaceName;
-    }
-  }
-#else
-  /* Get Network interfaces information */
-  char buf[16384];
-  struct ifconf ifconf;
-  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
-  ifconf.ifc_len = sizeof(buf);
-  ifconf.ifc_buf=buf;
-  if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
-    perror("ioctl(SIOCGIFCONF)");
-  }
+  const int working_buffer_size = 15000;
+  const int max_tries = 3;
 
-  /* Go through each interface */
-  struct ifreq *ifreq;
-  ifreq = ifconf.ifc_req;
-  for (int i = 0; i < ifconf.ifc_len; ) {
-    if (ifreq->ifr_addr.sa_family == AF_INET) {
-      /* Get the network interface ip */
-      char host[128] = { 0 };
-      const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
-                                    host, sizeof(host),
-                                    0, 0, NI_NUMERICHOST);
-      if (!error) {
-        std::string interfaceName = std::string(ifreq->ifr_name);
-        std::string interfaceIp = std::string(host);
-        /* Add to the map */
-        interfaces[interfaceName] = interfaceIp;
-      } else {
-        perror("getnameinfo()");
-      }
+  ULONG flags = GAA_FLAG_INCLUDE_PREFIX;
+
+  // default to unspecified address family (both)
+  ULONG family = AF_UNSPEC;
+
+  ULONG outBufLen = working_buffer_size;
+  ULONG Iterations = 0;
+  DWORD dwRetVal = 0;
+  PIP_ADAPTER_ADDRESSES interfacesHead = NULL;
+
+  // Successively allocate the required memory until GetAdaptersAddresses does not
+  // results in ERROR_BUFFER_OVERFLOW for a maximum of max_tries
+  do{
+    interfacesHead = (IP_ADAPTER_ADDRESSES *) malloc(outBufLen);
+    if (interfacesHead == NULL) {
+      std::cerr << "Memory allocation failed for IP_ADAPTER_ADDRESSES struct" << std::endl;
+      return interfaces;
     }
 
-    /* some systems have ifr_addr.sa_len and adjust the length that
-     * way, but not mine. weird */
-    size_t len;
-#ifndef __linux__
-    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
-#else
-    len = sizeof(*ifreq);
-#endif
-    ifreq = (struct ifreq*)((char*)ifreq+len);
-    i += len;
+    dwRetVal = GetAdaptersAddresses(family, flags, NULL, interfacesHead, &outBufLen);
+  } while ((dwRetVal == ERROR_BUFFER_OVERFLOW) && (Iterations < max_tries));
+
+  if (dwRetVal == NO_ERROR) {
+    PIP_ADAPTER_UNICAST_ADDRESS pUnicast = NULL;
+    unsigned int i = 0;
+    for (PIP_ADAPTER_ADDRESSES temp = interfacesHead; temp != NULL;
+    temp = temp->Next) {
+      pUnicast = temp->FirstUnicastAddress;
+            if (pUnicast != NULL) {
+                for (i = 0; pUnicast != NULL; i++){
+                  if (pUnicast->Address.lpSockaddr->sa_family == AF_INET)
+                  {
+                      sockaddr_in *si = (sockaddr_in *)(pUnicast->Address.lpSockaddr);
+                      char host[INET_ADDRSTRLEN]={0};
+                      inet_ntop(AF_INET, &(si->sin_addr), host, sizeof(host));
+                      interfaces[temp->AdapterName].addr=host;
+                  }
+                  else if (pUnicast->Address.lpSockaddr->sa_family == AF_INET6)
+                  {
+                      sockaddr_in6 *si = (sockaddr_in6 *)(pUnicast->Address.lpSockaddr);
+                      char host[INET6_ADDRSTRLEN]={0};
+                      inet_ntop(AF_INET6, &(si->sin6_addr), host, sizeof(host));
+                      if (!IN6_IS_ADDR_LINKLOCAL(&(si->sin6_addr)))
+                        interfaces[temp->AdapterName].addr6=host;
+                  }
+                  pUnicast = pUnicast->Next;
+                }
+            }
+    }
+  } else {
+    std::cerr << "Call to GetAdaptersAddresses failed with error: "<< dwRetVal << std::endl;
   }
-#endif
+
+  if (interfacesHead) free(interfacesHead);
+
   return interfaces;
 }
 
-std::string kiwix::getBestPublicIp() {
-  auto interfaces = getNetworkInterfaces();
+#else
 
+std::map<std::string, IpAddress> getNetworkInterfacesPosix() {
+  std::map<std::string, IpAddress> interfaces;
+
+  struct ifaddrs *interfacesHead;
+  if (getifaddrs(&interfacesHead) == -1) {
+      perror("getifaddrs");
+  }
+
+  for (ifaddrs *temp = interfacesHead; temp != NULL; temp = temp->ifa_next) {
+      if (temp->ifa_addr == NULL) continue;
+
+      if (temp->ifa_addr->sa_family == AF_INET) {
+          sockaddr_in *si = (sockaddr_in *)(temp->ifa_addr);
+          char host[INET_ADDRSTRLEN] = {0};
+          inet_ntop(AF_INET, &(si->sin_addr), host, sizeof(host));
+          interfaces[temp->ifa_name].addr=host;
+      } else if (temp->ifa_addr->sa_family == AF_INET6) {
+          sockaddr_in6 *si = (sockaddr_in6 *)(temp->ifa_addr);
+          char host[INET6_ADDRSTRLEN] = {0};
+          inet_ntop(AF_INET6, &(si->sin6_addr), host, sizeof(host));
+          if (!IN6_IS_ADDR_LINKLOCAL(&(si->sin6_addr)))
+            interfaces[temp->ifa_name].addr6=host;
+      }
+  }
+
+  freeifaddrs(interfacesHead);
+
+  return interfaces;
+}
+
+#endif
+
+} // unnamed namespace
+
+std::map<std::string, IpAddress> getNetworkInterfacesIPv4Or6() {
+#ifdef _WIN32
+  return getNetworkInterfacesWin();
+#else
+  return getNetworkInterfacesPosix();
+#endif
+}
+
+std::map<std::string, std::string> getNetworkInterfaces() {
+  std::map<std::string, std::string> result;
+  for ( const auto& kv : getNetworkInterfacesIPv4Or6() ) {
+      const std::string& interfaceName = kv.first;
+      const auto& ipAddresses = kv.second;
+      if ( !ipAddresses.addr.empty() ) {
+        result[interfaceName] = ipAddresses.addr;
+      }
+  }
+  return result;
+}
+
+IpAddress getBestPublicIps() {
+  IpAddress bestPublicIps;
+  std::map<std::string, IpAddress> interfaces = getNetworkInterfacesIPv4Or6();
 #ifndef _WIN32
-  const char* const prioritizedNames[] =
-      { "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
-  for(auto name: prioritizedNames) {
-    auto it = interfaces.find(name);
-    if(it != interfaces.end()) {
-      return it->second;
+  const char* const prioritizedNames[] = { "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
+  for (const auto& name : prioritizedNames) {
+    const auto it = interfaces.find(name);
+    if (it != interfaces.end()) {
+      updatePublicIpAddress(bestPublicIps, it->second);
     }
   }
 #endif
-
-  const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
-  for(auto prefix : prefixes){
-    for(auto& itr : interfaces) {
-      auto interfaceIp = itr.second;
-      if (interfaceIp.find(prefix) == 0) {
-        return interfaceIp;
+  const char* const v4prefixes[] = { "192.168", "172.16", "10.0" };
+  for (const auto& prefix : v4prefixes) {
+    for (const auto& kv : interfaces) {
+      const auto& interfaceIps = kv.second;
+      if (kiwix::startsWith(interfaceIps.addr, prefix)) {
+        updatePublicIpAddress(bestPublicIps, interfaceIps);
       }
     }
   }
 
-  return "127.0.0.1";
+  updatePublicIpAddress(bestPublicIps, {"127.0.0.1", "::1"});
+
+  return bestPublicIps;
 }
+
+std::string getBestPublicIp()
+{
+  return getBestPublicIps().addr;
+}
+
+} // namespace kiwix

src/tools/opdsParsingTools.cpp

@@ -0,0 +1,70 @@
+#include "tools.h"
+#include <pugixml.hpp>
+
+namespace kiwix
+{
+
+namespace
+{
+#define VALUE(name) entryNode.child(name).child_value()
+FeedLanguages parseLanguages(const pugi::xml_document& doc)
+{
+    pugi::xml_node feedNode = doc.child("feed");
+    FeedLanguages langs;
+
+    for (pugi::xml_node entryNode = feedNode.child("entry"); entryNode;
+        entryNode = entryNode.next_sibling("entry")) {
+            auto title = VALUE("title");
+            auto code = VALUE("dc:language");
+            langs.push_back({code, title});
+    }
+
+    return langs;
+}
+
+FeedCategories parseCategories(const pugi::xml_document& doc)
+{
+    pugi::xml_node feedNode = doc.child("feed");
+    FeedCategories categories;
+
+    for (pugi::xml_node entryNode = feedNode.child("entry"); entryNode;
+        entryNode = entryNode.next_sibling("entry")) {
+            auto title = VALUE("title");
+            categories.push_back(title);
+    }
+
+    return categories;
+}
+
+} // unnamed namespace
+
+FeedLanguages readLanguagesFromFeed(const std::string& content)
+{
+    pugi::xml_document doc;
+    pugi::xml_parse_result result
+        = doc.load_buffer((void*)content.data(), content.size());
+
+    if (result) {
+        auto langs = parseLanguages(doc);
+        return langs;
+    }
+
+    return FeedLanguages();
+}
+
+FeedCategories readCategoriesFromFeed(const std::string& content)
+{
+    pugi::xml_document doc;
+    pugi::xml_parse_result result
+        = doc.load_buffer((void*)content.data(), content.size());
+
+    FeedCategories categories;
+    if (result) {
+        categories = parseCategories(doc);
+        return categories;
+    }
+
+    return categories;
+}
+
+} // namespace kiwix

src/tools/otherTools.cpp

@@ -32,12 +32,15 @@
 #endif
 
 #include "tools/stringTools.h"
+#include "server/i18n_utils.h"
+#include "libkiwix-resources.h"
 
 #include <map>
 #include <sstream>
 #include <pugixml.hpp>
 
 #include <zim/uuid.h>
+#include <zim/suggestion_iterator.h>
 
 
 static std::map<std::string, std::string> codeisomapping {
@@ -288,67 +291,6 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
-namespace
-{
-// The counter metadata format is a list of item separated by a `;` :
-// item0;item1;item2
-// Each item is a "tuple" mimetype=number.
-// However, the mimetype may contains parameters:
-// text/html;raw=true;foo=bar
-// So the final format may be complex to parse:
-// key0=value0;key1;foo=bar=value1;key2=value2
-
-typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
-
-std::string readFullMimetypeAndCounterString(std::istream& in)
-{
-  std::string mtcStr, params;
-  getline(in, mtcStr, ';');
-  if ( mtcStr.find('=') == std::string::npos )
-  {
-    do
-    {
-      if ( !getline(in, params, ';' ) )
-        return std::string();
-      mtcStr += ";" + params;
-    }
-    while ( std::count(params.begin(), params.end(), '=') != 2 );
-  }
-  return mtcStr;
-}
-
-MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
-{
-  const std::string::size_type k = s.find_last_of("=");
-  if ( k != std::string::npos )
-  {
-    const std::string mimeType = s.substr(0, k);
-    std::istringstream counterSS(s.substr(k+1));
-    unsigned int counter;
-    if (counterSS >> counter && counterSS.eof())
-      return MimetypeAndCounter{mimeType, counter};
-  }
-  return MimetypeAndCounter{"", 0};
-}
-
-} // unnamed namespace
-
-kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
-{
-  kiwix::MimeCounterType counters;
-  std::istringstream ss(counterData);
-
-  while (ss)
-  {
-    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
-    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
-    if ( !mtc.first.empty() )
-      counters.insert(mtc);
-  }
-
-  return counters;
-}
-
 std::string kiwix::gen_date_str()
 {
   auto now = std::time(0);
@@ -380,10 +322,91 @@ kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
   std::stringstream ss;
   tmpl.render(data, [&ss](const std::string& str) { ss << str; });
   return ss.str();
 }
+
+// The escapeQuote parameter of escapeForJSON() defaults to true.
+// This constant makes the calls to escapeForJSON() where the quote symbol
+// should not be escaped (as it is later replaced with the HTML character entity
+// &quot;) more readable.
+static const bool DONT_ESCAPE_QUOTE = false;
+
+std::string kiwix::escapeForJSON(const std::string& s, bool escapeQuote)
+{
+  std::ostringstream oss;
+  for (char c : s) {
+    if ( c == '\\' ) {
+      oss << "\\\\";
+    } else if ( unsigned(c) < 0x20U ) {
+      switch ( c ) {
+      case '\n': oss << "\\n"; break;
+      case '\r': oss << "\\r"; break;
+      case '\t': oss << "\\t"; break;
+      default:   oss << "\\u" << std::setw(4) << std::setfill('0') << unsigned(c);
+      }
+    } else if ( c == '"' && escapeQuote ) {
+      oss << "\\\"";
+    } else {
+      oss << c;
+    }
+  }
+  return oss.str();
+}
+
+namespace
+{
+
+std::string makeFulltextSearchSuggestion(const std::string& lang,
+                                         const std::string& queryString)
+{
+  return kiwix::i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+} // unnamed namespace
+
+kiwix::Suggestions::Suggestions()
+  : m_data(kainjow::mustache::data::type::list)
+{
+}
+
+void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
+{
+  kainjow::mustache::data result;
+
+  const std::string label = suggestion.hasSnippet()
+                          ? suggestion.getSnippet()
+                          : suggestion.getTitle();
+
+  result.set("label", escapeForJSON(label, DONT_ESCAPE_QUOTE));
+  result.set("value", escapeForJSON(suggestion.getTitle(), DONT_ESCAPE_QUOTE));
+  result.set("kind", "path");
+  result.set("path", escapeForJSON(suggestion.getPath(), DONT_ESCAPE_QUOTE));
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
+                                               const std::string& queryString)
+{
+  kainjow::mustache::data result;
+  const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
+  result.set("label", escapeForJSON(label, DONT_ESCAPE_QUOTE));
+  result.set("value", escapeForJSON(queryString + " ", DONT_ESCAPE_QUOTE));
+  result.set("kind", "pattern");
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+std::string kiwix::Suggestions::getJSON() const
+{
+  kainjow::mustache::data data;
+  data.set("suggestions", m_data);
+
+  return render_template(RESOURCE::templates::suggestion_json, data);
+}

src/tools/otherTools.h

@@ -33,6 +33,10 @@ namespace pugi {
   class xml_node;
 }
 
+namespace zim {
+  class SuggestionItem;
+}
+
 namespace kiwix
 {
   std::string nodeToString(const pugi::xml_node& node);
@@ -45,9 +49,6 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
-  using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
-  MimeCounterType parseMimetypeCounter(const std::string& counterData);
-
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
@@ -70,6 +71,22 @@ namespace kiwix
 
     return defaultValue;
   }
+
+  class Suggestions
+  {
+  public:
+    Suggestions();
+
+    void add(const zim::SuggestionItem& suggestion);
+
+    void addFTSearchSuggestion(const std::string& uiLang,
+                               const std::string& query);
+
+    std::string getJSON() const;
+
+  private:
+    kainjow::mustache::data m_data;
+  };
 }
 
 #endif

src/tools/pathTools.cpp

@@ -320,16 +320,6 @@ bool kiwix::fileReadable(const std::string& path)
 #endif
 }
 
-bool makeDirectory(const std::string& path)
-{
-#ifdef _WIN32
-  int status = _wmkdir(Utf8ToWide(path).c_str());
-#else
-  int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-#endif
-  return status == 0;
-}
-
 std::string makeTmpDirectory()
 {
 #ifdef _WIN32
@@ -438,52 +428,6 @@ std::string kiwix::getCurrentDirectory()
   return ret;
 }
 
-std::string kiwix::getDataDirectory()
-{
-// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
-#ifdef _WIN32
-  wchar_t* cDataDir = ::_wgetenv(L"KIWIX_DATA_DIR");
-  if (cDataDir != nullptr) {
-    return WideToUtf8(cDataDir);
-  }
-#else
-  char* cDataDir = ::getenv("KIWIX_DATA_DIR");
-  if (cDataDir != nullptr) {
-    return cDataDir;
-  }
-#endif
-
-// Compute the dataDir from the user directory.
-  std::string dataDir;
-#ifdef _WIN32
-  cDataDir = ::_wgetenv(L"APPDATA");
-  if (cDataDir == nullptr)
-    cDataDir = ::_wgetenv(L"USERPROFILE");
-  if (cDataDir != nullptr)
-    dataDir = WideToUtf8(cDataDir);
-#else
-  cDataDir = ::getenv("XDG_DATA_HOME");
-  if (cDataDir != nullptr) {
-    dataDir = cDataDir;
-  } else {
-    cDataDir = ::getenv("HOME");
-    if (cDataDir != nullptr) {
-      dataDir = cDataDir;
-      dataDir = appendToDirectory(dataDir, ".local");
-      dataDir = appendToDirectory(dataDir, "share");
-    }
-  }
-#endif
-  if (!dataDir.empty()) {
-    dataDir = appendToDirectory(dataDir, "kiwix");
-    makeDirectory(dataDir);
-    return dataDir;
-  }
-
-// Let's use the currentDirectory
-  return getCurrentDirectory();
-}
-
 static std::map<std::string, std::string> extMimeTypes = {
  { "html", "text/html"},
  { "htm", "text/html"},
@@ -493,12 +437,14 @@ static std::map<std::string, std::string> extMimeTypes = {
  { "jpeg", "image/jpeg"},
  { "jpg", "image/jpeg"},
  { "gif", "image/gif"},
+ { "ico", "image/x-icon"},
  { "svg", "image/svg+xml"},
  { "txt", "text/plain"},
  { "xml", "text/xml"},
  { "pdf", "application/pdf"},
  { "ogg", "application/ogg"},
  { "js", "application/javascript"},
+ { "json", "application/json"},
  { "css", "text/css"},
  { "otf", "application/vnd.ms-opentype"},
  { "ttf", "application/font-ttf"},

src/tools/pathTools.h

@@ -29,7 +29,6 @@ std::wstring Utf8ToWide(const std::string& str);
 
 unsigned int getFileSize(const std::string& path);
 std::string getFileSizeAsString(const std::string& path);
-bool makeDirectory(const std::string& path);
 std::string makeTmpDirectory();
 bool copyFile(const std::string& sourcePath, const std::string& destPath);
 bool writeTextFile(const std::string& path, const std::string& content);

src/tools/regexTools.cpp

@@ -75,41 +75,3 @@ std::string replaceRegex(const std::string& content,
   uresult.toUTF8String(tmp);
   return tmp;
 }
-
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->end(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}
-
-std::string prependToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->start(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}

src/tools/regexTools.h

@@ -26,11 +26,5 @@ bool matchRegex(const std::string& content, const std::string& regex);
 std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement);
-std::string prependToFirstOccurence(const std::string& content,
-                                    const std::string& regex,
-                                    const std::string& replacement);
 
 #endif

src/tools/stringTools.cpp

@@ -32,6 +32,7 @@
 
 #include <iostream>
 #include <iomanip>
+#include <regex>
 
 /* tell ICU where to find its dat file (tables) */
 void kiwix::loadICUExternalTables()
@@ -161,15 +162,14 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-/* urlEncode() based on javascript encodeURI() &
-   encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
+namespace
+{
 
 bool isReservedUrlChar(char c)
 {
   switch (c) {
   case ';':
   case ',':
-  case '/':
   case '?':
   case ':':
   case '@':
@@ -177,22 +177,22 @@ bool isReservedUrlChar(char c)
   case '=':
   case '+':
   case '$':
+  case '#':
     return true;
   default:
     return false;
   }
 }
 
-bool needsEscape(char c, bool encodeReserved)
+bool isHarmlessUriChar(char c)
 {
   if (c >= 'a' && c <= 'z')
-    return false;
+    return true;
   if (c >= 'A' && c <= 'Z')
-    return false;
+    return true;
   if (c >= '0' && c <= '9')
-    return false;
-  if (isReservedUrlChar(c))
-    return encodeReserved;
+    return true;
+
   switch (c) {
   case '-':
   case '_':
@@ -203,9 +203,10 @@ bool needsEscape(char c, bool encodeReserved)
   case '\'':
   case '(':
   case ')':
-    return false;
+  case '/':
+    return true;
   }
-  return true;
+  return false;
 }
 
 int hexToInt(char c) {
@@ -230,18 +231,18 @@ int hexToInt(char c) {
   }
 }
 
-std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
+} // unnamed namespace
+
+std::string kiwix::urlEncode(const std::string& value)
 {
   std::ostringstream os;
   os << std::hex << std::uppercase;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
-
-    if (!needsEscape(*it, encodeReserved)) {
-      os << *it;
+  for (const char c : value) {
+    if (isHarmlessUriChar(c)) {
+      os << c;
     } else {
-      os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
+      const unsigned int charVal = static_cast<unsigned char>(c);
+      os << '%' << std::setw(2) << std::setfill('0') << charVal;
     }
   }
   return os.str();
@@ -256,7 +257,7 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 
     // If there aren't enough characters left for this to be a
     // valid escape code, just use the character and move on
-    if (it > value.end() - 3) {
+    if (value.end() - it < 3) {
       os << *it;
       continue;
     }
@@ -267,15 +268,15 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
       int iHi = hexToInt(hi);
       int iLo = hexToInt(lo);
       if (iHi < 0 || iLo < 0) {
-	// Invalid escape sequence
-	os << '%' << hi << lo;
-	continue;
+        // Invalid escape sequence
+        os << '%' << hi << lo;
+        continue;
       }
       char c = (char)(iHi << 4 | iLo);
       if (!component && isReservedUrlChar(c)) {
-	os << '%' << hi << lo;
+        os << '%' << hi << lo;
       } else {
-	os << c;
+        os << c;
       }
     } else {
       os << *it;
@@ -415,6 +416,17 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
         && std::equal(start.begin(), start.end(), base.begin());
 }
 
+std::string kiwix::stripSuffix(const std::string& str, const std::string& suffix)
+{
+  if (str.size() > suffix.size()) {
+    const auto subStr = str.substr(str.size() - suffix.size(), str.size());
+    if (subStr == suffix) {
+      return str.substr(0, str.size() - suffix.size());
+    }
+  }
+  return str;
+}
+
 std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   std::vector<std::string> variants;
   variants.push_back(title);
@@ -428,3 +440,13 @@ template<>
 std::string kiwix::extractFromString(const std::string& str) {
   return str;
 }
+
+std::string kiwix::getSlugifiedFileName(const std::string& filename)
+{
+#ifdef _WIN32
+  const std::regex reservedCharsReg(R"([<>:"/\\|?*])");
+#else
+  const std::regex reservedCharsReg("/");
+#endif
+  return std::regex_replace(filename, reservedCharsReg, "_"); 
+}

src/tools/stringTools.h

@@ -31,7 +31,6 @@
 namespace kiwix
 {
 std::string beautifyInteger(uint64_t number);
-std::string beautifyFileSize(uint64_t number);
 void printStringInHexadecimal(const char* s);
 void printStringInHexadecimal(icu::UnicodeString s);
 void stringReplacement(std::string& str,
@@ -54,8 +53,11 @@ private:
   const icu::Locale locale;
 };
 
+std::string escapeForJSON(const std::string& s, bool escapeQuote = true);
 
-std::string urlEncode(const std::string& value, bool encodeReserved = false);
+/* urlEncode() is the equivalent of JS encodeURIComponent(), with the only
+ * difference that the slash (/) symbol is NOT encoded. */
+std::string urlEncode(const std::string& value);
 std::string urlDecode(const std::string& value, bool component = false);
 
 std::string join(const std::vector<std::string>& list, const std::string& sep);
@@ -64,6 +66,9 @@ std::string ucAll(const std::string& word);
 std::string lcAll(const std::string& word);
 std::string ucFirst(const std::string& word);
 std::string lcFirst(const std::string& word);
+
+/* This function is broken, related Github issue 
+ * https://github.com/kiwix/libkiwix/issues/1188 */ 
 std::string toTitle(const std::string& word);
 
 std::string normalize(const std::string& word);
@@ -91,6 +96,8 @@ std::string extractFromString(const std::string& str);
 
 bool startsWith(const std::string& base, const std::string& start);
 
+std::string stripSuffix(const std::string& str, const std::string& suffix);
+
 std::vector<std::string> getTitleVariants(const std::string& title);
 } //namespace kiwix
 #endif

static/generate_i18n_resources_list.py

@@ -17,15 +17,44 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from pathlib import Path
+import json
 
 script_path = Path(__file__)
 
 resource_file = script_path.parent / "i18n_resources_list.txt"
-translation_dir = script_path.parent / "i18n"
+translation_dir = script_path.parent / "skin/i18n"
+language_list_relpath = "skin/languages.js"
 
+def get_translation_info(filepath):
+    lang_code = Path(filepath).stem
+    with open(filepath, 'r', encoding="utf-8") as f:
+        content = json.load(f)
+        lang_name = content.get("name")
+        translation_count = len(content)
+        return dict(iso_code=lang_code,
+                    self_name=lang_name,
+                    translation_count=translation_count)
+
+language_list = []
 json_files = translation_dir.glob("*.json")
 with open(resource_file, 'w', encoding="utf-8") as f:
-    for json in sorted(translation_dir.glob("*.json")):
-        if json.name == "qqq.json":
+    for i18n_file in sorted(translation_dir.glob("*.json")):
+        if i18n_file.name == "qqq.json":
             continue
-        f.write(str(json.relative_to(script_path.parent)) + '\n')
+        print("Processing", i18n_file.name)
+        if i18n_file.name != "test.json":
+            translation_info = get_translation_info(i18n_file)
+            lang_name = translation_info["self_name"]
+            if lang_name:
+                language_list.append(translation_info)
+            else:
+                print(f"Warning: missing 'name' in {i18n_file.name}")
+        f.write(str(i18n_file.relative_to(script_path.parent)) + '\n')
+
+language_list_jsobj_str = json.dumps(language_list,
+                                     indent=2,
+                                     ensure_ascii=False)
+print("Saving", language_list_relpath)
+fullpath = script_path.parent / language_list_relpath
+with open(fullpath, 'w', encoding="utf-8") as f:
+    f.write("const uiLanguages = " + language_list_jsobj_str)

static/i18n/bn.json

@@ -1,14 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"আফতাবুজ্জামান"
-		]
-	},
-	"name": "বাংলা",
-	"404-page-heading": "পাওয়া যায়নি",
-	"500-page-title": "অভ্যন্তরীণ সার্ভার ত্রুটি",
-	"500-page-heading": "অভ্যন্তরীণ সার্ভার ত্রুটি",
-	"library-button-text": "স্বাগত পাতায় চলুন",
-	"home-button-text": "'{{BOOK_TITLE}}'-এর প্রধান পাতায় চলুন",
-	"searchbox-tooltip": "'{{BOOK_TITLE}}' অনুসন্ধান করুন"
-}

static/i18n/en.json

@@ -1,30 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-		]
-	},
-	"name":"English",
-	"suggest-full-text-search" : "containing '{{{SEARCH_TERMS}}}'..."
-	, "no-such-book" : "No such book: {{BOOK_NAME}}"
-	, "too-many-books" : "Too many books requested ({{NB_BOOKS}}) where limit is {{LIMIT}}"
-	, "no-book-found" : "No book matches selection criteria"
-	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
-	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
-	, "random-article-failure" :  "Oops! Failed to pick a random article :("
-	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
-	, "no-value-for-arg": "No value provided for argument {{ARGUMENT}}"
-	, "no-query" : "No query provided."
-	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
-	, "400-page-title" : "Invalid request"
-	, "400-page-heading" : "Invalid request"
-	, "404-page-title" : "Content not found"
-	, "404-page-heading" : "Not Found"
-	, "500-page-title" : "Internal Server Error"
-	, "500-page-heading" : "Internal Server Error"
-	, "fulltext-search-unavailable" : "Fulltext search unavailable"
-	, "no-search-results": "The fulltext search engine is not available for this content."
-	, "library-button-text": "Go to welcome page"
-	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
-	, "random-page-button-text": "Go to a randomly selected page"
-	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
-}

static/i18n/fr.json

@@ -1,33 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Gomoko",
-			"Thibaut120094",
-			"Verdy p"
-		]
-	},
-	"name": "français",
-	"suggest-full-text-search": "contenant « {{{SEARCH_TERMS}}} »...",
-	"no-such-book": "Aucun livre avec ce nom : {{BOOK_NAME}}",
-	"too-many-books": "Trop de livres demandés ({{NB_BOOKS}}) alors que la limite est de {{LIMIT}}",
-	"no-book-found": "Aucun livre ne correspond à ces critères de sélection",
-	"url-not-found": "L’URL demandée « {{url}} » est introuvable sur ce serveur.",
-	"suggest-search": "Faire une recherche en texte intégral de « <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> »",
-	"random-article-failure": "Oups ! Échec de sélection d’un article aléatoire :(",
-	"invalid-raw-data-type": "{{DATATYPE}} n’est pas une requête valide pour du contenu brut.",
-	"no-value-for-arg": "Aucune valeur fournie pour l’argument {{ARGUMENT}}",
-	"no-query": "Aucune requête fournie.",
-	"raw-entry-not-found": "Impossible de trouver l’entrée « {{ENTRY}} » de type « {{DATATYPE}} »",
-	"400-page-title": "Requête non valide",
-	"400-page-heading": "Requête non valide",
-	"404-page-title": "Contenu non trouvé",
-	"404-page-heading": "Non trouvé",
-	"500-page-title": "Erreur interne du serveur",
-	"500-page-heading": "Erreur interne du serveur",
-	"fulltext-search-unavailable": "Recherche en texte intégral non disponible",
-	"no-search-results": "Le moteur de recherche en texte intégral n’est pas disponible pour ce contenu.",
-	"library-button-text": "Aller à la page de bienvenue",
-	"home-button-text": "Aller à la page principale de « {{BOOK_TITLE}} »",
-	"random-page-button-text": "Aller à une page sélectionnée aléatoirement",
-	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »"
-}

static/i18n/he.json

@@ -1,31 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Amire80"
-		]
-	},
-	"name": "עברית",
-	"suggest-full-text-search": "מכיל '{{{SEARCH_TERMS}}}'...",
-	"no-such-book": "אין ספר כזה: {{BOOK_NAME}}",
-	"too-many-books": "נתבקשו יותר ספרים ({{NB_BOOKS}}) והמגבלה היא {{LIMIT}}",
-	"no-book-found": "אין ספר שמתאים לתנאים שנבחרו",
-	"url-not-found": "הכתובת המבוקשת \"{{url}}\" לא נמצאה בשרת הזה.",
-	"suggest-search": "לעשות חיפוש טקסט מלא עבור <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
-	"random-article-failure": "אוי! לא עבדה בחירת ערך אקראי :(",
-	"invalid-raw-data-type": "{{DATATYPE}} הוא לא בקשה תקינה של תוכן גולמי.",
-	"no-value-for-arg": "לא סופק ערך לארגומנט {{ARGUMENT}}",
-	"no-query": "לא סופקה שאילתה.",
-	"raw-entry-not-found": "לא ניתן למצוא את רשומת ה־{{DATATYPE}} בשם {{ENTRY}}",
-	"400-page-title": "בקשה בלתי־תקינה",
-	"400-page-heading": "בקשה בלתי־תקינה",
-	"404-page-title": "התוכן לא נמצא",
-	"404-page-heading": "לא נמצא",
-	"500-page-title": "שגיאת שרת פנימית",
-	"500-page-heading": "שגיאת שרת פנימית",
-	"fulltext-search-unavailable": "חיפוש בטקסט מלא אינו זמין",
-	"no-search-results": "מנוע החיפוש בטקסט מלא אינו זמין עבור התוכן הזה.",
-	"library-button-text": "מעבר לדף הבית \"ברוך בואך\"",
-	"home-button-text": "מעבר לדף הראשי של \"{{BOOK_TITLE}}\"",
-	"random-page-button-text": "מעבר לדף שנבחר אקראית",
-	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\""
-}

static/i18n/hy.json

@@ -1,16 +0,0 @@
-{
-	"@metadata": {
-		"authors": []
-	},
-	"name": "Հայերեն",
-	"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'...",
-	"no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}",
-	"url-not-found": "Սխալ հասցե՝ {{url}}",
-	"suggest-search": "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
-	"404-page-title": "Սխալ հասցե",
-	"404-page-heading": "Սխալ հասցե",
-	"library-button-text": "Գրադարանի էջ",
-	"home-button-text": "Դեպի '{{BOOK_TITLE}}'֊ի գլխավոր էջը",
-	"random-page-button-text": "Բացել պատահական էջ",
-	"searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում"
-}

static/i18n/it.json

@@ -1,15 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Beta16"
-		]
-	},
-	"name": "italiano",
-	"suggest-full-text-search": "contenente '{{{SEARCH_TERMS}}}'...",
-	"url-not-found": "L'URL richiesto \"{{url}}\" non è stato trovato in questo server.",
-	"400-page-title": "Richiesta non valida",
-	"400-page-heading": "Richiesta non valida",
-	"404-page-title": "Contenuto non trovato",
-	"404-page-heading": "Non trovato",
-	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'"
-}

static/i18n/ja.json

@@ -1,18 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"MathXplore"
-		]
-	},
-	"no-query": "クエリを指定していません。",
-	"400-page-title": "無効なリクエストです",
-	"400-page-heading": "無効なリクエストです",
-	"404-page-title": "コンテンツが見つかりませんでした",
-	"404-page-heading": "見つかりません",
-	"500-page-title": "内部サーバーエラー",
-	"500-page-heading": "内部サーバーエラー",
-	"fulltext-search-unavailable": "全文検索は利用できません",
-	"no-search-results": "このコンテンツでは全文検索エンジンが利用できません",
-	"library-button-text": "ウェルカムページに移動",
-	"random-page-button-text": "無作為に選ばれたページに移動する"
-}

static/i18n/ko.json

@@ -1,18 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Ykhwong"
-		]
-	},
-	"name": "한국어",
-	"suggest-full-text-search": "'{{{SEARCH_TERMS}}}' 포함...",
-	"no-such-book": "해당 책이 없습니다: {{BOOK_NAME}}",
-	"400-page-title": "잘못된 요청",
-	"400-page-heading": "잘못된 요청",
-	"404-page-title": "내용이 없습니다",
-	"404-page-heading": "찾을 수 없음",
-	"500-page-title": "내부 서버 오류",
-	"500-page-heading": "내부 서버 오류",
-	"fulltext-search-unavailable": "전문 검색을 사용할 수 없습니다",
-	"random-page-button-text": "무작위로 선택된 문서로 이동"
-}

static/i18n/mk.json

@@ -1,31 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Bjankuloski06"
-		]
-	},
-	"name": "македонски",
-	"suggest-full-text-search": "содржи „{{{SEARCH_TERMS}}}“...",
-	"no-such-book": "Нема книга нарчена {{BOOK_NAME}}",
-	"too-many-books": "Побаравте премногу книги ({{NB_BOOKS}}). Ограничени сте на {{LIMIT}}",
-	"no-book-found": "Ниедна книга не одговара на избраното",
-	"url-not-found": "Не ја пронајдов побараната адреса „{{url}}“ на опслужувачот.",
-	"suggest-search": "Побарајте го <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> по целиот текст",
-	"random-article-failure": "Упс! Не успеав да изберам случајна статија :(",
-	"invalid-raw-data-type": "{{DATATYPE}} не претставува важечко барање за сирова содржина.",
-	"no-value-for-arg": "Нема укажано вредност за аргументот {{ARGUMENT}}",
-	"no-query": "Не е укажано барање.",
-	"raw-entry-not-found": "Не можам да ја најдам {{DATATYPE}}-ставката {{ENTRY}}",
-	"400-page-title": "Неважечко барање",
-	"400-page-heading": "Неважечко барање",
-	"404-page-title": "Содржината не е најдена",
-	"404-page-heading": "Не е најдено",
-	"500-page-title": "Внатрешна грешка во опслужувачот",
-	"500-page-heading": "Внатрешна грешка во опслужувачот",
-	"fulltext-search-unavailable": "Целотекстното пребарување е недостапно",
-	"no-search-results": "Погонот за целотекстно пребарување не е достапен за оваа содржина.",
-	"library-button-text": "Оди на воведната страница",
-	"home-button-text": "Оди на главната страница на „{{BOOK_TITLE}}“",
-	"random-page-button-text": "Оди на случајно избрана страница",
-	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“"
-}