fixup! Fixed external links in the viewer iframe

Before this fix clicking an external link in the viewer iframe had no
effect (other than an error being reported in the browser dev tools
console) because the attempt to navigate the top browser context was
suppressed due to sandboxing.

.github/move.yml

@@ -1,27 +0,0 @@
-# Configuration for Move Issues - https://github.com/dessant/move-issues
-
-# Delete the command comment when it contains no other content
-deleteCommand: true
-
-# Close the source issue after moving
-closeSourceIssue: true
-
-# Lock the source issue after moving
-lockSourceIssue: false
-
-# Mention issue and comment authors
-mentionAuthors: true
-
-# Preserve mentions in the issue content
-keepContentMentions: true
-
-# Move labels that also exist on the target repository
-moveLabels: true
-
-# Set custom aliases for targets
-# aliases:
-#   r: repo
-#   or: owner/repo
-
-# Repository to extend settings from
-# _extends: repo

.github/workflows/ci.yml

@@ -3,46 +3,42 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
 
 jobs:
-  Macos:
-    runs-on: macos-latest
+  macOS:
+    runs-on: macos-12
+    env:
+      HOME: /Users/runner
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Setup python 3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
+      - name: Retrieve source code
+        uses: actions/checkout@v3
+
       - name: Install packages
         run: |
           brew update
-          brew install gcovr pkg-config ninja
-      - name: Install python modules
-        run: pip3 install meson==0.49.2 pytest
-      - name: Install deps
-        shell: bash
+          brew install pkg-config ninja meson
+
+      - name: Install dependencies
+        env:
+          ARCHIVE_NAME: deps2_osx_native_dyn_libkiwix.tar.xz
         run: |
-          ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
-          wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
-      - name: Compile
-        shell: bash
+          wget -O- https://tmp.kiwix.org/ci/${{env.ARCHIVE_NAME}} | tar -xJ -C ${{env.HOME}}
+
+      - name: Compile source code
+        env:
+          PKG_CONFIG_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib/pkgconfig
+          CPPFLAGS: -I${{env.HOME}}/BUILD_native_dyn/INSTALL/include
         run: |
-          export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
-          export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
           meson . build --default-library=shared -Db_coverage=true
-          cd build
-          ninja
-      - name: Test
-        shell: bash
-        run: |
-          export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
-          cd build
-          meson test --verbose
+          ninja -C build
+
+      - name: Test libkiwix
         env:
           SKIP_BIG_MEMORY_TEST: 1
+          LD_LIBRARY_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib:${{env.HOME}}/BUILD_native_dyn/INSTALL/lib64
+        run: meson test -C build --verbose
 
   Linux:
     strategy:
@@ -84,7 +80,7 @@ jobs:
       HOME: /home/runner
     runs-on: ubuntu-latest
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:36"
     steps:
     - name: Checkout code
       shell: python
@@ -137,13 +133,9 @@ jobs:
       env:
         LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
         SKIP_BIG_MEMORY_TEST: 1
+
     - name: Publish coverage
-      shell: bash
-      run: |
-        cd $HOME/libkiwix
-        curl https://codecov.io/bash -o codecov.sh
-        bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
-        rm codecov.sh
       if: startsWith(matrix.target, 'native_')
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -1,5 +1,10 @@
 name: Packages
-on: [push, pull_request]
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   build-deb:
@@ -8,12 +13,12 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-kinetic
           - ubuntu-jammy
-          - ubuntu-impish
           - ubuntu-focal
           - ubuntu-bionic
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Determine which PPA we should upload to
       - name: PPA
@@ -34,18 +39,18 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
-        if: matrix.distro == 'ubuntu-jammy'
-        name: Build package for ubuntu-jammy
-        id: build-ubuntu-jammy
+      - uses: legoktm/gh-action-build-deb@ubuntu-kinetic
+        if: matrix.distro == 'ubuntu-kinetic'
+        name: Build package for ubuntu-kinetic
+        id: build-ubuntu-kinetic
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
@@ -66,15 +71,15 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: Packages for ${{ matrix.distro }}
           path: output
 
       - uses: legoktm/gh-action-dput@master
         name: Upload dev package
-        # Only upload on pushes to master
-        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/dev

ChangeLog

@@ -1,3 +1,43 @@
+libkiwix 12.0.0
+===============
+
+ * [API Break] Remove wrapper around libzim (@mgautierfr #789)
+ * Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
+ * Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
+ * Prevent search on multi language content (@veloman-yunkan #838)
+ * Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
+ * Catalog:
+  - Include tags in free text catalog search (@veloman-yunkan #802)
+  - Illustration's url is based on book's uuid (@veloman-yunkan #804)
+  - Cleanup of the opds-dumper (@veloman-yunkan #829)
+  - Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
+  - Make opds-dumper respect the namemapper (@mgautierfr #837)
+ * Server:
+  - Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
+  - Better http caching (@veloman-yunkan #833)
+  - Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
+  - Better redirection of main page (@veloman-yunkan #827)
+  - Remove jquery (@mgautierfr @juuz0 #796)
+  - Better Viewer of zim content :
+    . Introduce `/content` endpoints (@veloman-yunkan #806)
+    . Switch to iframe based content viewer (@veloman-yunkan #716)
+  - Optimised design of the welcome page:
+    . Alignement (@juuz0 @kelson42 #786)
+    . Exit download modal on pressing escape key (@juzz0 #800)
+    . Add favicon for different devices (@juzz0 #805)
+    . Fix auto hidding of the toolbar (@veloman-yunkan #821)
+    . Allow user to filter books by tags in the front page (@juuz0 #711)
+ * CI :
+   - Trigger CI on pull_request (@kelson42 #791)
+   - Drop Ubuntu Impish packaging (@legoktm #825)
+   - Add Ubuntu Kinetic packaging (@legoktm #801)
+ * Testing:
+   - Test ICULanguageInfo (@veloman-yunkan #795)
+   - Introduce fake `test` language to test i18n (@veloman-yunkan #848)
+ * Fix documentation (@kelson42 #816)
+ * Udpate translation (#787 #839 #847)
+
+
 libkiwix 11.0.0
 ===============
 
@@ -5,7 +45,7 @@ libkiwix 11.0.0
  * [server] Use gzip compression instead of deflat (mgautierfr #757)
  * [server] Version the static resources. This allow better invalidating
    browser cache when resources are changed (@veloman-yunkan #712)
- * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server|front] Use integer to query the host for page length (@juuz0 #772)
  * [server] Improve multizim search API:
    - Improvement of the cache system
    - Better API to select on which books to search in.

README.md

@@ -7,10 +7,10 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
-[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
 [![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
-[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -101,6 +101,33 @@ meson . build -Dwrapper=android -Dwerror=false
 ninja -C build
 ```
 
+Static files compilation
+------------------------
+
+Libkiwix has a few static files 'compiled' within the binary
+code. This is mostly Javascript/HTML/pictures necessary for the HTTP
+daemon.
+
+These static files are available in the `static` directory and are
+compiled by custom Python code available in this repository `scripts`
+directory. This happens automatically at compilation time without any
+additional command to run.
+
+To avoid HTTP caching issues, the URLs (to the static content) are
+appended with a `cacheid` parameter (this is called "cache
+busting"). This `cacheid` value derived from the
+[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
+static file. As a consequence, each time you change a static file, the
+corresponding `cacheid` value will change.
+
+To properly test this feature, this `cacheid` needs to be added
+manually to the automated tests and has to be commited. After
+modifying the needed static file, [run the automated
+tests](#Testing). They will fail, but the inspection of the testing
+log will give you the new `cacheid` value(s). Finally update
+`test/server.cpp` with the appropriate `cacheid` value(s) which have
+changed.
+
 Testing
 -------
 
@@ -124,7 +151,7 @@ where you want to install the libraries. After the installation
 succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
-------------
+--------------
 
 If you want to uninstall the Kiwix library:
 ```bash
@@ -134,28 +161,6 @@ ninja -C build uninstall
 Like for the installation, you might need to run the command as `root`
 (or using `sudo`).
 
-Troubleshooting
----------------
-
-If you need to install Meson "manually":
-```bash
-virtualenv -p python3 ./ # Create virtualenv
-source bin/activate      # Activate the virtualenv
-pip3 install meson       # Install Meson
-hash -r                  # Refresh bash paths
-```
-
-If you need to install Ninja "manually":
-```bash
-git clone git://github.com/ninja-build/ninja.git
-cd ninja
-git checkout release
-./configure.py --bootstrap
-mkdir ../bin
-cp ninja ../bin
-cd ..
-```
-
 Custom Index Page
 -----------------
 
@@ -185,7 +190,7 @@ To use JS provided by kiwix-serve you can use the following template to start wi
 - To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
 - To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
 - To add language select box add - `<select id="languageFilter"></select>` under body tag.
-- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add category select box add - `<select id="categoryFilter"></select>` under body tag.
 - To add search box for books use following form -
     ```
         <form id='kiwixSearchForm'>
@@ -205,6 +210,28 @@ distribution. Try then with a source tarball distributed by the
 problematic upstream project or even directly from the source code
 repository.
 
+Troubleshooting
+---------------
+
+If you need to install Meson "manually":
+```bash
+virtualenv -p python3 ./ # Create virtualenv
+source bin/activate      # Activate the virtualenv
+pip3 install meson       # Install Meson
+hash -r                  # Refresh bash paths
+```
+
+If you need to install Ninja "manually":
+```bash
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+mkdir ../bin
+cp ninja ../bin
+cd ..
+```
+
 License
 -------
 

android-kiwix-lib-publisher/.gitignore

@@ -1,13 +0,0 @@
-*.iml
-.gradle
-/local.properties
-/.idea/caches
-/.idea/libraries
-/.idea/modules.xml
-/.idea/workspace.xml
-/.idea/navEditor.xml
-/.idea/assetWizardSettings.xml
-.DS_Store
-/build
-/captures
-.externalNativeBuild

android-kiwix-lib-publisher/build.gradle

@@ -1,25 +0,0 @@
-// Top-level build file where you can add configuration options common to all sub-projects/modules.
-
-buildscript {
-    repositories {
-        google()
-        jcenter()
-
-    }
-    dependencies {
-        classpath 'com.android.tools.build:gradle:3.4.1'
-        // NOTE: Do not place your application dependencies here; they belong
-        // in the individual module build.gradle files
-    }
-}
-
-allprojects {
-    repositories {
-        google()
-        jcenter()
-    }
-}
-
-task clean(type: Delete) {
-    delete rootProject.buildDir
-}

android-kiwix-lib-publisher/gradle.properties

@@ -1,15 +0,0 @@
-# Project-wide Gradle settings.
-# IDE (e.g. Android Studio) users:
-# Gradle settings configured through the IDE *will override*
-# any settings specified in this file.
-# For more details on how to configure your build environment visit
-# http://www.gradle.org/docs/current/userguide/build_environment.html
-# Specifies the JVM arguments used for the daemon process.
-# The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx1536m
-# When configured, Gradle will run in incubating parallel mode.
-# This option should only be used with decoupled projects. More details, visit
-# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
-# org.gradle.parallel=true
-# Kotlin code style for this project: "official" or "obsolete":
-kotlin.code.style=official

android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +0,0 @@
-#Wed Jun 19 15:28:39 BST 2019
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

android-kiwix-lib-publisher/gradlew

@@ -1,172 +0,0 @@
-#!/usr/bin/env sh
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
-exec "$JAVACMD" "$@"

android-kiwix-lib-publisher/gradlew.bat

@@ -1,84 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore

@@ -1 +0,0 @@
-/build

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -1,64 +0,0 @@
-apply plugin: 'com.android.library'
-apply plugin: 'maven'
-
-android {
-    compileSdkVersion 28
-    defaultConfig {
-        minSdkVersion 15
-        targetSdkVersion 28
-        versionCode 1
-        versionName "1.0"
-    }
-    buildTypes {
-        release {
-            minifyEnabled false
-            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
-        }
-    }
-}
-
-dependencies {
-    implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
-}
-
-task writePom {
-  pom {
-    project {
-        groupId 'org.kiwix.kiwixlib'
-        artifactId 'kiwixlib'
-        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
-        packaging 'aar'
-        name 'kiwixlib'
-        url 'https://github.com/kiwix/libkiwix'
-        licenses {
-            license {
-                name 'GPLv3'
-                url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
-            }
-        }
-      developers {
-          developer {
-              id 'kiwix'
-              name 'kiwix'
-              email 'contact@kiwix.org'
-          }
-      }
-      scm {
-          connection 'https://github.com/kiwix/libkiwix.git'
-          developerConnection 'https://github.com/kiwix/libkiwix.git'
-          url 'https://github.com/kiwix/libkiwix'
-      }
-    }
-  }.withXml {
-    def dependenciesNode = asNode().appendNode('dependencies')
-
-    //Iterate over the implementation dependencies, adding a <dependency> node for each
-    configurations.implementation.allDependencies.each {
-      def dependencyNode = dependenciesNode.appendNode('dependency')
-      dependencyNode.appendNode('groupId', it.group)
-      dependencyNode.appendNode('artifactId', it.name)
-      dependencyNode.appendNode('version', it.version)
-    }
-  }.writeTo("$buildDir/pom.xml")
-}
-

android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro

@@ -1,21 +0,0 @@
-# Add project specific ProGuard rules here.
-# You can control the set of applied configuration files using the
-# proguardFiles setting in build.gradle.
-#
-# For more details, see
-#   http://developer.android.com/guide/developing/tools/proguard.html
-
-# If your project uses WebView with JS, uncomment the following
-# and specify the fully qualified class name to the JavaScript interface
-# class:
-#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
-#   public *;
-#}
-
-# Uncomment this to preserve the line number information for
-# debugging stack traces.
-#-keepattributes SourceFile,LineNumberTable
-
-# If you keep the line number information, uncomment this to
-# hide the original source file name.
-#-renamesourcefileattribute SourceFile

android-kiwix-lib-publisher/kiwixLibAndroid/src/main/AndroidManifest.xml

@@ -1,10 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="org.kiwix.kiwixlib">
-
-    <application
-        android:allowBackup="true"
-        android:supportsRtl="true">
-
-    </application>
-
-</manifest>

android-kiwix-lib-publisher/settings.gradle

@@ -1 +0,0 @@
-include ':kiwixLibAndroid'

docs/index.rst

@@ -12,4 +12,3 @@ Welcome to libkiwix's documentation!
 
    usage
    api/ref_api
-   widget

docs/widget.rst

@@ -1,82 +0,0 @@
-Kiwix serve widget
-====================
-
-Introduction
-------------
-
-The kiwix-serve widget provides an easy to embed way to show the `kiwix-serve` homepage.
-
-Usage
------
-
-To use the widget, simply add an iframe with its `src` attribute set to the `widget` endpoint.
-Example HTML Page ::
-
-    <!DOCTYPE html>
-    <html lang="en">
-    <head>
-    <title>Widget Test</title>
-    </head>
-    <body>
-    <iframe src="http://192.168.18.8:8080/widget?disabledesc&disablefilter&disabledownload" width=1000 height=1000></iframe>
-    </body>
-    </html>
-
-This creates an iframe with the kiwix-serve homepage contents.
-
-Arguments are explained below.
-
-Possible Arguments
--------------------
-
-Currently, the following arguments are supported.
-
-disabledesc (value = N/A)
-    Disables the description part of a tile.
-
-disablefilter (value = N/A)
-    Disables the search filters: language, category, tag and search function.
-
-disableclick (value = N/A)
-    Disables clicking the book to open it for reading.
-
-disabledownload (value = N/A)
-    Disables the download button (if avaialable at all) on the tile.
-
-
-Custom CSS and JS
------------------
-
-You can add your custom CSS rules and Javascript code to the widget.
-
-To do that, use the following code as template::
-
-    <iframe id="receiver" src="http://192.168.18.8:8080/widget?disabledesc=&disablefilter=&disabledownload=" width="1000" height="1000">
-    <p>Your browser does not support iframes.</p>
-    </iframe>
-
-    <script>
-    window.onload = function() {
-    var receiver = document.getElementById('receiver').contentWindow;
-    function sendMessage() {
-        let msg = {
-        css: `
-        .book__header {
-            color:red;
-        }`,
-        js: `
-        function widgetTest() {
-        console.log("Testing widget");
-        }
-        widgetTest();
-        `
-        }
-        receiver.postMessage(msg, 'http://192.168.18.8:8080/widget');
-    }
-    sendMessage();
-    }
-    </script>
-
-
-The CSS/JS fields are optional, you may send both or only one.
-

format_code.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/bash
-
-files=(
-"include/library.h"
-"include/common/stringTools.h"
-"include/common/pathTools.h"
-"include/common/otherTools.h"
-"include/common/regexTools.h"
-"include/common/networkTools.h"
-"include/common/archiveTools.h"
-"include/manager.h"
-"include/reader.h"
-"include/kiwix.h"
-"include/xapianSearcher.h"
-"include/searcher.h"
-"src/library.cpp"
-"src/android/kiwix.cpp"
-"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
-"src/searcher.cpp"
-"src/common/pathTools.cpp"
-"src/common/regexTools.cpp"
-"src/common/otherTools.cpp"
-"src/common/archiveTools.cpp"
-"src/common/networkTools.cpp"
-"src/common/stringTools.cpp"
-"src/xapianSearcher.cpp"
-"src/manager.cpp"
-"src/reader.cpp"
-)
-
-for i in "${files[@]}"
-do
-  echo $i
-  clang-format -i -style=file $i
-done

include/book.h

@@ -79,7 +79,9 @@ class Book
   bool isPathValid() const { return m_pathValid; }
   const std::string& getTitle() const { return m_title; }
   const std::string& getDescription() const { return m_description; }
-  const std::string& getLanguage() const { return m_language; }
+  DEPRECATED const std::string& getLanguage() const { return m_language; }
+  const std::string& getCommaSeparatedLanguages() const { return m_language; }
+  const std::vector<std::string> getLanguages() const;
   const std::string& getCreator() const { return m_creator; }
   const std::string& getPublisher() const { return m_publisher; }
   const std::string& getDate() const { return m_date; }

include/downloader.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <memory>
 #include <stdexcept>
+#include <mutex>
 
 namespace kiwix
 {
@@ -43,6 +44,14 @@ class AriaError : public std::runtime_error {
 };
 
 
+/**
+ * A representation of a current download.
+ *
+ * `Download` is not thread safe. User must care to not call method on a
+ * same download from different threads.
+ * However, it is safe to use different `Download`s from different threads.
+ */
+
 class Download {
  public:
   typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
@@ -53,19 +62,89 @@ class Download {
     : mp_aria(p_aria),
       m_status(K_UNKNOWN),
       m_did(did) {};
-  void updateStatus(bool follow=false);
+
+  /**
+   * Update the status of the download.
+   *
+   * This call make an aria rpc call and is blocking.
+   * Some download (started with a metalink) are in fact several downloads.
+   * - A first one to download the metadlink.
+   * - A second one to download the real file.
+   *
+   * If `follow` is true, updateStatus tries to detect that and tracks
+   * the second download when the first one is finished.
+   * By passing false to `follow`, `Download` will only track the first download.
+   *
+   * `getFoo` methods are based on the last statusUpdate.
+   *
+   * @param follow: Do we have to follow following downloads.
+   */
+  void updateStatus(bool follow);
+
+  /**
+   * Pause the download (and call updateStatus)
+   */
   void pauseDownload();
+
+  /**
+   * Resume the download (and call updateStatus)
+   */
   void resumeDownload();
+
+  /**
+   * Cancel the download.
+   *
+   * A canceled downlod cannot be resume and updateStatus does nothing.
+   * However, you can still get information based on the last known information.
+   */
   void cancelDownload();
-  StatusResult getStatus()          { return m_status; }
-  std::string  getDid()             { return m_did; }
-  std::string  getFollowedBy()      { return m_followedBy; }
-  uint64_t     getTotalLength()     { return m_totalLength; }
-  uint64_t     getCompletedLength() { return m_completedLength; }
-  uint64_t     getDownloadSpeed()   { return m_downloadSpeed; }
-  uint64_t     getVerifiedLength()  { return m_verifiedLength; }
-  std::string  getPath()            { return m_path; }
-  std::vector<std::string>&  getUris()             { return m_uris; }
+
+  /*
+   * Get the status of the download.
+   */
+  StatusResult getStatus() const             { return m_status; }
+
+  /*
+   * Get the id of the download.
+   */
+  const std::string&  getDid() const         { return m_did; }
+
+  /*
+   * Get the id of the "second" download.
+   *
+   * Set only if the "first" download is a metalink and is complete.
+   */
+  const std::string&  getFollowedBy() const  { return m_followedBy; }
+
+  /*
+   * Get the total length of the download.
+   */
+  uint64_t     getTotalLength() const        { return m_totalLength; }
+
+  /*
+   * Get the completed length of the download.
+   */
+  uint64_t     getCompletedLength() const    { return m_completedLength; }
+
+  /*
+   * Get the download speed of the download.
+   */
+  uint64_t     getDownloadSpeed() const      { return m_downloadSpeed; }
+
+  /*
+   * Get the verified length of the download.
+   */
+  uint64_t     getVerifiedLength() const     { return m_verifiedLength; }
+
+  /*
+   * Get the path (local file) of the download.
+   */
+  const std::string&  getPath() const        { return m_path; }
+
+  /*
+   * Get the download uris of the download.
+   */
+  const std::vector<std::string>&  getUris() const { return m_uris; }
 
  protected:
   std::shared_ptr<Aria2> mp_aria;
@@ -83,6 +162,9 @@ class Download {
 /**
  * A tool to download things.
  *
+ * A Downloader manages `Download` using aria2 in the background.
+ * `Downloader` is threadsafe.
+ * However, the returned `Download`s are NOT threadsafe.
  */
 class Downloader
 {
@@ -92,14 +174,41 @@ class Downloader
 
   void close();
 
-  Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
-  Download* getDownload(const std::string& did);
+  /**
+   * Start a new download.
+   *
+   * This method is thread safe and return a pointer to a newly created `Download`.
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param uri: The uri of the thing to download.
+   * @param options: A series of pair <option_name, option_value> to pass to aria.
+   * @return: The newly created Download.
+   */
+  std::shared_ptr<Download> startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
 
-  size_t getNbDownload() { return m_knownDownloads.size(); }
-  std::vector<std::string> getDownloadIds();
+  /**
+   * Get a download corrsponding to a download id (did)
+   * User should call `update` on the returned `Download` to have an accurate status.
+   *
+   * @param did: The download id to search for.
+   * @return: The Download corresponding to did.
+   * @throw: Throw std::out_of_range if did is not found.
+   */
+  std::shared_ptr<Download> getDownload(const std::string& did);
+
+  /**
+   * Get the number of downloads currently managed.
+   */
+  size_t getNbDownload() const;
+
+  /**
+   * Get the ids of the managed downloads.
+   */
+  std::vector<std::string> getDownloadIds() const;
 
  private:
-  std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
+  mutable std::mutex m_lock;
+  std::map<std::string, std::shared_ptr<Download>> m_knownDownloads;
   std::shared_ptr<Aria2> mp_aria;
 };
 }

include/html_dumper.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_HTML_DUMPER_H
+#define KIWIX_HTML_DUMPER_H
+
+#include <string>
+
+#include "library_dumper.h"
+
+namespace kiwix
+{
+
+/**
+ * A class to dump Library in HTML format.
+ */
+class HTMLDumper : public LibraryDumper
+{
+ public:
+  HTMLDumper(const Library* library, const NameMapper* NameMapper);
+  ~HTMLDumper();
+
+
+  /**
+   * Dump library in HTML
+   *
+   * @return HTML content
+   */
+  std::string dumpPlainHTML(kiwix::Filter filter) const;
+};
+
+}
+
+#endif // KIWIX_HTML_DUMPER_H

include/library.h

@@ -106,12 +106,22 @@ class Filter {
     Filter& rejectTags(const Tags& tags);
 
     Filter& category(std::string category);
+
+    /**
+     *  Set the filter to only accept books in the specified language.
+     *
+     *  Multiple languages can be specified as a comma-separated list (in
+     *  which case a book in any of those languages will match).
+     */
     Filter& lang(std::string lang);
+
     Filter& publisher(std::string publisher);
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
     Filter& query(std::string query, bool partial=true);
     Filter& name(std::string name);
+    Filter& clearLang();
+    Filter& clearCategory();
 
     bool hasQuery() const;
     const std::string& getQuery() const { return _query; }
@@ -332,8 +342,8 @@ class Library
   /**
    * Return the current revision of the library.
    *
-   * The revision of the library is updated (incremented by one) only by
-   * the addBook() operation.
+   * The revision of the library is updated (incremented by one) by
+   * the addBook() and removeBookById() operations.
    *
    * @return Current revision of the library.
    */

include/library_dumper.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
+ * Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_LIBRARY_DUMPER_H
+#define KIWIX_LIBRARY_DUMPER_H
+
+#include <string>
+
+
+#include "library.h"
+#include "name_mapper.h"
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+/**
+ * A base class to dump Library in various formats.
+ *
+ */
+class LibraryDumper
+{
+ public:
+  LibraryDumper(const Library* library, const NameMapper* NameMapper);
+  ~LibraryDumper();
+
+  void setLibraryId(const std::string& id) { this->libraryId = id;}
+
+  /**
+   * Set the root location used when generating url.
+   *
+   * @param rootLocation the root location to use.
+   */
+  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
+
+  /**
+   * Set some informations about the search results.
+   *
+   * @param totalResult the total number of results of the search.
+   * @param startIndex the start index of the result.
+   * @param count the number of result of the current set (or page).
+   */
+  void setOpenSearchInfo(int totalResult, int startIndex, int count);
+  
+  /**
+   * Sets user default language
+   * 
+   * @param userLang the user language to be set
+   */
+  void setUserLanguage(std::string userLang) { this->m_userLang = userLang; }
+
+  /**
+   * Get the data of categories
+   */
+  kainjow::mustache::list getCategoryData() const;
+
+  /**
+   * Get the data of languages
+   */
+  kainjow::mustache::list getLanguageData() const;
+
+ protected:
+  const kiwix::Library* const library;
+  const kiwix::NameMapper* const nameMapper;
+  std::string libraryId;
+  std::string rootLocation;
+  std::string m_userLang;
+  int m_totalResults;
+  int m_startIndex;
+  int m_count;
+};
+}
+
+#endif // KIWIX_LIBRARY_DUMPER_H

include/opds_dumper.h

@@ -27,6 +27,8 @@
 #include <pugixml.hpp>
 
 #include "library.h"
+#include "name_mapper.h"
+#include "library_dumper.h"
 
 using namespace std;
 
@@ -37,11 +39,10 @@ namespace kiwix
  * A tool to dump a `Library` into a opds stream.
  *
  */
-class OPDSDumper
+class OPDSDumper : public LibraryDumper
 {
  public:
-  OPDSDumper() = default;
-  OPDSDumper(Library* library);
+  OPDSDumper(const Library* library, const NameMapper* NameMapper);
   ~OPDSDumper();
 
   /**
@@ -84,37 +85,6 @@ class OPDSDumper
    * @return The OPDS feed.
    */
   std::string languagesOPDSFeed() const;
-
-  /**
-   * Set the id of the library.
-   *
-   * @param id the id to use.
-   */
-  void setLibraryId(const std::string& id) { this->libraryId = id;}
-
-  /**
-   * Set the root location used when generating url.
-   *
-   * @param rootLocation the root location to use.
-   */
-  void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
-
-  /**
-   * Set some informations about the search results.
-   *
-   * @param totalResult the total number of results of the search.
-   * @param startIndex the start index of the result.
-   * @param count the number of result of the current set (or page).
-   */
-  void setOpenSearchInfo(int totalResult, int startIndex, int count);
-
- protected:
-  kiwix::Library* library;
-  std::string libraryId;
-  std::string rootLocation;
-  int m_totalResults;
-  int m_startIndex;
-  int m_count;
 };
 }
 

meson.build

@@ -1,5 +1,5 @@
 project('libkiwix', 'cpp',
-  version : '11.0.0',
+  version : '12.0.0',
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -35,7 +35,7 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif

scripts/format_code.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+
+# Compute 'src' path
+SCRIPT_DIR=$(dirname "$0")
+REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
+DIRS="src include"
+
+# Apply formating to all *.cpp and *.h files
+cd "$REPO_DIR"
+for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
+do
+  echo $FILE
+  clang-format -i -style=file "$FILE"
+done

scripts/kiwix-compile-resources

@@ -52,15 +52,21 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
 class Resource:
-    def __init__(self, base_dirs, filename):
-        filename = filename.strip()
+    def __init__(self, base_dirs, filename, cacheid=None):
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
@@ -71,7 +77,7 @@ class Resource:
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -93,6 +99,12 @@ class Resource:
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
             namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
@@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -182,15 +202,17 @@ if __name__ == "__main__":
     parser.add_argument('--source_dir',
                         help="Additional directory where to look for resources.",
                         action='append')
-    parser.add_argument('resource_file',
+    parser.add_argument('resource_files', nargs='+',
                         help='The list of resources to compile.')
     args = parser.parse_args()
 
-    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
-    with open(args.resource_file, 'r') as f:
-        resources = [Resource([base_dir]+source_dir, filename)
-                        for filename in f.readlines()]
+    resources = []
+    for resfile in args.resource_files:
+        base_dir = os.path.dirname(os.path.realpath(resfile))
+        with open(resfile, 'r') as f:
+            resources += [Resource([base_dir]+source_dir, *line.strip().split())
+                            for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-compile-resources.1

@@ -2,7 +2,7 @@
 .SH NAME
 kiwix-compile-resources \- helper to compile and generate some Kiwix resources
 .SH SYNOPSIS
-\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
 .SH DESCRIPTION
 .TP
 resource_file

scripts/kiwix-resources

@@ -99,16 +99,24 @@ def preprocess_resource(resource_path):
             print(preprocessed_content, end='', file=target)
 
 
-def copy_file(src_path, dst_path):
-    with open(src_path, 'rb') as src:
-        with open(dst_path, 'wb') as dst:
-            dst.write(src.read())
+def copy_resource_list_file(src_path, dst_path):
+    with open(src_path, 'r') as src:
+        with open(dst_path, 'w') as dst:
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
 
 def preprocess_resources(resource_file_path):
     resource_filename = os.path.basename(resource_file_path)
     for resource in read_resource_file(resource_file_path):
-        preprocess_resource(resource)
-    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+        if resource.startswith('skin/'):
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

src/aria2.cpp

@@ -24,7 +24,7 @@
 #define LOG_ARIA_ERROR() \
   { \
     std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
-    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+    std::cerr << (curlErrorBuffer[0] ? curlErrorBuffer : curl_easy_strerror(res)) << std::endl; \
   }
 
 namespace kiwix {
@@ -32,9 +32,7 @@ namespace kiwix {
 Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
-  m_secret(getNewRpcSecret()),
-  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
-  mp_curl(nullptr)
+  m_secret(getNewRpcSecret())
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -91,36 +89,32 @@ Aria2::Aria2():
       launchCmd.append(cmd).append(" ");
   }
   mp_aria = Subprocess::run(callCmd);
-  mp_curl = curl_easy_init();
 
-  curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
-  curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
-  curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
+  CURL* p_curl = curl_easy_init();
+  char curlErrorBuffer[CURL_ERROR_SIZE];
+
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
-    m_curlErrorBuffer[0] = 0;
-    auto res = curl_easy_perform(mp_curl);
+    curlErrorBuffer[0] = 0;
+    auto res = curl_easy_perform(p_curl);
     if (res == CURLE_OK) {
       break;
     } else if (watchdog == 1) {
       LOG_ARIA_ERROR();
     }
   }
+  curl_easy_cleanup(p_curl);
   if (!watchdog) {
-    curl_easy_cleanup(mp_curl);
     throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
   }
 }
 
-Aria2::~Aria2()
-{
-  std::unique_lock<std::mutex> lock(m_lock);
-  curl_easy_cleanup(mp_curl);
-}
-
 void Aria2::close()
 {
   saveSession();
@@ -140,20 +134,25 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
   std::stringstream outStream;
   CURLcode res;
   long response_code;
-  {
-    std::unique_lock<std::mutex> lock(m_lock);
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
-    m_curlErrorBuffer[0] = 0;
-    res = curl_easy_perform(mp_curl);
-    if (res != CURLE_OK) {
-      LOG_ARIA_ERROR();
-      throw std::runtime_error("Cannot perform request");
-    }
-    curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  char curlErrorBuffer[CURL_ERROR_SIZE];
+  CURL* p_curl = curl_easy_init();
+  curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
+  curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
+  curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+  curl_easy_setopt(p_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+  curl_easy_setopt(p_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+  curl_easy_setopt(p_curl, CURLOPT_WRITEDATA, &outStream);
+  curlErrorBuffer[0] = 0;
+  res = curl_easy_perform(p_curl);
+  if (res != CURLE_OK) {
+    LOG_ARIA_ERROR();
+    curl_easy_cleanup(p_curl);
+    throw std::runtime_error("Cannot perform request");
   }
+  curl_easy_getinfo(p_curl, CURLINFO_RESPONSE_CODE, &response_code);
+  curl_easy_cleanup(p_curl);
 
   auto responseContent = outStream.str();
   if (response_code != 200) {

src/aria2.h

@@ -12,7 +12,6 @@
 #include "xmlrpc.h"
 
 #include <memory>
-#include <mutex>
 #include <curl/curl.h>
 
 namespace kiwix {
@@ -24,15 +23,11 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
-    std::unique_ptr<char[]> m_curlErrorBuffer;
-    CURL* mp_curl;
-    std::mutex m_lock;
-
     std::string doRequest(const MethodCall& methodCall);
 
   public:
     Aria2();
-    virtual ~Aria2();
+    virtual ~Aria2() = default;
     void close();
 
     std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});

src/book.cpp

@@ -66,7 +66,7 @@ bool Book::update(const kiwix::Book& other)
 void Book::update(const zim::Archive& archive) {
   m_path = archive.getFilename();
   m_pathValid = true;
-  m_id = getArchiveId(archive);
+  m_id = std::string(archive.getUuid());
   m_title = getArchiveTitle(archive);
   m_description = getMetaDescription(archive);
   m_language = getMetaLanguage(archive);
@@ -77,8 +77,8 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = getArchiveArticleCount(archive);
-  m_mediaCount = getArchiveMediaCount(archive);
+  m_articleCount = archive.getArticleCount();
+  m_mediaCount = archive.getMediaCount();
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
   m_illustrations.clear();
@@ -286,4 +286,9 @@ std::string Book::getCategoryFromTags() const
   }
 }
 
+const std::vector<std::string> Book::getLanguages() const
+{
+  return kiwix::split(m_language, ",");
+}
+
 }

src/downloader.cpp

@@ -127,22 +127,24 @@ void Download::cancelDownload()
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  try {
-    for (auto gid : mp_aria->tellActive()) {
-      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
-    }
-  } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
-  }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
       m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-      m_knownDownloads[gid]->updateStatus();
+      m_knownDownloads[gid]->updateStatus(false);
     }
   } catch (std::exception& e) {
     std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      if( m_knownDownloads.find(gid) == m_knownDownloads.end()) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads[gid]->updateStatus(false);
+      }
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
+  }
 }
 
 /* Destructor */
@@ -155,7 +157,8 @@ void Downloader::close()
   mp_aria->close();
 }
 
-std::vector<std::string> Downloader::getDownloadIds() {
+std::vector<std::string> Downloader::getDownloadIds() const {
+  std::unique_lock<std::mutex> lock(m_lock);
   std::vector<std::string> ret;
   for(auto& p:m_knownDownloads) {
     ret.push_back(p.first);
@@ -163,42 +166,46 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
-Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
+std::shared_ptr<Download> Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   for (auto& p: m_knownDownloads) {
     auto& d = p.second;
     auto& uris = d->getUris();
     if (std::find(uris.begin(), uris.end(), uri) != uris.end())
-      return d.get();
+      return d;
   }
   std::vector<std::string> uris = {uri};
   auto gid = mp_aria->addUri(uris, options);
-  m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-  return m_knownDownloads[gid].get();
+  m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+  return m_knownDownloads[gid];
 }
 
-Download* Downloader::getDownload(const std::string& did)
+std::shared_ptr<Download> Downloader::getDownload(const std::string& did)
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   try {
-    m_knownDownloads.at(did).get()->updateStatus(true);
-    return m_knownDownloads.at(did).get();
+    return m_knownDownloads.at(did);
   } catch(std::exception& e) {
-    for (auto gid : mp_aria->tellActive()) {
-      if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
-      }
-    }
     for (auto gid : mp_aria->tellWaiting()) {
       if (gid == did) {
-        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-        m_knownDownloads.at(gid).get()->updateStatus(true);
-        return m_knownDownloads[gid].get();
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
       }
-  }
+    }
+    for (auto gid : mp_aria->tellActive()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
+        return m_knownDownloads[gid];
+      }
+    }
     throw e;
   }
 }
 
+size_t Downloader::getNbDownload() const {
+  std::unique_lock<std::mutex> lock(m_lock);
+  return m_knownDownloads.size();
+}
+
 }

src/html_dumper.cpp

@@ -0,0 +1,120 @@
+#include "html_dumper.h"
+#include "libkiwix-resources.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+#include "tools/regexTools.h"
+#include "server/i18n.h"
+
+namespace kiwix
+{
+
+/* Constructor */
+HTMLDumper::HTMLDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
+{
+}
+/* Destructor */
+HTMLDumper::~HTMLDumper()
+{
+}
+
+namespace {
+
+std::string humanFriendlyTitle(std::string title)
+{
+  std::string humanFriendlyString = replaceRegex(title, "_", " ");
+  humanFriendlyString[0] = toupper(humanFriendlyString[0]);
+  return humanFriendlyString;
+}
+
+kainjow::mustache::list getTagList(std::string tags)
+{
+  const auto tagsList = kiwix::split(tags, ";", true, false);
+  kainjow::mustache::list finalTagList;
+  for (auto tag : tagsList) {
+  if (tag[0] != '_')
+    finalTagList.push_back(kainjow::mustache::object{
+      {"tag", tag}
+    });
+  }
+  return finalTagList;
+}
+
+} // unnamed namespace
+
+std::string HTMLDumper::dumpPlainHTML(kiwix::Filter filter) const
+{
+  kainjow::mustache::list booksData;
+  const auto filteredBooks = library->filter(filter);
+  const auto searchQuery = filter.getQuery();
+  auto languages = getLanguageData();
+  auto categories = getCategoryData();
+
+  for (auto &category : categories) {
+    const auto categoryName = category.get("name")->string_value();
+    if (categoryName == filter.getCategory()) {
+      category["selected"] = true;
+    }
+    category["hf_name"] = humanFriendlyTitle(categoryName);
+  }
+
+  for (auto &language : languages) {
+    if (language.get("lang_code")->string_value() == filter.getLang()) {
+      language["selected"] = true;
+    }
+  }
+
+  for ( const auto& bookId : filteredBooks ) {
+    const auto bookObj = library->getBookById(bookId);
+    const auto bookTitle = bookObj.getTitle();
+    std::string contentId = "";
+    try {
+      contentId = urlEncode(nameMapper->getNameForId(bookId));
+    } catch (...) {}
+    const auto bookDescription = bookObj.getDescription();
+    const auto langCode = bookObj.getCommaSeparatedLanguages();
+    const auto bookIconUrl = rootLocation + "/catalog/v2/illustration/" + bookId +  "/?size=48";
+    const auto tags = bookObj.getTags();
+    const auto downloadAvailable = (bookObj.getUrl() != "");
+    std::string faviconAttr = "style=background-image:url(" + bookIconUrl + ")";
+    
+    booksData.push_back(kainjow::mustache::object{
+      {"id", contentId},
+      {"title", bookTitle},
+      {"description", bookDescription},
+      {"langCode", langCode},
+      {"faviconAttr", faviconAttr},
+      {"tagList", getTagList(tags)},
+      {"downloadAvailable", downloadAvailable}
+    });
+  }
+
+  auto getTranslation = i18n::GetTranslatedStringWithMsgId(m_userLang);
+
+  const auto translations = kainjow::mustache::object{
+                    getTranslation("search"),
+                    getTranslation("download"),
+                    getTranslation("count-of-matching-books", {{"COUNT", to_string(filteredBooks.size())}}),
+                    getTranslation("book-filtering-all-categories"),
+                    getTranslation("book-filtering-all-languages"),
+                    getTranslation("powered-by-kiwix-html"),
+                    getTranslation("welcome-to-kiwix-server"),
+                    getTranslation("preview-book"),
+                    getTranslation("welcome-page-overzealous-filter", {{"URL", "?lang="}})
+  };
+
+  return render_template(
+             RESOURCE::templates::no_js_library_page_html,
+             kainjow::mustache::object{
+               {"root", rootLocation},
+               {"books", booksData },
+               {"searchQuery", searchQuery},
+               {"languages", languages},
+               {"categories", categories},
+               {"noResults", filteredBooks.size() == 0},
+               {"translations", translations}
+             }
+  );
+}
+
+} // namespace kiwix

src/library.cpp

@@ -221,7 +221,11 @@ bool Library::removeBookById(const std::string& id)
   // Having a too big cache is not a problem here (or it would have been before)
   // (And setMaxSize doesn't actually reduce the cache size, extra cached items
   //  will be removed in put or getOrPut).
-  return mp_impl->m_books.erase(id) == 1;
+  const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
+  if ( bookWasRemoved ) {
+    ++mp_impl->m_revision;
+  }
+  return bookWasRemoved;
 }
 
 Library::Revision Library::getRevision() const
@@ -369,12 +373,27 @@ std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
 
 std::vector<std::string> Library::getBooksLanguages() const
 {
-  return getBookPropValueSet(&Book::getLanguage);
+  std::vector<std::string> langs;
+  for ( const auto& langAndCount : getBooksLanguagesWithCounts() ) {
+    langs.push_back(langAndCount.first);
+  }
+  return langs;
 }
 
 Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
 {
-  return getBookAttributeCounts(&Book::getLanguage);
+  std::lock_guard<std::mutex> lock(m_mutex);
+  AttributeCounts langsWithCounts;
+
+  for (const auto& pair: mp_impl->m_books) {
+    const auto& book = pair.second;
+    if (book.getOrigId().empty()) {
+      for ( const auto& lang : book.getLanguages() ) {
+        ++langsWithCounts[lang];
+      }
+    }
+  }
+  return langsWithCounts;
 }
 
 std::vector<std::string> Library::getBooksCategories() const
@@ -436,12 +455,14 @@ void Library::updateBookDB(const Book& book)
 {
   Xapian::Stem stemmer;
   Xapian::TermGenerator indexer;
-  const std::string lang = book.getLanguage();
-  try {
-    stemmer = Xapian::Stem(iso639_3ToXapian(lang));
-    indexer.set_stemmer(stemmer);
-    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
-  } catch (...) {}
+  const auto langs = book.getLanguages();
+  if ( langs.size() == 1 ) {
+    try {
+      stemmer = Xapian::Stem(iso639_3ToXapian(langs[0]));
+      indexer.set_stemmer(stemmer);
+      indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
+    } catch (...) {}
+  }
   Xapian::Document doc;
   indexer.set_document(doc);
 
@@ -456,7 +477,9 @@ void Library::updateBookDB(const Book& book)
   // Index all fields for field-based search
   indexer.index_text(title, 1, "S");
   indexer.index_text(desc,  1, "XD");
-  indexer.index_text(lang,  1, "L");
+  for ( const auto& lang : langs ) {
+    indexer.index_text(lang,  1, "L");
+  }
   indexer.index_text(normalizeText(book.getCreator()),   1, "A");
   indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
   indexer.index_text(normalizeText(book.getName()),      1, "XN");
@@ -531,9 +554,20 @@ Xapian::Query categoryQuery(const std::string& category)
   return Xapian::Query("XC" + normalizeText(category));
 }
 
-Xapian::Query langQuery(const std::string& lang)
+Xapian::Query langQuery(const std::string& commaSeparatedLanguageList)
 {
-  return Xapian::Query("L" + normalizeText(lang));
+  Xapian::Query q;
+  bool firstIteration = true;
+  for ( const auto& lang : kiwix::split(commaSeparatedLanguageList, ",") ) {
+    const Xapian::Query singleLangQuery("L" + normalizeText(lang));
+    if ( firstIteration ) {
+      q = singleLangQuery;
+      firstIteration = false;
+    } else {
+      q = Xapian::Query(Xapian::Query::OP_OR, q, singleLangQuery);
+    }
+  }
+  return q;
 }
 
 Xapian::Query publisherQuery(const std::string& publisher)
@@ -844,6 +878,18 @@ Filter& Filter::name(std::string name)
   return *this;
 }
 
+Filter& Filter::clearLang()
+{
+  activeFilters &= ~LANG;
+  return *this;
+}
+
+Filter& Filter::clearCategory()
+{
+  activeFilters &= ~CATEGORY;
+  return *this;
+}
+
 #define ACTIVE(X) (activeFilters & (X))
 #define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
 bool Filter::hasQuery() const

src/library_dumper.cpp

@@ -0,0 +1,120 @@
+#include "library_dumper.h"
+#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools.h"
+
+namespace kiwix
+{
+/* Constructor */
+LibraryDumper::LibraryDumper(const Library* library, const NameMapper* nameMapper)
+  : library(library),
+    nameMapper(nameMapper)
+{
+}
+/* Destructor */
+LibraryDumper::~LibraryDumper()
+{
+}
+
+void LibraryDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
+{
+  m_totalResults = totalResults;
+  m_startIndex = startIndex,
+  m_count = count;
+}
+
+namespace {
+
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"hbs", "srpskohrvatski"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"},
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
+  }
+}
+
+std::string getLanguageSelfName(const std::string& lang) {
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
+};
+
+} // unnamed namespace
+
+kainjow::mustache::list LibraryDumper::getCategoryData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list categoryData;
+  for ( const auto& category : library->getBooksCategories() ) {
+    const auto urlencodedCategoryName = urlEncode(category);
+    categoryData.push_back(kainjow::mustache::object{
+      {"name", category},
+      {"urlencoded_name",  urlencodedCategoryName},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
+    });
+  }
+  return categoryData;
+}
+
+kainjow::mustache::list LibraryDumper::getLanguageData() const
+{
+  const auto now = gen_date_str();
+  kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
+  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
+    const std::string languageCode = langAndBookCount.first;
+    const int bookCount = langAndBookCount.second;
+    const auto languageSelfName = getLanguageSelfName(languageCode);
+    languageData.push_back(kainjow::mustache::object{
+      {"lang_code",  languageCode},
+      {"lang_self_name", languageSelfName},
+      {"book_count", to_string(bookCount)},
+      {"updated", now},
+      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
+    });
+  }
+  return languageData;
+}
+
+} // namespace kiwix

src/libxml_dumper.cpp

@@ -54,7 +54,7 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
   if (book.getOrigId().empty()) {
     ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
     ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
-    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
+    ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getCommaSeparatedLanguages());
     ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
     ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
     ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
@@ -97,7 +97,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
     auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
     ADD_TEXT_ENTRY(book_node, "id", book.getId());
     ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
-    ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
+    ADD_TEXT_ENTRY(book_node, "language", book.getCommaSeparatedLanguages());
     ADD_TEXT_ENTRY(book_node, "date", book.getDate());
   } catch (...) {
     ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());

src/manager.cpp

@@ -238,7 +238,7 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
     }
 
     if (!checkMetaData
-        || (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
+        || (!book.getTitle().empty() && !book.getLanguages().empty()
             && !book.getDate().empty())) {
       book.setUrl(url);
       manipulator->addBookToLibrary(book);

src/meson.build

@@ -5,6 +5,8 @@ kiwix_sources = [
   'manager.cpp',
   'libxml_dumper.cpp',
   'opds_dumper.cpp',
+  'html_dumper.cpp',
+  'library_dumper.cpp',
   'downloader.cpp',
   'server.cpp',
   'search_renderer.cpp',
@@ -24,7 +26,7 @@ kiwix_sources = [
   'server/request_context.cpp',
   'server/response.cpp',
   'server/internalServer.cpp',
-  'server/internalServer_catalog_v2.cpp',
+  'server/internalServer_catalog.cpp',
   'server/i18n.cpp',
   'opds_catalog.cpp',
   'version.cpp'
@@ -45,7 +47,7 @@ config_h = configure_file(output : 'kiwix_config.h',
                input : 'config.h.in')
 install_headers(config_h, subdir:'kiwix')
 
-kiwixlib = library('kiwix',
+libkiwix = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,

src/opds_dumper.cpp

@@ -20,7 +20,7 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include <mustache.hpp>
 
 #include "tools/stringTools.h"
@@ -30,8 +30,8 @@ namespace kiwix
 {
 
 /* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
-  : library(library)
+OPDSDumper::OPDSDumper(const Library* library, const NameMapper* nameMapper)
+  : LibraryDumper(library, nameMapper)
 {
 }
 /* Destructor */
@@ -39,16 +39,11 @@ OPDSDumper::~OPDSDumper()
 {
 }
 
-void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
-{
-  m_totalResults = totalResults;
-  m_startIndex = startIndex,
-  m_count = count;
-}
-
 namespace
 {
 
+const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
+
 typedef kainjow::mustache::data MustacheData;
 typedef kainjow::mustache::list BooksData;
 typedef kainjow::mustache::list IllustrationInfo;
@@ -69,16 +64,17 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
     return illustrations;
 }
 
-kainjow::mustache::object getSingleBookData(const Book& book)
+std::string fullEntryXML(const Book& book, const std::string& rootLocation, const std::string& contentId)
 {
     const auto bookDate = book.getDate() + "T00:00:00Z";
-    return kainjow::mustache::object{
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
-      {"language", book.getLanguage()},
-      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"language", book.getCommaSeparatedLanguages()},
+      {"content_id",  urlEncode(contentId)},
       {"updated", bookDate}, // XXX: this should be the entry update datetime
       {"book_date", bookDate},
       {"category", book.getCategory()},
@@ -92,27 +88,34 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
+    return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+std::string partialEntryXML(const Book& book, const std::string& rootLocation)
 {
-  auto data = getSingleBookData(book);
-  data["with_xml_header"] = MustacheData(withXMLHeader);
-  data["dump_partial_entries"] = MustacheData(partial);
-  data["endpoint_root"] = endpointRoot;
-  data["root"] = rootLocation;
-  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+    const auto bookDate = book.getDate() + "T00:00:00Z";
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
+      {"endpoint_root", rootLocation + "/catalog/v2"},
+      {"id", book.getId()},
+      {"title", book.getTitle()},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+    };
+    const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
+    return render_template(xmlTemplate, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library, const NameMapper* nameMapper, const std::vector<std::string>& bookIds, const std::string& rootLocation, bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
     try {
       const Book book = library->getBookByIdThreadSafe(bookId);
-      booksData.push_back(kainjow::mustache::object{
-          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
-      });
+      const std::string contentId = nameMapper->getNameForId(bookId);
+      const auto entryXML = partial
+                          ? partialEntryXML(book, rootLocation)
+                          : fullEntryXML(book, rootLocation, contentId);
+      booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
     } catch ( const std::out_of_range& ) {
       // the book was removed from the library since its id was obtained
       // ignore it
@@ -122,64 +125,11 @@ BooksData getBooksData(const Library* library, const std::vector<std::string>& b
   return booksData;
 }
 
-std::map<std::string, std::string> iso639_3 = {
-  {"atj", "atikamekw"},
-  {"azb", "آذربایجان دیلی"},
-  {"bcl", "central bikol"},
-  {"bgs", "tagabawa"},
-  {"bxr", "буряад хэлэн"},
-  {"cbk", "chavacano"},
-  {"cdo", "閩東語"},
-  {"dag", "Dagbani"},
-  {"diq", "dimli"},
-  {"dty", "डोटेली"},
-  {"eml", "emiliân-rumagnōl"},
-  {"fbs", "српскохрватски"},
-  {"ido", "ido"},
-  {"kbp", "kabɩyɛ"},
-  {"kld", "Gamilaraay"},
-  {"lbe", "лакку маз"},
-  {"lbj", "ལ་དྭགས་སྐད་"},
-  {"map", "Austronesian"},
-  {"mhr", "марий йылме"},
-  {"mnw", "ဘာသာမန်"},
-  {"myn", "mayan"},
-  {"nah", "nahuatl"},
-  {"nai", "north American Indian"},
-  {"nds", "plattdütsch"},
-  {"nrm", "bhasa narom"},
-  {"olo", "livvi"},
-  {"pih", "Pitcairn-Norfolk"},
-  {"pnb", "Western Panjabi"},
-  {"rmr", "Caló"},
-  {"rmy", "romani shib"},
-  {"roa", "romance languages"},
-  {"twi", "twi"}
-};
-
-std::once_flag fillLanguagesFlag;
-
-void fillLanguagesMap()
-{
-  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
-    const ICULanguageInfo lang(*icuLangPtr);
-    iso639_3.insert({lang.iso3Code(), lang.selfName()});
-  }
-}
-
-std::string getLanguageSelfName(const std::string& lang) {
-  const auto itr = iso639_3.find(lang);
-  if (itr != iso639_3.end()) {
-    return itr->second;
-  }
-  return lang;
-};
-
 } // unnamed namespace
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
@@ -197,20 +147,20 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
+  const std::string url = endpoint + (query.empty() ? "" : "?" + query);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
      {"filter", onlyAsNonEmptyMustacheValue(query)},
-     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"self_url", url},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
-     {"books", booksData },
-     {"dump_partial_entries", MustacheData(partial)}
+     {"books", booksData }
   };
 
   return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
@@ -218,23 +168,17 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+  const auto book = library->getBookById(bookId);
+  const std::string contentId = nameMapper->getNameForId(bookId);
+  return XML_HEADER
+         + "\n"
+         + fullEntryXML(book, rootLocation, contentId);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list categoryData;
-  for ( const auto& category : library->getBooksCategories() ) {
-    const auto urlencodedCategoryName = urlEncode(category);
-    categoryData.push_back(kainjow::mustache::object{
-      {"name", category},
-      {"urlencoded_name",  urlencodedCategoryName},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
-    });
-  }
-
+  kainjow::mustache::list categoryData = getCategoryData();
   return render_template(
              RESOURCE::templates::catalog_v2_categories_xml,
              kainjow::mustache::object{
@@ -249,21 +193,7 @@ std::string OPDSDumper::categoriesOPDSFeed() const
 std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
-  kainjow::mustache::list languageData;
-  std::call_once(fillLanguagesFlag, fillLanguagesMap);
-  for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
-    const std::string languageCode = langAndBookCount.first;
-    const int bookCount = langAndBookCount.second;
-    const auto languageSelfName = getLanguageSelfName(languageCode);
-    languageData.push_back(kainjow::mustache::object{
-      {"lang_code",  languageCode},
-      {"lang_self_name", languageSelfName},
-      {"book_count", to_string(bookCount)},
-      {"updated", now},
-      {"id", gen_uuid(libraryId + "/languages/" + languageCode)}
-    });
-  }
-
+  kainjow::mustache::list languageData = getLanguageData();
   return render_template(
              RESOURCE::templates::catalog_v2_languages_xml,
              kainjow::mustache::object{

src/search_renderer.cpp

@@ -29,7 +29,7 @@
 #include <zim/search.h>
 
 #include <mustache.hpp>
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include "tools/stringTools.h"
 
 namespace kiwix
@@ -94,7 +94,7 @@ kainjow::mustache::data buildQueryData
   kainjow::mustache::data query;
   query.set("pattern", kiwix::encodeDiples(pattern));
   std::ostringstream ss;
-  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
   ss << "&" << bookQuery;
   query.set("unpaginatedQuery", ss.str());
   auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
@@ -166,14 +166,15 @@ kainjow::mustache::data buildPagination(
 
 std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
 {
-  const std::string absPathPrefix = protocolPrefix + "content/";
+  const std::string absPathPrefix = protocolPrefix;
   // Build the results list
   kainjow::mustache::data items{kainjow::mustache::data::type::list};
   for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
-    std::string zim_id(it.getZimId());
+    const std::string zim_id(it.getZimId());
+    const auto path = mp_nameMapper->getNameForId(zim_id) + "/" + it.getPath();
     result.set("title", it.getTitle());
-    result.set("absolutePath", absPathPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
+    result.set("absolutePath", absPathPrefix + urlEncode(path));
     result.set("snippet", it.getSnippet());
     if (mp_library) {
       result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
@@ -206,7 +207,7 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
 
 
   kainjow::mustache::data allData;
-  allData.set("protocolPrefix", protocolPrefix);
+  allData.set("searchProtocolPrefix", searchProtocolPrefix);
   allData.set("results", results);
   allData.set("pagination", pagination);
   allData.set("query", query);

src/server/etag.cpp

@@ -37,11 +37,11 @@ namespace {
 // into the ETag for ETag::Option opt.
 // IMPORTANT: The characters in all_options must come in sorted order (so that
 // IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
+const char all_options[] = "Zz";
 
 static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
 
-bool isValidServerId(const std::string& s)
+bool isValidETagBody(const std::string& s)
 {
   return !s.empty() && s.find_first_of("\"/") == std::string::npos;
 }
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
 
 std::string ETag::get_etag() const
 {
-  if ( m_serverId.empty() )
+  if ( m_body.empty() )
     return std::string();
 
-  return "\"" + m_serverId + "/" + m_options + "\"";
+  return "\"" + m_body + "/" + m_options + "\"";
 }
 
-ETag::ETag(const std::string& serverId, const std::string& options)
+ETag::ETag(const std::string& body, const std::string& options)
 {
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
   {
-    m_serverId = serverId;
+    m_body = body;
     m_options = options;
   }
 }
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
   return ETag(s.substr(0, i), s.substr(i+1));
 }
 
-ETag ETag::match(const std::string& etags, const std::string& server_id)
+ETag ETag::match(const std::string& etags, const std::string& body)
 {
   std::istringstream ss(etags);
   std::string etag_str;
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
       etag_str.pop_back();
 
     const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
+    if ( etag && etag.m_body == body )
       return etag;
   }
 

src/server/etag.h

@@ -28,10 +28,11 @@ namespace kiwix {
 // The ETag string used by Kiwix server (more precisely, its value inside the
 // double quotes) consists of two parts:
 //
-// 1. ServerId - The string obtained on server start up
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
 //
-// 2. Options -  Zero or more characters encoding the values of some of the
-//               headers of the response
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
+//               the values of some of the headers of the response
 //
 // The two parts are separated with a slash (/) symbol (which is always present,
 // even when the the options part is empty). Neither portion of a Kiwix ETag
@@ -40,7 +41,7 @@ namespace kiwix {
 //
 //   "abcdefghijklmn/"
 //   "1234567890/z"
-//   "1234567890/cz"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
 //
 // The options part of the Kiwix ETag allows to correctly set the required
 // headers when responding to a conditional If-None-Match request with a 304
@@ -51,7 +52,7 @@ class ETag
 {
   public: // types
     enum Option {
-      CACHEABLE_ENTITY,
+      ZIM_CONTENT,
       COMPRESSED_CONTENT,
       OPTION_COUNT
     };
@@ -59,10 +60,10 @@ class ETag
   public: // functions
     ETag() {}
 
-    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_body(const std::string& s) { m_body = s; }
     void set_option(Option opt);
 
-    explicit operator bool() const { return !m_serverId.empty(); }
+    explicit operator bool() const { return !m_body.empty(); }
 
     bool get_option(Option opt) const;
     std::string get_etag() const;
@@ -76,7 +77,7 @@ class ETag
     static ETag parse(std::string s);
 
   private: // data
-    std::string m_serverId;
+    std::string m_body;
     std::string m_options;
 };
 

src/server/i18n.cpp

@@ -70,6 +70,14 @@ public: // functions
     return s;
   }
 
+  size_t getStringCount(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang)->entryCount;
+    } catch(const std::out_of_range&) {
+      return 0;
+    }
+  }
+
 private: // functions
   const I18nStringTable* getStringsFor(const std::string& lang) const {
     try {
@@ -84,13 +92,17 @@ private: // data
   const I18nStringTable* enStrings;
 };
 
+const I18nStringDB& getStringDb()
+{
+  static const I18nStringDB stringDb;
+  return stringDb;
+}
+
 } // unnamed namespace
 
 std::string getTranslatedString(const std::string& lang, const std::string& key)
 {
-  static const I18nStringDB stringDb;
-
-  return stringDb.get(lang, key);
+  return getStringDb().get(lang, key);
 }
 
 namespace i18n
@@ -111,4 +123,70 @@ std::string ParameterizedMessage::getText(const std::string& lang) const
   return i18n::expandParameterizedString(lang, msgId, params);
 }
 
+namespace
+{
+
+LangPreference parseSingleLanguagePreference(const std::string& s)
+{
+  const size_t langStart = s.find_first_not_of(" \t\n");
+  if ( langStart == std::string::npos ) {
+    return {"", 0};
+  }
+
+  const size_t langEnd = s.find(';', langStart);
+  if ( langEnd == std::string::npos ) {
+    return {s.substr(langStart), 1};
+  }
+
+  const std::string lang = s.substr(langStart, langEnd - langStart);
+  // We don't care about langEnd == langStart which will result in an empty
+  // language name - it will be dismissed by parseUserLanguagePreferences()
+
+  float q = 1.0;
+  int nCharsScanned;
+  if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
+       && langEnd + 1 + nCharsScanned == s.size() ) {
+    return {lang, q};
+  }
+
+  return {"", 0};
+}
+
+} // unnamed namespace
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s)
+{
+  UserLangPreferences result;
+  std::istringstream iss(s);
+  std::string singleLangPrefStr;
+  while ( std::getline(iss, singleLangPrefStr, ',') )
+  {
+    const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
+    if ( !langPref.lang.empty() && langPref.preference > 0 ) {
+      result.push_back(langPref);
+    }
+  }
+
+  return result;
+}
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
+{
+  if ( prefs.empty() ) {
+    return "en";
+  }
+
+  std::string bestLangSoFar("en");
+  float bestScoreSoFar = 0;
+  const auto& stringDb = getStringDb();
+  for ( const auto& entry : prefs ) {
+    const float score = entry.preference * stringDb.getStringCount(entry.lang);
+    if ( score > bestScoreSoFar ) {
+      bestScoreSoFar = score;
+      bestLangSoFar = entry.lang;
+    }
+  }
+  return bestLangSoFar;
+}
+
 } // namespace kiwix

src/server/i18n.h

@@ -69,6 +69,28 @@ private:
   const std::string m_lang;
 };
 
+class GetTranslatedStringWithMsgId
+{
+  typedef kainjow::mustache::basic_data<std::string> MustacheString;
+  typedef std::pair<std::string, MustacheString> MsgIdAndTranslation;
+
+public:
+  explicit GetTranslatedStringWithMsgId(const std::string& lang) : m_lang(lang) {}
+
+  MsgIdAndTranslation operator()(const std::string& key) const
+  {
+    return {key, getTranslatedString(m_lang, key)};
+  }
+
+  MsgIdAndTranslation operator()(const std::string& key, const Parameters& params) const
+  {
+    return {key, expandParameterizedString(m_lang, key, params)};
+  }
+
+private:
+  const std::string m_lang;
+};
+
 } // namespace i18n
 
 struct ParameterizedMessage
@@ -89,6 +111,18 @@ private: // data
   const Parameters  params;
 };
 
+struct LangPreference
+{
+  const std::string lang;
+  const float preference;
+};
+
+typedef std::vector<LangPreference> UserLangPreferences;
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s);
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs);
+
 } // namespace kiwix
 
 #endif // KIWIX_SERVER_I18N

src/server/internalServer.cpp

@@ -53,6 +53,7 @@ extern "C" {
 #include "name_mapper.h"
 #include "search_renderer.h"
 #include "opds_dumper.h"
+#include "html_dumper.h"
 #include "i18n.h"
 
 #include <zim/uuid.h>
@@ -68,7 +69,7 @@ extern "C" {
 #include <vector>
 #include <chrono>
 #include <fstream>
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #ifndef _WIN32
 # include <arpa/inet.h>
@@ -77,7 +78,6 @@ extern "C" {
 #include "request_context.h"
 #include "response.h"
 
-#define MAX_SEARCH_LEN 140
 #define DEFAULT_CACHE_SIZE 2
 
 namespace kiwix {
@@ -95,6 +95,22 @@ inline std::string normalizeRootUrl(std::string rootUrl)
   return rootUrl.empty() ? rootUrl : "/" + rootUrl;
 }
 
+std::string
+fullURL2LocalURL(const std::string& fullUrl, const std::string& rootLocation)
+{
+  if ( kiwix::startsWith(fullUrl, rootLocation) ) {
+    return fullUrl.substr(rootLocation.size());
+  } else {
+    return "INVALID URL";
+  }
+}
+
+std::string getSearchComponent(const RequestContext& request)
+{
+    const std::string query = request.get_query();
+    return query.empty() ? query : "?" + query;
+}
+
 Filter get_search_filter(const RequestContext& request, const std::string& prefix="")
 {
     auto filter = kiwix::Filter().valid(true).local(true);
@@ -139,15 +155,6 @@ std::string renderUrl(const std::string& root, const std::string& urlTemplate)
   return url;
 }
 
-std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
-{
-  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
-               {
-                  {"SEARCH_TERMS", queryString}
-               }
-         );
-}
-
 ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
 {
   return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
@@ -212,12 +219,41 @@ void checkBookNumber(const Library::BookIdSet& bookIds, size_t limit) {
   }
 }
 
+typedef std::set<std::string> Languages;
+
+Languages getLanguages(const Library& lib, const Library::BookIdSet& bookIds) {
+  Languages langs;
+  for ( const auto& b : bookIds ) {
+    const auto bookLangs = lib.getBookById(b).getLanguages();
+    langs.insert(bookLangs.begin(), bookLangs.end());
+  }
+  return langs;
+}
+
 struct CustomizedResourceData
 {
   std::string mimeType;
   std::string resourceFilePath;
 };
 
+bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request)
+{
+  return response.getReturnCode() == MHD_HTTP_OK
+      && response.get_kind() == Response::DYNAMIC_CONTENT
+      && request.get_url() != "/random";
+}
+
+ETag
+get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody)
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, etagBody);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
 } // unnamed namespace
 
 std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
@@ -289,6 +325,10 @@ SearchInfo InternalServer::getSearchInfo(const RequestContext& request) const
 {
   auto bookIds = selectBooks(request);
   checkBookNumber(bookIds.second, m_multizimSearchLimit);
+  if ( getLanguages(*mp_library, bookIds.second).size() != 1 ) {
+    throw Error(nonParameterizedMessage("confusion-of-tongues"));
+  }
+
   auto pattern = request.get_optional_param<std::string>("pattern", "");
   GeoQuery geoQuery;
 
@@ -382,6 +422,7 @@ InternalServer::InternalServer(Library* library,
   m_addr(addr),
   m_port(port),
   m_root(normalizeRootUrl(root)),
+  m_rootPrefixOfDecodedURL(m_root),
   m_nbThreads(nbThreads),
   m_multizimSearchLimit(multizimSearchLimit),
   m_verbose(verbose),
@@ -396,7 +437,9 @@ InternalServer::InternalServer(Library* library,
   searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
   suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
   m_customizedResources(new CustomizedResources)
-{}
+{
+  m_root = urlEncode(m_root);
+}
 
 InternalServer::~InternalServer() = default;
 
@@ -443,7 +486,6 @@ bool InternalServer::start() {
   }
   auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
   m_server_id = kiwix::to_string(server_start_time.count());
-  m_library_id = m_server_id;
   return true;
 }
 
@@ -473,7 +515,7 @@ static MHD_Result staticHandlerCallback(void* cls,
 }
 
 MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                           const char* url,
+                                           const char* fullUrl,
                                            const char* method,
                                            const char* version,
                                            const char* upload_data,
@@ -484,8 +526,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
   if (m_verbose.load() ) {
     printf("======================\n");
     printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
+    printf("full_url  : %s\n", fullUrl);
   }
+
+  const auto url = fullURL2LocalURL(fullUrl, m_rootPrefixOfDecodedURL);
   RequestContext request(connection, m_root, url, method, version);
 
   if (m_verbose.load() ) {
@@ -506,13 +550,14 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
     printf("========== INTERNAL ERROR !! ============\n");
     if (!m_verbose.load()) {
       printf("Requesting : \n");
-      printf("full_url : %s\n", url);
+      printf("full_url : %s\n", fullUrl);
       request.print_debug_info();
     }
   }
 
-  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response->set_server_id(m_server_id);
+  if ( responseMustBeETaggedWithLibraryId(*response, request) ) {
+    response->set_etag_body(getLibraryId());
+  }
 
   auto ret = response->send(request, connection);
   auto end_time = std::chrono::steady_clock::now();
@@ -534,6 +579,11 @@ bool isEndpointUrl(const std::string& url, const std::string& endpoint)
 
 } // unnamed namespace
 
+std::string InternalServer::getLibraryId() const
+{
+  return m_server_id + "." + kiwix::to_string(mp_library->getRevision());
+}
+
 std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
 {
   try {
@@ -542,7 +592,14 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
              + urlNotFoundMsg;
     }
 
-    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( request.get_url() == "" ) {
+      // Redirect /ROOT_LOCATION to /ROOT_LOCATION/ (note the added slash)
+      // so that relative URLs are resolved correctly
+      const std::string query = getSearchComponent(request);
+      return Response::build_redirect(*this, m_root + "/" + query);
+    }
+
+    const ETag etag = get_matching_if_none_match_etag(request, getLibraryId());
     if ( etag )
       return Response::build_304(*this, etag);
 
@@ -553,9 +610,12 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (url == "/" )
       return build_homepage(request);
 
-    if (isEndpointUrl(url, "skin"))
+    if (isEndpointUrl(url, "viewer") || isEndpointUrl(url, "skin"))
       return handle_skin(request);
 
+    if (url == "/viewer_settings.js")
+      return handle_viewer_settings(request);
+
     if (isEndpointUrl(url, "content"))
       return handle_content(request);
 
@@ -568,6 +628,9 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (isEndpointUrl(url, "search"))
       return handle_search(request);
 
+    if (isEndpointUrl(url, "nojs"))
+      return handle_no_js(request);
+
     if (isEndpointUrl(url, "suggest"))
      return handle_suggest(request);
 
@@ -577,14 +640,9 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (isEndpointUrl(url, "catch"))
       return handle_catch(request);
 
-    if (isEndpointUrl(url, "widget"))
-      return handle_widget(request);
-
-    std::string contentUrl = m_root + "/content" + url;
-    const std::string query = request.get_query();
-    if ( ! query.empty() )
-      contentUrl += "?" + query;
-    return Response::build_redirect(*this, contentUrl);
+    const std::string contentUrl = m_root + "/content" + urlEncode(url);
+    const std::string query = getSearchComponent(request);
+    return Response::build_redirect(*this, contentUrl + query);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
     return HTTP500Response(*this, request)
@@ -603,36 +661,30 @@ MustacheData InternalServer::get_default_data() const
   return data;
 }
 
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
 std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
 {
-  return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8", true);
+  return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8");
 }
 
 /**
  * Archive and Zim handlers begin
  **/
 
+class InternalServer::LockableSuggestionSearcher : public zim::SuggestionSearcher
+{
+  public:
+    explicit LockableSuggestionSearcher(const zim::Archive& archive)
+      : zim::SuggestionSearcher(archive)
+    {}
+
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~LockableSuggestionSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
+
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -656,8 +708,7 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
 
   if (archive == nullptr) {
     return HTTP404Response(*this, request)
-           + noSuchBookErrorMsg(bookName)
-           + TaskbarInfo(bookName);
+           + noSuchBookErrorMsg(bookName);
   }
 
   const auto queryString = request.get_optional_param("term", std::string());
@@ -671,65 +722,149 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
     printf("Searching suggestions for: \"%s\"\n", queryString.c_str());
   }
 
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
+  Suggestions results;
 
   /* Get the suggestions */
   auto searcher = suggestionSearcherCache.getOrPut(bookId,
-    [=](){ return make_shared<zim::SuggestionSearcher>(*archive); }
+    [=](){ return make_shared<LockableSuggestionSearcher>(*archive); }
   );
+  const auto lock(searcher->getLock());
   auto search = searcher->suggest(queryString);
   auto srs = search.getResults(start, count);
 
   for(auto& suggestion: srs) {
-    MustacheData result;
-    result.set("label", suggestion.getTitle());
-
-    if (suggestion.hasSnippet()) {
-      result.set("label", suggestion.getSnippet());
-    }
-
-    result.set("value", suggestion.getTitle());
-    result.set("kind", "path");
-    result.set("path", suggestion.getPath());
-    result.set("first", first);
-    first = false;
-    results.push_back(result);
+    results.add(suggestion);
   }
 
 
   /* Propose the fulltext search if possible */
   if (archive->hasFulltextIndex()) {
-    MustacheData result;
-    const auto lang = request.get_user_language();
-    result.set("label", makeFulltextSearchSuggestion(lang, queryString));
-    result.set("value", queryString + " ");
-    result.set("kind", "pattern");
-    result.set("first", first);
-    results.push_back(result);
+    results.addFTSearchSuggestion(request.get_user_language(), queryString);
   }
 
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
-  return std::move(response);
+  return ContentResponse::build(*this, results.getJSON(), "application/json; charset=utf-8");
 }
 
+std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_viewer_settings\n");
+  }
+
+  const kainjow::mustache::object data{
+    {"enable_toolbar", m_withTaskbar ? "true" : "false" },
+    {"enable_link_blocking", m_blockExternalLinks ? "true" : "false" },
+    {"enable_library_button", m_withLibraryButton ? "true" : "false" }
+  };
+  return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
+}
+
+std::string InternalServer::getNoJSDownloadPageHTML(const std::string& bookId, const std::string& userLang) const
+{
+  const auto book = mp_library->getBookById(bookId);
+  auto bookUrl = kiwix::stripSuffix(book.getUrl(), ".meta4");
+  auto getTranslation = i18n::GetTranslatedStringWithMsgId(userLang);
+  const auto translations = kainjow::mustache::object{
+                            getTranslation("download-links-heading", {{"BOOK_TITLE", book.getTitle()}}),
+                            getTranslation("download-links-title"),
+                            getTranslation("direct-download-link-text"),
+                            getTranslation("hash-download-link-text"),
+                            getTranslation("magnet-link-text"),
+                            getTranslation("torrent-download-link-text")
+  };
+
+  return render_template(
+             RESOURCE::templates::no_js_download_html,
+             kainjow::mustache::object{
+               {"url", bookUrl},
+               {"translations", translations}
+             }
+  );
+}
+
+std::unique_ptr<Response> InternalServer::handle_no_js(const RequestContext& request)
+{
+  const auto url = request.get_url();
+  const auto urlParts = kiwix::split(url, "/", true, false);
+  HTMLDumper htmlDumper(mp_library, mp_nameMapper);
+  htmlDumper.setRootLocation(m_root);
+  htmlDumper.setLibraryId(getLibraryId());
+  auto userLang = request.get_user_language();
+  htmlDumper.setUserLanguage(userLang);
+  std::string content;
+
+  if (urlParts.size() == 1) {
+    auto filter = get_search_filter(request);
+    try {
+      if (request.get_argument("category") == "") {
+        filter.clearCategory();
+      }
+    } catch (...) {}
+    try {
+      if (request.get_argument("lang") == "") {
+        filter.clearLang();
+      }
+    } catch (...) {}
+    content = htmlDumper.dumpPlainHTML(filter);
+  } else if ((urlParts.size() == 3) && (urlParts[1] == "download")) {
+    try {
+      const auto bookId = mp_nameMapper->getIdForName(urlParts[2]);
+      content = getNoJSDownloadPageHTML(bookId, userLang);
+    } catch (const std::out_of_range&) {
+      return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+    }
+  } else {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  return ContentResponse::build(
+             *this,
+             content,
+             "text/html; charset=utf-8"
+  );
+}
+
+namespace
+{
+
+Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid)
+{
+  if ( expectedCacheid == nullptr )
+    return Response::DYNAMIC_CONTENT;
+
+  try {
+    if ( expectedCacheid != req.get_argument("cacheid") )
+      throw ResourceNotFound("Wrong cacheid");
+    return Response::STATIC_RESOURCE;
+  } catch( const std::out_of_range& ) {
+    return Response::DYNAMIC_CONTENT;
+  }
+}
+
+} // unnamed namespace
+
 std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_skin\n");
   }
 
-  auto resourceName = request.get_url().substr(1);
+  const bool isRequestForViewer = request.get_url() == "/viewer";
+  auto resourceName = isRequestForViewer
+                    ? "viewer.html"
+                    : request.get_url().substr(1);
+
+  const char* const resourceCacheId = getResourceCacheId(resourceName);
+
   try {
+    const auto accessType = staticResourceAccessType(request, resourceCacheId);
     auto response = ContentResponse::build(
         *this,
         getResource(resourceName),
         getMimeTypeForFile(resourceName));
-    response->set_cacheable();
+    response->set_kind(accessType);
     return std::move(response);
   } catch (const ResourceNotFound& e) {
     return HTTP404Response(*this, request)
@@ -756,75 +891,7 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
   }
 
   try {
-    auto searchInfo = getSearchInfo(request);
-    auto bookIds = searchInfo.getBookIds();
-
-    /* Make the search */
-    // Try to get a search from the searchInfo, else build it
-    auto searcher = mp_library->getSearcherByIds(bookIds);
-    auto lock(searcher->getLock());
-
-    std::shared_ptr<zim::Search> search;
-    try {
-      search = searchCache.getOrPut(searchInfo,
-        [=](){
-          return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
-        }
-      );
-    } catch(std::runtime_error& e) {
-      // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
-      // (in case of zim file not containing a index)
-      const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
-      HTTPErrorResponse response(*this, request, MHD_HTTP_NOT_FOUND,
-                                 "fulltext-search-unavailable",
-                                 "404-page-heading",
-                                 cssUrl);
-      response += nonParameterizedMessage("no-search-results");
-      if(bookIds.size() == 1) {
-        auto bookId = *bookIds.begin();
-        auto bookName = mp_nameMapper->getNameForId(bookId);
-        response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
-      }
-      return response;
-    }
-
-    auto start = 1;
-    try {
-      start = request.get_argument<unsigned int>("start");
-    } catch (const std::exception&) {}
-    start = max(1, start);
-
-    auto pageLength = 25;
-    try {
-      pageLength = request.get_argument<unsigned int>("pageLength");
-    } catch (const std::exception&) {}
-    if (pageLength > MAX_SEARCH_LEN) {
-      pageLength = MAX_SEARCH_LEN;
-    }
-    if (pageLength == 0) {
-      pageLength = 25;
-    }
-
-    /* Get the results */
-    SearchRenderer renderer(search->getResults(start-1, pageLength), mp_nameMapper, mp_library, start,
-                            search->getEstimatedMatches());
-    renderer.setSearchPattern(searchInfo.pattern);
-    renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search");
-    renderer.setPageLength(pageLength);
-    if (request.get_requested_format() == "xml") {
-      return ContentResponse::build(*this, renderer.getXml(), "application/rss+xml; charset=utf-8",
-      /*isHomePage =*/false,
-      /*raw =*/true);
-    }
-    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    if(bookIds.size() == 1) {
-      auto bookId = *bookIds.begin();
-      auto bookName = mp_nameMapper->getNameForId(bookId);
-      response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
-    }
-    return std::move(response);
+    return handle_search_request(request);
   } catch (const Error& e) {
     return HTTP400Response(*this, request)
       + invalidUrlMsg
@@ -832,6 +899,87 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
   }
 }
 
+namespace
+{
+
+unsigned getSearchPageSize(const RequestContext& r)
+{
+  const auto DEFAULT_PAGE_LEN = 25u;
+  const auto MAX_PAGE_LEN = 140u;
+
+  const auto pageLength = r.get_optional_param("pageLength", DEFAULT_PAGE_LEN);
+  return pageLength == 0
+       ? DEFAULT_PAGE_LEN
+       : min(MAX_PAGE_LEN, pageLength);
+}
+
+} // unnamed namespace
+
+std::unique_ptr<Response> InternalServer::handle_search_request(const RequestContext& request)
+{
+  auto searchInfo = getSearchInfo(request);
+  auto bookIds = searchInfo.getBookIds();
+
+  /* Make the search */
+  // Try to get a search from the searchInfo, else build it
+  auto searcher = mp_library->getSearcherByIds(bookIds);
+  auto lock(searcher->getLock());
+
+  std::shared_ptr<zim::Search> search;
+  try {
+    search = searchCache.getOrPut(searchInfo,
+      [=](){
+        return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
+      }
+    );
+  } catch(std::runtime_error& e) {
+    // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
+    // (in case of zim file not containing a index)
+    const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
+    HTTPErrorResponse response(*this, request, MHD_HTTP_NOT_FOUND,
+                               "fulltext-search-unavailable",
+                               "404-page-heading",
+                               cssUrl);
+    response += nonParameterizedMessage("no-search-results");
+    // XXX: Now this has to be handled by the iframe-based viewer which
+    // XXX: has to resolve if the book selection resulted in a single book.
+    /*
+    if(bookIds.size() == 1) {
+      auto bookId = *bookIds.begin();
+      auto bookName = mp_nameMapper->getNameForId(bookId);
+      response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
+    }
+    */
+    return response;
+  }
+
+  const auto start = max(1u, request.get_optional_param("start", 1u));
+  const auto pageLength = getSearchPageSize(request);
+
+  /* Get the results */
+  SearchRenderer renderer(search->getResults(start-1, pageLength), mp_nameMapper, mp_library, start,
+                          search->getEstimatedMatches());
+  renderer.setSearchPattern(searchInfo.pattern);
+  renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
+  renderer.setProtocolPrefix(m_root + "/content/");
+  renderer.setSearchProtocolPrefix(m_root + "/search");
+  renderer.setPageLength(pageLength);
+  if (request.get_requested_format() == "xml") {
+    return ContentResponse::build(*this, renderer.getXml(), "application/rss+xml; charset=utf-8");
+  }
+  auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+  // XXX: Now this has to be handled by the iframe-based viewer which
+  // XXX: has to resolve if the book selection resulted in a single book.
+  /*
+  if(bookIds.size() == 1) {
+    auto bookId = *bookIds.begin();
+    auto bookName = mp_nameMapper->getNameForId(bookId);
+    response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
+  }
+  */
+  return std::move(response);
+}
+
 std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -855,8 +1003,7 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
 
   if (archive == nullptr) {
     return HTTP404Response(*this, request)
-           + noSuchBookErrorMsg(bookName)
-           + TaskbarInfo(bookName);
+           + noSuchBookErrorMsg(bookName);
   }
 
   try {
@@ -864,16 +1011,10 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
     return build_redirect(bookName, getFinalItem(*archive, entry));
   } catch(zim::EntryNotFound& e) {
     return HTTP404Response(*this, request)
-           + nonParameterizedMessage("random-article-failure")
-           + TaskbarInfo(bookName, archive.get());
+           + nonParameterizedMessage("random-article-failure");
   }
 }
 
-std::unique_ptr<Response> InternalServer::handle_widget(const RequestContext& request)
-{
-  return ContentResponse::build(*this, RESOURCE::templates::widget_html, get_default_data(), "text/html; charset=utf-8", true);
-}
-
 std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
 {
   std::string source = "";
@@ -905,69 +1046,16 @@ std::unique_ptr<Response> InternalServer::handle_catch(const RequestContext& req
          + urlNotFoundMsg;
 }
 
-std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
-  }
-
-  if (url == "v2") {
-    return handle_catalog_v2(request);
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return HTTP404Response(*this, request)
-           + urlNotFoundMsg;
-  }
-
-  if (url == "searchdescription.xml") {
-    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
-    return std::move(response);
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper(mp_library);
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    bookIdsToDump = search_catalog(request, opdsDumper);
-    uuid = zim::Uuid::generate();
-  }
-
-  auto response = ContentResponse::build(
-      *this,
-      opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()),
-      "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
-  return std::move(response);
-}
-
 std::vector<std::string>
 InternalServer::search_catalog(const RequestContext& request,
                                kiwix::OPDSDumper& opdsDumper)
 {
     const auto filter = get_search_filter(request);
-    const std::string q = filter.hasQuery()
-                        ? filter.getQuery()
-                        : "<Empty query>";
     std::vector<std::string> bookIdsToDump = mp_library->filter(filter);
     const auto totalResults = bookIdsToDump.size();
-    const size_t count = request.get_optional_param("count", 10UL);
+    const long count = request.get_optional_param("count", 10L);
     const size_t startIndex = request.get_optional_param("start", 0UL);
-    const size_t intendedCount = count > 0 ? count : bookIdsToDump.size();
+    const size_t intendedCount = count >= 0 ? count : bookIdsToDump.size();
     bookIdsToDump = subrange(bookIdsToDump, startIndex, intendedCount);
     opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
     return bookIdsToDump;
@@ -985,14 +1073,38 @@ ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::s
                               });
 }
 
+///////////////////////////////////////////////////////////////////////////////
+// The content security policy below is set on responses to the /content
+// endpoint in order to prevent the ZIM content from interfering with the
+// viewer (e.g. breaking out of the viewer iframe by performing top-level
+// navigation).
+const std::string CONTENT_CSP_HEADER =
+    "default-src 'self' "
+                "data: "
+                "blob: "
+                "about: "
+                "'unsafe-inline' "
+                "'unsafe-eval'; "
+
+    "sandbox allow-scripts "
+            "allow-same-origin "
+            "allow-top-navigation-by-user-activation "
+            "allow-modals "
+            "allow-popups "
+            "allow-forms "
+            "allow-downloads;";
+
+// End of content security policy
+///////////////////////////////////////////////////////////////////////////////
+
 } // unnamed namespace
 
 std::unique_ptr<Response>
 InternalServer::build_redirect(const std::string& bookName, const zim::Item& item) const
 {
-  const auto path = kiwix::urlEncode(item.getPath());
-  const auto redirectUrl = m_root + "/content/" + bookName + "/" + path;
-  return Response::build_redirect(*this, redirectUrl);
+  const auto contentPath = "/content/" + bookName + "/" + item.getPath();
+  const auto url = m_root + kiwix::urlEncode(contentPath);
+  return Response::build_redirect(*this, url);
 }
 
 std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
@@ -1015,13 +1127,17 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   } catch (const std::out_of_range& e) {}
 
   if (archive == nullptr) {
-    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
+    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern);
     return HTTP404Response(*this, request)
            + urlNotFoundMsg
-           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
-           + TaskbarInfo(bookName);
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
   }
 
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
   auto urlStr = url.substr(prefixLength + bookName.size());
   if (urlStr[0] == '/') {
     urlStr = urlStr.substr(1);
@@ -1029,15 +1145,25 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
 
   try {
     auto entry = getEntryFromPath(*archive, urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
+    if (entry.isRedirect() || urlStr != entry.getPath()) {
+      // In the condition above, the second case (an entry with a different
+      // URL was returned) can occur in the following situations:
+      // 1. urlStr is empty or equal to "/" and the ZIM file doesn't contain
+      //    such an entry, in which case the main entry is returned instead.
+      // 2. The ZIM file uses old namespace scheme, and the resource at urlStr
+      //    is not present but can be found under one of the 'A', 'I', 'J' or
+      //    '-' namespaces, in which case that resource is returned instead.
       return build_redirect(bookName, getFinalItem(*archive, entry));
     }
     auto response = ItemResponse::build(*this, request, entry.getItem());
-    try {
-      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, archive.get());
-    } catch (std::bad_cast& e) {}
+    response->set_etag_body(archiveUuid);
+
+    if ( !startsWith(entry.getItem().getMimetype(), "application/pdf") ) {
+      // NOTE: Content security policy is not applied to PDF content so that
+      // NOTE: it can be displayed in the viewer in Chromium-based browsers.
+      response->add_header("Content-Security-Policy", CONTENT_CSP_HEADER);
+      response->add_header("Referrer-Policy", "no-referrer");
+    }
 
     if (m_verbose.load()) {
       printf("Found %s\n", entry.getPath().c_str());
@@ -1049,11 +1175,10 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     if (m_verbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
 
-    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
+    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern);
     return HTTP404Response(*this, request)
            + urlNotFoundMsg
-           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
-           + TaskbarInfo(bookName, archive.get());
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
   }
 }
 
@@ -1092,6 +1217,11 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
            + noSuchBookErrorMsg(bookName);
   }
 
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
   // Remove the beggining of the path:
   // /raw/<bookName>/<kind>/foo
   // ^^^^^          ^      ^
@@ -1101,13 +1231,17 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   try {
     if (kind == "meta") {
       auto item = archive->getMetadataItem(itemPath);
-      return ItemResponse::build(*this, request, item, /*raw=*/true);
+      auto response = ItemResponse::build(*this, request, item);
+      response->set_etag_body(archiveUuid);
+      return response;
     } else {
       auto entry = archive->getEntryByPath(itemPath);
       if (entry.isRedirect()) {
         return build_redirect(bookName, entry.getItem(true));
       }
-      return ItemResponse::build(*this, request, entry.getItem(), /*raw=*/true);
+      auto response = ItemResponse::build(*this, request, entry.getItem());
+      response->set_etag_body(archiveUuid);
+      return response;
     }
   } catch (zim::EntryNotFound& e ) {
     if (m_verbose.load()) {
@@ -1144,9 +1278,7 @@ std::unique_ptr<Response> InternalServer::handle_locally_customized_resource(con
 
   return ContentResponse::build(*this,
                                 resourceData,
-                                crd.mimeType,
-                                /*isHomePage=*/false,
-                                /*raw=*/true);
+                                crd.mimeType);
 }
 
 }

src/server/internalServer.h

@@ -88,9 +88,6 @@ class SearchInfo {
 
 
 typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
-typedef ConcurrentCache<std::string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
-
 class OPDSDumper;
 
 class InternalServer {
@@ -126,6 +123,7 @@ class InternalServer {
     std::unique_ptr<Response> handle_request(const RequestContext& request);
     std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
     std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_viewer_settings(const RequestContext& request);
     std::unique_ptr<Response> handle_skin(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
@@ -133,9 +131,11 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
     std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
     std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
+    std::unique_ptr<Response> handle_no_js(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_search_request(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
     std::unique_ptr<Response> handle_catch(const RequestContext& request);
@@ -143,24 +143,31 @@ class InternalServer {
     std::unique_ptr<Response> handle_content(const RequestContext& request);
     std::unique_ptr<Response> handle_raw(const RequestContext& request);
     std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
-    std::unique_ptr<Response> handle_widget(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
 
     MustacheData get_default_data() const;
 
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
     std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
     SearchInfo getSearchInfo(const RequestContext& r) const;
 
     bool isLocallyCustomizedResource(const std::string& url) const;
 
+    std::string getLibraryId() const;
+
+    std::string getNoJSDownloadPageHTML(const std::string& bookId, const std::string& userLang) const;
+
+  private: // types
+    class LockableSuggestionSearcher;
+    typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+    typedef ConcurrentCache<std::string, std::shared_ptr<LockableSuggestionSearcher>> SuggestionSearcherCache;
+
   private: // data
     std::string m_addr;
     int m_port;
-    std::string m_root;
+    std::string m_root;                   // URI-encoded
+    std::string m_rootPrefixOfDecodedURL; // URI-decoded
     int m_nbThreads;
     unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
@@ -178,14 +185,13 @@ class InternalServer {
     SuggestionSearcherCache suggestionSearcherCache;
 
     std::string m_server_id;
-    std::string m_library_id;
 
     class CustomizedResources;
     std::unique_ptr<CustomizedResources> m_customizedResources;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 };
 
 }

src/server/internalServer_catalog.cpp

@@ -24,7 +24,7 @@
 #include "request_context.h"
 #include "response.h"
 #include "tools/otherTools.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include <mustache.hpp>
 
@@ -33,6 +33,74 @@
 
 namespace kiwix {
 
+namespace
+{
+
+enum OPDSResponseKind
+{
+  OPDS_ENTRY,
+  OPDS_NAVIGATION_FEED,
+  OPDS_ACQUISITION_FEED
+};
+
+const std::string opdsMimeType[] = {
+  "application/atom+xml;type=entry;profile=opds-catalog;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=navigation;charset=utf-8",
+  "application/atom+xml;profile=opds-catalog;kind=acquisition;charset=utf-8"
+};
+
+} // unnamed namespace
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  if (url == "v2") {
+    return handle_catalog_v2(request);
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper(mp_library, mp_nameMapper);
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setLibraryId(getLibraryId());
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    bookIdsToDump = search_catalog(request, opdsDumper);
+    uuid = zim::Uuid::generate();
+  }
+
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()),
+      opdsMimeType[OPDS_ACQUISITION_FEED]);
+  return std::move(response);
+}
+
 std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -77,33 +145,34 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
 {
+  const std::string libraryId = getLibraryId();
   return ContentResponse::build(
              *this,
              RESOURCE::templates::catalog_v2_root_xml,
              kainjow::mustache::object{
                {"date", gen_date_str()},
                {"endpoint_root", m_root + "/catalog/v2"},
-               {"feed_id", gen_uuid(m_library_id)},
-               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
-               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
-               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
+               {"feed_id", gen_uuid(libraryId)},
+               {"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
+               {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
+               {"category_list_feed_id", gen_uuid(libraryId + "/categories")},
+               {"language_list_feed_id", gen_uuid(libraryId + "/languages")}
              },
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto bookIds = search_catalog(request, opdsDumper);
   const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
              *this,
              opdsFeed,
-             "application/atom+xml;profile=opds-catalog;kind=acquisition"
+             opdsMimeType[OPDS_ACQUISITION_FEED]
   );
 }
 
@@ -116,38 +185,38 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
            + urlNotFoundMsg;
   }
 
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
   return ContentResponse::build(
              *this,
              opdsFeed,
-             "application/atom+xml;type=entry;profile=opds-catalog"
+             opdsMimeType[OPDS_ENTRY]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.categoriesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.languagesOPDSFeed(),
-             "application/atom+xml;profile=opds-catalog;kind=navigation"
+             opdsMimeType[OPDS_NAVIGATION_FEED]
   );
 }
 
@@ -158,7 +227,11 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const R
     auto book = mp_library->getBookByIdThreadSafe(bookId);
     auto size = request.get_argument<unsigned int>("size");
     auto illustration = book.getIllustration(size);
-    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+    return ContentResponse::build(
+               *this,
+               illustration->getData(),
+               illustration->mimeType
+    );
   } catch(...) {
     return HTTP404Response(*this, request)
            + urlNotFoundMsg;

src/server/request_context.cpp

@@ -25,8 +25,10 @@
 #include <sstream>
 #include <cstdio>
 #include <atomic>
+#include <cctype>
 
 #include "tools/stringTools.h"
+#include "i18n.h"
 
 namespace kiwix {
 
@@ -47,31 +49,15 @@ RequestMethod str2RequestMethod(const std::string& method) {
   else                          return RequestMethod::OTHER;
 }
 
-std::string
-fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
-{
-  if (rootLocation.empty()) {
-    // nothing special to handle.
-    return full_url;
-  } else if (full_url == rootLocation) {
-    return "/";
-  } else if (full_url.size() > rootLocation.size() &&
-             full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
-    return full_url.substr(rootLocation.size());
-  } else {
-    return "";
-  }
-}
-
 } // unnamed namespace
 
 RequestContext::RequestContext(struct MHD_Connection* connection,
-                               std::string rootLocation,
-                               const std::string& _url,
+                               const std::string& _rootLocation, // URI-encoded
+                               const std::string& unrootedUrl,   // URI-decoded
                                const std::string& _method,
                                const std::string& version) :
-  full_url(_url),
-  url(fullURL2LocalURL(_url, rootLocation)),
+  rootLocation(_rootLocation),
+  url(unrootedUrl),
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
@@ -80,6 +66,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
+  MHD_get_connection_values(connection, MHD_COOKIE_KIND, &RequestContext::fill_cookie, this);
 
   try {
     acceptEncodingGzip =
@@ -89,6 +76,8 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   try {
     byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
+
+  userlang = determine_user_language();
 }
 
 RequestContext::~RequestContext()
@@ -107,6 +96,22 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key].push_back(value == nullptr ? "" : value);
+  if ( ! _this->queryString.empty() ) {
+    _this->queryString += "&";
+  }
+  _this->queryString += urlEncode(key);
+  if ( value ) {
+    _this->queryString += "=";
+    _this->queryString += urlEncode(value);
+  }
+  return MHD_YES;
+}
+
+MHD_Result RequestContext::fill_cookie(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
+{
+  RequestContext *_this = static_cast<RequestContext*>(__this);
+  _this->cookies[key] = value == nullptr ? "" : value;
   return MHD_YES;
 }
 
@@ -131,7 +136,6 @@ void RequestContext::print_debug_info() const {
     printf("\n");
   }
   printf("Parsed : \n");
-  printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
   printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
@@ -169,11 +173,15 @@ std::string RequestContext::get_url_part(int number) const {
 }
 
 std::string RequestContext::get_full_url() const {
-  return full_url;
+  return rootLocation + urlEncode(url);
+}
+
+std::string RequestContext::get_root_path() const {
+  return rootLocation.empty() ? "/" : rootLocation;
 }
 
 bool RequestContext::is_valid_url() const {
-  return !url.empty();
+  return url.empty() || url[0] == '/';
 }
 
 ByteRange RequestContext::get_range() const {
@@ -190,16 +198,33 @@ std::string RequestContext::get_header(const std::string& name) const {
 }
 
 std::string RequestContext::get_user_language() const
+{
+  return userlang.lang;
+}
+
+bool RequestContext::user_language_comes_from_cookie() const
+{
+  return userlang.selectedBy == UserLanguage::SelectorKind::COOKIE;
+}
+
+RequestContext::UserLanguage RequestContext::determine_user_language() const
 {
   try {
-    return get_argument("userlang");
+    return {UserLanguage::SelectorKind::QUERY_PARAM, get_argument("userlang")};
   } catch(const std::out_of_range&) {}
 
   try {
-    return get_header("Accept-Language");
+     return {UserLanguage::SelectorKind::COOKIE, cookies.at("userlang")};
   } catch(const std::out_of_range&) {}
 
-  return "en";
+  try {
+    const std::string acceptLanguage = get_header("Accept-Language");
+    const auto userLangPrefs = parseUserLanguagePreferences(acceptLanguage);
+    const auto lang = selectMostSuitableLanguage(userLangPrefs);
+    return {UserLanguage::SelectorKind::ACCEPT_LANGUAGE_HEADER, lang};
+  } catch(const std::out_of_range&) {}
+
+  return {UserLanguage::SelectorKind::DEFAULT, "en"};
 }
 
 std::string RequestContext::get_requested_format() const

src/server/request_context.h

@@ -57,8 +57,8 @@ class IndexError: public std::runtime_error {};
 class RequestContext {
   public: // functions
     RequestContext(struct MHD_Connection* connection,
-                   std::string rootLocation,
-                   const std::string& url,
+                   const std::string& rootLocation, // URI-encoded
+                   const std::string& unrootedUrl,  // URI-decoded
                    const std::string& method,
                    const std::string& version);
     ~RequestContext();
@@ -91,16 +91,15 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
+    std::string get_root_path() const;
 
-    std::string get_query(bool mustEncode = false) const {
-      return get_query([](const std::string& key) {return true;}, mustEncode);
-    }
+    std::string get_query() const { return queryString; }
 
     template<class F>
     std::string get_query(F filter, bool mustEncode) const {
       std::string q;
       const char* sep = "";
-      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value):value; };
       for ( const auto& a : arguments ) {
         if (!filter(a.first)) {
           continue;
@@ -120,8 +119,25 @@ class RequestContext {
     std::string get_user_language() const;
     std::string get_requested_format() const;
 
+    bool user_language_comes_from_cookie() const;
+
+  private: // types
+    struct UserLanguage
+    {
+      enum SelectorKind
+      {
+        QUERY_PARAM,
+        COOKIE,
+        ACCEPT_LANGUAGE_HEADER,
+        DEFAULT
+      };
+
+      SelectorKind selectedBy;
+      std::string  lang;
+    };
+
   private: // data
-    std::string full_url;
+    std::string rootLocation;
     std::string url;
     RequestMethod method;
     std::string version;
@@ -132,9 +148,15 @@ class RequestContext {
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::vector<std::string>> arguments;
+    std::map<std::string, std::string> cookies;
+    std::string queryString;
+    UserLanguage userlang;
 
   private: // functions
+    UserLanguage determine_user_language() const;
+
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_cookie(void *, enum MHD_ValueKind, const char*, const char*);
     static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 

src/server/response.cpp

@@ -20,7 +20,7 @@
 #include "response.h"
 #include "request_context.h"
 #include "internalServer.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
@@ -64,7 +64,13 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/javascript") != std::string::npos
       || mimeType.find("application/atom") != std::string::npos
       || mimeType.find("application/opensearchdescription") != std::string::npos
-      || mimeType.find("application/json") != std::string::npos;
+      || mimeType.find("application/json") != std::string::npos
+
+      // Web fonts
+      || mimeType.find("application/font-") != std::string::npos
+      || mimeType.find("application/x-font-") != std::string::npos
+      || mimeType.find("application/vnd.ms-fontobject") != std::string::npos
+      || mimeType.find("font/") != std::string::npos;
 }
 
 bool compress(std::string &content) {
@@ -102,6 +108,14 @@ bool compress(std::string &content) {
 }
 
 
+const char* getCacheControlHeader(Response::Kind k)
+{
+  switch(k) {
+    case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
+    case Response::ZIM_CONTENT:     return "max-age=3600, must-revalidate";
+    default:                        return "max-age=0, must-revalidate";
+  }
+}
 
 } // unnamed namespace
 
@@ -112,6 +126,13 @@ Response::Response(bool verbose)
   add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
+void Response::set_kind(Kind k)
+{
+  m_kind = k;
+  if ( k == ZIM_CONTENT )
+    m_etag.set_option(ETag::ZIM_CONTENT);
+}
+
 std::unique_ptr<Response> Response::build(const InternalServer& server)
 {
   return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
@@ -122,6 +143,9 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   auto response = Response::build(server);
   response->set_code(MHD_HTTP_NOT_MODIFIED);
   response->m_etag = etag;
+  if ( etag.get_option(ETag::ZIM_CONTENT) ) {
+    response->set_kind(Response::ZIM_CONTENT);
+  }
   if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
     response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
   }
@@ -140,9 +164,6 @@ std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObjec
 {
   auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
   r->set_code(m_httpStatusCode);
-  if ( m_taskbarInfo ) {
-    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
-  }
   return r;
 }
 
@@ -179,7 +200,7 @@ HTTP404Response::HTTP404Response(const InternalServer& server,
 
 HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
 {
-  const std::string requestUrl = m_request.get_full_url();
+  const std::string requestUrl = urlDecode(m_request.get_full_url(), false);
   return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
 }
 
@@ -213,7 +234,7 @@ HTTP400Response::HTTP400Response(const InternalServer& server,
 
 HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
 {
-  std::string requestUrl = m_request.get_full_url();
+  std::string requestUrl = urlDecode(m_request.get_full_url(), false);
   const auto query = m_request.get_query();
   if (!query.empty()) {
     requestUrl += "?" + encodeDiples(query);
@@ -236,29 +257,12 @@ HTTP500Response::HTTP500Response(const InternalServer& server,
 
 std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
 {
-  // We want a 500 response to be a minimalistic one (so that the server doesn't
-  // have to provide additional resources required for its proper rendering)
-  // ";raw=true" in the MIME-type below disables response decoration
-  // (see ContentResponse::contentDecorationAllowed())
-  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  const std::string mimeType = "text/html;charset=utf-8";
   auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
   r->set_code(m_httpStatusCode);
   return r;
 }
 
-ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
-{
-  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
-  return *this;
-}
-
-ContentResponseBlueprint& ContentResponseBlueprint::operator+=(const TaskbarInfo& taskbarInfo)
-{
-  // operator+() is already a state-modifying operator (akin to operator+=)
-  return *this + taskbarInfo;
-}
-
-
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
 {
   auto response = Response::build(server);
@@ -337,52 +341,6 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar(const std::string& lang)
-{
-  i18n::GetTranslatedString t(lang);
-  kainjow::mustache::object data{
-    {"root", m_root},
-    {"content", m_bookName},
-    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
-    {"title", m_bookTitle},
-    {"withlibrarybutton", m_withLibraryButton},
-    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
-    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
-    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
-    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
-  };
-  auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    head_content);
-
-  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<body[^>]*>",
-    taskbar_part);
-}
-
-
-void ContentResponse::inject_externallinks_blocker()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    script_tag);
-}
-
-void ContentResponse::inject_root_link(){
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    "<link type=\"root\" href=\"" + m_root + "\">");
-}
-
 bool
 ContentResponse::can_compress(const RequestContext& request) const
 {
@@ -391,16 +349,6 @@ ContentResponse::can_compress(const RequestContext& request) const
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
-bool
-ContentResponse::contentDecorationAllowed() const
-{
-  if (m_raw) {
-    return false;
-  }
-  return (startsWith(m_mimeType, "text/html")
-    && m_mimeType.find(";raw=true") == std::string::npos);
-}
-
 MHD_Response*
 Response::create_mhd_response(const RequestContext& request)
 {
@@ -411,17 +359,6 @@ Response::create_mhd_response(const RequestContext& request)
 MHD_Response*
 ContentResponse::create_mhd_response(const RequestContext& request)
 {
-  if (contentDecorationAllowed()) {
-    inject_root_link();
-
-    if (m_withTaskbar) {
-      introduce_taskbar(request.get_user_language());
-    }
-    if (m_blockExternalLinks) {
-      inject_externallinks_blocker();
-    }
-  }
-
   const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
@@ -442,7 +379,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   MHD_Response* response = create_mhd_response(request);
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+                          getCacheControlHeader(m_kind));
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
     MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
@@ -450,6 +387,13 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
     MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
   }
 
+  if ( ! request.user_language_comes_from_cookie() ) {
+    const std::string cookie = "userlang=" + request.get_user_language()
+                               + ";Path=" + request.get_root_path()
+                               + ";Max-Age=31536000";
+    MHD_add_response_header(response, MHD_HTTP_HEADER_SET_COOKIE, cookie.c_str());
+  }
+
   if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
@@ -461,24 +405,11 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
-{
-  m_bookName = bookName;
-  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
-}
-
-
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
-  m_mimeType(mimetype),
-  m_raw(raw),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  m_bookName(""),
-  m_bookTitle("")
+  m_mimeType(mimetype)
 {
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
@@ -486,17 +417,11 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw
 std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& content,
-  const std::string& mimetype,
-  bool isHomePage,
-  bool raw)
+  const std::string& mimetype)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
-        raw,
-        server.m_withTaskbar && !isHomePage,
-        server.m_withLibraryButton,
-        server.m_blockExternalLinks,
         content,
         mimetype));
 }
@@ -505,11 +430,10 @@ std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& template_str,
   kainjow::mustache::data data,
-  const std::string& mimetype,
-  bool isHomePage)
+  const std::string& mimetype)
 {
   auto content = render_template(template_str, data);
-  return ContentResponse::build(server, content, mimetype, isHomePage);
+  return ContentResponse::build(server, content, mimetype);
 }
 
 ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
@@ -518,26 +442,26 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   m_mimeType(mimetype)
 {
   m_byteRange = byterange;
-  set_cacheable();
+  set_kind(Response::ZIM_CONTENT);
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
-    response->set_cacheable();
+    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    response->set_kind(Response::ZIM_CONTENT);
     response->m_byteRange = byteRange;
     return std::move(response);
   }
 
   if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
     auto response = Response::build_416(server, item.getSize());
-    response->set_cacheable();
+    response->set_kind(Response::ZIM_CONTENT);
     return response;
   }
 

src/server/response.h

@@ -45,6 +45,14 @@ class InternalServer;
 class RequestContext;
 
 class Response {
+  public:
+    enum Kind
+    {
+      STATIC_RESOURCE,
+      ZIM_CONTENT,
+      DYNAMIC_CONTENT
+    };
+
   public:
     Response(bool verbose);
     virtual ~Response() = default;
@@ -57,8 +65,9 @@ class Response {
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
     void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_kind(Kind k);
+    Kind get_kind() const { return m_kind; }
+    void set_etag_body(const std::string& id) { m_etag.set_body(id); }
     void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
@@ -68,6 +77,7 @@ class Response {
     MHD_Response* create_error_response(const RequestContext& request) const;
 
   protected: // data
+    Kind m_kind = DYNAMIC_CONTENT;
     bool m_verbose;
     int m_returnCode;
     ByteRange m_byteRange;
@@ -83,60 +93,32 @@ class ContentResponse : public Response {
     ContentResponse(
       const std::string& root,
       bool verbose,
-      bool raw,
-      bool withTaskbar,
-      bool withLibraryButton,
-      bool blockExternalLinks,
       const std::string& content,
       const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& content,
-      const std::string& mimetype,
-      bool isHomePage = false,
-      bool raw = false);
+      const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& template_str,
       kainjow::mustache::data data,
-      const std::string& mimetype,
-      bool isHomePage = false);
-
-    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
+      const std::string& mimetype);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar(const std::string& lang);
-    void inject_externallinks_blocker();
-    void inject_root_link();
     bool can_compress(const RequestContext& request) const;
-    bool contentDecorationAllowed() const;
 
 
   private:
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
-    bool m_raw;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    std::string m_bookName;
-    std::string m_bookTitle;
  };
 
-struct TaskbarInfo
-{
-  const std::string bookName;
-  const zim::Archive* const archive;
-
-  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
-    : bookName(bookName)
-    , archive(a)
-  {}
-};
-
 class ContentResponseBlueprint
 {
 public: // functions
@@ -165,9 +147,6 @@ public: // functions
   }
 
 
-  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
-  ContentResponseBlueprint& operator+=(const TaskbarInfo& taskbarInfo);
-
 protected: // functions
   std::string getMessage(const std::string& msgId) const;
   virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
@@ -179,7 +158,6 @@ public: //data
   const std::string m_mimeType;
   const std::string m_template;
   kainjow::mustache::data m_data;
-  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
 };
 
 struct HTTPErrorResponse : ContentResponseBlueprint
@@ -191,8 +169,6 @@ struct HTTPErrorResponse : ContentResponseBlueprint
                     const std::string& headingMsgId,
                     const std::string& cssUrl = "");
 
-  using ContentResponseBlueprint::operator+;
-  using ContentResponseBlueprint::operator+=;
   HTTPErrorResponse& operator+(const std::string& msg);
   HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
   HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
@@ -238,7 +214,7 @@ private: // overrides
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -93,10 +93,6 @@ std::string getMetaFlavour(const zim::Archive& archive) {
   return getMetadata(archive, "Flavour");
 }
 
-std::string getArchiveId(const zim::Archive& archive) {
-  return (std::string) archive.getUuid();
-}
-
 bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType){
   try {
@@ -109,46 +105,6 @@ bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
   return false;
 }
 
-// should this be in libzim
-unsigned int getArchiveMediaCount(const zim::Archive& archive) {
-  std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
-  unsigned int counter = 0;
-
-  for (auto &pair:counterMap) {
-    if (startsWith(pair.first, "image/") ||
-        startsWith(pair.first, "video/") ||
-        startsWith(pair.first, "audio/")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-unsigned int getArchiveArticleCount(const zim::Archive& archive) {
-  // [HACK]
-  // getArticleCount() returns different things depending of the "version" of the zim.
-  // On old zim (<=6), it returns the number of entry in `A` namespace
-  // On recent zim (>=7), it returns:
-  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
-  //  - the number of front article if a frontArticleIndex is present
-  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
-  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
-  if (archive.hasNewNamespaceScheme()) {
-    //The articleCount is "good"
-    return archive.getArticleCount();
-  } else {
-    // We have to parse the `M/Counter` metadata
-    unsigned int counter = 0;
-    for(const auto& pair:parseArchiveCounter(archive)) {
-      if (startsWith(pair.first, "text/html")) {
-        counter += pair.second;
-      }
-    }
-    return counter;
-  }
-}
-
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }
@@ -169,14 +125,4 @@ zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path
   }
   throw zim::EntryNotFound("Cannot find entry for non empty path");
 }
-
-MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
-  try {
-    auto counterContent = archive.getMetadata("Counter");
-    return parseMimetypeCounter(counterContent);
-  } catch (zim::EntryNotFound& e) {
-    return {};
-  }
-}
-
 } // kiwix

src/tools/archiveTools.h

@@ -40,7 +40,6 @@ namespace kiwix
     std::string getMetaCreator(const zim::Archive& archive);
     std::string getMetaPublisher(const zim::Archive& archive);
     std::string getMetaFlavour(const zim::Archive& archive);
-    std::string getArchiveId(const zim::Archive& archive);
 
     bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType);
@@ -52,9 +51,6 @@ namespace kiwix
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
 
     zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
-
-    MimeCounterType parseArchiveCounter(const zim::Archive& archive);
-
 }
 
 #endif

src/tools/otherTools.cpp

@@ -32,12 +32,15 @@
 #endif
 
 #include "tools/stringTools.h"
+#include "server/i18n.h"
+#include "libkiwix-resources.h"
 
 #include <map>
 #include <sstream>
 #include <pugixml.hpp>
 
 #include <zim/uuid.h>
+#include <zim/suggestion_iterator.h>
 
 
 static std::map<std::string, std::string> codeisomapping {
@@ -288,67 +291,6 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
-namespace
-{
-// The counter metadata format is a list of item separated by a `;` :
-// item0;item1;item2
-// Each item is a "tuple" mimetype=number.
-// However, the mimetype may contains parameters:
-// text/html;raw=true;foo=bar
-// So the final format may be complex to parse:
-// key0=value0;key1;foo=bar=value1;key2=value2
-
-typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
-
-std::string readFullMimetypeAndCounterString(std::istream& in)
-{
-  std::string mtcStr, params;
-  getline(in, mtcStr, ';');
-  if ( mtcStr.find('=') == std::string::npos )
-  {
-    do
-    {
-      if ( !getline(in, params, ';' ) )
-        return std::string();
-      mtcStr += ";" + params;
-    }
-    while ( std::count(params.begin(), params.end(), '=') != 2 );
-  }
-  return mtcStr;
-}
-
-MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
-{
-  const std::string::size_type k = s.find_last_of("=");
-  if ( k != std::string::npos )
-  {
-    const std::string mimeType = s.substr(0, k);
-    std::istringstream counterSS(s.substr(k+1));
-    unsigned int counter;
-    if (counterSS >> counter && counterSS.eof())
-      return MimetypeAndCounter{mimeType, counter};
-  }
-  return MimetypeAndCounter{"", 0};
-}
-
-} // unnamed namespace
-
-kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
-{
-  kiwix::MimeCounterType counters;
-  std::istringstream ss(counterData);
-
-  while (ss)
-  {
-    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
-    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
-    if ( !mtc.first.empty() )
-      counters.insert(mtc);
-  }
-
-  return counters;
-}
-
 std::string kiwix::gen_date_str()
 {
   auto now = std::time(0);
@@ -380,10 +322,76 @@ kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
   std::stringstream ss;
   tmpl.render(data, [&ss](const std::string& str) { ss << str; });
   return ss.str();
 }
+
+namespace
+{
+
+std::string escapeBackslashes(const std::string& s)
+{
+  std::string es;
+  es.reserve(s.size());
+  for (char c : s) {
+    if ( c == '\\' ) {
+      es.push_back('\\');
+    }
+    es.push_back(c);
+  }
+  return es;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang,
+                                         const std::string& queryString)
+{
+  return kiwix::i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+} // unnamed namespace
+
+kiwix::Suggestions::Suggestions()
+  : m_data(kainjow::mustache::data::type::list)
+{
+}
+
+void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
+{
+  kainjow::mustache::data result;
+
+  const std::string label = suggestion.hasSnippet()
+                          ? suggestion.getSnippet()
+                          : suggestion.getTitle();
+
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(suggestion.getTitle()));
+  result.set("kind", "path");
+  result.set("path", escapeBackslashes(suggestion.getPath()));
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
+                                               const std::string& queryString)
+{
+  kainjow::mustache::data result;
+  const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(queryString + " "));
+  result.set("kind", "pattern");
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+std::string kiwix::Suggestions::getJSON() const
+{
+  kainjow::mustache::data data;
+  data.set("suggestions", m_data);
+
+  return render_template(RESOURCE::templates::suggestion_json, data);
+}

src/tools/otherTools.h

@@ -33,6 +33,10 @@ namespace pugi {
   class xml_node;
 }
 
+namespace zim {
+  class SuggestionItem;
+}
+
 namespace kiwix
 {
   std::string nodeToString(const pugi::xml_node& node);
@@ -45,9 +49,6 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
-  using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
-  MimeCounterType parseMimetypeCounter(const std::string& counterData);
-
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
@@ -70,6 +71,22 @@ namespace kiwix
 
     return defaultValue;
   }
+
+  class Suggestions
+  {
+  public:
+    Suggestions();
+
+    void add(const zim::SuggestionItem& suggestion);
+
+    void addFTSearchSuggestion(const std::string& uiLang,
+                               const std::string& query);
+
+    std::string getJSON() const;
+
+  private:
+    kainjow::mustache::data m_data;
+  };
 }
 
 #endif

src/tools/pathTools.cpp

@@ -493,12 +493,14 @@ static std::map<std::string, std::string> extMimeTypes = {
  { "jpeg", "image/jpeg"},
  { "jpg", "image/jpeg"},
  { "gif", "image/gif"},
+ { "ico", "image/x-icon"},
  { "svg", "image/svg+xml"},
  { "txt", "text/plain"},
  { "xml", "text/xml"},
  { "pdf", "application/pdf"},
  { "ogg", "application/ogg"},
  { "js", "application/javascript"},
+ { "json", "application/json"},
  { "css", "text/css"},
  { "otf", "application/vnd.ms-opentype"},
  { "ttf", "application/font-ttf"},

src/tools/regexTools.cpp

@@ -75,41 +75,3 @@ std::string replaceRegex(const std::string& content,
   uresult.toUTF8String(tmp);
   return tmp;
 }
-
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->end(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}
-
-std::string prependToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->start(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}

src/tools/regexTools.h

@@ -26,11 +26,5 @@ bool matchRegex(const std::string& content, const std::string& regex);
 std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement);
-std::string prependToFirstOccurence(const std::string& content,
-                                    const std::string& regex,
-                                    const std::string& replacement);
 
 #endif

src/tools/stringTools.cpp

@@ -161,15 +161,14 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-/* urlEncode() based on javascript encodeURI() &
-   encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
+namespace
+{
 
 bool isReservedUrlChar(char c)
 {
   switch (c) {
   case ';':
   case ',':
-  case '/':
   case '?':
   case ':':
   case '@':
@@ -177,22 +176,22 @@ bool isReservedUrlChar(char c)
   case '=':
   case '+':
   case '$':
+  case '#':
     return true;
   default:
     return false;
   }
 }
 
-bool needsEscape(char c, bool encodeReserved)
+bool isHarmlessUriChar(char c)
 {
   if (c >= 'a' && c <= 'z')
-    return false;
+    return true;
   if (c >= 'A' && c <= 'Z')
-    return false;
+    return true;
   if (c >= '0' && c <= '9')
-    return false;
-  if (isReservedUrlChar(c))
-    return encodeReserved;
+    return true;
+
   switch (c) {
   case '-':
   case '_':
@@ -203,9 +202,10 @@ bool needsEscape(char c, bool encodeReserved)
   case '\'':
   case '(':
   case ')':
-    return false;
+  case '/':
+    return true;
   }
-  return true;
+  return false;
 }
 
 int hexToInt(char c) {
@@ -230,18 +230,18 @@ int hexToInt(char c) {
   }
 }
 
-std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
+} // unnamed namespace
+
+std::string kiwix::urlEncode(const std::string& value)
 {
   std::ostringstream os;
   os << std::hex << std::uppercase;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
-
-    if (!needsEscape(*it, encodeReserved)) {
-      os << *it;
+  for (const char c : value) {
+    if (isHarmlessUriChar(c)) {
+      os << c;
     } else {
-      os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
+      const unsigned int charVal = static_cast<unsigned char>(c);
+      os << '%' << std::setw(2) << std::setfill('0') << charVal;
     }
   }
   return os.str();
@@ -267,15 +267,15 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
       int iHi = hexToInt(hi);
       int iLo = hexToInt(lo);
       if (iHi < 0 || iLo < 0) {
-	// Invalid escape sequence
-	os << '%' << hi << lo;
-	continue;
+        // Invalid escape sequence
+        os << '%' << hi << lo;
+        continue;
       }
       char c = (char)(iHi << 4 | iLo);
       if (!component && isReservedUrlChar(c)) {
-	os << '%' << hi << lo;
+        os << '%' << hi << lo;
       } else {
-	os << c;
+        os << c;
       }
     } else {
       os << *it;
@@ -415,6 +415,17 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
         && std::equal(start.begin(), start.end(), base.begin());
 }
 
+std::string kiwix::stripSuffix(const std::string& str, const std::string& suffix)
+{
+  if (str.size() > suffix.size()) {
+    const auto subStr = str.substr(str.size() - suffix.size(), str.size());
+    if (subStr == suffix) {
+      return str.substr(0, str.size() - suffix.size());
+    }
+  }
+  return str;
+}
+
 std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   std::vector<std::string> variants;
   variants.push_back(title);

src/tools/stringTools.h

@@ -55,7 +55,9 @@ private:
 };
 
 
-std::string urlEncode(const std::string& value, bool encodeReserved = false);
+/* urlEncode() is the equivalent of JS encodeURIComponent(), with the only
+ * difference that the slash (/) symbol is NOT encoded. */
+std::string urlEncode(const std::string& value);
 std::string urlDecode(const std::string& value, bool component = false);
 
 std::string join(const std::vector<std::string>& list, const std::string& sep);
@@ -91,6 +93,8 @@ std::string extractFromString(const std::string& str);
 
 bool startsWith(const std::string& base, const std::string& start);
 
+std::string stripSuffix(const std::string& str, const std::string& suffix);
+
 std::vector<std::string> getTitleVariants(const std::string& title);
 } //namespace kiwix
 #endif

static/generate_i18n_resources_list.py

@@ -17,15 +17,41 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from pathlib import Path
+import json
 
 script_path = Path(__file__)
 
 resource_file = script_path.parent / "i18n_resources_list.txt"
-translation_dir = script_path.parent / "i18n"
+translation_dir = script_path.parent / "skin/i18n"
+language_list_relpath = "skin/languages.js"
 
+def get_translation_info(filepath):
+    lang_code = Path(filepath).stem
+    with open(filepath, 'r', encoding="utf-8") as f:
+        content = json.load(f)
+        lang_name = content.get("name")
+        return lang_code, lang_name
+
+language_list = []
 json_files = translation_dir.glob("*.json")
 with open(resource_file, 'w', encoding="utf-8") as f:
-    for json in sorted(translation_dir.glob("*.json")):
-        if json.name == "qqq.json":
+    for i18n_file in sorted(translation_dir.glob("*.json")):
+        if i18n_file.name == "qqq.json":
             continue
-        f.write(str(json.relative_to(script_path.parent)) + '\n')
+        print("Processing", i18n_file.name)
+        if i18n_file.name != "test.json":
+            lang_code, lang_name = get_translation_info(i18n_file)
+            if lang_name:
+                language_list.append((lang_code, lang_name))
+            else:
+                print(f"Warning: missing 'name' in {i18n_file.name}")
+        f.write(str(i18n_file.relative_to(script_path.parent)) + '\n')
+
+language_list = [{name: code} for code, name in sorted(language_list)]
+language_list_jsobj_str = json.dumps(language_list,
+                                     indent=2,
+                                     ensure_ascii=False)
+print("Saving", language_list_relpath)
+fullpath = script_path.parent / language_list_relpath
+with open(fullpath, 'w', encoding="utf-8") as f:
+    f.write("const uiLanguages = " + language_list_jsobj_str)

static/i18n/en.json

@@ -1,30 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-		]
-	},
-	"name":"English",
-	"suggest-full-text-search" : "containing '{{{SEARCH_TERMS}}}'..."
-	, "no-such-book" : "No such book: {{BOOK_NAME}}"
-	, "too-many-books" : "Too many books requested ({{NB_BOOKS}}) where limit is {{LIMIT}}"
-	, "no-book-found" : "No book matches selection criteria"
-	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
-	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
-	, "random-article-failure" :  "Oops! Failed to pick a random article :("
-	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
-	, "no-value-for-arg": "No value provided for argument {{ARGUMENT}}"
-	, "no-query" : "No query provided."
-	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
-	, "400-page-title" : "Invalid request"
-	, "400-page-heading" : "Invalid request"
-	, "404-page-title" : "Content not found"
-	, "404-page-heading" : "Not Found"
-	, "500-page-title" : "Internal Server Error"
-	, "500-page-heading" : "Internal Server Error"
-	, "fulltext-search-unavailable" : "Fulltext search unavailable"
-	, "no-search-results": "The fulltext search engine is not available for this content."
-	, "library-button-text": "Go to welcome page"
-	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
-	, "random-page-button-text": "Go to a randomly selected page"
-	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
-}

static/i18n/it.json

@@ -1,15 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Beta16"
-		]
-	},
-	"name": "italiano",
-	"suggest-full-text-search": "contenente '{{{SEARCH_TERMS}}}'...",
-	"url-not-found": "L'URL richiesto \"{{url}}\" non è stato trovato in questo server.",
-	"400-page-title": "Richiesta non valida",
-	"400-page-heading": "Richiesta non valida",
-	"404-page-title": "Contenuto non trovato",
-	"404-page-heading": "Non trovato",
-	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'"
-}

static/i18n/zh-hant.json

@@ -1,15 +0,0 @@
-{
-	"@metadata": {
-		"authors": [
-			"Winston Sung"
-		]
-	},
-	"name": "繁體中文",
-	"400-page-title": "無效請求",
-	"400-page-heading": "無效請求",
-	"404-page-title": "查無內容",
-	"404-page-heading": "查無頁面",
-	"500-page-title": "內部伺服器錯誤",
-	"500-page-heading": "內部伺服器錯誤",
-	"searchbox-tooltip": "在{{BOOK_TITLE}}搜尋"
-}

static/i18n_resources_list.txt

@@ -1,17 +1,29 @@
-i18n/bn.json
-i18n/en.json
-i18n/fr.json
-i18n/he.json
-i18n/hy.json
-i18n/it.json
-i18n/ja.json
-i18n/ko.json
-i18n/ku-latn.json
-i18n/mk.json
-i18n/pl.json
-i18n/ru.json
-i18n/sc.json
-i18n/sk.json
-i18n/sv.json
-i18n/tr.json
-i18n/zh-hant.json
+skin/i18n/ar.json
+skin/i18n/bn.json
+skin/i18n/cs.json
+skin/i18n/de.json
+skin/i18n/dga.json
+skin/i18n/el.json
+skin/i18n/en.json
+skin/i18n/fr.json
+skin/i18n/he.json
+skin/i18n/hy.json
+skin/i18n/ia.json
+skin/i18n/it.json
+skin/i18n/ja.json
+skin/i18n/ko.json
+skin/i18n/ku-latn.json
+skin/i18n/lb.json
+skin/i18n/mk.json
+skin/i18n/nl.json
+skin/i18n/nqo.json
+skin/i18n/pl.json
+skin/i18n/ru.json
+skin/i18n/sc.json
+skin/i18n/sk.json
+skin/i18n/sl.json
+skin/i18n/sv.json
+skin/i18n/test.json
+skin/i18n/tr.json
+skin/i18n/zh-hans.json
+skin/i18n/zh-hant.json

static/meson.build

@@ -1,7 +1,18 @@
-resource_files = run_command(res_manager,
-                    '--list-all',
-                    files('resources_list.txt')
-                 ).stdout().strip().split('\n')
+if meson.version().version_compare('>=0.47.0')
+    resource_files = run_command(
+        res_manager,
+        '--list-all',
+        files('resources_list.txt'),
+        check: true
+    ).stdout().strip().split('\n')
+else
+    resource_files = run_command(
+        res_manager,
+        '--list-all',
+        files('resources_list.txt')
+    ).stdout().strip().split('\n')
+endif
+
 
 preprocessed_resources = custom_target('preprocessed_resource_files',
     input: 'resources_list.txt',
@@ -14,8 +25,8 @@ preprocessed_resources = custom_target('preprocessed_resource_files',
 )
 
 lib_resources = custom_target('resources',
-    input: preprocessed_resources,
-    output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
+    input: [preprocessed_resources, 'i18n_resources_list.txt'],
+    output: ['libkiwix-resources.cpp', 'libkiwix-resources.h'],
     command:[res_compiler,
              '--cxxfile', '@OUTPUT0@',
              '--hfile', '@OUTPUT1@',
@@ -30,11 +41,24 @@ lib_resources = custom_target('resources',
 # i18n_resource_files = fs.read('i18n_resources_list.txt').strip().split('\n')
 # ```
 # once we move to meson >= 0.57.0
-i18n_resource_files = run_command(find_program('python3'),
-                                    '-c',
-                                    'import sys; f=open(sys.argv[1]); print(f.read())',
-                                    files('i18n_resources_list.txt')
-                                 ).stdout().strip().split('\n')
+
+if meson.version().version_compare('>=0.47.0')
+    i18n_resource_files = run_command(
+        find_program('python3'),
+        '-c',
+        'import sys; f=open(sys.argv[1]); print(f.read())',
+        files('i18n_resources_list.txt'),
+        check: true
+    ).stdout().strip().split('\n')
+else
+    i18n_resource_files = run_command(
+        find_program('python3'),
+        '-c',
+        'import sys; f=open(sys.argv[1]); print(f.read())',
+        files('i18n_resources_list.txt'),
+    ).stdout().strip().split('\n')
+endif
+
 
 i18n_resources = custom_target('i18n_resources',
     input: i18n_resource_files,

static/resources_list.txt

@@ -1,44 +1,48 @@
 skin/caret.png
 skin/bittorrent.png
 skin/magnet.png
+skin/feed.svg
+skin/langSelector.svg
 skin/download.png
 skin/hash.png
 skin/search-icon.svg
-skin/taskbar.js
 skin/iso6391To3.js
 skin/isotope.pkgd.min.js
 skin/index.js
 skin/autoComplete.min.js
-skin/widget.js
 skin/taskbar.css
 skin/index.css
 skin/fonts/Poppins.ttf
 skin/fonts/Roboto.ttf
-skin/block_external.js
 skin/search_results.css
+skin/blank.html
+skin/viewer.js
+skin/i18n.js
+skin/languages.js
+skin/mustache.min.js
+viewer.html
 templates/search_result.html
 templates/search_result.xml
 templates/error.html
 templates/error.xml
 templates/index.html
-templates/widget.html
 templates/suggestion.json
-templates/head_taskbar.html
-templates/taskbar_part.html
-templates/external_blocker_part.html
 templates/captured_external.html
 templates/catalog_entries.xml
 templates/catalog_v2_root.xml
 templates/catalog_v2_entries.xml
 templates/catalog_v2_entry.xml
+templates/catalog_v2_partial_entry.xml
 templates/catalog_v2_categories.xml
 templates/catalog_v2_languages.xml
 templates/url_of_search_results_css
+templates/viewer_settings.js
+templates/no_js_library_page.html
+templates/no_js_download.html
 opensearchdescription.xml
 ft_opensearchdescription.xml
 catalog_v2_searchdescription.xml
 skin/css/autoComplete.css
-skin/css/images/search.svg
 skin/favicon/android-chrome-192x192.png
 skin/favicon/android-chrome-512x512.png
 skin/favicon/apple-touch-icon.png

static/skin/blank.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <title>Blank page</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  </head>
+
+  <body>
+  </body>
+</html>

static/skin/block_external.js

@@ -1,74 +0,0 @@
-const root = document.querySelector( `link[type='root']` ).getAttribute("href");
-// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
-var block_path = `${root}/catch/external`;
-// called only on external links
-function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
-
-// called on all link clicks. filters external and call capture_event
-function on_click_event(e) {
-  var target = findParent("a", e.target);
-  if (target !== null && "href" in target) {
-    var href = target.href;
-    if (window.location.pathname.indexOf(block_path) == 0) // already in catch page
-      return;
-    if (href.indexOf(window.location.origin) == 0)
-      return;
-    if (href.substr(0, 2) == "//")
-      return capture_event(e, target);
-    if (href.substr(0, 5) == "http:")
-      return capture_event(e, target);
-    if (href.substr(0, 6) == "https:")
-      return capture_event(e, target);
-    return;
-  }
-}
-
-// script entrypoint (called on document ready)
-function run() { live('a', 'click', on_click_event); }
-
-// find first parent with tagname
-function findParent(tagname, el) {
-  while (el) {
-    if ((el.nodeName || el.tagName).toLowerCase() === tagname.toLowerCase()) {
-      return el;
-    }
-    el = el.parentNode;
-  }
-  return null;
-}
-
-// matches polyfill
-this.Element && function(ElementPrototype) {
-    ElementPrototype.matches = ElementPrototype.matches ||
-    ElementPrototype.matchesSelector ||
-    ElementPrototype.webkitMatchesSelector ||
-    ElementPrototype.msMatchesSelector ||
-    function(selector) {
-        var node = this, nodes = (node.parentNode || node.document).querySelectorAll(selector), i = -1;
-        while (nodes[++i] && nodes[i] != node);
-        return !!nodes[i];
-    }
-}(Element.prototype);
-
-// helper for enabling IE 8 event bindings
-function addEvent(el, type, handler) {
-    if (el.attachEvent) el.attachEvent('on'+type, handler); else el.addEventListener(type, handler);
-}
-
-// live binding helper using matchesSelector
-function live(selector, event, callback, context) {
-    addEvent(context || document, event, function(e) {
-        var found, el = e.target || e.srcElement;
-        while (el && el.matches && el !== context && !(found = el.matches(selector))) el = el.parentElement;
-        if (found) callback.call(el, e);
-    });
-}
-
-// in case the document is already rendered
-if (document.readyState!='loading') run();
-// modern browsers
-else if (document.addEventListener) document.addEventListener('DOMContentLoaded', run);
-// IE <= 8
-else document.attachEvent('onreadystatechange', function(){
-    if (document.readyState=='complete') run();
-});

static/skin/css/images/search.svg

@@ -1,8 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" focusable="false" x="0px" y="0px" width="30" height="30" viewBox="0 0 171 171" style=" fill:#000000;">
-    <g fill="none" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal">
-        <path d="M0,171.99609v-171.99609h171.99609v171.99609z" fill="none"></path>
-        <g fill="#ff7a7a">
-            <path d="M74.1,17.1c-31.41272,0 -57,25.58728 -57,57c0,31.41272 25.58728,57 57,57c13.6601,0 26.20509,-4.85078 36.03692,-12.90293l34.03301,34.03301c1.42965,1.48907 3.55262,2.08891 5.55014,1.56818c1.99752,-0.52073 3.55746,-2.08067 4.07819,-4.07819c0.52073,-1.99752 -0.0791,-4.12049 -1.56818,-5.55014l-34.03301,-34.03301c8.05215,-9.83182 12.90293,-22.37682 12.90293,-36.03692c0,-31.41272 -25.58728,-57 -57,-57zM74.1,28.5c25.2517,0 45.6,20.3483 45.6,45.6c0,25.2517 -20.3483,45.6 -45.6,45.6c-25.2517,0 -45.6,-20.3483 -45.6,-45.6c0,-25.2517 20.3483,-45.6 45.6,-45.6z"></path>
-        </g>
-    </g>
-</svg>

static/skin/feed.svg

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 27.3.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="uiLanguageSelectorButton" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
+	 x="0px" y="0px" viewBox="0 0 30 30" style="enable-background:new 0 0 30 30;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#CCCCCC;}
+	.st2{fill:#F78422;}
+</style>
+<g>
+	<path class="st0" d="M2.9,29.6c-1.4,0-2.6-1.1-2.6-2.6V2.9c0-1.4,1.1-2.6,2.6-2.6h24.1c1.4,0,2.6,1.1,2.6,2.6v24.1
+		c0,1.4-1.1,2.6-2.6,2.6H2.9z"/>
+	<path class="st1" d="M27.1,0.6c1.3,0,2.3,1,2.3,2.3v24.1c0,1.3-1,2.3-2.3,2.3H2.9c-1.3,0-2.3-1-2.3-2.3V2.9c0-1.3,1-2.3,2.3-2.3
+		H27.1 M27.1,0.1H2.9c-1.6,0-2.8,1.3-2.8,2.8v24.1c0,1.6,1.3,2.8,2.8,2.8h24.1c1.6,0,2.8-1.3,2.8-2.8V2.9
+		C29.9,1.4,28.6,0.1,27.1,0.1L27.1,0.1z"/>
+</g>
+<g>
+	<path class="st2" d="M18,24h-3c0-5.2-4.2-9.4-9.4-9.4v-3C12.4,11.6,18,17.2,18,24z"/>
+	<path class="st2" d="M24.5,24h-3c-0.1-8.7-7.2-15.9-16-15.9v-3C16,5.1,24.5,13.6,24.5,24z"/>
+	<circle class="st2" cx="8.1" cy="21.6" r="2.6"/>
+</g>
+</svg>

static/skin/i18n.js

@@ -0,0 +1,158 @@
+import mustache from '../skin/mustache.min.js?KIWIXCACHEID'
+
+const Translations = {
+  defaultLanguage: null,
+  currentLanguage: null,
+  promises: {},
+  data: {},
+
+  load: function(lang, asDefault=false) {
+    if ( asDefault ) {
+      this.defaultLanguage = lang;
+      this.loadTranslationsJSON(lang);
+    } else {
+      this.currentLanguage = lang;
+      if ( lang != this.defaultLanguage ) {
+        this.loadTranslationsJSON(lang);
+      }
+    }
+  },
+
+  loadTranslationsJSON: function(lang) {
+    if ( this.promises[lang] )
+      return;
+
+    const errorMsg = `Error loading translations for language '${lang}': `;
+    this.promises[lang] = fetch(`./skin/i18n/${lang}.json`).then(async (resp) => {
+      if ( resp.ok ) {
+        this.data[lang] = JSON.parse(await resp.text());
+      } else {
+        console.log(errorMsg + resp.statusText);
+      }
+    }).catch((err) => {
+      console.log(errorMsg + err);
+    });
+  },
+
+  whenReady: function(callback) {
+    const defaultLangPromise = this.promises[this.defaultLanguage];
+    const currentLangPromise = this.promises[this.currentLanguage];
+    Promise.all([defaultLangPromise, currentLangPromise]).then(callback);
+  },
+
+  get: function(msgId) {
+    const activeTranslation = this.data[this.currentLanguage];
+
+    const r = activeTranslation && activeTranslation[msgId];
+    if ( r )
+      return r;
+
+    const defaultMsgs = this.data[this.defaultLanguage];
+    if ( defaultMsgs )
+      return defaultMsgs[msgId];
+
+    throw "Translations are not loaded";
+  }
+}
+
+
+function $t(msgId, params={}) {
+  try {
+    const msgTemplate = Translations.get(msgId);
+    if ( ! msgTemplate ) {
+      return "Invalid message id: " + msgId;
+    }
+
+    return mustache.render(msgTemplate, params);
+  } catch (err) {
+    return "ERROR: " + err;
+  }
+}
+
+function getCookie(cookieName) {
+    const name = cookieName + "=";
+    let result;
+    decodeURIComponent(document.cookie).split('; ').forEach(val => {
+        if (val.indexOf(name) === 0) {
+            result = val.substring(name.length);
+        }
+    });
+    return result;
+}
+
+
+const DEFAULT_UI_LANGUAGE = 'en';
+
+Translations.load(DEFAULT_UI_LANGUAGE, /*asDefault=*/true);
+
+function getUserLanguage() {
+  return new URLSearchParams(window.location.search).get('userlang')
+      || getCookie('userlang')
+      || DEFAULT_UI_LANGUAGE;
+}
+
+function setUserLanguage(lang, callback) {
+  setPermanentGlobalCookie('userlang', lang);
+  Translations.load(lang);
+  Translations.whenReady(callback);
+}
+
+function createModalUILanguageSelector() {
+  document.body.insertAdjacentHTML('beforeend',
+    `<div id="uiLanguageSelector" class="modal-wrapper">
+      <div class="modal">
+        <div class="modal-heading">
+          <div class="modal-title">
+            <div>
+                Select UI language
+            </div>
+          </div>
+          <div onclick="window.modalUILanguageSelector.close()" class="modal-close-button">
+            <div>
+              <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 14 14" fill="none">
+                  <path fill-rule="evenodd" clip-rule="evenodd" d="M13.7071 1.70711C14.0976 1.31658 14.0976
+                  0.683417 13.7071 0.292893C13.3166 -0.0976311 12.6834 -0.0976311 12.2929 0.292893L7 5.58579L1.70711
+                  0.292893C1.31658 -0.0976311 0.683417 -0.0976311 0.292893 0.292893C-0.0976311 0.683417
+                  -0.0976311 1.31658 0.292893 1.70711L5.58579 7L0.292893 12.2929C-0.0976311 12.6834
+                  -0.0976311 13.3166 0.292893 13.7071C0.683417 14.0976 1.31658 14.0976 1.70711 13.7071L7
+                  8.41421L12.2929 13.7071C12.6834 14.0976 13.3166 14.0976 13.7071 13.7071C14.0976 13.3166
+                  14.0976 12.6834 13.7071 12.2929L8.41421 7L13.7071 1.70711Z" fill="black" />
+              </svg>
+            </div>
+          </div>
+        </div>
+        <div class="modal-content">
+          <select id="ui_language"></select>
+        </div>
+      </div>
+    </div>`);
+
+  window.modalUILanguageSelector = {
+    show: () => {
+      document.getElementById('uiLanguageSelector').style.display = 'flex';
+    },
+
+    close: () => {
+      document.getElementById('uiLanguageSelector').style.display = 'none';
+    }
+  };
+}
+
+function initUILanguageSelector(activeLanguage, languageChangeCallback) {
+  if ( document.getElementById("ui_language") == null ) {
+    createModalUILanguageSelector();
+  }
+  const languageSelector = document.getElementById("ui_language");
+  for (const lang of uiLanguages ) {
+    const lang_name = Object.getOwnPropertyNames(lang)[0];
+    const lang_code = lang[lang_name];
+    const is_selected = lang_code == activeLanguage;
+    languageSelector.appendChild(new Option(lang_name, lang_code, is_selected, is_selected));
+  }
+  languageSelector.onchange = languageChangeCallback;
+}
+
+window.$t = $t;
+window.getUserLanguage = getUserLanguage;
+window.setUserLanguage = setUserLanguage;
+window.initUILanguageSelector = initUILanguageSelector;

static/skin/i18n/ar.json

@@ -0,0 +1,33 @@
+{
+	"@metadata": {
+		"authors": [
+			"Asma",
+			"Ravan",
+			"محمد أحمد عبد الفتاح"
+		]
+	},
+	"name": "الإنجليزية",
+	"no-such-book": "لا يوجد مثل هذا الكتاب: {{BOOK_NAME}}",
+	"too-many-books": "طلب العديد من الكتب {{NB_BOOKS}} حيث الحد {{LIMIT}}",
+	"no-book-found": "لا يوجد كتاب يطابق معايير الاختيار",
+	"url-not-found": "لم يتم العثور على عنوان URL المطلوب \"{{url}}\" على هذا الخادم.",
+	"suggest-search": "قم بإجراء بحث عن النص الكامل لـ <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "مع الأسف! فشل اختيار مقال عشوائي :(",
+	"invalid-raw-data-type": "{{DATATYPE}} ليس طلبًا صالحًا للمحتوى الأولي.",
+	"no-value-for-arg": "لم يتم تقديم قيمة للوسيطة {{ARGUMENT}}",
+	"no-query": "لم يتم تقديم ملخص.",
+	"raw-entry-not-found": "لا يمكن العثور على إدخال {{DATATYPE}} {{ENTRY}}",
+	"400-page-title": "طلب غير صالح",
+	"400-page-heading": "طلب غير صالح",
+	"404-page-title": "المحتوى غير موجود",
+	"404-page-heading": "لم يتم العثور عليه",
+	"500-page-title": "خطأ في الخادم الداخلي",
+	"500-page-heading": "خطأ في الخادم الداخلي",
+	"fulltext-search-unavailable": "البحث عن النص الكامل غير متاح",
+	"no-search-results": "محرك البحث عن النص الكامل غير متاح لهذا المحتوى.",
+	"library-button-text": "اذهب لصفحة الترحيب",
+	"home-button-text": "انتقل إلى الصفحة الرئيسية لـ \"{{BOOK_TITLE}}\"",
+	"random-page-button-text": "اذهب إلى صفحة عشوائية",
+	"searchbox-tooltip": "بحث \"{{BOOK_TITLE}}\"",
+	"confusion-of-tongues": "قد يشارك في البحث كتابان أو أكثر بلغات مختلفة، مما قد يؤدي إلى نتائج محيرة."
+}

static/skin/i18n/bn.json

@@ -10,5 +10,6 @@
 	"500-page-heading": "অভ্যন্তরীণ সার্ভার ত্রুটি",
 	"library-button-text": "স্বাগত পাতায় চলুন",
 	"home-button-text": "'{{BOOK_TITLE}}'-এর প্রধান পাতায় চলুন",
-	"searchbox-tooltip": "'{{BOOK_TITLE}}' অনুসন্ধান করুন"
+	"searchbox-tooltip": "'{{BOOK_TITLE}}' অনুসন্ধান করুন",
+	"search": "অনুসন্ধান"
 }

static/skin/i18n/cs.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Spotter"
+		]
+	},
+	"name": "Čeština",
+	"suggest-full-text-search": "obsahující '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Žádná taková kniha: {{BOOK_NAME}}",
+	"too-many-books": "Bylo požadováno příliš mnoho knih ({{NB_BOOKS}}), kde je limit {{LIMIT}}",
+	"no-book-found": "Výběrovým kritériím nevyhovuje žádná kniha",
+	"url-not-found": "Požadovaná adresa URL \"{{url}}\" nebyla na tomto serveru nalezena.",
+	"suggest-search": "Proveďte fulltextové vyhledávání <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Jejda! Nepodařilo se vybrat náhodný článek :(",
+	"invalid-raw-data-type": "{{DATATYPE}} není platný požadavek na nezpracovaný obsah.",
+	"no-value-for-arg": "Pro argument {{ARGUMENT}} nebyla zadána žádná hodnota",
+	"no-query": "Nebyl poskytnut žádný dotaz.",
+	"raw-entry-not-found": "Nelze najít položku {{DATATYPE}} {{ENTRY}}",
+	"400-page-title": "Neplatný požadavek",
+	"400-page-heading": "Neplatný požadavek",
+	"404-page-title": "Obsah nenalezen",
+	"404-page-heading": "Nenalezeno",
+	"500-page-title": "Interní chyba serveru",
+	"500-page-heading": "Interní chyba serveru",
+	"fulltext-search-unavailable": "Fulltextové vyhledávání není k dispozici",
+	"no-search-results": "Fulltextový vyhledávač není pro tento obsah dostupný.",
+	"library-button-text": "Přejít na uvítací stránku",
+	"home-button-text": "Přejít na hlavní stránku '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Přejít na náhodně vybranou stránku",
+	"searchbox-tooltip": "Hledat '{{BOOK_TITLE}}'"
+}

static/skin/i18n/de.json

@@ -0,0 +1,20 @@
+{
+	"@metadata": {
+		"authors": [
+			"Lucas Werkmeister",
+			"ThisCarthing"
+		]
+	},
+	"name": "Deutsch",
+	"random-article-failure": "Hoppla! Konnte keinen zufälligen Artikel auswählen :(",
+	"400-page-title": "Ungültige Anfrage",
+	"400-page-heading": "Ungültige Anfrage",
+	"404-page-title": "Inhalt nicht gefunden",
+	"404-page-heading": "Nicht gefunden",
+	"500-page-title": "Interner Server-Fehler",
+	"500-page-heading": "Interner Server-Fehler",
+	"library-button-text": "Zur Willkommensseite gehen",
+	"home-button-text": "Zur Hauptseite von '{{BOOK_TITLE}}' gehen",
+	"random-page-button-text": "Zu einer zufällig ausgewählten Seite gehen",
+	"searchbox-tooltip": "Nach '{{BOOK_TITLE}}' suchen"
+}

static/skin/i18n/dga.json

@@ -0,0 +1,21 @@
+{
+	"@metadata": {
+		"authors": [
+			"Alhaji Yakubu"
+		]
+	},
+	"welcome-page-overzealous-filter": "Duoro kyebe. E na boɔra ka fo <a href=\"?lang=\"></a>",
+	"search": "Bo",
+	"book-filtering-all-categories": "Zagre zaa",
+	"book-filtering-all-languages": "Kɔkɔrɛɛ zaa",
+	"count-of-matching-books": "{{COUNT}}  gama",
+	"download": "Tagebo",
+	"direct-download-link-text": "Toribu",
+	"direct-download-alt-text": "Toribu tagebo",
+	"hash-download-alt-text": "Tage bonmannaa",
+	"magnet-link-text": "Kurimaraa sobie",
+	"magnet-alt-text": "Tage kurimaraa",
+	"library-opds-feed": "Gamadie OPDS diibu",
+	"filter-by-tag": "Guy yi kpuli {{TAG}}",
+	"stop-filtering-by-tag": "Bare gyɛɛbo kpuli {{TAG}}"
+}

static/skin/i18n/el.json

@@ -0,0 +1,21 @@
+{
+	"@metadata": {
+		"authors": [
+			"Norhorn"
+		]
+	},
+	"welcome-page-overzealous-filter": "Κανένα αποτέλεσμα. Θέλετε να <a href=\"?lang=\">επαναφέρετε το φίλτρο</a>;",
+	"powered-by-kiwix-html": "Με την υποστήριξη by&nbsp;<a href=\"https://kiwix.org\">Kiwix</a>",
+	"search": "Αναζήτηση",
+	"book-filtering-all-categories": "Όλες οι κατηγορίες",
+	"book-filtering-all-languages": "Όλες οι γλώσσες",
+	"count-of-matching-books": "{{COUNT}} βιβλίο(α)",
+	"download": "Λήψη",
+	"direct-download-link-text": "Απευθείας",
+	"direct-download-alt-text": "άμεση λήψη",
+	"hash-download-alt-text": "λήψη αναγνωριστικού",
+	"torrent-download-link-text": "Αρχείο torrent",
+	"torrent-download-alt-text": "λήψη torrent",
+	"filter-by-tag": "Φίλτρο ανά ετικέτα \"{{TAG}}\"",
+	"stop-filtering-by-tag": "Διακοπή φίλτρου ανά ετικέτα \"{{TAG}}\""
+}

static/skin/i18n/en.json

@@ -0,0 +1,54 @@
+{
+	"@metadata": {
+		"authors": [
+		]
+	},
+	"name":"English",
+	"suggest-full-text-search" : "containing '{{{SEARCH_TERMS}}}'..."
+	, "no-such-book" : "No such book: {{BOOK_NAME}}"
+	, "too-many-books" : "Too many books requested ({{NB_BOOKS}}) where limit is {{LIMIT}}"
+	, "no-book-found" : "No book matches selection criteria"
+	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
+	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
+	, "random-article-failure" :  "Oops! Failed to pick a random article :("
+	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
+	, "no-value-for-arg": "No value provided for argument {{ARGUMENT}}"
+	, "no-query" : "No query provided."
+	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
+	, "400-page-title" : "Invalid request"
+	, "400-page-heading" : "Invalid request"
+	, "404-page-title" : "Content not found"
+	, "404-page-heading" : "Not Found"
+	, "500-page-title" : "Internal Server Error"
+	, "500-page-heading" : "Internal Server Error"
+	, "fulltext-search-unavailable" : "Fulltext search unavailable"
+	, "no-search-results": "The fulltext search engine is not available for this content."
+	, "library-button-text": "Go to welcome page"
+	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
+	, "random-page-button-text": "Go to a randomly selected page"
+	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
+	, "confusion-of-tongues": "Two or more books in different languages would participate in search, which may lead to confusing results."
+	, "welcome-page-overzealous-filter": "No result. Would you like to <a href=\"{{URL}}\">reset filter</a>?"
+	, "powered-by-kiwix-html": "Powered by&nbsp;<a href=\"https://kiwix.org\">Kiwix</a>"
+	, "search": "Search"
+	, "book-filtering-all-categories": "All categories"
+	, "book-filtering-all-languages": "All languages"
+	, "count-of-matching-books": "{{COUNT}} book(s)"
+	, "download": "Download"
+	, "direct-download-link-text": "Direct"
+	, "direct-download-alt-text": "direct download"
+	, "hash-download-link-text": "Sha256 hash"
+	, "hash-download-alt-text": "download hash"
+	, "magnet-link-text": "Magnet link"
+	, "magnet-alt-text": "download magnet"
+	, "torrent-download-link-text": "Torrent file"
+	, "torrent-download-alt-text": "download torrent"
+	, "library-opds-feed-all-entries": "Library OPDS Feed - All entries"
+	, "filter-by-tag": "Filter by tag \"{{TAG}}\""
+	, "stop-filtering-by-tag": "Stop filtering by tag \"{{TAG}}\""
+	, "library-opds-feed-parameterised": "Library OPDS Feed - entries matching {{#LANG}}\nLanguage: {{LANG}} {{/LANG}}{{#CATEGORY}}\nCategory: {{CATEGORY}} {{/CATEGORY}}{{#TAG}}\nTag: {{TAG}} {{/TAG}}{{#Q}}\nQuery: {{Q}} {{/Q}}"
+	, "welcome-to-kiwix-server": "Welcome to Kiwix Server"
+	, "download-links-heading": "Download links for <b><i>{{BOOK_TITLE}}</i></b>"
+	, "download-links-title": "Download book"
+	, "preview-book": "Preview"
+}

static/skin/i18n/fr.json

@@ -29,5 +29,24 @@
 	"library-button-text": "Aller à la page de bienvenue",
 	"home-button-text": "Aller à la page principale de « {{BOOK_TITLE}} »",
 	"random-page-button-text": "Aller à une page sélectionnée aléatoirement",
-	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »"
+	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »",
+	"confusion-of-tongues": "Deux livres ou plus dans des langues différentes participeraient à la recherche, ce qui pourrait conduire à des résultats confus.",
+	"welcome-page-overzealous-filter": "Aucun résultat. Souhaitez-vous <a href=\"?lang=\">réinitialiser le filtre</a> ?",
+	"powered-by-kiwix-html": "Propulsé par <a href=\"https://kiwix.org/\">Kiwix</a>",
+	"search": "Rechercher",
+	"book-filtering-all-categories": "Toutes les catégories",
+	"book-filtering-all-languages": "Toutes les langues",
+	"count-of-matching-books": "{{COUNT}} livre(s)",
+	"download": "Télécharger",
+	"direct-download-link-text": "Direct",
+	"direct-download-alt-text": "téléchargement direct",
+	"hash-download-link-text": "Hachage sha256",
+	"hash-download-alt-text": "télécharger le hachage",
+	"magnet-link-text": "Lien Magnet",
+	"magnet-alt-text": "télécharger le lien Magnet",
+	"torrent-download-link-text": "Fichier torrent",
+	"torrent-download-alt-text": "télécharger le torrent",
+	"library-opds-feed": "Flux OPDS de la bibliothèque",
+	"filter-by-tag": "Filtrer par la balise « {{TAG}} »",
+	"stop-filtering-by-tag": "Arrêter le filtrage par la balise « {{TAG}} »"
 }

static/skin/i18n/he.json

@@ -1,7 +1,8 @@
 {
 	"@metadata": {
 		"authors": [
-			"Amire80"
+			"Amire80",
+			"YaronSh"
 		]
 	},
 	"name": "עברית",
@@ -27,5 +28,24 @@
 	"library-button-text": "מעבר לדף הבית \"ברוך בואך\"",
 	"home-button-text": "מעבר לדף הראשי של \"{{BOOK_TITLE}}\"",
 	"random-page-button-text": "מעבר לדף שנבחר אקראית",
-	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\""
+	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\"",
+	"confusion-of-tongues": "שני ספרים או יותר בשפות שונות ישתתפו בחיפוש, מה שעלול להוביל לתוצאות מבלבלות.",
+	"welcome-page-overzealous-filter": "אין תוצאות. האם <a href=\"?lang=\">לאפס את המסנן</a>?",
+	"powered-by-kiwix-html": "מופעל על־ידי&nbsp;<a href=\"https://kiwix.org\">Kiwix</a>",
+	"search": "חיפוש",
+	"book-filtering-all-categories": "כל הקטגוריות",
+	"book-filtering-all-languages": "כל השפות",
+	"count-of-matching-books": "{{COUNT}} ספרים",
+	"download": "הורדה",
+	"direct-download-link-text": "ישירה",
+	"direct-download-alt-text": "הורדה ישירה",
+	"hash-download-link-text": "גיבוב Sha256",
+	"hash-download-alt-text": "הורדת גיבוב",
+	"magnet-link-text": "קישור Magnet",
+	"magnet-alt-text": "הורדת magnet",
+	"torrent-download-link-text": "קובץ טורנט",
+	"torrent-download-alt-text": "הורדת טורנט",
+	"library-opds-feed": "הזנת OPDS של ספרייה",
+	"filter-by-tag": "סינון לפי התג \"{{TAG}}\"",
+	"stop-filtering-by-tag": "להפסיק סינון לפי התג \"{{TAG}}\""
 }

static/skin/i18n/hy.json

@@ -1,16 +1,21 @@
 {
 	"@metadata": {
-		"authors": []
+		"authors": [
+			"Kareyac"
+		]
 	},
 	"name": "Հայերեն",
 	"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}",
 	"url-not-found": "Սխալ հասցե՝ {{url}}",
 	"suggest-search": "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"400-page-title": "Անվավեր հարցում",
+	"400-page-heading": "Անվավեր հարցում",
 	"404-page-title": "Սխալ հասցե",
 	"404-page-heading": "Սխալ հասցե",
 	"library-button-text": "Գրադարանի էջ",
 	"home-button-text": "Դեպի '{{BOOK_TITLE}}'֊ի գլխավոր էջը",
 	"random-page-button-text": "Բացել պատահական էջ",
-	"searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում"
+	"searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում",
+	"book-filtering-all-categories": "Բոլոր կատեգորիաներ"
 }

static/skin/i18n/ia.json

@@ -0,0 +1,25 @@
+{
+	"@metadata": {
+		"authors": [
+			"McDutchie"
+		]
+	},
+	"welcome-page-overzealous-filter": "Nulle resultato. Vole tu <a href=\"?lang=\">reinitialisar le filtro</a>?",
+	"powered-by-kiwix-html": "Actionate per&nbsp;<a href=\"https://kiwix.org\">Kiwix</a>",
+	"search": "Cercar",
+	"book-filtering-all-categories": "Tote le categorias",
+	"book-filtering-all-languages": "Tote le linguas",
+	"count-of-matching-books": "{{COUNT}} libro(s)",
+	"download": "Discargar",
+	"direct-download-link-text": "Directe",
+	"direct-download-alt-text": "discargamento directe",
+	"hash-download-link-text": "Hash SHA256",
+	"hash-download-alt-text": "hash del discargamento",
+	"magnet-link-text": "Ligamine Magnet",
+	"magnet-alt-text": "ligamine \"magnet\" de discargamento",
+	"torrent-download-link-text": "File Torrent",
+	"torrent-download-alt-text": "discargar Torrent",
+	"library-opds-feed": "Fluxo OPDS del bibliotheca",
+	"filter-by-tag": "Filtrar per etiquetta \"{{TAG}}\"",
+	"stop-filtering-by-tag": "Non plus filtrar per etiquetta \"{{TAG}}\""
+}

static/skin/i18n/it.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Albano",
+			"Beta16"
+		]
+	},
+	"name": "italiano",
+	"suggest-full-text-search": "contenente '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Nessun libro del genere: {{BOOK_NAME}}",
+	"too-many-books": "Troppi libri richiesti ({{NB_BOOKS}}) dove il limite è {{LIMIT}}",
+	"no-book-found": "Nessun libro corrisponde ai criteri di selezione",
+	"url-not-found": "L'URL richiesto \"{{url}}\" non è stato trovato in questo server.",
+	"suggest-search": "Effettua una ricerca di testo completo per <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ops! Impossibile selezionare un articolo casuale :(",
+	"no-value-for-arg": "Nessun valore fornito per l'argomento {{ARGUMENT}}",
+	"400-page-title": "Richiesta non valida",
+	"400-page-heading": "Richiesta non valida",
+	"404-page-title": "Contenuto non trovato",
+	"404-page-heading": "Non trovato",
+	"500-page-title": "Errore interno del server",
+	"500-page-heading": "Errore interno del server",
+	"library-button-text": "Vai alla pagina di benvenuto",
+	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Vai a una pagina selezionata casualmente",
+	"searchbox-tooltip": "Cerca '{{BOOK_TITLE}}'",
+	"book-filtering-all-categories": "Tutte le categorie",
+	"book-filtering-all-languages": "Tutte le lingue",
+	"count-of-matching-books": "{{COUNT}} libro/i",
+	"download": "Scarica"
+}

static/skin/i18n/ja.json

@@ -1,9 +1,11 @@
 {
 	"@metadata": {
 		"authors": [
-			"MathXplore"
+			"MathXplore",
+			"もなー(偽物)"
 		]
 	},
+	"name": "日本語",
 	"no-query": "クエリを指定していません。",
 	"400-page-title": "無効なリクエストです",
 	"400-page-heading": "無効なリクエストです",
@@ -14,5 +16,17 @@
 	"fulltext-search-unavailable": "全文検索は利用できません",
 	"no-search-results": "このコンテンツでは全文検索エンジンが利用できません",
 	"library-button-text": "ウェルカムページに移動",
-	"random-page-button-text": "無作為に選ばれたページに移動する"
+	"random-page-button-text": "無作為に選ばれたページに移動する",
+	"search": "検索",
+	"book-filtering-all-categories": "すべてのカテゴリー",
+	"book-filtering-all-languages": "すべての言語",
+	"download": "ダウンロード",
+	"direct-download-link-text": "直ダウンロードリンク",
+	"direct-download-alt-text": "直ダウンロード",
+	"hash-download-link-text": "Sha256 ハッシュ",
+	"hash-download-alt-text": "ハッシュをダウンロード",
+	"magnet-link-text": "マグネットリンク",
+	"magnet-alt-text": "マグネットをダウンロード",
+	"torrent-download-link-text": "Torrentファイル",
+	"torrent-download-alt-text": "Torrentをダウンロード"
 }

static/skin/i18n/ku-latn.json

@@ -27,5 +27,6 @@
 	"library-button-text": "Here rûpela xêrhatinê",
 	"home-button-text": "Here rûpela destpêkê yê {{BOOK_TITLE}}",
 	"random-page-button-text": "Here rûpeleke ketober bijartî",
-	"searchbox-tooltip": "Li {{BOOK_TITLE}} bigere"
+	"searchbox-tooltip": "Li {{BOOK_TITLE}} bigere",
+	"confusion-of-tongues": "Du an zêdetir kitêbên bi zimanên cihê wê beşdarî lêgerînê bibin, ev jî dibe ku bibe sedema tevliheviya encaman."
 }

static/skin/i18n/lb.json

@@ -0,0 +1,26 @@
+{
+	"@metadata": {
+		"authors": [
+			"Robby",
+			"Volvox"
+		]
+	},
+	"name": "Lëtzebuergesch",
+	"suggest-search": "Maacht eng Volltext-Sich fir <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ups! Et konnt keen zoufällegen Artikel ausgewielt ginn :(",
+	"404-page-title": "Inhalt net fonnt",
+	"404-page-heading": "Net fonnt",
+	"500-page-title": "Interne Feeler um Server",
+	"500-page-heading": "Interne Feeler um Server",
+	"fulltext-search-unavailable": "Volltext-Sich net verfügbar",
+	"home-button-text": "Gitt op d'Haaptsäit vun '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Gitt op eng zoufälleg gewielte Säit",
+	"searchbox-tooltip": "No '{{BOOK_TITLE}}' sichen",
+	"welcome-page-overzealous-filter": "Kee Resultat. Wëllt Dir <a href=\"?lang=\">de Filter zrécksetzen</a>?",
+	"search": "Sichen",
+	"book-filtering-all-categories": "All Kategorien",
+	"book-filtering-all-languages": "All Sproochen",
+	"count-of-matching-books": "{{COUNT}} Buch/Bicher",
+	"download": "Eroflueden",
+	"direct-download-link-text": "Direkt"
+}

static/skin/i18n/mk.json

@@ -27,5 +27,24 @@
 	"library-button-text": "Оди на воведната страница",
 	"home-button-text": "Оди на главната страница на „{{BOOK_TITLE}}“",
 	"random-page-button-text": "Оди на случајно избрана страница",
-	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“"
+	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“",
+	"confusion-of-tongues": "Во пребарувањето ќе учествуваат две или повеќе книги на различни јазици, што може да довете до збунувачки исход.",
+	"welcome-page-overzealous-filter": "Нема исход. Дали би сакале да го <a href=\"?lang=\">поништите филтерот</a>?",
+	"powered-by-kiwix-html": "Овозможено од&nbsp;<a href=\"https://kiwix.org\">Кивикс</a>",
+	"search": "Пребарај",
+	"book-filtering-all-categories": "Сите категории",
+	"book-filtering-all-languages": "Сите јазици",
+	"count-of-matching-books": "{{COUNT}} книги",
+	"download": "Преземи",
+	"direct-download-link-text": "Непосредно",
+	"direct-download-alt-text": "непосредно преземање",
+	"hash-download-link-text": "Sha256-тараба",
+	"hash-download-alt-text": "преземи тараба",
+	"magnet-link-text": "Магнетна врска",
+	"magnet-alt-text": "преземи магнет",
+	"torrent-download-link-text": "Торентна податотека",
+	"torrent-download-alt-text": "преземи торент",
+	"library-opds-feed": "Библиотечен OPDS-тековник",
+	"filter-by-tag": "Филтрирај по ознаката „{{TAG}}“",
+	"stop-filtering-by-tag": "Запри филтрирање по ознаката „{{TAG}}“"
 }

static/skin/i18n/nl.json

@@ -0,0 +1,30 @@
+{
+	"@metadata": {
+		"authors": [
+			"McDutchie",
+			"Vistaus"
+		]
+	},
+	"too-many-books": "Er zijn teveel boeken opgevraagd ({{NB_BOOKS}}). Het limiet is {{LIMIT}}.",
+	"no-book-found": "Er zijn geen boeken die overeenkomen met de zoekcriteria",
+	"no-value-for-arg": "Er is geen waarde opgegeven bij {{ARGUMENT}}",
+	"no-query": "Er is geen zoekterm opgegeven.",
+	"welcome-page-overzealous-filter": "Geen resultaat. Wilt u <a href=\"?lang=\">het filter resetten</a>?",
+	"powered-by-kiwix-html": "Mogelijk gemaakt door <a href=\"https://kiwix.org\">Kiwix</a>",
+	"search": "Zoeken",
+	"book-filtering-all-categories": "Alle categorieën",
+	"book-filtering-all-languages": "Alle talen",
+	"count-of-matching-books": "{{COUNT}} boek(en)",
+	"download": "Downloaden",
+	"direct-download-link-text": "Direct",
+	"direct-download-alt-text": "directe download",
+	"hash-download-link-text": "SHA256-hash",
+	"hash-download-alt-text": "controlesom (hash) van de download",
+	"magnet-link-text": "Magnet-link",
+	"magnet-alt-text": "magnet-link van de download",
+	"torrent-download-link-text": "Torrent-bestand",
+	"torrent-download-alt-text": "torrent downloaden",
+	"library-opds-feed": "OPDS-feed van de bibliotheek",
+	"filter-by-tag": "Filteren op tag \"{{TAG}}\"",
+	"stop-filtering-by-tag": "Stoppen met filteren op tag \"{{TAG}}\""
+}

static/skin/i18n/nqo.json

@@ -0,0 +1,33 @@
+{
+	"@metadata": {
+		"authors": [
+			"Lancine.kounfantoh.fofana"
+		]
+	},
+	"name": "ߒߞߏ",
+	"suggest-full-text-search": "ߞߣߐߞߍߣߍ߲߫ ߦߋ߫ '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "ߞߊ߬ߝߊ߫ ߛߎ߮ ߏ߬ ߕߴߦߋ߲߬: {{BOOK_NAME}}",
+	"too-many-books": "ߞߝߊ߬ ߛߌߦߊߡߊ߲߫ ߡߊߢߌ߬ߣߌ߲߬ߞߊ߬ߣߍ߲߫ ߞߏߖߎ߰ {{NB_BOOKS}} ߡߍ߲ ߞߐߘߊ߲ ߦߋ߫ {{LIMIT}} ߘߌ߫",
+	"no-book-found": "ߞߝߊ߬ ߛߌ߫ ߓߍ߬ߣߍ߲߬ ߕߍ߫ ߛߎߥߊ߲ߘߟߌ ߛߙߊߕߌ ߏ߬ ߡߊ߬",
+	"url-not-found": "ߍ߲߬ߤߍ߲߫ ''URL ''{{url}} ߡߊߢߌ߬ߣߌ߲߬ߞߊ߬ߣߍ߲ ߡߊ߫ ߛߐ߬ߘߐ߲߫ ߡߊ߬ߟߐ߬ߟߊ߲ ߣߌ߲߬ ߠߊ߫.",
+	"suggest-search": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲ ߞߍ߫ <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "ߏ߯߹ ߞߢߊ߬ ߓߘߊ߫ ߞߍ߫ ߓߍ߲߬ߛߋ߲߬ߡߊ߬ ߞߎߡߘߊ ߛߐ߬ߘߐ߲߬ ߠߊ߫:(",
+	"invalid-raw-data-type": "{{DATATYPE}} ߕߍ߫ ߡߞߊ߬ߣߌ߲߬ߣߍ߲߫ ߓߍ߲߬ߣߍ߲߫ ߘߌ߫ ߞߣߐߘߐ߫ ߘߝߊߣߍ߲ ߞߊ߲ߡߊ߬.",
+	"no-value-for-arg": "ߡߐ߬ߟߐ߲߬ ߡߊ߫ ߡߊߛߐ߫ ߘߊߘߐߡߌߣߊߞߎ߲ߧߊ {{ARGUMENT}} ߞߊ߲ߡߊ߬",
+	"no-query": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߛߌ߫ ߡߊ߫ ߡߊߛߐ߫.",
+	"raw-entry-not-found": "ߟߊ߬ߘߏ߲߬ߣߍ߲ {{ENTRY}} {{DATATYPE}} ߕߍ߫ ߣߊ߬ ߡߊߛߐ߬ߘߐ߲߫ ߠߊ߫",
+	"400-page-title": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߓߍ߲߬ߓߊߟߌ",
+	"400-page-heading": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߓߍ߲߬ߓߊߟߌ",
+	"404-page-title": "ߞߣߐߘߐ ߡߊ߫ ߛߐ߬ߘߐ߲߫",
+	"404-page-heading": "ߡߊ߫ ߛߐ߬ߘߐ߲߫",
+	"500-page-title": "ߞߣߐߟߊ ߡߊ߬ߛߐ߬ߟߊ߲ ߝߎ߬ߕߎ߲߬ߕߌ",
+	"500-page-heading": "ߞߣߐߟߊ ߡߊ߬ߛߐ߬ߟߊ߲ ߝߎ߬ߕߎ߲߬ߕߌ",
+	"fulltext-search-unavailable": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲ ߕߍ߫ ߦߋ߲߬",
+	"no-search-results": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲߫ ߣߌߘߐ ߕߍ߫ ߦߋ߲߬ ߞߣߐߘߐ ߣߌ߲߬ ߢߍ߫.",
+	"library-button-text": "ߕߊ߯ ߟߊ߬ߛߣߍ߬ߟߌ߫ ߞߐߜߍ ߞߊ߲߬",
+	"home-button-text": "ߕߊ߯ {{BOOK_TITLE}} ߓߏ߬ߟߏ߲߬ߘߊ ߞߐߜߍ ߞߊ߲߬",
+	"random-page-button-text": "ߕߊ߯ ߓߍ߲߬ߛߋ߲߬ߡߊ߬ ߞߐߜߍ߫ ߛߎߥߊ߲ߘߌߣߍ߲ ߠߎ߬ ߞߊ߲߬",
+	"searchbox-tooltip": "ߕߌߙߌ߲ߠߌ߲ {{BOOK_TITLE}}",
+	"confusion-of-tongues": "ߞߊ߬ߝߊ߫ ߝߌ߬ߟߊ߬ ߥߟߊ߫ ߦߙߌߞߊ ߞߊ߲߫ ߜߘߍ ߟߎ߬ ߘߐ߫߸ ߏ߬ ߟߎ߫ ߘߌߣߊ߬ ߕߘߍ߬ ߢߌߣߌ߲ߠߌ߲ ߘߐ߫߸ ߡߍ߲ ߠߎ߬ ߛߌ߫ ߞߣߐ߬ߝߟߌ ߟߊߘߏ߲߬ ߠߊ߫ ߞߐߝߟߌ ߘߐ߫.",
+	"welcome-page-overzealous-filter": "ߞߐߝߟߌ߫ ߕߴߦߋ߲߬. ߊ߬ ߝߐ߫ ߌ ߦߴߊ߬ ߝߍ߬ ߞߊ߬ <a href=\"?lang=\">ߛߍ߲ߛߍ߲ߟߊ߲ ߘߐߛߌ߰ ߕߎ߲߯</a>؟"
+}

static/skin/i18n/qqq.json

@@ -29,5 +29,28 @@
 	"library-button-text": "Tooltip of the button leading to the welcome page",
 	"home-button-text": "Tooltip of the button leading to the main page of a book",
 	"random-page-button-text": "Tooltip of the button opening a randomly selected page",
-	"searchbox-tooltip": "Tooltip displayed for the search box"
+	"searchbox-tooltip": "Tooltip displayed for the search box",
+	"welcome-page-overzealous-filter": "Text shown when book filtering on the welcome page produces zero results",
+	"powered-by-kiwix-html": "Link to Kiwix website",
+	"search": "A general search action (text displayed on search buttons or as aplaceholder in searchboxes)",
+	"book-filtering-all-categories": "Choosing this filter will disable filtering of books by category",
+	"book-filtering-all-languages": "Choosing this filter will disable filtering of books by language",
+	"count-of-matching-books": "Reporting the count of books matching the filter",
+	"download": "A general download action",
+	"direct-download-link-text": "Link text for a direct download",
+	"direct-download-alt-text": "Hint for a direct download icon",
+	"hash-download-link-text": "Link text for downloading the hash",
+	"hash-download-alt-text": "Hint for the icon of hash download",
+	"magnet-link-text": "Link text for a magnet link",
+	"magnet-alt-text": "Hint for the icon of a magnet link",
+	"torrent-download-link-text": "Link text for downloading the torrent file",
+	"torrent-download-alt-text": "Hint for the icon of torrent download",
+	"filter-by-tag": "Hint for a link that would load results filtered by a single tag",
+	"stop-filtering-by-tag": "Tooltip for the button that cancels filtering by tag",
+	"library-opds-feed-all-entries": "Hint for the library OPDS feed for all entries",
+	"library-opds-feed-parameterised": "Hint for the library OPDS feed for filtered entries",
+	"welcome-to-kiwix-server": "Title shown in browser's title bar/page tab",
+	"download-links-heading": "Heading for no-js download page",
+	"download-links-title": "Title for no-js download page",
+	"preview-book": "Tooltip of book-tile leading to the book"
 }

static/skin/i18n/ru.json

@@ -3,15 +3,22 @@
 		"authors": [
 			"Fenixs-ru",
 			"Kareyac",
+			"Okras",
 			"Pacha Tchernof"
 		]
 	},
 	"name": "русский",
 	"suggest-full-text-search": "содержащее '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Такой книги нет: {{BOOK_NAME}}",
+	"too-many-books": "Запрошено слишком много книг ({{NB_BOOKS}}), максимальное количество — {{LIMIT}}.",
+	"no-book-found": "Ни одна книга не соответствует критериям отбора",
 	"url-not-found": "Запрошенный URL \"{{url}}\" не найден на этом сервере.",
 	"suggest-search": "Выполните полнотекстовый поиск для <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Ой! Не удалось выбрать случайную статью :(",
+	"invalid-raw-data-type": "{{DATATYPE}} не является допустимым запросом необработанного контента.",
+	"no-value-for-arg": "Не указано значение для аргумента {{ARGUMENT}}",
+	"no-query": "Не предоставлен запрос.",
+	"raw-entry-not-found": "Не удаётся найти запись {{ENTRY}} типа {{DATATYPE}}",
 	"400-page-title": "Недействительный запрос",
 	"400-page-heading": "Недействительный запрос",
 	"404-page-title": "Содержание не найдено",
@@ -23,5 +30,9 @@
 	"library-button-text": "Перейти на страницу-приветствие",
 	"home-button-text": "Перейти на главную страницу '{{BOOK_TITLE}}'",
 	"random-page-button-text": "Перейти на случайно выбранную страницу",
-	"searchbox-tooltip": "Искать '{{BOOK_TITLE}}'"
+	"searchbox-tooltip": "Искать '{{BOOK_TITLE}}'",
+	"confusion-of-tongues": "В поиске будут участвовать две или более книг на разных языках, что может привести к запутанным результатам.",
+	"book-filtering-all-categories": "Все категории",
+	"book-filtering-all-languages": "Все языки",
+	"download": "Скачать"
 }

static/skin/i18n/sc.json

@@ -7,10 +7,14 @@
 	"name": "Sardu",
 	"suggest-full-text-search": "chi cuntenet '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Perunu libru cun custu nùmene: {{BOOK_NAME}}",
+	"too-many-books": "Tropu libros pedidos, {{NB_BOOKS}} cando su lìmite est de {{LIMIT}}",
+	"no-book-found": "Perunu libru currispondet a sos critèrios de seletzione",
 	"url-not-found": "S'URL pedidu \"{{url}}\" non s'est atzapadu in custu serbidore.",
 	"suggest-search": "Faghe una chirca de testu intreu pro <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Oops! Sa seletzione de un'artìculu a casu est fallida :(",
 	"invalid-raw-data-type": "{{DATATYPE}} no est una rechesta vàlida pro cuntenutu puru.",
+	"no-value-for-arg": "Perunu valore frunidu pro s'argumentu {{ARGUMENT}}",
+	"no-query": "Peruna chirca frunida.",
 	"raw-entry-not-found": "Non faghet a atzapare s'elementu {{ENTRY}} de genia {{DATATYPE}}",
 	"400-page-title": "Rechesta non vàlida",
 	"400-page-heading": "Rechesta non vàlida",
@@ -23,5 +27,6 @@
 	"library-button-text": "Bae a sa pàgina de bene bènnidu",
 	"home-button-text": "Bae a sa pàgina printzipale de '{{BOOK_TITLE}}'",
 	"random-page-button-text": "Bae a una pàgina seletzionada a manera casuale",
-	"searchbox-tooltip": "Chirca '{{BOOK_TITLE}}'"
+	"searchbox-tooltip": "Chirca '{{BOOK_TITLE}}'",
+	"confusion-of-tongues": "Duos o prus libros in limbas diferentes diant pigare parte a sa chirca, cosa chi diat pòdere causare resurtados confusionosos."
 }