Enter uriEncode()

`urlEncode(str)` is now equivalent to the previous `urlEncode(str, true)`.

.github/move.yml

@@ -1,27 +0,0 @@
-# Configuration for Move Issues - https://github.com/dessant/move-issues
-
-# Delete the command comment when it contains no other content
-deleteCommand: true
-
-# Close the source issue after moving
-closeSourceIssue: true
-
-# Lock the source issue after moving
-lockSourceIssue: false
-
-# Mention issue and comment authors
-mentionAuthors: true
-
-# Preserve mentions in the issue content
-keepContentMentions: true
-
-# Move labels that also exist on the target repository
-moveLabels: true
-
-# Set custom aliases for targets
-# aliases:
-#   r: repo
-#   or: owner/repo
-
-# Repository to extend settings from
-# _extends: repo

.github/workflows/ci.yml

@@ -1,21 +1,25 @@
 name: CI
 
-on: [push]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   Macos:
     runs-on: macos-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
-      - name: Setup python 3.10
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v2
+      - name: Setup python 3.9
+        uses: actions/setup-python@v1
         with:
-          python-version: '3.10'
+          python-version: '3.9'
       - name: Install packages
         run: |
           brew update
-          brew install gcovr pkg-config ninja
+          brew install gcovr pkg-config ninja || brew link --overwrite python
       - name: Install python modules
         run: pip3 install meson==0.49.2 pytest
       - name: Install deps
@@ -82,10 +86,6 @@ jobs:
     container:
       image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
     steps:
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
     - name: Checkout code
       shell: python
       run: |
@@ -95,7 +95,7 @@ jobs:
           'git', 'clone',
           'https://github.com/${{github.repository}}',
           '--depth=1',
-          '--branch', '${{steps.extract_branch.outputs.branch}}'
+          '--branch', '${{ github.head_ref || github.ref_name }}'
         ]
         check_call(command, cwd=environ['HOME'])
     - name: Install deps

.github/workflows/package.yml

@@ -8,8 +8,8 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-kinetic
           - ubuntu-jammy
-          - ubuntu-impish
           - ubuntu-focal
           - ubuntu-bionic
     steps:
@@ -34,18 +34,18 @@ jobs:
           email: release+launchpad@kiwix.org
           distro: ${{ matrix.distro }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
-        if: matrix.distro == 'ubuntu-jammy'
-        name: Build package for ubuntu-jammy
-        id: build-ubuntu-jammy
+      - uses: legoktm/gh-action-build-deb@ubuntu-kinetic
+        if: matrix.distro == 'ubuntu-kinetic'
+        name: Build package for ubuntu-kinetic
+        id: build-ubuntu-kinetic
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
@@ -73,8 +73,8 @@ jobs:
 
       - uses: legoktm/gh-action-dput@master
         name: Upload dev package
-        # Only upload on pushes to master
-        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/dev

ChangeLog

@@ -1,3 +1,43 @@
+libkiwix 12.0.0
+===============
+
+ * [API Break] Remove wrapper around libzim (@mgautierfr #789)
+ * Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
+ * Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
+ * Prevent search on multi language content (@veloman-yunkan #838)
+ * Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
+ * Catalog:
+  - Include tags in free text catalog search (@veloman-yunkan #802)
+  - Illustration's url is based on book's uuid (@veloman-yunkan #804)
+  - Cleanup of the opds-dumper (@veloman-yunkan #829)
+  - Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
+  - Make opds-dumper respect the namemapper (@mgautierfr #837)
+ * Server:
+  - Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
+  - Better http caching (@veloman-yunkan #833)
+  - Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
+  - Better redirection of main page (@veloman-yunkan #827)
+  - Remove jquery (@mgautierfr @juuz0 #796)
+  - Better Viewer of zim content :
+    . Introduce `/content` endpoints (@veloman-yunkan #806)
+    . Switch to iframe based content viewer (@veloman-yunkan #716)
+  - Optimised design of the welcome page:
+    . Alignement (@juuz0 @kelson42 #786)
+    . Exit download modal on pressing escape key (@juzz0 #800)
+    . Add favicon for different devices (@juzz0 #805)
+    . Fix auto hidding of the toolbar (@veloman-yunkan #821)
+    . Allow user to filter books by tags in the front page (@juuz0 #711)
+ * CI :
+   - Trigger CI on pull_request (@kelson42 #791)
+   - Drop Ubuntu Impish packaging (@legoktm #825)
+   - Add Ubuntu Kinetic packaging (@legoktm #801)
+ * Testing:
+   - Test ICULanguageInfo (@veloman-yunkan #795)
+   - Introduce fake `test` language to test i18n (@veloman-yunkan #848)
+ * Fix documentation (@kelson42 #816)
+ * Udpate translation (#787 #839 #847)
+
+
 libkiwix 11.0.0
 ===============
 
@@ -5,7 +45,7 @@ libkiwix 11.0.0
  * [server] Use gzip compression instead of deflat (mgautierfr #757)
  * [server] Version the static resources. This allow better invalidating
    browser cache when resources are changed (@veloman-yunkan #712)
- * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server|front] Use integer to query the host for page length (@juuz0 #772)
  * [server] Improve multizim search API:
    - Improvement of the cache system
    - Better API to select on which books to search in.

README.md

@@ -7,10 +7,10 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
-[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
 [![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
-[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
+[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Disclaimer
@@ -101,6 +101,33 @@ meson . build -Dwrapper=android -Dwerror=false
 ninja -C build
 ```
 
+Static files compilation
+------------------------
+
+Libkiwix has a few static files 'compiled' within the binary
+code. This is mostly Javascript/HTML/pictures necessary for the HTTP
+daemon.
+
+These static files are available in the `static` directory and are
+compiled by custom Python code available in this repository `scripts`
+directory. This happens automatically at compilation time without any
+additional command to run.
+
+To avoid HTTP caching issues, the URLs (to the static content) are
+appended with a `cacheid` parameter (this is called "cache
+busting"). This `cacheid` value derived from the
+[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
+static file. As a consequence, each time you change a static file, the
+corresponding `cacheid` value will change.
+
+To properly test this feature, this `cacheid` needs to be added
+manually to the automated tests and has to be commited. After
+modifying the needed static file, [run the automated
+tests](#Testing). They will fail, but the inspection of the testing
+log will give you the new `cacheid` value(s). Finally update
+`test/server.cpp` with the appropriate `cacheid` value(s) which have
+changed.
+
 Testing
 -------
 
@@ -124,7 +151,7 @@ where you want to install the libraries. After the installation
 succeeded, you may need to run `ldconfig` (as `root`).
 
 Uninstallation
-------------
+--------------
 
 If you want to uninstall the Kiwix library:
 ```bash
@@ -134,6 +161,55 @@ ninja -C build uninstall
 Like for the installation, you might need to run the command as `root`
 (or using `sudo`).
 
+Custom Index Page
+-----------------
+
+to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
+(note - while using custom html file please mention all external links as absolute path.)
+
+to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
+
+To use JS provided by kiwix-serve you can use the following template to start with ->
+
+```
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title><-- Custom Tittle --></title>
+    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+  </head>
+  <body>
+  </body>
+</html>
+```
+
+- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
+- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
+- To add language select box add - `<select id="languageFilter"></select>` under body tag.
+- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
+- To add search box for books use following form -
+    ```
+        <form id='kiwixSearchForm'>
+        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
+        <input type="submit" class="kiwixButton" value="Search"/>
+        </form>
+    ```
+
+
+If you compile manually Libmicrohttpd, you might need to compile it
+without GNU TLS, a bug here will empeach further compilation
+otherwise.
+
+If the compilation still fails, you might need to get a more recent
+version of a dependency than the one packaged by your Linux
+distribution. Try then with a source tarball distributed by the
+problematic upstream project or even directly from the source code
+repository.
+
 Troubleshooting
 ---------------
 
@@ -156,63 +232,6 @@ cp ninja ../bin
 cd ..
 ```
 
-Custom Index Page
------------------
-
-to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
-(note - while using custom html file please mention all external links as absolute path.)
-
-to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
-
-To use JS provided by kiwix-serve you can use the following template to start with ->
-
-```
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width,initial-scale=1" />
-    <title><-- Custom Tittle --></title>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
-    ></script>
-    <script
-      type="text/javascript"
-      src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
-    ></script>
-    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
-    <script src="{{root}}/skin/iso6391To3.js"></script>
-    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
-  </head>
-  <body>
-  </body>
-</html>
-```
-
-- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
-- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
-- To add language select box add - `<select id="languageFilter"></select>` under body tag.
-- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
-- To add search box for books use following form -
-    ```
-        <form id='kiwixSearchForm'>
-        <input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
-        <input type="submit" class="searchButton" value="Search"/>
-        </form>
-    ```
-
-
-If you compile manually Libmicrohttpd, you might need to compile it
-without GNU TLS, a bug here will empeach further compilation
-otherwise.
-
-If the compilation still fails, you might need to get a more recent
-version of a dependency than the one packaged by your Linux
-distribution. Try then with a source tarball distributed by the
-problematic upstream project or even directly from the source code
-repository.
-
 License
 -------
 

docs/conf.py

@@ -19,7 +19,7 @@ import os
 
 project = 'libkiwix'
 copyright = '2022, libkiwix-team'
-author = 'libzim-team'
+author = 'libkiwix-team'
 
 
 # -- General configuration ---------------------------------------------------

docs/index.rst

@@ -3,7 +3,7 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to libzim's documentation!
+Welcome to libkiwix's documentation!
 ==================================
 
 .. toctree::

docs/usage.rst

@@ -7,11 +7,9 @@ Introduction
 libkiwix is written in C++. To use the library, you need the include files of libkiwix have
 to link against libzim.
 
-Errors are handled with exceptions. When something goes wrong, libzim throws an error,
+Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
 which is always derived from std::exception.
 
 All classes are defined in the namespace kiwix.
 
 libkiwix is a set of tools to manage zim files and provide some common functionnality.
-While libkiwix has some wrappers around libzim classes, they are deprecated and will be removed
-in the future.

format_code.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/bash
-
-files=(
-"include/library.h"
-"include/common/stringTools.h"
-"include/common/pathTools.h"
-"include/common/otherTools.h"
-"include/common/regexTools.h"
-"include/common/networkTools.h"
-"include/common/archiveTools.h"
-"include/manager.h"
-"include/reader.h"
-"include/kiwix.h"
-"include/xapianSearcher.h"
-"include/searcher.h"
-"src/library.cpp"
-"src/android/kiwix.cpp"
-"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
-"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
-"src/searcher.cpp"
-"src/common/pathTools.cpp"
-"src/common/regexTools.cpp"
-"src/common/otherTools.cpp"
-"src/common/archiveTools.cpp"
-"src/common/networkTools.cpp"
-"src/common/stringTools.cpp"
-"src/xapianSearcher.cpp"
-"src/manager.cpp"
-"src/reader.cpp"
-)
-
-for i in "${files[@]}"
-do
-  echo $i
-  clang-format -i -style=file $i
-done

include/book.h

@@ -38,7 +38,6 @@ namespace kiwix
 {
 
 class OPDSDumper;
-class Reader;
 
 /**
  * A class to store information about a book (a zim file)
@@ -69,7 +68,6 @@ class Book
   ~Book();
 
   bool update(const Book& other);
-  DEPRECATED void update(const Reader& reader);
   void update(const zim::Archive& archive);
   void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
   void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);

include/entry.h

@@ -1,193 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_ENTRY_H
-#define KIWIX_ENTRY_H
-
-#include <stdio.h>
-#include <zim/entry.h>
-#include <zim/item.h>
-#include <exception>
-#include <string>
-
-#include "common.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-
-class NoEntry : public std::exception {};
-
-/**
- * A entry represent an.. entry in a zim file.
- */
-class Entry
-{
-  public:
-    /**
-     * Construct an entry making reference to an zim article.
-     *
-     * @param article a zim::Article object
-     */
-    DEPRECATED Entry(zim::Entry entry) : Entry(entry, true) {};
-    virtual ~Entry() = default;
-
-    /**
-     * Get the path of the entry.
-     *
-     * The path is the "key" of an entry.
-     *
-     * @return the path of the entry.
-     */
-    std::string getPath() const { return entry.getPath(); }
-
-    /**
-     * Get the title of the entry.
-     *
-     * @return the title of the entry.
-     */
-    std::string getTitle() const { return entry.getTitle(); }
-
-    /**
-     * Get the content of the entry.
-     *
-     * The string is a copy of the content.
-     * If you don't want to do a copy, use get_blob.
-     *
-     * @return the content of the entry.
-     */
-    std::string getContent() const { return entry.getItem().getData(); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
-
-    /**
-     * Get the blob of the entry.
-     *
-     * A blob make reference to the content without copying it.
-     *
-     * @param offset The starting offset of the blob.
-     * @param size The size of the blob.
-     * @return the blob of the entry.
-     */
-    zim::Blob   getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
-
-    /**
-     * Get the info for direct access to the content of the entry.
-     *
-     * Some entry (ie binary ones) have their content plain stored
-     * in the zim file. Knowing the offset where the content is stored
-     * an user can directly read the content in the zim file bypassing the
-     * libkiwix/libzim.
-     *
-     * @return A pair specifying where to read the content.
-     *         The string is the real file to read (may be different that .zim
-     *         file if zim is cut).
-     *         The offset is the offset to read in the file.
-     *         Return <"",0> if is not possible to read directly.
-     */
-    zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
-
-    /**
-     * Get the size of the entry.
-     *
-     * @return the size of the entry.
-     */
-    size_type   getSize() const;
-
-    /**
-     * Get the mime_type of the entry.
-     *
-     * @return the mime_type of the entry.
-     */
-    std::string getMimetype() const;
-
-
-    /**
-     * Get if the entry is a redirect entry.
-     *
-     * @return True if the entry is a redirect.
-     */
-    bool isRedirect() const;
-
-    /**
-     * Get if the entry is a link target entry.
-     *
-     * @return True if the entry is a link target.
-     */
-    bool isLinkTarget() const;
-
-    /**
-     * Get if the entry is a deleted entry.
-     *
-     * @return True if the entry is a deleted entry.
-     */
-    bool isDeleted() const;
-
-    /**
-     * Get the entry pointed by this entry.
-     *
-     * @return the entry pointed.
-     * @throw NoEntry if the entry is not a redirected entry.
-     */
-    Entry getRedirectEntry() const;
-
-    /**
-     * Get the final entry pointed by this entry.
-     *
-     * Follow the redirection until a "not redirecting" entry is found.
-     * If the entry is not a redirected entry, return the entry itself.
-     *
-     * @return the final entry.
-     */
-    Entry getFinalEntry() const;
-
-    /**
-     * Get the zim entry wrapped by this (kiwix) entry
-     *
-     * @return the zim entry
-     */
-    const zim::Entry& getZimEntry() const { return entry; }
-
-  private:
-    zim::Entry entry;
-
-  private:
-    // Entry is deprecated, so we've marked the constructor as deprecated.
-    // But we still need to construct the entry (in our deprecated code)
-    // To avoid warning because we use deprecated function, we create a second
-    // constructor not deprecated. The `bool marker` is unused, it sole purpose
-    // is to change the signature to have two different constructor.
-    // This one is not deprecated and we must use it in our private code.
-    Entry(zim::Entry entry, bool marker);
-    friend class Reader;
-};
-
-}
-
-#endif // KIWIX_ENTRY_H

include/library.h

@@ -106,7 +106,15 @@ class Filter {
     Filter& rejectTags(const Tags& tags);
 
     Filter& category(std::string category);
+
+    /**
+     *  Set the filter to only accept books in the specified language.
+     *
+     *  Multiple languages can be specified as a comma-separated list (in
+     *  which case a book in any of those languages will match).
+     */
     Filter& lang(std::string lang);
+
     Filter& publisher(std::string publisher);
     Filter& creator(std::string creator);
     Filter& maxSize(size_t size);
@@ -223,7 +231,6 @@ class Library
 
   Book getBookByIdThreadSafe(const std::string& id) const;
 
-  DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
   std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
     return getSearcherByIds(BookIdSet{id});
@@ -333,8 +340,8 @@ class Library
   /**
    * Return the current revision of the library.
    *
-   * The revision of the library is updated (incremented by one) only by
-   * the addBook() operation.
+   * The revision of the library is updated (incremented by one) by
+   * the addBook() and removeBookById() operations.
    *
    * @return Current revision of the library.
    */

include/manager.h

@@ -22,7 +22,6 @@
 
 #include "book.h"
 #include "library.h"
-#include "reader.h"
 
 #include <string>
 #include <vector>

include/meson.build

@@ -7,9 +7,6 @@ headers = [
   'libxml_dumper.h',
   'opds_dumper.h',
   'downloader.h',
-  'reader.h',
-  'entry.h',
-  'searcher.h',
   'search_renderer.h',
   'server.h',
   'kiwixserve.h',

include/opds_dumper.h

@@ -27,7 +27,7 @@
 #include <pugixml.hpp>
 
 #include "library.h"
-#include "reader.h"
+#include "name_mapper.h"
 
 using namespace std;
 
@@ -42,7 +42,7 @@ class OPDSDumper
 {
  public:
   OPDSDumper() = default;
-  OPDSDumper(Library* library);
+  OPDSDumper(Library* library, NameMapper* NameMapper);
   ~OPDSDumper();
 
   /**
@@ -111,6 +111,7 @@ class OPDSDumper
 
  protected:
   kiwix::Library* library;
+  kiwix::NameMapper* nameMapper;
   std::string libraryId;
   std::string rootLocation;
   int m_totalResults;

include/reader.h

@@ -1,506 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_READER_H
-#define KIWIX_READER_H
-
-#include <stdio.h>
-#include <zim/zim.h>
-#include <zim/archive.h>
-#include <exception>
-#include <map>
-#include <sstream>
-#include <string>
-#include "common.h"
-#include "entry.h"
-
-using namespace std;
-
-namespace kiwix
-{
-
-/**
- * The SuggestionItem is a helper class that contains the info about a single
- * suggestion item.
- */
-class SuggestionItem
-{
-  // Functions
-  public:
-    // Create a sugggestion item.
-    explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
-                            const std::string& path, const std::string& snippet = "") :
-      title(title),
-      normalizedTitle(normalizedTitle),
-      path(path),
-      snippet(snippet) {}
-
-  public:
-    const std::string& getTitle() const           { return title;}
-    const std::string& getNormalizedTitle() const { return normalizedTitle;}
-    const std::string& getPath() const            { return path;}
-    const std::string& getSnippet() const         { return snippet;}
-
-    bool hasSnippet() const                { return !snippet.empty();}
-
-  // Data
-  private:
-    std::string title;
-    std::string normalizedTitle;
-    std::string path;
-    std::string snippet;
-};
-
-/**
- * The Reader class is the class who allow to get an entry content from a zim
- * file.
- *
- * Reader is now deprecated. Directly use `zim::Archive`.
- */
-
-using SuggestionsList_t = std::vector<SuggestionItem>;
-class Reader
-{
- public:
-  /**
-   * Create a Reader to read a zim file specified by zimFilePath.
-   *
-   * @param zimFilePath The path to the zim file to read.
-   *                    The zim file can be splitted (.zimaa, .zimab, ...).
-   *                    In this case, the file path must still point to the
-   *                    unsplitted path as if the file were not splitted
-   *                    (.zim extesion).
-   */
-  explicit DEPRECATED Reader(const string zimFilePath);
-
-   /**
-   * Create a Reader to read a zim file given by the Archive.
-   *
-   * @param archive The shared pointer to the Archive object.
-   */
-  explicit DEPRECATED Reader(const std::shared_ptr<zim::Archive> archive)
-   : Reader(archive, true) {};
-#ifndef _WIN32
-  explicit DEPRECATED Reader(int fd);
-  DEPRECATED Reader(int fd, zim::offset_type offset, zim::size_type size);
-#endif
-  ~Reader() = default;
-
-  /**
-   * Get the number of "displayable" entries in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'text/html' MIMEtype specified in the metadata.
-   *         Else return the number of entries in the 'A' namespace.
-   */
-  unsigned int getArticleCount() const;
-
-  /**
-   * Get the number of media in the zim file.
-   *
-   * @return If the zim file has a /M/Counter metadata, return the number of
-   *         entries with the 'image/jpeg', 'image/gif' and 'image/png' in
-   *         the metadata.
-   *         Else return the number of entries in the 'I' namespace.
-   */
-  unsigned int getMediaCount() const;
-
-  /**
-   * Get the number of all entries in the zim file.
-   *
-   * @return Return the number of all the entries, whatever their MIMEtype or
-   *         their namespace.
-   */
-  unsigned int getGlobalCount() const;
-
-  /**
-   * Get the path of the zim file.
-   *
-   * @return the path of the zim file as given in the constructor.
-   */
-  string getZimFilePath() const;
-
-  /**
-   * Get the Id of the zim file.
-   *
-   * @return The uuid stored in the zim file.
-   */
-  string getId() const;
-
-  /**
-   * Get a random page.
-   *
-   * @return A random Entry. The entry is picked from all entries in
-   *         the 'A' namespace.
-   *         The main entry is excluded from the potential results.
-   */
-  Entry getRandomPage() const;
-
-  /**
-   * Get the entry of the main page.
-   *
-   * @return Entry of the main page as specified in the zim file.
-   */
-  Entry getMainPage() const;
-
-  /**
-   * Get the content of a metadata.
-   *
-   * @param[in]  name The name of the metadata.
-   * @param[out] value The value will be set to the content of the metadata.
-   * @return True if it was possible to get the content of the metadata.
-   */
-  bool getMetadata(const string& name, string& value) const;
-
-  /**
-   * Get the name of the zim file.
-   *
-   * @return The name of the zim file as specified in the zim metadata.
-   */
-  string getName() const;
-
-  /**
-   * Get the title of the zim file.
-   *
-   * @return The title of zim file as specified in the zim metadata.
-   *         If no title has been set, return a title computed from the
-   *         file path.
-   */
-  string getTitle() const;
-
-  /**
-   * Get the creator of the zim file.
-   *
-   * @return The creator of the zim file as specified in the zim metadata.
-   */
-  string getCreator() const;
-
-  /**
-   * Get the publisher of the zim file.
-   *
-   * @return The publisher of the zim file as specified in the zim metadata.
-   */
-  string getPublisher() const;
-
-  /**
-   * Get the date of the zim file.
-   *
-   * @return The date of the zim file as specified in the zim metadata.
-   */
-  string getDate() const;
-
-  /**
-   * Get the description of the zim file.
-   *
-   * @return The description of the zim file as specified in the zim metadata.
-   *         If no description has been set, return the subtitle.
-   */
-  string getDescription() const;
-
-  /**
-   * Get the long description of the zim file.
-   *
-   * @return The long description of the zim file as specifed in the zim metadata.
-   */
-  string getLongDescription() const;
-
-  /**
-   * Get the language of the zim file.
-   *
-   * @return The language of the zim file as specified in the zim metadata.
-   */
-  string getLanguage() const;
-
-  /**
-   * Get the license of the zim file.
-   *
-   * @return The license of the zim file as specified in the zim metadata.
-   */
-  string getLicense() const;
-
-  /**
-   * Get the tags of the zim file.
-   *
-   * @param original If true, return the original tags as specified in the zim metadata.
-   *                 Else, try to convert it to the new 'normalized' format.
-   * @return The tags of the zim file.
-   */
-  string getTags(bool original=false) const;
-
-  /**
-   * Get the value (as a string) of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   */
-  string getTagStr(const std::string& tagName) const;
-
-  /**
-   * Get the boolean value of a specific tag.
-   *
-   * According to https://wiki.openzim.org/wiki/Tags
-   *
-   * @return The boolean value of the specified tag.
-   * @throw  std::out_of_range if the specified tag is not found.
-   *         std::domain_error if the value of the tag cannot be convert to bool.
-   */
-  bool getTagBool(const std::string& tagName) const;
-
-  /**
-   * Get the relations of the zim file.
-   *
-   * @return The relation of the zim file as specified in the zim metadata.
-   */
-  string getRelation() const;
-
-  /**
-   * Get the flavour of the zim file.
-   *
-   * @return The flavour of the zim file as specified in the zim metadata.
-   */
-  string getFlavour() const;
-
-  /**
-   * Get the source of the zim file.
-   *
-   * @return The source of the zim file as specified in the zim metadata.
-   */
-  string getSource() const;
-
-  /**
-   * Get the scraper of the zim file.
-   *
-   * @return The scraper of the zim file as specified in the zim metadata.
-   */
-  string getScraper() const;
-
-  /**
-   * Get the favicon of the zim file.
-   *
-   * @param[out] content The content of the favicon.
-   * @param[out] mimeType The mimeType of the favicon.
-   * @return True if a favicon has been found.
-   */
-  bool getFavicon(string& content, string& mimeType) const;
-
-  /**
-   * Get an entry associated to an path.
-   *
-   * @param path The path of the entry.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromPath(const std::string& path) const;
-
-  /**
-   * Get an entry associated to an url encoded path.
-   *
-   * Equivalent to `getEntryFromPath(urlDecode(path));`
-   *
-   * @param path The url encoded path.
-   * @return The entry.
-   * @throw NoEntry If no entry correspond to the path.
-   */
-  Entry getEntryFromEncodedPath(const std::string& path) const;
-
-  /**
-   * Get un entry associated to a title.
-   *
-   * @param title The title.
-   * @return The entry
-   * throw NoEntry If no entry correspond to the url.
-   */
-  Entry getEntryFromTitle(const std::string& title) const;
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are stored in an internal vector and can be retrieved using
-   * `getNextSuggestion` method.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestions(const string& prefix,
-   *                        unsigned int suggestionsCount,
-   *                        SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param reset If true, remove previous suggestions in the internal vector.
-   *              If false, add suggestions to the internal vector
-   *              (until internal vector size is suggestionCount (or no more
-   *               suggestion))
-   * @return True if some suggestions have been added to the internal vector.
-   */
-  DEPRECATED bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         const bool reset = true);
-
-  /**
-   * Search for entries with title starting with prefix (case sensitive).
-   *
-   * Suggestions are added to the `result` vector.
-   *
-   * @param prefix The prefix to search.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param result The vector where to store the suggestions.
-   * @return True if some suggestions have been added to the vector.
-   */
-
-  bool searchSuggestions(const string& prefix,
-                         unsigned int suggestionsCount,
-                         SuggestionsList_t& resuls);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   * This method is not thread safe and is deprecated. Use :
-   * bool searchSuggestionsSmart(const string& prefix,
-   *                             unsigned int suggestionsCount,
-   *                             SuggestionsList_t& results);
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   */
-  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
-
-  /**
-   * Search for entries for the given prefix.
-   *
-   * If the zim file has a internal fulltext index, the suggestions will be
-   * searched using it.
-   * Else the suggestions will be search using `searchSuggestions` while trying
-   * to be smart about case sensitivity (using `getTitleVariants`).
-   *
-   * In any case, suggestions are stored in an internal vector and can be
-   * retrieved using `getNextSuggestion` method.
-   * The internal vector will be reset.
-   *
-   * @param prefix The prefix to search for.
-   * @param suggestionsCount How many suggestions to search for.
-   * @param results The vector where to store the suggestions
-   * @return True if some suggestions have been added to the results.
-   */
-   bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount,
-                              SuggestionsList_t& results);
-
-
-  /**
-   * Check if the path exists in the zim file.
-   *
-   * @param path the path to check.
-   * @return True if the path exists in the zim file.
-   */
-  bool pathExists(const string& path) const;
-
-  /**
-   * Check if the zim file has a embedded fulltext index.
-   *
-   * @return True if the zim file has a embedded fulltext index
-   *         and is not split (else the fulltext is not accessible).
-   */
-  bool hasFulltextIndex() const;
-
-  /**
-   * Get potential case title variations for a title.
-   *
-   * @param title a title.
-   * @return the list of variantions.
-   */
-  std::vector<std::string> getTitleVariants(const std::string& title) const;
-
-  /**
-   * Get the next suggestion title.
-   *
-   * @param[out] title the title of the suggestion.
-   * @return True if title has been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title);
-
-  /**
-   * Get the next suggestion title and url.
-   *
-   * @param[out] title the title of the suggestion.
-   * @param[out] url the url of the suggestion.
-   * @return True if title and url have been set.
-   */
-  DEPRECATED bool getNextSuggestion(string& title, string& url);
-
-  /**
-   * Get if we can check zim file integrity (has a checksum).
-   *
-   * @return True if zim file have a checksum.
-   */
-  bool canCheckIntegrity() const;
-
-  /**
-   * Check is zim file is corrupted.
-   *
-   * @return True if zim file is corrupted.
-   */
-  bool isCorrupted() const;
-
-  /**
-   * Return the total size of the zim file.
-   *
-   * If zim file is split, return the sum of all parts' size.
-   *
-   * @return Size of the size file is KiB.
-   */
-  unsigned int getFileSize() const;
-
-  /**
-   * Get the zim file handler.
-   *
-   * @return The libzim file handler.
-   */
-  zim::Archive* getZimArchive() const;
-
- protected:
-  std::shared_ptr<zim::Archive> zimArchive;
-  std::string zimFilePath;
-
-  SuggestionsList_t suggestions;
-  SuggestionsList_t::iterator suggestionsOffset;
-
- private:
-  std::map<const std::string, unsigned int> parseCounterMetadata() const;
-
-  // Reader is deprecated, so we've marked the constructor as deprecated.
-  // But we still need to construct the reader (in our deprecated code)
-  // To avoid warning because we use deprecated function, we create a
-  // constructor not deprecated. The `bool marker` is unused, it sole purpose
-  // is to change the signature to have a different constructor.
-  // This one is not deprecated and we must use it in our private code.
-  Reader(const std::shared_ptr<zim::Archive> archive, bool marker);
-  friend class Library;
-};
-}
-
-#endif

include/search_renderer.h

@@ -27,7 +27,6 @@
 namespace kiwix
 {
 
-class Searcher;
 class NameMapper;
 /**
  * The SearcherRenderer class is used to render a search result to a html page.
@@ -35,17 +34,6 @@ class NameMapper;
 class SearchRenderer
 {
  public:
-  /**
-   * Construct a SearchRenderer from a Searcher.
-   *
-   * This method is now deprecated. Construct the renderer from a
-   * `zim::SearchResultSet`
-   *
-   * @param searcher The `Searcher` to render.
-   * @param mapper The `NameMapper` to use to do the rendering.
-   */
-  DEPRECATED SearchRenderer(Searcher* searcher, NameMapper* mapper);
-
   /**
    * Construct a SearchRenderer from a SearchResultSet.
    *

include/searcher.h

@@ -1,180 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#ifndef KIWIX_SEARCHER_H
-#define KIWIX_SEARCHER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unicode/putil.h>
-#include <algorithm>
-#include <cctype>
-#include <locale>
-#include <string>
-#include <memory>
-#include <vector>
-
-#include <zim/search.h>
-
-using namespace std;
-
-namespace kiwix
-{
-class Reader;
-class Result
-{
- public:
-  virtual ~Result(){};
-  virtual std::string get_url() = 0;
-  virtual std::string get_title() = 0;
-  virtual int get_score() = 0;
-  virtual std::string get_snippet() = 0;
-  virtual std::string get_content() = 0;
-  virtual int get_wordCount() = 0;
-  virtual int get_size() = 0;
-  virtual std::string get_zimId() = 0;
-};
-
-struct SearcherInternal;
-struct SuggestionInternal;
-/**
- * The Searcher class is reponsible to do different kind of search using the
- * fulltext index.
- *
- * The Searcher is now deprecated. Use libzim search feature.
- */
-class Searcher
-{
- public:
-  /**
-   * The default constructor.
-   */
-  DEPRECATED Searcher();
-
-  ~Searcher();
-
-  /**
-   * Add a reader (containing embedded fulltext index) to the search.
-   *
-   * @param reader The Reader for the zim containing the fulltext index.
-   * @return true if the reader has been added.
-   *         false if the reader cannot be added (no embedded fulltext index present)
-   */
-  bool add_reader(std::shared_ptr<Reader> reader);
-
-
-  std::shared_ptr<Reader> get_reader(int index);
-
-  /**
-   * Start a search on the zim associated to the Searcher.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param search The search query.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void search(const std::string& search,
-              unsigned int resultStart,
-              unsigned int maxResultCount,
-              const bool verbose = false);
-
-  /**
-   * Start a geographique search.
-   * The search return result for entry in a disc of center latitude/longitude
-   * and radius distance.
-   *
-   * Search results should be retrived using the getNextResult method.
-   *
-   * @param latitude The latitude of the center point.
-   * @param longitude The longitude of the center point.
-   * @param distance The radius of the disc.
-   * @param resultStart the start offset of the search results (used for pagination).
-   * @param maxResultCount Maximum number of results to get from start (used for pagination).
-   * @param verbose print some info on stdout if true.
-   */
-  void geo_search(float latitude, float longitude, float distance,
-                  unsigned int resultStart,
-                  unsigned int maxResultCount,
-                  const bool verbose = false);
-
-  /**
-   * Start a suggestion search.
-   * The search made depend of the "version" of the embedded index.
-   *  - If the index is newer enough and have a title namespace, the search is
-   *    made in the titles only.
-   *  - Else the search is made on the whole article content.
-   * In any case, the search is made "partial" (as adding '*' at the end of the query)
-   *
-   * @param search The search query.
-   * @param verbose print some info on stdout if true.
-   */
-  void suggestions(std::string& search, const bool verbose = false);
-
-  /**
-   * Get the next result of a started search.
-   * This is the method to use to loop hover the search results.
-   */
-  Result* getNextResult();
-
-  /**
-   * Restart the previous search.
-   * Next call to getNextResult will return the first result.
-   */
-  void restart_search();
-
-  /**
-   * Get a estimation of the result count.
-   */
-  unsigned int getEstimatedResultCount();
-
-  /**
-   * Get a SearchResultSet object for current search
-   */
-  zim::SearchResultSet getSearchResultSet();
-
-  unsigned int getResultStart() { return resultStart; }
-  unsigned int getMaxResultCount() { return maxResultCount; }
-
- protected:
-  std::string beautifyInteger(const unsigned int number);
-  void closeIndex();
-  void searchInIndex(string& search,
-                     const unsigned int resultStart,
-                     const unsigned int maxResultCount,
-                     const bool verbose = false);
-
-  std::vector<std::shared_ptr<Reader>> readers;
-  std::unique_ptr<SearcherInternal> internal;
-  std::unique_ptr<SuggestionInternal> suggestionInternal;
-  std::string searchPattern;
-  unsigned int estimatedResultCount;
-  unsigned int resultStart;
-  unsigned int maxResultCount;
-
- private:
-  void reset();
-
-};
-
-
-}
-
-#endif

include/version.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <vector>
+#include <iostream>
 
 namespace kiwix
 {

meson.build

@@ -1,5 +1,5 @@
 project('libkiwix', 'cpp',
-  version : '11.0.0',
+  version : '12.0.0',
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -35,7 +35,7 @@ else
   error('Cannot found header mustache.hpp')
 endif
 
-libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
 if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
   error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
 endif

scripts/format_code.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+
+# Compute 'src' path
+SCRIPT_DIR=$(dirname "$0")
+REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
+DIRS="src include"
+
+# Apply formating to all *.cpp and *.h files
+cd "$REPO_DIR"
+for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
+do
+  echo $FILE
+  clang-format -i -style=file "$FILE"
+done

scripts/kiwix-compile-resources

@@ -52,15 +52,21 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
 class Resource:
-    def __init__(self, base_dirs, filename):
-        filename = filename.strip()
+    def __init__(self, base_dirs, filename, cacheid=None):
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
@@ -71,7 +77,7 @@ class Resource:
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -93,6 +99,12 @@ class Resource:
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
             namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
@@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -189,8 +209,8 @@ if __name__ == "__main__":
     base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
     with open(args.resource_file, 'r') as f:
-        resources = [Resource([base_dir]+source_dir, filename)
-                        for filename in f.readlines()]
+        resources = [Resource([base_dir]+source_dir, *line.strip().split())
+                        for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-resources

@@ -99,16 +99,24 @@ def preprocess_resource(resource_path):
             print(preprocessed_content, end='', file=target)
 
 
-def copy_file(src_path, dst_path):
-    with open(src_path, 'rb') as src:
-        with open(dst_path, 'wb') as dst:
-            dst.write(src.read())
+def copy_resource_list_file(src_path, dst_path):
+    with open(src_path, 'r') as src:
+        with open(dst_path, 'w') as dst:
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
 
 def preprocess_resources(resource_file_path):
     resource_filename = os.path.basename(resource_file_path)
     for resource in read_resource_file(resource_file_path):
-        preprocess_resource(resource)
-    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+        if resource.startswith('skin/'):
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

src/book.cpp

@@ -18,7 +18,6 @@
  */
 
 #include "book.h"
-#include "reader.h"
 
 #include "tools.h"
 #include "tools/base64.h"
@@ -30,7 +29,7 @@
 #include "tools/archiveTools.h"
 
 #include <zim/archive.h>
-
+#include <zim/item.h>
 #include <pugixml.hpp>
 
 namespace kiwix
@@ -64,15 +63,10 @@ bool Book::update(const kiwix::Book& other)
   return true;
 }
 
-void Book::update(const kiwix::Reader& reader)
-{
-  update(*reader.getZimArchive());
-}
-
 void Book::update(const zim::Archive& archive) {
   m_path = archive.getFilename();
   m_pathValid = true;
-  m_id = getArchiveId(archive);
+  m_id = std::string(archive.getUuid());
   m_title = getArchiveTitle(archive);
   m_description = getMetaDescription(archive);
   m_language = getMetaLanguage(archive);
@@ -83,8 +77,8 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = getArchiveArticleCount(archive);
-  m_mediaCount = getArchiveMediaCount(archive);
+  m_articleCount = archive.getArticleCount();
+  m_mediaCount = archive.getMediaCount();
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
   m_illustrations.clear();

src/entry.cpp

@@ -1,73 +0,0 @@
-/*
- * Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-namespace kiwix
-{
-
-Entry::Entry(zim::Entry entry, bool _marker)
-  : entry(entry)
-{
-}
-
-size_type Entry::getSize() const
-{
-  if (entry.isRedirect()) {
-    return 0;
-  } else {
-    return entry.getItem().getSize();
-  }
-}
-
-std::string Entry::getMimetype() const
-{
-  return entry.getItem(true).getMimetype();
-}
-
-bool Entry::isRedirect() const
-{
-  return entry.isRedirect();
-}
-
-Entry Entry::getRedirectEntry() const
-{
-  if ( !entry.isRedirect() ) {
-    throw NoEntry();
-  }
-
-  return Entry(entry.getRedirectEntry(), true);
-}
-
-Entry Entry::getFinalEntry() const
-{
-  int loopCounter = 42;
-  auto final_entry = entry;
-  while (final_entry.isRedirect() && loopCounter--) {
-    final_entry = final_entry.getRedirectEntry();
-  }
-  // Prevent infinite loops.
-  if (final_entry.isRedirect()) {
-    throw NoEntry();
-  }
-  return Entry(final_entry, true);
-}
-
-}

src/library.cpp

@@ -19,7 +19,6 @@
 
 #include "library.h"
 #include "book.h"
-#include "reader.h"
 #include "libxml_dumper.h"
 
 #include "tools.h"
@@ -222,7 +221,11 @@ bool Library::removeBookById(const std::string& id)
   // Having a too big cache is not a problem here (or it would have been before)
   // (And setMaxSize doesn't actually reduce the cache size, extra cached items
   //  will be removed in put or getOrPut).
-  return mp_impl->m_books.erase(id) == 1;
+  const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
+  if ( bookWasRemoved ) {
+    ++mp_impl->m_revision;
+  }
+  return bookWasRemoved;
 }
 
 Library::Revision Library::getRevision() const
@@ -278,16 +281,6 @@ const Book& Library::getBookByPath(const std::string& path) const
   throw std::out_of_range(ss.str());
 }
 
-std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
-{
-  auto archive = getArchiveById(id);
-  if(archive) {
-    return std::shared_ptr<Reader>(new Reader(archive, true));
-  } else {
-    return nullptr;
-  }
-}
-
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
   try {
@@ -473,8 +466,13 @@ void Library::updateBookDB(const Book& book)
   indexer.index_text(normalizeText(book.getName()),      1, "XN");
   indexer.index_text(normalizeText(book.getCategory()),  1, "XC");
 
-  for ( const auto& tag : split(normalizeText(book.getTags()), ";") )
+  for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
     doc.add_boolean_term("XT" + tag);
+    if ( tag[0] != '_' ) {
+      indexer.increase_termpos();
+      indexer.index_text(tag);
+    }
+  }
 
   const std::string idterm = "Q" + book.getId();
   doc.add_boolean_term(idterm);
@@ -537,9 +535,20 @@ Xapian::Query categoryQuery(const std::string& category)
   return Xapian::Query("XC" + normalizeText(category));
 }
 
-Xapian::Query langQuery(const std::string& lang)
+Xapian::Query langQuery(const std::string& commaSeparatedLanguageList)
 {
-  return Xapian::Query("L" + normalizeText(lang));
+  Xapian::Query q;
+  bool firstIteration = true;
+  for ( const auto& lang : kiwix::split(commaSeparatedLanguageList, ",") ) {
+    const Xapian::Query singleLangQuery("L" + normalizeText(lang));
+    if ( firstIteration ) {
+      q = singleLangQuery;
+      firstIteration = false;
+    } else {
+      q = Xapian::Query(Xapian::Query::OP_OR, q, singleLangQuery);
+    }
+  }
+  return q;
 }
 
 Xapian::Query publisherQuery(const std::string& publisher)

src/meson.build

@@ -6,10 +6,7 @@ kiwix_sources = [
   'libxml_dumper.cpp',
   'opds_dumper.cpp',
   'downloader.cpp',
-  'reader.cpp',
-  'entry.cpp',
   'server.cpp',
-  'searcher.cpp',
   'search_renderer.cpp',
   'subprocess.cpp',
   'aria2.cpp',
@@ -48,7 +45,7 @@ config_h = configure_file(output : 'kiwix_config.h',
                input : 'config.h.in')
 install_headers(config_h, subdir:'kiwix')
 
-kiwixlib = library('kiwix',
+libkiwix = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,

src/opds_dumper.cpp

@@ -20,9 +20,8 @@
 #include "opds_dumper.h"
 #include "book.h"
 
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include <mustache.hpp>
-#include <unicode/locid.h>
 
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
@@ -31,8 +30,9 @@ namespace kiwix
 {
 
 /* Constructor */
-OPDSDumper::OPDSDumper(Library* library)
-  : library(library)
+OPDSDumper::OPDSDumper(Library* library, NameMapper* nameMapper)
+  : library(library),
+    nameMapper(nameMapper)
 {
 }
 /* Destructor */
@@ -50,6 +50,8 @@ void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
 namespace
 {
 
+const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
+
 typedef kainjow::mustache::data MustacheData;
 typedef kainjow::mustache::list BooksData;
 typedef kainjow::mustache::list IllustrationInfo;
@@ -70,16 +72,17 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
     return illustrations;
 }
 
-kainjow::mustache::object getSingleBookData(const Book& book)
+std::string fullEntryXML(const Book& book, const std::string& rootLocation, const std::string& contentId)
 {
     const auto bookDate = book.getDate() + "T00:00:00Z";
-    return kainjow::mustache::object{
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
       {"id", book.getId()},
       {"name", book.getName()},
       {"title", book.getTitle()},
       {"description", book.getDescription()},
       {"language", book.getLanguage()},
-      {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
+      {"content_id",  urlEncode(contentId)},
       {"updated", bookDate}, // XXX: this should be the entry update datetime
       {"book_date", bookDate},
       {"category", book.getCategory()},
@@ -93,27 +96,34 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
+    return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
 }
 
-std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+std::string partialEntryXML(const Book& book, const std::string& rootLocation)
 {
-  auto data = getSingleBookData(book);
-  data["with_xml_header"] = MustacheData(withXMLHeader);
-  data["dump_partial_entries"] = MustacheData(partial);
-  data["endpoint_root"] = endpointRoot;
-  data["root"] = rootLocation;
-  return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
+    const auto bookDate = book.getDate() + "T00:00:00Z";
+    const kainjow::mustache::object data{
+      {"root",  rootLocation},
+      {"endpoint_root", rootLocation + "/catalog/v2"},
+      {"id", book.getId()},
+      {"title", book.getTitle()},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+    };
+    const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
+    return render_template(xmlTemplate, data);
 }
 
-BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
+BooksData getBooksData(const Library* library, const NameMapper* nameMapper, const std::vector<std::string>& bookIds, const std::string& rootLocation, bool partial)
 {
   BooksData booksData;
   for ( const auto& bookId : bookIds ) {
     try {
       const Book book = library->getBookByIdThreadSafe(bookId);
-      booksData.push_back(kainjow::mustache::object{
-          {"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
-      });
+      const std::string contentId = nameMapper->getNameForId(bookId);
+      const auto entryXML = partial
+                          ? partialEntryXML(book, rootLocation)
+                          : fullEntryXML(book, rootLocation, contentId);
+      booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
     } catch ( const std::out_of_range& ) {
       // the book was removed from the library since its id was obtained
       // ignore it
@@ -163,14 +173,8 @@ std::once_flag fillLanguagesFlag;
 void fillLanguagesMap()
 {
   for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
-    auto lang = *icuLangPtr;
-    const icu::Locale locale(lang);
-    icu::UnicodeString ustring;
-    locale.getDisplayLanguage(locale, ustring);
-    std::string displayLanguage;
-    ustring.toUTF8String(displayLanguage);
-    std::string iso3LangCode = locale.getISO3Language();
-    iso639_3.insert({iso3LangCode, displayLanguage});
+    const ICULanguageInfo lang(*icuLangPtr);
+    iso639_3.insert({lang.iso3Code(), lang.selfName()});
   }
 }
 
@@ -186,7 +190,7 @@ std::string getLanguageSelfName(const std::string& lang) {
 
 string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
 {
-  const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, false);
   const kainjow::mustache::object template_data{
      {"date", gen_date_str()},
      {"root", rootLocation},
@@ -204,7 +208,7 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
 string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
 {
   const auto endpointRoot = rootLocation + "/catalog/v2";
-  const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
+  const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, partial);
 
   const char* const endpoint = partial ? "/partial_entries" : "/entries";
   const kainjow::mustache::object template_data{
@@ -212,7 +216,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
      {"filter", onlyAsNonEmptyMustacheValue(query)},
-     {"query", query.empty() ? "" : "?" + urlEncode(query)},
+     {"query", query.empty() ? "" : "?" + query},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
@@ -225,7 +229,11 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
 
 std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
 {
-  return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
+  const auto book = library->getBookById(bookId);
+  const std::string contentId = nameMapper->getNameForId(bookId);
+  return XML_HEADER
+         + "\n"
+         + fullEntryXML(book, rootLocation, contentId);
 }
 
 std::string OPDSDumper::categoriesOPDSFeed() const

src/reader.cpp

@@ -1,472 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include "reader.h"
-#include <time.h>
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-#include <zim/item.h>
-#include <zim/error.h>
-
-#include "tools.h"
-#include "tools/stringTools.h"
-#include "tools/otherTools.h"
-#include "tools/archiveTools.h"
-
-namespace kiwix
-{
-/* Constructor */
-Reader::Reader(const string zimFilePath)
-  :  zimArchive(nullptr),
-     zimFilePath(zimFilePath)
-{
-  string tmpZimFilePath = zimFilePath;
-
-  /* Remove potential trailing zimaa */
-  size_t found = tmpZimFilePath.rfind("zimaa");
-  if (found != string::npos && tmpZimFilePath.size() > 5
-      && found == tmpZimFilePath.size() - 5) {
-    tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
-  }
-
-  zimArchive.reset(new zim::Archive(tmpZimFilePath));
-
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(const std::shared_ptr<zim::Archive> archive, bool _marker)
-  : zimArchive(archive),
-    zimFilePath(archive->getFilename())
-  {}
-
-#ifndef _WIN32
-Reader::Reader(int fd)
-  :  zimArchive(new zim::Archive(fd)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-
-Reader::Reader(int fd, zim::offset_type offset, zim::size_type size)
-  :  zimArchive(new zim::Archive(fd, offset, size)),
-     zimFilePath("")
-{
-  /* initialize random seed: */
-  srand(time(nullptr));
-}
-#endif // #ifndef _WIN32
-
-zim::Archive* Reader::getZimArchive() const
-{
-  return zimArchive.get();
-}
-
-MimeCounterType Reader::parseCounterMetadata() const
-{
-  return kiwix::parseArchiveCounter(*zimArchive);
-}
-
-/* Get the count of articles which can be indexed/displayed */
-unsigned int Reader::getArticleCount() const
-{
-  std::map<const std::string, unsigned int> counterMap
-      = this->parseCounterMetadata();
-  unsigned int counter = 0;
-
-  for(auto &pair:counterMap) {
-    if (startsWith(pair.first, "text/html")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-/* Get the count of medias content in the ZIM file */
-unsigned int Reader::getMediaCount() const
-{
-  return kiwix::getArchiveMediaCount(*zimArchive);
-}
-
-/* Get the total of all items of a ZIM file, redirects included */
-unsigned int Reader::getGlobalCount() const
-{
-  return zimArchive->getEntryCount();
-}
-
-/* Return the UID of the ZIM file */
-string Reader::getId() const
-{
-  return kiwix::getArchiveId(*zimArchive);
-}
-
-Entry Reader::getRandomPage() const
-{
-  try {
-    return Entry(zimArchive->getRandomEntry(), true);
-  } catch(...) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getMainPage() const
-{
-  return Entry(zimArchive->getMainEntry(), true);
-}
-
-bool Reader::getFavicon(string& content, string& mimeType) const
-{
-  return kiwix::getArchiveFavicon(*zimArchive, 48, content, mimeType);
-}
-
-string Reader::getZimFilePath() const
-{
-  return zimFilePath;
-}
-/* Return a metatag value */
-bool Reader::getMetadata(const string& name, string& value) const
-{
-  try {
-    value = zimArchive->getMetadata(name);
-    return true;
-  } catch(zim::EntryNotFound& e) {
-    return false;
-  }
-}
-
-#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
-
-string Reader::getName() const
-{
-  return kiwix::getMetaName(*zimArchive);
-}
-
-string Reader::getTitle() const
-{
-  return kiwix::getArchiveTitle(*zimArchive);
-}
-
-string Reader::getCreator() const
-{
-  return kiwix::getMetaCreator(*zimArchive);
-}
-
-string Reader::getPublisher() const
-{
-  return kiwix::getMetaPublisher(*zimArchive);
-}
-
-string Reader::getDate() const
-{
-  return kiwix::getMetaDate(*zimArchive);
-}
-
-string Reader::getDescription() const
-{
-  return kiwix::getMetaDescription(*zimArchive);
-}
-
-string Reader::getLongDescription() const
-{
-  METADATA("LongDescription")
-}
-
-string Reader::getLanguage() const
-{
-  return kiwix::getMetaLanguage(*zimArchive);
-}
-
-string Reader::getLicense() const
-{
-  METADATA("License")
-}
-
-string Reader::getTags(bool original) const
-{
-  return kiwix::getMetaTags(*zimArchive, original);
-}
-
-
-string Reader::getTagStr(const std::string& tagName) const
-{
-  string tags_str;
-  getMetadata("Tags", tags_str);
-  return getTagValueFromTagList(convertTags(tags_str), tagName);
-}
-
-bool Reader::getTagBool(const std::string& tagName) const
-{
-  return convertStrToBool(getTagStr(tagName));
-}
-
-string Reader::getRelation() const
-{
-  METADATA("Relation")
-}
-
-string Reader::getFlavour() const
-{
-  return kiwix::getMetaFlavour(*zimArchive);
-}
-
-string Reader::getSource() const
-{
-  METADATA("Source")
-}
-
-string Reader::getScraper() const
-{
-  METADATA("Scraper")
-}
-#undef METADATA
-
-Entry Reader::getEntryFromPath(const std::string& path) const
-{
-  try {
-    return Entry(kiwix::getEntryFromPath(*zimArchive, path), true);
-  } catch (zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-Entry Reader::getEntryFromEncodedPath(const std::string& path) const
-{
-  return getEntryFromPath(urlDecode(path, true));
-}
-
-Entry Reader::getEntryFromTitle(const std::string& title) const
-{
-  try {
-    return Entry(zimArchive->getEntryByTitle(title), true);
-  } catch(zim::EntryNotFound& e) {
-    throw NoEntry();
-  }
-}
-
-bool Reader::pathExists(const string& path) const
-{
-  return zimArchive->hasEntryByPath(path);
-}
-
-/* Does the ZIM file has a fulltext index */
-bool Reader::hasFulltextIndex() const
-{
-  return zimArchive->hasFulltextIndex();
-}
-
-/* Search titles by prefix */
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               const bool reset)
-{
-  /* Reset the suggestions otherwise check if the suggestions number is less
-   * than the suggestionsCount */
-  if (reset) {
-    this->suggestions.clear();
-    this->suggestionsOffset = this->suggestions.begin();
-  } else {
-    if (this->suggestions.size() > suggestionsCount) {
-      return false;
-    }
-  }
-
-  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
-
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-
-bool Reader::searchSuggestions(const string& prefix,
-                               unsigned int suggestionsCount,
-                               SuggestionsList_t& results)
-{
-  bool retVal = false;
-
-  /* Return if no prefix */
-  if (prefix.size() == 0) {
-    return false;
-  }
-
-  for (auto& entry: zimArchive->findByTitle(prefix)) {
-    if (results.size() >= suggestionsCount) {
-      break;
-    }
-    /* Extract the interesting part of article title & url */
-    std::string normalizedArticleTitle
-        = kiwix::normalize(entry.getTitle());
-
-    // Get the final path.
-    auto item = entry.getItem(true);
-    std::string articleFinalUrl = item.getPath();
-
-    /* Go through all already found suggestions and skip if this
-       article is already in the suggestions list (with an other
-       title) */
-    bool insert = true;
-    std::vector<SuggestionItem>::iterator suggestionItr;
-    for (suggestionItr = results.begin();
-         suggestionItr != results.end();
-         suggestionItr++) {
-      int result = normalizedArticleTitle.compare((*suggestionItr).getNormalizedTitle());
-      if (result == 0 && articleFinalUrl.compare((*suggestionItr).getPath()) == 0) {
-        insert = false;
-        break;
-      } else if (result < 0) {
-        break;
-      }
-    }
-
-    /* Insert if possible */
-    if (insert) {
-      SuggestionItem suggestion(entry.getTitle(), normalizedArticleTitle, articleFinalUrl);
-      results.insert(suggestionItr, suggestion);
-    }
-
-    /* Suggestions where found */
-    retVal = true;
-  }
-
-  return retVal;
-}
-
-std::vector<std::string> Reader::getTitleVariants(
-    const std::string& title) const
-{
-  return kiwix::getTitleVariants(title);
-}
-
-
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount)
-{
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
-
-  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
-
-  this->suggestionsOffset = this->suggestions.begin();
-
-  return ret;
-}
-
-/* Try also a few variations of the prefix to have better results */
-bool Reader::searchSuggestionsSmart(const string& prefix,
-                                    unsigned int suggestionsCount,
-                                    SuggestionsList_t& results)
-{
-  std::vector<std::string> variants = this->getTitleVariants(prefix);
-
-  auto suggestionSearcher = zim::SuggestionSearcher(*zimArchive);
-
-  if (zimArchive->hasTitleIndex()) {
-    auto suggestionSearch = suggestionSearcher.suggest(prefix);
-    const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
-    for (auto current : suggestions) {
-      SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-      results.push_back(suggestion);
-    }
-  } else {
-    // Check some of the variants of the prefix
-    for (std::vector<std::string>::iterator variantsItr = variants.begin();
-         variantsItr != variants.end();
-         variantsItr++) {
-      auto suggestionSearch = suggestionSearcher.suggest(*variantsItr);
-      for (auto current : suggestionSearch.getResults(0, suggestionsCount)) {
-        if (results.size() >= suggestionsCount) {
-          break;
-        }
-
-        SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
-                                current.getPath(), current.getSnippet());
-        results.push_back(suggestion);
-      }
-    }
-  }
-
-  return results.size() > 0;
-}
-
-/* Get next suggestion */
-bool Reader::getNextSuggestion(string& title)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-bool Reader::getNextSuggestion(string& title, string& url)
-{
-  if (this->suggestionsOffset != this->suggestions.end()) {
-    /* title */
-    title = (*(this->suggestionsOffset)).getTitle();
-    url = (*(this->suggestionsOffset)).getPath();
-
-    /* increment the cursor for the next call */
-    this->suggestionsOffset++;
-
-    return true;
-  }
-
-  return false;
-}
-
-/* Check if the file has as checksum */
-bool Reader::canCheckIntegrity() const
-{
-  return zimArchive->hasChecksum();
-}
-
-/* Return true if corrupted, false otherwise */
-bool Reader::isCorrupted() const
-{
-  try {
-    if (zimArchive->check() == true) {
-      return false;
-    }
-  } catch (exception& e) {
-    cerr << e.what() << endl;
-    return true;
-  }
-
-  return true;
-}
-
-/* Return the file size, works also for splitted files */
-unsigned int Reader::getFileSize() const
-{
-  return kiwix::getArchiveFileSize(*zimArchive);
-}
-
-}

src/search_renderer.cpp

@@ -21,8 +21,6 @@
 #include <cmath>
 
 #include "search_renderer.h"
-#include "searcher.h"
-#include "reader.h"
 #include "library.h"
 #include "name_mapper.h"
 
@@ -31,23 +29,13 @@
 #include <zim/search.h>
 
 #include <mustache.hpp>
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 #include "tools/stringTools.h"
 
 namespace kiwix
 {
 
 /* Constructor */
-SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : SearchRenderer(
-        /* srs */ searcher->getSearchResultSet(),
-        /* mapper */ mapper,
-        /* library */ nullptr,
-        /* start */ searcher->getResultStart(),
-        /* estimatedResultCount */ searcher->getEstimatedResultCount()
-        )
-{}
-
 SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                       unsigned int start, unsigned int estimatedResultCount)
     : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
@@ -106,7 +94,7 @@ kainjow::mustache::data buildQueryData
   kainjow::mustache::data query;
   query.set("pattern", kiwix::encodeDiples(pattern));
   std::ostringstream ss;
-  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
   ss << "&" << bookQuery;
   query.set("unpaginatedQuery", ss.str());
   auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
@@ -142,7 +130,7 @@ kainjow::mustache::data buildPagination(
   auto nbPages = lastPage + 1;
 
   auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
-  auto lastPageGenerated = min(currentPage+4, lastPage);
+  auto lastPageGenerated = std::min(currentPage+4, lastPage);
 
   if (nbPages != 1) {
     if (firstPageGenerated!=0) {
@@ -178,13 +166,15 @@ kainjow::mustache::data buildPagination(
 
 std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
 {
+  const std::string absPathPrefix = protocolPrefix;
   // Build the results list
   kainjow::mustache::data items{kainjow::mustache::data::type::list};
   for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
-    std::string zim_id(it.getZimId());
+    const std::string zim_id(it.getZimId());
+    const auto path = mp_nameMapper->getNameForId(zim_id) + "/" + it.getPath();
     result.set("title", it.getTitle());
-    result.set("absolutePath", protocolPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
+    result.set("absolutePath", absPathPrefix + urlEncode(path));
     result.set("snippet", it.getSnippet());
     if (mp_library) {
       result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
@@ -200,7 +190,7 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
   results.set("count", kiwix::beautifyInteger(estimatedResultCount));
   results.set("hasResults", estimatedResultCount != 0);
   results.set("start", kiwix::beautifyInteger(resultStart));
-  results.set("end", kiwix::beautifyInteger(min(resultStart+pageLength-1, estimatedResultCount)));
+  results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
 
   // pagination
   auto pagination = buildPagination(
@@ -217,7 +207,7 @@ std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
 
 
   kainjow::mustache::data allData;
-  allData.set("protocolPrefix", protocolPrefix);
+  allData.set("searchProtocolPrefix", searchProtocolPrefix);
   allData.set("results", results);
   allData.set("pagination", pagination);
   allData.set("query", query);

src/searcher.cpp

@@ -1,330 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-
-#include "searcher.h"
-#include "reader.h"
-
-#include <zim/search.h>
-#include <zim/suggestion.h>
-
-#include <mustache.hpp>
-#include <cmath>
-#include "tools/stringTools.h"
-#include "kiwixlib-resources.h"
-
-#define MAX_SEARCH_LEN 140
-
-namespace kiwix
-{
-class _Result : public Result
-{
- public:
-  _Result(zim::SearchResultSet::iterator iterator);
-  _Result(SuggestionItem suggestionItem);
-  virtual ~_Result(){};
-
-  virtual std::string get_url();
-  virtual std::string get_title();
-  virtual int get_score();
-  virtual std::string get_snippet();
-  virtual std::string get_content();
-  virtual int get_wordCount();
-  virtual int get_size();
-  virtual std::string get_zimId();
-
- private:
-  zim::SearchResultSet::iterator iterator;
-  SuggestionItem suggestionItem;
-  bool isSuggestion;
-};
-
-struct SearcherInternal : zim::SearchResultSet {
-  explicit SearcherInternal(const zim::SearchResultSet& srs)
-    : zim::SearchResultSet(srs)
-    , current_iterator(srs.begin())
-  {
-  }
-
-  zim::SearchResultSet::iterator current_iterator;
-};
-
-struct SuggestionInternal : zim::SuggestionResultSet {
-  explicit SuggestionInternal(const zim::SuggestionResultSet& srs)
-    : zim::SuggestionResultSet(srs),
-      currentIterator(srs.begin()) {}
-
-  zim::SuggestionResultSet::iterator currentIterator;
-};
-
-/* Constructor */
-Searcher::Searcher()
-    : searchPattern(""),
-      estimatedResultCount(0),
-      resultStart(0),
-      maxResultCount(0)
-{
-  loadICUExternalTables();
-}
-
-/* Destructor */
-Searcher::~Searcher()
-{
-}
-
-bool Searcher::add_reader(std::shared_ptr<Reader> reader)
-{
-  if (!reader->hasFulltextIndex()) {
-      return false;
-  }
-
-  for ( auto existing_reader : readers ) {
-    if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
-      return false;
-  }
-
-  this->readers.push_back(reader);
-  return true;
-}
-
-
-std::shared_ptr<Reader> Searcher::get_reader(int readerIndex)
-{
-  return readers.at(readerIndex);
-}
-
-/* Search strings in the database */
-void Searcher::search(const std::string& search,
-                      unsigned int resultStart,
-                      unsigned int maxResultCount,
-                      const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing query `" << search << "'" << endl;
-  }
-
-  this->searchPattern = search;
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-  /* Try to find results */
-  if (maxResultCount != 0) {
-    /* Perform the search */
-    string unaccentedSearch = removeAccents(search);
-    std::vector<zim::Archive> archives;
-    for (auto current = this->readers.begin(); current != this->readers.end();
-         current++) {
-      if ( (*current)->hasFulltextIndex() ) {
-          archives.push_back(*(*current)->getZimArchive());
-      }
-    }
-    zim::Searcher searcher(archives);
-    searcher.setVerbose(verbose);
-    zim::Query query;
-    query.setQuery(unaccentedSearch);
-    zim::Search search = searcher.search(query);
-    internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-    this->estimatedResultCount = search.getEstimatedMatches();
-  }
-
-  return;
-}
-
-
-void Searcher::geo_search(float latitude, float longitude, float distance,
-                          unsigned int resultStart,
-                          unsigned int maxResultCount,
-                          const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-  }
-
-  /* Perform the search */
-  std::ostringstream oss;
-  oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
-  this->searchPattern = oss.str();
-  this->resultStart = resultStart;
-  this->maxResultCount = maxResultCount;
-
-  /* Try to find results */
-  if (maxResultCount == 0) {
-    return;
-  }
-
-  std::vector<zim::Archive> archives;
-  for (auto current = this->readers.begin(); current != this->readers.end();
-       current++) {
-    archives.push_back(*(*current)->getZimArchive());
-  }
-  zim::Searcher searcher(archives);
-  searcher.setVerbose(verbose);
-  zim::Query query;
-  query.setQuery("");
-  query.setGeorange(latitude, longitude, distance);
-  zim::Search search = searcher.search(query);
-  internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-
-void Searcher::restart_search()
-{
-  if (internal.get()) {
-    internal->current_iterator = internal->begin();
-  }
-}
-
-Result* Searcher::getNextResult()
-{
-  if (internal.get() && internal->current_iterator != internal->end()) {
-    Result* result = new _Result(internal->current_iterator);
-    internal->current_iterator++;
-    return result;
-  } else if (suggestionInternal.get() &&
-             suggestionInternal->currentIterator != suggestionInternal->end()) {
-    SuggestionItem item(
-                        suggestionInternal->currentIterator->getTitle(),
-                        normalize(suggestionInternal->currentIterator->getTitle()),
-                        suggestionInternal->currentIterator->getPath(),
-                        suggestionInternal->currentIterator->getSnippet()
-                      );
-    Result* result = new _Result(item);
-    suggestionInternal->currentIterator++;
-    return result;
-  }
-  return NULL;
-}
-
-/* Reset the results */
-void Searcher::reset()
-{
-  this->estimatedResultCount = 0;
-  this->searchPattern = "";
-  return;
-}
-
-void Searcher::suggestions(std::string& searchPattern, const bool verbose)
-{
-  this->reset();
-
-  if (verbose == true) {
-    cout << "Performing suggestion query `" << searchPattern << "`" << endl;
-  }
-
-  this->searchPattern = searchPattern;
-  this->resultStart = 0;
-  this->maxResultCount = 10;
-  string unaccentedSearch = removeAccents(searchPattern);
-
-  // Multizim suggestion is not supported as of now! taking only one archive
-  zim::Archive archive = *(*this->readers.begin())->getZimArchive();
-  zim::SuggestionSearcher searcher(archive);
-  searcher.setVerbose(verbose);
-  zim::SuggestionSearch search = searcher.suggest(searchPattern);
-  suggestionInternal.reset(new SuggestionInternal(search.getResults(resultStart, maxResultCount)));
-  this->estimatedResultCount = search.getEstimatedMatches();
-}
-
-/* Return the result count estimation */
-unsigned int Searcher::getEstimatedResultCount()
-{
-  return this->estimatedResultCount;
-}
-
-zim::SearchResultSet Searcher::getSearchResultSet()
-{
-  return *(this->internal);
-}
-
-_Result::_Result(zim::SearchResultSet::iterator iterator)
-    : iterator(iterator),
-      suggestionItem("", "", ""),
-      isSuggestion(false)
-{}
-
-_Result::_Result(SuggestionItem item)
-    : iterator(),
-      suggestionItem(item.getTitle(), item.getNormalizedTitle(), item.getPath(), item.getSnippet()),
-      isSuggestion(true)
-{}
-
-std::string _Result::get_url()
-{
-  if (isSuggestion) {
-    return suggestionItem.getPath();
-  }
-  return iterator.getPath();
-}
-std::string _Result::get_title()
-{
-  if (isSuggestion) {
-    return suggestionItem.getTitle();
-  }
-  return iterator.getTitle();
-}
-int _Result::get_score()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getScore();
-}
-std::string _Result::get_snippet()
-{
-  if (isSuggestion) {
-    return suggestionItem.getSnippet();
-  }
-  return iterator.getSnippet();
-}
-std::string _Result::get_content()
-{
-  if (isSuggestion) return "";
-  return iterator->getItem(true).getData();
-}
-int _Result::get_size()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getSize();
-}
-int _Result::get_wordCount()
-{
-  if (isSuggestion) {
-    return 0;
-  }
-  return iterator.getWordCount();
-}
-std::string _Result::get_zimId()
-{
-  if (isSuggestion) {
-    return "";
-  }
-  std::ostringstream s;
-  s << iterator.getZimId();
-  return s.str();
-}
-
-
-}

src/server.cpp

@@ -24,6 +24,7 @@
 
 #include <string>
 
+#include <zim/item.h>
 #include "server/internalServer.h"
 
 namespace kiwix {

src/server/etag.cpp

@@ -37,11 +37,11 @@ namespace {
 // into the ETag for ETag::Option opt.
 // IMPORTANT: The characters in all_options must come in sorted order (so that
 // IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
+const char all_options[] = "Zz";
 
 static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
 
-bool isValidServerId(const std::string& s)
+bool isValidETagBody(const std::string& s)
 {
   return !s.empty() && s.find_first_of("\"/") == std::string::npos;
 }
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
 
 std::string ETag::get_etag() const
 {
-  if ( m_serverId.empty() )
+  if ( m_body.empty() )
     return std::string();
 
-  return "\"" + m_serverId + "/" + m_options + "\"";
+  return "\"" + m_body + "/" + m_options + "\"";
 }
 
-ETag::ETag(const std::string& serverId, const std::string& options)
+ETag::ETag(const std::string& body, const std::string& options)
 {
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
   {
-    m_serverId = serverId;
+    m_body = body;
     m_options = options;
   }
 }
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
   return ETag(s.substr(0, i), s.substr(i+1));
 }
 
-ETag ETag::match(const std::string& etags, const std::string& server_id)
+ETag ETag::match(const std::string& etags, const std::string& body)
 {
   std::istringstream ss(etags);
   std::string etag_str;
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
       etag_str.pop_back();
 
     const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
+    if ( etag && etag.m_body == body )
       return etag;
   }
 

src/server/etag.h

@@ -28,10 +28,11 @@ namespace kiwix {
 // The ETag string used by Kiwix server (more precisely, its value inside the
 // double quotes) consists of two parts:
 //
-// 1. ServerId - The string obtained on server start up
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
 //
-// 2. Options -  Zero or more characters encoding the values of some of the
-//               headers of the response
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
+//               the values of some of the headers of the response
 //
 // The two parts are separated with a slash (/) symbol (which is always present,
 // even when the the options part is empty). Neither portion of a Kiwix ETag
@@ -40,7 +41,7 @@ namespace kiwix {
 //
 //   "abcdefghijklmn/"
 //   "1234567890/z"
-//   "1234567890/cz"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
 //
 // The options part of the Kiwix ETag allows to correctly set the required
 // headers when responding to a conditional If-None-Match request with a 304
@@ -51,7 +52,7 @@ class ETag
 {
   public: // types
     enum Option {
-      CACHEABLE_ENTITY,
+      ZIM_CONTENT,
       COMPRESSED_CONTENT,
       OPTION_COUNT
     };
@@ -59,10 +60,10 @@ class ETag
   public: // functions
     ETag() {}
 
-    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_body(const std::string& s) { m_body = s; }
     void set_option(Option opt);
 
-    explicit operator bool() const { return !m_serverId.empty(); }
+    explicit operator bool() const { return !m_body.empty(); }
 
     bool get_option(Option opt) const;
     std::string get_etag() const;
@@ -76,7 +77,7 @@ class ETag
     static ETag parse(std::string s);
 
   private: // data
-    std::string m_serverId;
+    std::string m_body;
     std::string m_options;
 };
 

src/server/i18n.cpp

@@ -70,6 +70,14 @@ public: // functions
     return s;
   }
 
+  size_t getStringCount(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang)->entryCount;
+    } catch(const std::out_of_range&) {
+      return 0;
+    }
+  }
+
 private: // functions
   const I18nStringTable* getStringsFor(const std::string& lang) const {
     try {
@@ -84,13 +92,17 @@ private: // data
   const I18nStringTable* enStrings;
 };
 
+const I18nStringDB& getStringDb()
+{
+  static const I18nStringDB stringDb;
+  return stringDb;
+}
+
 } // unnamed namespace
 
 std::string getTranslatedString(const std::string& lang, const std::string& key)
 {
-  static const I18nStringDB stringDb;
-
-  return stringDb.get(lang, key);
+  return getStringDb().get(lang, key);
 }
 
 namespace i18n
@@ -111,4 +123,70 @@ std::string ParameterizedMessage::getText(const std::string& lang) const
   return i18n::expandParameterizedString(lang, msgId, params);
 }
 
+namespace
+{
+
+LangPreference parseSingleLanguagePreference(const std::string& s)
+{
+  const size_t langStart = s.find_first_not_of(" \t\n");
+  if ( langStart == std::string::npos ) {
+    return {"", 0};
+  }
+
+  const size_t langEnd = s.find(';', langStart);
+  if ( langEnd == std::string::npos ) {
+    return {s.substr(langStart), 1};
+  }
+
+  const std::string lang = s.substr(langStart, langEnd - langStart);
+  // We don't care about langEnd == langStart which will result in an empty
+  // language name - it will be dismissed by parseUserLanguagePreferences()
+
+  float q = 1.0;
+  int nCharsScanned;
+  if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
+       && langEnd + 1 + nCharsScanned == s.size() ) {
+    return {lang, q};
+  }
+
+  return {"", 0};
+}
+
+} // unnamed namespace
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s)
+{
+  UserLangPreferences result;
+  std::istringstream iss(s);
+  std::string singleLangPrefStr;
+  while ( std::getline(iss, singleLangPrefStr, ',') )
+  {
+    const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
+    if ( !langPref.lang.empty() && langPref.preference > 0 ) {
+      result.push_back(langPref);
+    }
+  }
+
+  return result;
+}
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
+{
+  if ( prefs.empty() ) {
+    return "en";
+  }
+
+  std::string bestLangSoFar("en");
+  float bestScoreSoFar = 0;
+  const auto& stringDb = getStringDb();
+  for ( const auto& entry : prefs ) {
+    const float score = entry.preference * stringDb.getStringCount(entry.lang);
+    if ( score > bestScoreSoFar ) {
+      bestScoreSoFar = score;
+      bestLangSoFar = entry.lang;
+    }
+  }
+  return bestLangSoFar;
+}
+
 } // namespace kiwix

src/server/i18n.h

@@ -89,6 +89,18 @@ private: // data
   const Parameters  params;
 };
 
+struct LangPreference
+{
+  const std::string lang;
+  const float preference;
+};
+
+typedef std::vector<LangPreference> UserLangPreferences;
+
+UserLangPreferences parseUserLanguagePreferences(const std::string& s);
+
+std::string selectMostSuitableLanguage(const UserLangPreferences& prefs);
+
 } // namespace kiwix
 
 #endif // KIWIX_SERVER_I18N

src/server/internalServer.cpp

@@ -51,8 +51,6 @@ extern "C" {
 #include "tools/networkTools.h"
 #include "library.h"
 #include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
 #include "search_renderer.h"
 #include "opds_dumper.h"
 #include "i18n.h"
@@ -61,6 +59,7 @@ extern "C" {
 #include <zim/error.h>
 #include <zim/entry.h>
 #include <zim/item.h>
+#include <zim/suggestion.h>
 
 #include <mustache.hpp>
 
@@ -68,7 +67,8 @@ extern "C" {
 #include <string>
 #include <vector>
 #include <chrono>
-#include "kiwixlib-resources.h"
+#include <fstream>
+#include "libkiwix-resources.h"
 
 #ifndef _WIN32
 # include <arpa/inet.h>
@@ -77,7 +77,6 @@ extern "C" {
 #include "request_context.h"
 #include "response.h"
 
-#define MAX_SEARCH_LEN 140
 #define DEFAULT_CACHE_SIZE 2
 
 namespace kiwix {
@@ -139,15 +138,6 @@ std::string renderUrl(const std::string& root, const std::string& urlTemplate)
   return url;
 }
 
-std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
-{
-  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
-               {
-                  {"SEARCH_TERMS", queryString}
-               }
-         );
-}
-
 ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
 {
   return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
@@ -212,6 +202,40 @@ void checkBookNumber(const Library::BookIdSet& bookIds, size_t limit) {
   }
 }
 
+typedef std::set<std::string> Languages;
+
+Languages getLanguages(const Library& lib, const Library::BookIdSet& bookIds) {
+  Languages langs;
+  for ( const auto& b : bookIds ) {
+    langs.insert(lib.getBookById(b).getLanguage());
+  }
+  return langs;
+}
+
+struct CustomizedResourceData
+{
+  std::string mimeType;
+  std::string resourceFilePath;
+};
+
+bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request)
+{
+  return response.getReturnCode() == MHD_HTTP_OK
+      && response.get_kind() == Response::DYNAMIC_CONTENT
+      && request.get_url() != "/random";
+}
+
+ETag
+get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody)
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, etagBody);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
 } // unnamed namespace
 
 std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
@@ -221,7 +245,7 @@ std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const Req
     auto bookName = request.get_argument("content");
     try {
       const auto bookIds = Library::BookIdSet{mp_nameMapper->getIdForName(bookName)};
-      const auto queryString = request.get_query([&](const std::string& key){return key == "content";}, true);
+      const auto queryString = request.get_query([&](const std::string& key){return key == "content";});
       return {queryString, bookIds};
     } catch (const std::out_of_range&) {
       throw Error(noSuchBookErrorMsg(bookName));
@@ -246,7 +270,7 @@ std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const Req
       }
     }
     const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
-    const auto queryString = request.get_query([&](const std::string& key){return key == "books.id";}, true);
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.id";});
     return {queryString, bookIds};
   } catch(const std::out_of_range&) {}
 
@@ -264,7 +288,7 @@ std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const Req
         throw Error(noSuchBookErrorMsg(bookName));
       }
     }
-    const auto queryString = request.get_query([&](const std::string& key){return key == "books.name";}, true);
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.name";});
     return {queryString, bookIds};
   } catch(const std::out_of_range&) {}
 
@@ -275,7 +299,7 @@ std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const Req
     throw Error(nonParameterizedMessage("no-book-found"));
   }
   const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
-  const auto queryString = request.get_query([&](const std::string& key){return startsWith(key, "books.filter.");}, true);
+  const auto queryString = request.get_query([&](const std::string& key){return startsWith(key, "books.filter.");});
   return {queryString, bookIds};
 }
 
@@ -283,6 +307,10 @@ SearchInfo InternalServer::getSearchInfo(const RequestContext& request) const
 {
   auto bookIds = selectBooks(request);
   checkBookNumber(bookIds.second, m_multizimSearchLimit);
+  if ( getLanguages(*mp_library, bookIds.second).size() != 1 ) {
+    throw Error(nonParameterizedMessage("confusion-of-tongues"));
+  }
+
   auto pattern = request.get_optional_param<std::string>("pattern", "");
   GeoQuery geoQuery;
 
@@ -327,7 +355,6 @@ zim::Query SearchInfo::getZimQuery(bool verbose) const {
   return query;
 }
 
-
 static IdNameMapper defaultNameMapper;
 
 static MHD_Result staticHandlerCallback(void* cls,
@@ -339,6 +366,27 @@ static MHD_Result staticHandlerCallback(void* cls,
                                         size_t* upload_data_size,
                                         void** cont_cls);
 
+class InternalServer::CustomizedResources : public std::map<std::string, CustomizedResourceData>
+{
+public:
+  CustomizedResources()
+  {
+    const char* fname = ::getenv("KIWIX_SERVE_CUSTOMIZED_RESOURCES");
+    if ( fname )
+    {
+      std::cout << "Populating customized resources" << std::endl;
+      std::ifstream file(fname);
+      std::string url, mimeType, resourceFilePath;
+      while ( file >> url >> mimeType >> resourceFilePath )
+      {
+        std::cout << "Got " << url << " " << mimeType << " " << resourceFilePath << std::endl;
+        (*this)[url] = CustomizedResourceData{mimeType, resourceFilePath};
+      }
+      std::cout << "Done populating customized resources" << std::endl;
+    }
+  }
+};
+
 
 InternalServer::InternalServer(Library* library,
                                NameMapper* nameMapper,
@@ -368,9 +416,12 @@ InternalServer::InternalServer(Library* library,
   mp_library(library),
   mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
   searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
-  suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
+  suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
+  m_customizedResources(new CustomizedResources)
 {}
 
+InternalServer::~InternalServer() = default;
+
 bool InternalServer::start() {
 #ifdef _WIN32
   int flags = MHD_USE_SELECT_INTERNALLY;
@@ -414,7 +465,6 @@ bool InternalServer::start() {
   }
   auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
   m_server_id = kiwix::to_string(server_start_time.count());
-  m_library_id = m_server_id;
   return true;
 }
 
@@ -482,8 +532,9 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
     }
   }
 
-  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response->set_server_id(m_server_id);
+  if ( responseMustBeETaggedWithLibraryId(*response, request) ) {
+    response->set_etag_body(getLibraryId());
+  }
 
   auto ret = response->send(request, connection);
   auto end_time = std::chrono::steady_clock::now();
@@ -495,6 +546,21 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
   return ret;
 }
 
+namespace
+{
+
+bool isEndpointUrl(const std::string& url, const std::string& endpoint)
+{
+  return startsWith(url, "/" + endpoint + "/") || url == "/" + endpoint;
+};
+
+} // unnamed namespace
+
+std::string InternalServer::getLibraryId() const
+{
+  return m_server_id + "." + kiwix::to_string(mp_library->getRevision());
+}
+
 std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
 {
   try {
@@ -503,40 +569,49 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
              + urlNotFoundMsg;
     }
 
-    const ETag etag = get_matching_if_none_match_etag(request);
+    const ETag etag = get_matching_if_none_match_etag(request, getLibraryId());
     if ( etag )
       return Response::build_304(*this, etag);
 
-    if (startsWith(request.get_url(), "/skin/"))
+    const auto url = request.get_url();
+    if ( isLocallyCustomizedResource(url) )
+      return handle_locally_customized_resource(request);
+
+    if (url == "/" )
+      return build_homepage(request);
+
+    if (isEndpointUrl(url, "viewer") || isEndpointUrl(url, "skin"))
       return handle_skin(request);
 
-    if (startsWith(request.get_url(), "/catalog/"))
+    if (url == "/viewer_settings.js")
+      return handle_viewer_settings(request);
+
+    if (isEndpointUrl(url, "content"))
+      return handle_content(request);
+
+    if (isEndpointUrl(url, "catalog"))
       return handle_catalog(request);
 
-    if (startsWith(request.get_url(), "/raw/"))
+    if (isEndpointUrl(url, "raw"))
       return handle_raw(request);
 
-    if (request.get_url() == "/search")
+    if (isEndpointUrl(url, "search"))
       return handle_search(request);
 
-    if (request.get_url() == "/search/searchdescription.xml") {
-      return ContentResponse::build(
-        *this,
-        RESOURCE::ft_opensearchdescription_xml,
-        get_default_data(),
-        "application/opensearchdescription+xml");
-    }
-
-    if (request.get_url() == "/suggest")
+    if (isEndpointUrl(url, "suggest"))
      return handle_suggest(request);
 
-    if (request.get_url() == "/random")
+    if (isEndpointUrl(url, "random"))
       return handle_random(request);
 
-    if (request.get_url() == "/catch/external")
-      return handle_captured_external(request);
+    if (isEndpointUrl(url, "catch"))
+      return handle_catch(request);
 
-    return handle_content(request);
+    std::string contentUrl = m_root + "/content" + url;
+    const std::string query = request.get_query();
+    if ( ! query.empty() )
+      contentUrl += "?" + query;
+    return Response::build_redirect(*this, contentUrl);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
     return HTTP500Response(*this, request)
@@ -555,69 +630,29 @@ MustacheData InternalServer::get_default_data() const
   return data;
 }
 
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
 std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
 {
-  return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8", true);
+  return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8");
 }
 
 /**
  * Archive and Zim handlers begin
  **/
 
-SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Archive* const archive,
-                  const std::string& bookId, const std::string& queryString, int start, int suggestionCount)
+class InternalServer::LockableSuggestionSearcher : public zim::SuggestionSearcher
 {
-  SuggestionsList_t suggestions;
-  std::shared_ptr<zim::SuggestionSearcher> searcher;
-  searcher = cache.getOrPut(bookId, [=](){ return make_shared<zim::SuggestionSearcher>(*archive); });
+  public:
+    explicit LockableSuggestionSearcher(const zim::Archive& archive)
+      : zim::SuggestionSearcher(archive)
+    {}
 
-  if (archive->hasTitleIndex()) {
-    auto search = searcher->suggest(queryString);
-    auto srs = search.getResults(start, suggestionCount);
-
-    for (auto it : srs) {
-      SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
-                                it.getPath(), it.getSnippet());
-      suggestions.push_back(suggestion);
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
     }
-  } else {
-    // TODO: This case should be handled by libzim
-    std::vector<std::string> variants = getTitleVariants(queryString);
-    int currCount = 0;
-    for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
-      auto search = searcher->suggest(queryString);
-      auto srs = search.getResults(0, suggestionCount);
-      for (auto it : srs) {
-        SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
-                                  it.getPath());
-        suggestions.push_back(suggestion);
-        currCount++;
-      }
-    }
-  }
-  return suggestions;
-}
+    virtual ~LockableSuggestionSearcher() = default;
+  private:
+    std::mutex m_mutex;
+};
 
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
@@ -625,6 +660,11 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
     printf("** running handle_suggest\n");
   }
 
+  if ( startsWith(request.get_url(), "/suggest/") ) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
   std::string bookName, bookId;
   std::shared_ptr<zim::Archive> archive;
   try {
@@ -637,8 +677,7 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
 
   if (archive == nullptr) {
     return HTTP404Response(*this, request)
-           + noSuchBookErrorMsg(bookName)
-           + TaskbarInfo(bookName);
+           + noSuchBookErrorMsg(bookName);
   }
 
   const auto queryString = request.get_optional_param("term", std::string());
@@ -652,61 +691,82 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
     printf("Searching suggestions for: \"%s\"\n", queryString.c_str());
   }
 
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
+  Suggestions results;
 
   /* Get the suggestions */
-  SuggestionsList_t suggestions = getSuggestions(suggestionSearcherCache, archive.get(),
-                                                  bookId, queryString, start, count);
-  for(auto& suggestion:suggestions) {
-    MustacheData result;
-    result.set("label", suggestion.getTitle());
+  auto searcher = suggestionSearcherCache.getOrPut(bookId,
+    [=](){ return make_shared<LockableSuggestionSearcher>(*archive); }
+  );
+  const auto lock(searcher->getLock());
+  auto search = searcher->suggest(queryString);
+  auto srs = search.getResults(start, count);
 
-    if (suggestion.hasSnippet()) {
-      result.set("label", suggestion.getSnippet());
-    }
-
-    result.set("value", suggestion.getTitle());
-    result.set("kind", "path");
-    result.set("path", suggestion.getPath());
-    result.set("first", first);
-    first = false;
-    results.push_back(result);
+  for(auto& suggestion: srs) {
+    results.add(suggestion);
   }
 
 
   /* Propose the fulltext search if possible */
   if (archive->hasFulltextIndex()) {
-    MustacheData result;
-    const auto lang = request.get_user_language();
-    result.set("label", makeFulltextSearchSuggestion(lang, queryString));
-    result.set("value", queryString + " ");
-    result.set("kind", "pattern");
-    result.set("first", first);
-    results.push_back(result);
+    results.addFTSearchSuggestion(request.get_user_language(), queryString);
   }
 
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
-  return std::move(response);
+  return ContentResponse::build(*this, results.getJSON(), "application/json; charset=utf-8");
 }
 
+std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_viewer_settings\n");
+  }
+
+  const kainjow::mustache::object data{
+    {"enable_toolbar", m_withTaskbar ? "true" : "false" },
+    {"enable_link_blocking", m_blockExternalLinks ? "true" : "false" },
+    {"enable_library_button", m_withLibraryButton ? "true" : "false" }
+  };
+  return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
+}
+
+namespace
+{
+
+Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid)
+{
+  if ( expectedCacheid == nullptr )
+    return Response::DYNAMIC_CONTENT;
+
+  try {
+    if ( expectedCacheid != req.get_argument("cacheid") )
+      throw ResourceNotFound("Wrong cacheid");
+    return Response::STATIC_RESOURCE;
+  } catch( const std::out_of_range& ) {
+    return Response::DYNAMIC_CONTENT;
+  }
+}
+
+} // unnamed namespace
+
 std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_skin\n");
   }
 
-  auto resourceName = request.get_url().substr(1);
+  const bool isRequestForViewer = request.get_url() == "/viewer";
+  auto resourceName = isRequestForViewer
+                    ? "viewer.html"
+                    : request.get_url().substr(1);
+
+  const char* const resourceCacheId = getResourceCacheId(resourceName);
+
   try {
+    const auto accessType = staticResourceAccessType(request, resourceCacheId);
     auto response = ContentResponse::build(
         *this,
         getResource(resourceName),
         getMimeTypeForFile(resourceName));
-    response->set_cacheable();
+    response->set_kind(accessType);
     return std::move(response);
   } catch (const ResourceNotFound& e) {
     return HTTP404Response(*this, request)
@@ -720,76 +780,20 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     printf("** running handle_search\n");
   }
 
+  if ( startsWith(request.get_url(), "/search/") ) {
+    if (request.get_url() == "/search/searchdescription.xml") {
+      return ContentResponse::build(
+        *this,
+        RESOURCE::ft_opensearchdescription_xml,
+        get_default_data(),
+        "application/opensearchdescription+xml");
+    }
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
   try {
-    auto searchInfo = getSearchInfo(request);
-    auto bookIds = searchInfo.getBookIds();
-
-    /* Make the search */
-    // Try to get a search from the searchInfo, else build it
-    auto searcher = mp_library->getSearcherByIds(bookIds);
-    auto lock(searcher->getLock());
-
-    std::shared_ptr<zim::Search> search;
-    try {
-      search = searchCache.getOrPut(searchInfo,
-        [=](){
-          return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
-        }
-      );
-    } catch(std::runtime_error& e) {
-      // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
-      // (in case of zim file not containing a index)
-      const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
-      HTTPErrorResponse response(*this, request, MHD_HTTP_NOT_FOUND,
-                                 "fulltext-search-unavailable",
-                                 "404-page-heading",
-                                 cssUrl);
-      response += nonParameterizedMessage("no-search-results");
-      if(bookIds.size() == 1) {
-        auto bookId = *bookIds.begin();
-        auto bookName = mp_nameMapper->getNameForId(bookId);
-        response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
-      }
-      return response;
-    }
-
-    auto start = 1;
-    try {
-      start = request.get_argument<unsigned int>("start");
-    } catch (const std::exception&) {}
-    start = max(1, start);
-
-    auto pageLength = 25;
-    try {
-      pageLength = request.get_argument<unsigned int>("pageLength");
-    } catch (const std::exception&) {}
-    if (pageLength > MAX_SEARCH_LEN) {
-      pageLength = MAX_SEARCH_LEN;
-    }
-    if (pageLength == 0) {
-      pageLength = 25;
-    }
-
-    /* Get the results */
-    SearchRenderer renderer(search->getResults(start-1, pageLength), mp_nameMapper, mp_library, start,
-                            search->getEstimatedMatches());
-    renderer.setSearchPattern(searchInfo.pattern);
-    renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search");
-    renderer.setPageLength(pageLength);
-    if (request.get_requested_format() == "xml") {
-      return ContentResponse::build(*this, renderer.getXml(), "application/rss+xml; charset=utf-8",
-      /*isHomePage =*/false,
-      /*raw =*/true);
-    }
-    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    if(bookIds.size() == 1) {
-      auto bookId = *bookIds.begin();
-      auto bookName = mp_nameMapper->getNameForId(bookId);
-      response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
-    }
-    return std::move(response);
+    return handle_search_request(request);
   } catch (const Error& e) {
     return HTTP400Response(*this, request)
       + invalidUrlMsg
@@ -797,12 +801,98 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
   }
 }
 
+namespace
+{
+
+unsigned getSearchPageSize(const RequestContext& r)
+{
+  const auto DEFAULT_PAGE_LEN = 25u;
+  const auto MAX_PAGE_LEN = 140u;
+
+  const auto pageLength = r.get_optional_param("pageLength", DEFAULT_PAGE_LEN);
+  return pageLength == 0
+       ? DEFAULT_PAGE_LEN
+       : min(MAX_PAGE_LEN, pageLength);
+}
+
+} // unnamed namespace
+
+std::unique_ptr<Response> InternalServer::handle_search_request(const RequestContext& request)
+{
+  auto searchInfo = getSearchInfo(request);
+  auto bookIds = searchInfo.getBookIds();
+
+  /* Make the search */
+  // Try to get a search from the searchInfo, else build it
+  auto searcher = mp_library->getSearcherByIds(bookIds);
+  auto lock(searcher->getLock());
+
+  std::shared_ptr<zim::Search> search;
+  try {
+    search = searchCache.getOrPut(searchInfo,
+      [=](){
+        return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
+      }
+    );
+  } catch(std::runtime_error& e) {
+    // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
+    // (in case of zim file not containing a index)
+    const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
+    HTTPErrorResponse response(*this, request, MHD_HTTP_NOT_FOUND,
+                               "fulltext-search-unavailable",
+                               "404-page-heading",
+                               cssUrl);
+    response += nonParameterizedMessage("no-search-results");
+    // XXX: Now this has to be handled by the iframe-based viewer which
+    // XXX: has to resolve if the book selection resulted in a single book.
+    /*
+    if(bookIds.size() == 1) {
+      auto bookId = *bookIds.begin();
+      auto bookName = mp_nameMapper->getNameForId(bookId);
+      response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
+    }
+    */
+    return response;
+  }
+
+  const auto start = max(1u, request.get_optional_param("start", 1u));
+  const auto pageLength = getSearchPageSize(request);
+
+  /* Get the results */
+  SearchRenderer renderer(search->getResults(start-1, pageLength), mp_nameMapper, mp_library, start,
+                          search->getEstimatedMatches());
+  renderer.setSearchPattern(searchInfo.pattern);
+  renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
+  renderer.setProtocolPrefix(m_root + "/content/");
+  renderer.setSearchProtocolPrefix(m_root + "/search");
+  renderer.setPageLength(pageLength);
+  if (request.get_requested_format() == "xml") {
+    return ContentResponse::build(*this, renderer.getXml(), "application/rss+xml; charset=utf-8");
+  }
+  auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+  // XXX: Now this has to be handled by the iframe-based viewer which
+  // XXX: has to resolve if the book selection resulted in a single book.
+  /*
+  if(bookIds.size() == 1) {
+    auto bookId = *bookIds.begin();
+    auto bookName = mp_nameMapper->getNameForId(bookId);
+    response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
+  }
+  */
+  return std::move(response);
+}
+
 std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_random\n");
   }
 
+  if ( startsWith(request.get_url(), "/random/") ) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
+
   std::string bookName;
   std::shared_ptr<zim::Archive> archive;
   try {
@@ -815,8 +905,7 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
 
   if (archive == nullptr) {
     return HTTP404Response(*this, request)
-           + noSuchBookErrorMsg(bookName)
-           + TaskbarInfo(bookName);
+           + noSuchBookErrorMsg(bookName);
   }
 
   try {
@@ -824,8 +913,7 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
     return build_redirect(bookName, getFinalItem(*archive, entry));
   } catch(zim::EntryNotFound& e) {
     return HTTP404Response(*this, request)
-           + nonParameterizedMessage("random-article-failure")
-           + TaskbarInfo(bookName, archive.get());
+           + nonParameterizedMessage("random-article-failure");
   }
 }
 
@@ -846,6 +934,20 @@ std::unique_ptr<Response> InternalServer::handle_captured_external(const Request
   return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
 }
 
+std::unique_ptr<Response> InternalServer::handle_catch(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catch\n");
+  }
+
+  if ( request.get_url() == "/catch/external" ) {
+    return handle_captured_external(request);
+  }
+
+  return HTTP404Response(*this, request)
+         + urlNotFoundMsg;
+}
+
 std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -877,9 +979,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   }
 
   zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper(mp_library);
+  kiwix::OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   std::vector<std::string> bookIdsToDump;
   if (url == "root.xml") {
     uuid = zim::Uuid::generate(host);
@@ -901,9 +1003,6 @@ InternalServer::search_catalog(const RequestContext& request,
                                kiwix::OPDSDumper& opdsDumper)
 {
     const auto filter = get_search_filter(request);
-    const std::string q = filter.hasQuery()
-                        ? filter.getQuery()
-                        : "<Empty query>";
     std::vector<std::string> bookIdsToDump = mp_library->filter(filter);
     const auto totalResults = bookIdsToDump.size();
     const size_t count = request.get_optional_param("count", 10UL);
@@ -917,15 +1016,6 @@ InternalServer::search_catalog(const RequestContext& request,
 namespace
 {
 
-std::string get_book_name(const RequestContext& request)
-{
-  try {
-    return request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return std::string();
-  }
-}
-
 ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::string& pattern)
 {
   return ParameterizedMessage("suggest-search",
@@ -940,7 +1030,8 @@ ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::s
 std::unique_ptr<Response>
 InternalServer::build_redirect(const std::string& bookName, const zim::Item& item) const
 {
-  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(item.getPath());
+  const auto path = kiwix::urlEncode(item.getPath());
+  const auto redirectUrl = m_root + "/content/" + bookName + "/" + path;
   return Response::build_redirect(*this, redirectUrl);
 }
 
@@ -952,9 +1043,10 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     printf("** running handle_content\n");
   }
 
-  const std::string bookName = get_book_name(request);
-  if (bookName.empty())
-    return build_homepage(request);
+  const std::string contentPrefix = "/content/";
+  const bool isContentPrefixedUrl = startsWith(url, contentPrefix);
+  const size_t prefixLength = isContentPrefixedUrl ? contentPrefix.size() : 1;
+  const std::string bookName = request.get_url_part(isContentPrefixedUrl);
 
   std::shared_ptr<zim::Archive> archive;
   try {
@@ -963,29 +1055,36 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   } catch (const std::out_of_range& e) {}
 
   if (archive == nullptr) {
-    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
+    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern);
     return HTTP404Response(*this, request)
            + urlNotFoundMsg
-           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
-           + TaskbarInfo(bookName);
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
   }
 
-  auto urlStr = request.get_url().substr(bookName.size()+1);
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
+  auto urlStr = url.substr(prefixLength + bookName.size());
   if (urlStr[0] == '/') {
     urlStr = urlStr.substr(1);
   }
 
   try {
     auto entry = getEntryFromPath(*archive, urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
+    if (entry.isRedirect() || urlStr != entry.getPath()) {
+      // In the condition above, the second case (an entry with a different
+      // URL was returned) can occur in the following situations:
+      // 1. urlStr is empty or equal to "/" and the ZIM file doesn't contain
+      //    such an entry, in which case the main entry is returned instead.
+      // 2. The ZIM file uses old namespace scheme, and the resource at urlStr
+      //    is not present but can be found under one of the 'A', 'I', 'J' or
+      //    '-' namespaces, in which case that resource is returned instead.
       return build_redirect(bookName, getFinalItem(*archive, entry));
     }
     auto response = ItemResponse::build(*this, request, entry.getItem());
-    try {
-      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, archive.get());
-    } catch (std::bad_cast& e) {}
+    response->set_etag_body(archiveUuid);
 
     if (m_verbose.load()) {
       printf("Found %s\n", entry.getPath().c_str());
@@ -997,11 +1096,10 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     if (m_verbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
 
-    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
+    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern);
     return HTTP404Response(*this, request)
            + urlNotFoundMsg
-           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
-           + TaskbarInfo(bookName, archive.get());
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
   }
 }
 
@@ -1040,6 +1138,11 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
            + noSuchBookErrorMsg(bookName);
   }
 
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
   // Remove the beggining of the path:
   // /raw/<bookName>/<kind>/foo
   // ^^^^^          ^      ^
@@ -1049,13 +1152,17 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   try {
     if (kind == "meta") {
       auto item = archive->getMetadataItem(itemPath);
-      return ItemResponse::build(*this, request, item, /*raw=*/true);
+      auto response = ItemResponse::build(*this, request, item);
+      response->set_etag_body(archiveUuid);
+      return response;
     } else {
       auto entry = archive->getEntryByPath(itemPath);
       if (entry.isRedirect()) {
         return build_redirect(bookName, entry.getItem(true));
       }
-      return ItemResponse::build(*this, request, entry.getItem(), /*raw=*/true);
+      auto response = ItemResponse::build(*this, request, entry.getItem());
+      response->set_etag_body(archiveUuid);
+      return response;
     }
   } catch (zim::EntryNotFound& e ) {
     if (m_verbose.load()) {
@@ -1067,4 +1174,32 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   }
 }
 
+bool InternalServer::isLocallyCustomizedResource(const std::string& url) const
+{
+  return m_customizedResources->find(url) != m_customizedResources->end();
+}
+
+std::unique_ptr<Response> InternalServer::handle_locally_customized_resource(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_locally_customized_resource\n");
+  }
+
+  const CustomizedResourceData& crd = m_customizedResources->at(request.get_url());
+
+  if (m_verbose.load()) {
+    std::cout << "Reading " <<  crd.resourceFilePath << std::endl;
+  }
+  const auto resourceData = getFileContent(crd.resourceFilePath);
+
+  auto byteRange = request.get_range().resolve(resourceData.size());
+  if (byteRange.kind() != ByteRange::RESOLVED_FULL_CONTENT) {
+    return Response::build_416(*this, resourceData.size());
+  }
+
+  return ContentResponse::build(*this,
+                                resourceData,
+                                crd.mimeType);
+}
+
 }

src/server/internalServer.h

@@ -88,10 +88,6 @@ class SearchInfo {
 
 
 typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
-typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
-
-class Entry;
 class OPDSDumper;
 
 class InternalServer {
@@ -109,7 +105,7 @@ class InternalServer {
                    bool blockExternalLinks,
                    std::string indexTemplateString,
                    int ipConnectionLimit);
-    virtual ~InternalServer() = default;
+    virtual ~InternalServer();
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
                                const char* url,
@@ -127,6 +123,7 @@ class InternalServer {
     std::unique_ptr<Response> handle_request(const RequestContext& request);
     std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
     std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_viewer_settings(const RequestContext& request);
     std::unique_ptr<Response> handle_skin(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
@@ -137,22 +134,32 @@ class InternalServer {
     std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
     std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
     std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_search_request(const RequestContext& request);
     std::unique_ptr<Response> handle_suggest(const RequestContext& request);
     std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_catch(const RequestContext& request);
     std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
     std::unique_ptr<Response> handle_content(const RequestContext& request);
     std::unique_ptr<Response> handle_raw(const RequestContext& request);
+    std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
 
     std::vector<std::string> search_catalog(const RequestContext& request,
                                             kiwix::OPDSDumper& opdsDumper);
 
     MustacheData get_default_data() const;
 
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
     std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
     SearchInfo getSearchInfo(const RequestContext& r) const;
 
+    bool isLocallyCustomizedResource(const std::string& url) const;
+
+    std::string getLibraryId() const;
+
+  private: // types
+    class LockableSuggestionSearcher;
+    typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+    typedef ConcurrentCache<std::string, std::shared_ptr<LockableSuggestionSearcher>> SuggestionSearcherCache;
+
   private: // data
     std::string m_addr;
     int m_port;
@@ -174,11 +181,13 @@ class InternalServer {
     SuggestionSearcherCache suggestionSearcherCache;
 
     std::string m_server_id;
-    std::string m_library_id;
+
+    class CustomizedResources;
+    std::unique_ptr<CustomizedResources> m_customizedResources;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
-    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
-    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 };
 
 }

src/server/internalServer_catalog_v2.cpp

@@ -24,7 +24,7 @@
 #include "request_context.h"
 #include "response.h"
 #include "tools/otherTools.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include <mustache.hpp>
 
@@ -77,17 +77,18 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
 {
+  const std::string libraryId = getLibraryId();
   return ContentResponse::build(
              *this,
              RESOURCE::templates::catalog_v2_root_xml,
              kainjow::mustache::object{
                {"date", gen_date_str()},
                {"endpoint_root", m_root + "/catalog/v2"},
-               {"feed_id", gen_uuid(m_library_id)},
-               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
-               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
-               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
+               {"feed_id", gen_uuid(libraryId)},
+               {"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
+               {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
+               {"category_list_feed_id", gen_uuid(libraryId + "/categories")},
+               {"language_list_feed_id", gen_uuid(libraryId + "/languages")}
              },
              "application/atom+xml;profile=opds-catalog;kind=navigation"
   );
@@ -95,9 +96,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto bookIds = search_catalog(request, opdsDumper);
   const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
@@ -116,9 +117,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
            + urlNotFoundMsg;
   }
 
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
   return ContentResponse::build(
              *this,
@@ -129,9 +130,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.categoriesOPDSFeed(),
@@ -141,9 +142,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
 {
-  OPDSDumper opdsDumper(mp_library);
+  OPDSDumper opdsDumper(mp_library, mp_nameMapper);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.languagesOPDSFeed(),
@@ -154,12 +155,15 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const Requ
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
 {
   try {
-    const auto bookName  = request.get_url_part(3);
-    const auto bookId = mp_nameMapper->getIdForName(bookName);
+    const auto bookId  = request.get_url_part(3);
     auto book = mp_library->getBookByIdThreadSafe(bookId);
     auto size = request.get_argument<unsigned int>("size");
     auto illustration = book.getIllustration(size);
-    return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
+    return ContentResponse::build(
+               *this,
+               illustration->getData(),
+               illustration->mimeType
+    );
   } catch(...) {
     return HTTP404Response(*this, request)
            + urlNotFoundMsg;

src/server/request_context.cpp

@@ -25,8 +25,10 @@
 #include <sstream>
 #include <cstdio>
 #include <atomic>
+#include <cctype>
 
 #include "tools/stringTools.h"
+#include "i18n.h"
 
 namespace kiwix {
 
@@ -66,12 +68,13 @@ fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
 } // unnamed namespace
 
 RequestContext::RequestContext(struct MHD_Connection* connection,
-                               std::string rootLocation,
+                               std::string _rootLocation,
                                const std::string& _url,
                                const std::string& _method,
                                const std::string& version) :
+  rootLocation(_rootLocation),
   full_url(_url),
-  url(fullURL2LocalURL(_url, rootLocation)),
+  url(fullURL2LocalURL(_url, _rootLocation)),
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
@@ -80,6 +83,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
+  MHD_get_connection_values(connection, MHD_COOKIE_KIND, &RequestContext::fill_cookie, this);
 
   try {
     acceptEncodingGzip =
@@ -89,6 +93,8 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   try {
     byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
+
+  userlang = determine_user_language();
 }
 
 RequestContext::~RequestContext()
@@ -107,6 +113,22 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key].push_back(value == nullptr ? "" : value);
+  if ( ! _this->queryString.empty() ) {
+    _this->queryString += "&";
+  }
+  _this->queryString += urlEncode(key);
+  if ( value ) {
+    _this->queryString += "=";
+    _this->queryString += urlEncode(value);
+  }
+  return MHD_YES;
+}
+
+MHD_Result RequestContext::fill_cookie(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
+{
+  RequestContext *_this = static_cast<RequestContext*>(__this);
+  _this->cookies[key] = value == nullptr ? "" : value;
   return MHD_YES;
 }
 
@@ -172,6 +194,10 @@ std::string RequestContext::get_full_url() const {
   return full_url;
 }
 
+std::string RequestContext::get_root_path() const {
+  return rootLocation.empty() ? "/" : rootLocation;
+}
+
 bool RequestContext::is_valid_url() const {
   return !url.empty();
 }
@@ -190,16 +216,33 @@ std::string RequestContext::get_header(const std::string& name) const {
 }
 
 std::string RequestContext::get_user_language() const
+{
+  return userlang.lang;
+}
+
+bool RequestContext::user_language_comes_from_cookie() const
+{
+  return userlang.selectedBy == UserLanguage::SelectorKind::COOKIE;
+}
+
+RequestContext::UserLanguage RequestContext::determine_user_language() const
 {
   try {
-    return get_argument("userlang");
+    return {UserLanguage::SelectorKind::QUERY_PARAM, get_argument("userlang")};
   } catch(const std::out_of_range&) {}
 
   try {
-    return get_header("Accept-Language");
+     return {UserLanguage::SelectorKind::COOKIE, cookies.at("userlang")};
   } catch(const std::out_of_range&) {}
 
-  return "en";
+  try {
+    const std::string acceptLanguage = get_header("Accept-Language");
+    const auto userLangPrefs = parseUserLanguagePreferences(acceptLanguage);
+    const auto lang = selectMostSuitableLanguage(userLangPrefs);
+    return {UserLanguage::SelectorKind::ACCEPT_LANGUAGE_HEADER, lang};
+  } catch(const std::out_of_range&) {}
+
+  return {UserLanguage::SelectorKind::DEFAULT, "en"};
 }
 
 std::string RequestContext::get_requested_format() const

src/server/request_context.h

@@ -91,22 +91,20 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
+    std::string get_root_path() const;
 
-    std::string get_query(bool mustEncode = false) const {
-      return get_query([](const std::string& key) {return true;}, mustEncode);
-    }
+    std::string get_query() const { return queryString; }
 
     template<class F>
-    std::string get_query(F filter, bool mustEncode) const {
+    std::string get_query(F filter) const {
       std::string q;
       const char* sep = "";
-      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
       for ( const auto& a : arguments ) {
         if (!filter(a.first)) {
           continue;
         }
         for (const auto& v: a.second) {
-          q += sep + encode(a.first) + '=' + encode(v);
+          q += sep + urlEncode(a.first) + '=' + urlEncode(v);
           sep = "&";
         }
       }
@@ -120,7 +118,25 @@ class RequestContext {
     std::string get_user_language() const;
     std::string get_requested_format() const;
 
+    bool user_language_comes_from_cookie() const;
+
+  private: // types
+    struct UserLanguage
+    {
+      enum SelectorKind
+      {
+        QUERY_PARAM,
+        COOKIE,
+        ACCEPT_LANGUAGE_HEADER,
+        DEFAULT
+      };
+
+      SelectorKind selectedBy;
+      std::string  lang;
+    };
+
   private: // data
+    std::string rootLocation;
     std::string full_url;
     std::string url;
     RequestMethod method;
@@ -132,9 +148,15 @@ class RequestContext {
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::vector<std::string>> arguments;
+    std::map<std::string, std::string> cookies;
+    std::string queryString;
+    UserLanguage userlang;
 
   private: // functions
+    UserLanguage determine_user_language() const;
+
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_cookie(void *, enum MHD_ValueKind, const char*, const char*);
     static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 

src/server/response.cpp

@@ -20,7 +20,7 @@
 #include "response.h"
 #include "request_context.h"
 #include "internalServer.h"
-#include "kiwixlib-resources.h"
+#include "libkiwix-resources.h"
 
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
@@ -53,18 +53,24 @@ std::string get_mime_type(const zim::Item& item)
 {
   try {
     return item.getMimetype();
-  } catch (exception& e) {
+  } catch (std::exception& e) {
     return "application/octet-stream";
   }
 }
 
 bool is_compressible_mime_type(const std::string& mimeType)
 {
-  return mimeType.find("text/") != string::npos
-      || mimeType.find("application/javascript") != string::npos
-      || mimeType.find("application/atom") != string::npos
-      || mimeType.find("application/opensearchdescription") != string::npos
-      || mimeType.find("application/json") != string::npos;
+  return mimeType.find("text/") != std::string::npos
+      || mimeType.find("application/javascript") != std::string::npos
+      || mimeType.find("application/atom") != std::string::npos
+      || mimeType.find("application/opensearchdescription") != std::string::npos
+      || mimeType.find("application/json") != std::string::npos
+
+      // Web fonts
+      || mimeType.find("application/font-") != std::string::npos
+      || mimeType.find("application/x-font-") != std::string::npos
+      || mimeType.find("application/vnd.ms-fontobject") != std::string::npos
+      || mimeType.find("font/") != std::string::npos;
 }
 
 bool compress(std::string &content) {
@@ -102,6 +108,14 @@ bool compress(std::string &content) {
 }
 
 
+const char* getCacheControlHeader(Response::Kind k)
+{
+  switch(k) {
+    case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
+    case Response::ZIM_CONTENT:     return "max-age=3600, must-revalidate";
+    default:                        return "max-age=0, must-revalidate";
+  }
+}
 
 } // unnamed namespace
 
@@ -112,6 +126,13 @@ Response::Response(bool verbose)
   add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
+void Response::set_kind(Kind k)
+{
+  m_kind = k;
+  if ( k == ZIM_CONTENT )
+    m_etag.set_option(ETag::ZIM_CONTENT);
+}
+
 std::unique_ptr<Response> Response::build(const InternalServer& server)
 {
   return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
@@ -122,6 +143,9 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   auto response = Response::build(server);
   response->set_code(MHD_HTTP_NOT_MODIFIED);
   response->m_etag = etag;
+  if ( etag.get_option(ETag::ZIM_CONTENT) ) {
+    response->set_kind(Response::ZIM_CONTENT);
+  }
   if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
     response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
   }
@@ -140,9 +164,6 @@ std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObjec
 {
   auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
   r->set_code(m_httpStatusCode);
-  if ( m_taskbarInfo ) {
-    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
-  }
   return r;
 }
 
@@ -236,29 +257,12 @@ HTTP500Response::HTTP500Response(const InternalServer& server,
 
 std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
 {
-  // We want a 500 response to be a minimalistic one (so that the server doesn't
-  // have to provide additional resources required for its proper rendering)
-  // ";raw=true" in the MIME-type below disables response decoration
-  // (see ContentResponse::contentDecorationAllowed())
-  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  const std::string mimeType = "text/html;charset=utf-8";
   auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
   r->set_code(m_httpStatusCode);
   return r;
 }
 
-ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
-{
-  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
-  return *this;
-}
-
-ContentResponseBlueprint& ContentResponseBlueprint::operator+=(const TaskbarInfo& taskbarInfo)
-{
-  // operator+() is already a state-modifying operator (akin to operator+=)
-  return *this + taskbarInfo;
-}
-
-
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
 {
   auto response = Response::build(server);
@@ -307,7 +311,7 @@ static ssize_t callback_reader_from_item(void* cls,
 {
   RunningResponse* response = static_cast<RunningResponse*>(cls);
 
-  size_t max_size_to_set = min<size_t>(
+  size_t max_size_to_set = std::min<size_t>(
     max,
     response->item.getSize() - pos - response->range_start);
 
@@ -337,52 +341,6 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar(const std::string& lang)
-{
-  i18n::GetTranslatedString t(lang);
-  kainjow::mustache::object data{
-    {"root", m_root},
-    {"content", m_bookName},
-    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
-    {"title", m_bookTitle},
-    {"withlibrarybutton", m_withLibraryButton},
-    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
-    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
-    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
-    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
-  };
-  auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    head_content);
-
-  auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
-  m_content = appendToFirstOccurence(
-    m_content,
-    "<body[^>]*>",
-    taskbar_part);
-}
-
-
-void ContentResponse::inject_externallinks_blocker()
-{
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    script_tag);
-}
-
-void ContentResponse::inject_root_link(){
-  m_content = prependToFirstOccurence(
-    m_content,
-    "</head[ \\t]*>",
-    "<link type=\"root\" href=\"" + m_root + "\">");
-}
-
 bool
 ContentResponse::can_compress(const RequestContext& request) const
 {
@@ -391,16 +349,6 @@ ContentResponse::can_compress(const RequestContext& request) const
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
-bool
-ContentResponse::contentDecorationAllowed() const
-{
-  if (m_raw) {
-    return false;
-  }
-  return (startsWith(m_mimeType, "text/html")
-    && m_mimeType.find(";raw=true") == std::string::npos);
-}
-
 MHD_Response*
 Response::create_mhd_response(const RequestContext& request)
 {
@@ -411,17 +359,6 @@ Response::create_mhd_response(const RequestContext& request)
 MHD_Response*
 ContentResponse::create_mhd_response(const RequestContext& request)
 {
-  if (contentDecorationAllowed()) {
-    inject_root_link();
-
-    if (m_withTaskbar) {
-      introduce_taskbar(request.get_user_language());
-    }
-    if (m_blockExternalLinks) {
-      inject_externallinks_blocker();
-    }
-  }
-
   const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
@@ -442,7 +379,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   MHD_Response* response = create_mhd_response(request);
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+                          getCacheControlHeader(m_kind));
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
     MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
@@ -450,6 +387,13 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
     MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
   }
 
+  if ( ! request.user_language_comes_from_cookie() ) {
+    const std::string cookie = "userlang=" + request.get_user_language()
+                               + ";Path=" + request.get_root_path()
+                               + ";Max-Age=31536000";
+    MHD_add_response_header(response, MHD_HTTP_HEADER_SET_COOKIE, cookie.c_str());
+  }
+
   if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
@@ -461,24 +405,11 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
-{
-  m_bookName = bookName;
-  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
-}
-
-
-ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+ContentResponse::ContentResponse(const std::string& root, bool verbose, const std::string& content, const std::string& mimetype) :
   Response(verbose),
   m_root(root),
   m_content(content),
-  m_mimeType(mimetype),
-  m_raw(raw),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  m_bookName(""),
-  m_bookTitle("")
+  m_mimeType(mimetype)
 {
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
@@ -486,17 +417,11 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw
 std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& content,
-  const std::string& mimetype,
-  bool isHomePage,
-  bool raw)
+  const std::string& mimetype)
 {
    return std::unique_ptr<ContentResponse>(new ContentResponse(
         server.m_root,
         server.m_verbose.load(),
-        raw,
-        server.m_withTaskbar && !isHomePage,
-        server.m_withLibraryButton,
-        server.m_blockExternalLinks,
         content,
         mimetype));
 }
@@ -505,11 +430,10 @@ std::unique_ptr<ContentResponse> ContentResponse::build(
   const InternalServer& server,
   const std::string& template_str,
   kainjow::mustache::data data,
-  const std::string& mimetype,
-  bool isHomePage)
+  const std::string& mimetype)
 {
   auto content = render_template(template_str, data);
-  return ContentResponse::build(server, content, mimetype, isHomePage);
+  return ContentResponse::build(server, content, mimetype);
 }
 
 ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
@@ -518,26 +442,26 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   m_mimeType(mimetype)
 {
   m_byteRange = byterange;
-  set_cacheable();
+  set_kind(Response::ZIM_CONTENT);
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
-std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
+std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
 {
   const std::string mimetype = get_mime_type(item);
   auto byteRange = request.get_range().resolve(item.getSize());
   const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
-    auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
-    response->set_cacheable();
+    auto response = ContentResponse::build(server, item.getData(), mimetype);
+    response->set_kind(Response::ZIM_CONTENT);
     response->m_byteRange = byteRange;
     return std::move(response);
   }
 
   if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
     auto response = Response::build_416(server, item.getSize());
-    response->set_cacheable();
+    response->set_kind(Response::ZIM_CONTENT);
     return response;
   }
 

src/server/response.h

@@ -26,10 +26,11 @@
 
 #include <mustache.hpp>
 #include "byte_range.h"
-#include "entry.h"
 #include "etag.h"
 #include "i18n.h"
 
+#include <zim/item.h>
+
 extern "C" {
 #include "microhttpd_wrapper.h"
 }
@@ -44,6 +45,14 @@ class InternalServer;
 class RequestContext;
 
 class Response {
+  public:
+    enum Kind
+    {
+      STATIC_RESOURCE,
+      ZIM_CONTENT,
+      DYNAMIC_CONTENT
+    };
+
   public:
     Response(bool verbose);
     virtual ~Response() = default;
@@ -56,8 +65,9 @@ class Response {
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
     void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    void set_kind(Kind k);
+    Kind get_kind() const { return m_kind; }
+    void set_etag_body(const std::string& id) { m_etag.set_body(id); }
     void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
@@ -67,6 +77,7 @@ class Response {
     MHD_Response* create_error_response(const RequestContext& request) const;
 
   protected: // data
+    Kind m_kind = DYNAMIC_CONTENT;
     bool m_verbose;
     int m_returnCode;
     ByteRange m_byteRange;
@@ -82,60 +93,32 @@ class ContentResponse : public Response {
     ContentResponse(
       const std::string& root,
       bool verbose,
-      bool raw,
-      bool withTaskbar,
-      bool withLibraryButton,
-      bool blockExternalLinks,
       const std::string& content,
       const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& content,
-      const std::string& mimetype,
-      bool isHomePage = false,
-      bool raw = false);
+      const std::string& mimetype);
+
     static std::unique_ptr<ContentResponse> build(
       const InternalServer& server,
       const std::string& template_str,
       kainjow::mustache::data data,
-      const std::string& mimetype,
-      bool isHomePage = false);
-
-    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
+      const std::string& mimetype);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar(const std::string& lang);
-    void inject_externallinks_blocker();
-    void inject_root_link();
     bool can_compress(const RequestContext& request) const;
-    bool contentDecorationAllowed() const;
 
 
   private:
     std::string m_root;
     std::string m_content;
     std::string m_mimeType;
-    bool m_raw;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    std::string m_bookName;
-    std::string m_bookTitle;
  };
 
-struct TaskbarInfo
-{
-  const std::string bookName;
-  const zim::Archive* const archive;
-
-  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
-    : bookName(bookName)
-    , archive(a)
-  {}
-};
-
 class ContentResponseBlueprint
 {
 public: // functions
@@ -164,9 +147,6 @@ public: // functions
   }
 
 
-  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
-  ContentResponseBlueprint& operator+=(const TaskbarInfo& taskbarInfo);
-
 protected: // functions
   std::string getMessage(const std::string& msgId) const;
   virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
@@ -178,7 +158,6 @@ public: //data
   const std::string m_mimeType;
   const std::string m_template;
   kainjow::mustache::data m_data;
-  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
 };
 
 struct HTTPErrorResponse : ContentResponseBlueprint
@@ -190,8 +169,6 @@ struct HTTPErrorResponse : ContentResponseBlueprint
                     const std::string& headingMsgId,
                     const std::string& cssUrl = "");
 
-  using ContentResponseBlueprint::operator+;
-  using ContentResponseBlueprint::operator+=;
   HTTPErrorResponse& operator+(const std::string& msg);
   HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
   HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
@@ -237,7 +214,7 @@ private: // overrides
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
-    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);

src/tools/archiveTools.cpp

@@ -93,10 +93,6 @@ std::string getMetaFlavour(const zim::Archive& archive) {
   return getMetadata(archive, "Flavour");
 }
 
-std::string getArchiveId(const zim::Archive& archive) {
-  return (std::string) archive.getUuid();
-}
-
 bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType){
   try {
@@ -109,46 +105,6 @@ bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
   return false;
 }
 
-// should this be in libzim
-unsigned int getArchiveMediaCount(const zim::Archive& archive) {
-  std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
-  unsigned int counter = 0;
-
-  for (auto &pair:counterMap) {
-    if (startsWith(pair.first, "image/") ||
-        startsWith(pair.first, "video/") ||
-        startsWith(pair.first, "audio/")) {
-      counter += pair.second;
-    }
-  }
-
-  return counter;
-}
-
-unsigned int getArchiveArticleCount(const zim::Archive& archive) {
-  // [HACK]
-  // getArticleCount() returns different things depending of the "version" of the zim.
-  // On old zim (<=6), it returns the number of entry in `A` namespace
-  // On recent zim (>=7), it returns:
-  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
-  //  - the number of front article if a frontArticleIndex is present
-  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
-  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
-  if (archive.hasNewNamespaceScheme()) {
-    //The articleCount is "good"
-    return archive.getArticleCount();
-  } else {
-    // We have to parse the `M/Counter` metadata
-    unsigned int counter = 0;
-    for(const auto& pair:parseArchiveCounter(archive)) {
-      if (startsWith(pair.first, "text/html")) {
-        counter += pair.second;
-      }
-    }
-    return counter;
-  }
-}
-
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }
@@ -169,14 +125,4 @@ zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path
   }
   throw zim::EntryNotFound("Cannot find entry for non empty path");
 }
-
-MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
-  try {
-    auto counterContent = archive.getMetadata("Counter");
-    return parseMimetypeCounter(counterContent);
-  } catch (zim::EntryNotFound& e) {
-    return {};
-  }
-}
-
 } // kiwix

src/tools/archiveTools.h

@@ -40,7 +40,6 @@ namespace kiwix
     std::string getMetaCreator(const zim::Archive& archive);
     std::string getMetaPublisher(const zim::Archive& archive);
     std::string getMetaFlavour(const zim::Archive& archive);
-    std::string getArchiveId(const zim::Archive& archive);
 
     bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
                            std::string& content, std::string& mimeType);
@@ -52,9 +51,6 @@ namespace kiwix
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
 
     zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
-
-    MimeCounterType parseArchiveCounter(const zim::Archive& archive);
-
 }
 
 #endif

src/tools/otherTools.cpp

@@ -32,12 +32,15 @@
 #endif
 
 #include "tools/stringTools.h"
+#include "server/i18n.h"
+#include "libkiwix-resources.h"
 
 #include <map>
 #include <sstream>
 #include <pugixml.hpp>
 
 #include <zim/uuid.h>
+#include <zim/suggestion_iterator.h>
 
 
 static std::map<std::string, std::string> codeisomapping {
@@ -288,67 +291,6 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
-namespace
-{
-// The counter metadata format is a list of item separated by a `;` :
-// item0;item1;item2
-// Each item is a "tuple" mimetype=number.
-// However, the mimetype may contains parameters:
-// text/html;raw=true;foo=bar
-// So the final format may be complex to parse:
-// key0=value0;key1;foo=bar=value1;key2=value2
-
-typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
-
-std::string readFullMimetypeAndCounterString(std::istream& in)
-{
-  std::string mtcStr, params;
-  getline(in, mtcStr, ';');
-  if ( mtcStr.find('=') == std::string::npos )
-  {
-    do
-    {
-      if ( !getline(in, params, ';' ) )
-        return std::string();
-      mtcStr += ";" + params;
-    }
-    while ( std::count(params.begin(), params.end(), '=') != 2 );
-  }
-  return mtcStr;
-}
-
-MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
-{
-  const std::string::size_type k = s.find_last_of("=");
-  if ( k != std::string::npos )
-  {
-    const std::string mimeType = s.substr(0, k);
-    std::istringstream counterSS(s.substr(k+1));
-    unsigned int counter;
-    if (counterSS >> counter && counterSS.eof())
-      return MimetypeAndCounter{mimeType, counter};
-  }
-  return MimetypeAndCounter{"", 0};
-}
-
-} // unnamed namespace
-
-kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
-{
-  kiwix::MimeCounterType counters;
-  std::istringstream ss(counterData);
-
-  while (ss)
-  {
-    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
-    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
-    if ( !mtc.first.empty() )
-      counters.insert(mtc);
-  }
-
-  return counters;
-}
-
 std::string kiwix::gen_date_str()
 {
   auto now = std::time(0);
@@ -380,10 +322,76 @@ kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
   std::stringstream ss;
   tmpl.render(data, [&ss](const std::string& str) { ss << str; });
   return ss.str();
 }
+
+namespace
+{
+
+std::string escapeBackslashes(const std::string& s)
+{
+  std::string es;
+  es.reserve(s.size());
+  for (char c : s) {
+    if ( c == '\\' ) {
+      es.push_back('\\');
+    }
+    es.push_back(c);
+  }
+  return es;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang,
+                                         const std::string& queryString)
+{
+  return kiwix::i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+} // unnamed namespace
+
+kiwix::Suggestions::Suggestions()
+  : m_data(kainjow::mustache::data::type::list)
+{
+}
+
+void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
+{
+  kainjow::mustache::data result;
+
+  const std::string label = suggestion.hasSnippet()
+                          ? suggestion.getSnippet()
+                          : suggestion.getTitle();
+
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(suggestion.getTitle()));
+  result.set("kind", "path");
+  result.set("path", escapeBackslashes(suggestion.getPath()));
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
+                                               const std::string& queryString)
+{
+  kainjow::mustache::data result;
+  const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
+  result.set("label", escapeBackslashes(label));
+  result.set("value", escapeBackslashes(queryString + " "));
+  result.set("kind", "pattern");
+  result.set("first", m_data.is_empty_list());
+  m_data.push_back(result);
+}
+
+std::string kiwix::Suggestions::getJSON() const
+{
+  kainjow::mustache::data data;
+  data.set("suggestions", m_data);
+
+  return render_template(RESOURCE::templates::suggestion_json, data);
+}

src/tools/otherTools.h

@@ -33,6 +33,10 @@ namespace pugi {
   class xml_node;
 }
 
+namespace zim {
+  class SuggestionItem;
+}
+
 namespace kiwix
 {
   std::string nodeToString(const pugi::xml_node& node);
@@ -45,9 +49,6 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
-  using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
-  MimeCounterType parseMimetypeCounter(const std::string& counterData);
-
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
@@ -70,6 +71,22 @@ namespace kiwix
 
     return defaultValue;
   }
+
+  class Suggestions
+  {
+  public:
+    Suggestions();
+
+    void add(const zim::SuggestionItem& suggestion);
+
+    void addFTSearchSuggestion(const std::string& uiLang,
+                               const std::string& query);
+
+    std::string getJSON() const;
+
+  private:
+    kainjow::mustache::data m_data;
+  };
 }
 
 #endif

src/tools/regexTools.cpp

@@ -75,41 +75,3 @@ std::string replaceRegex(const std::string& content,
   uresult.toUTF8String(tmp);
   return tmp;
 }
-
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->end(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}
-
-std::string prependToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement)
-{
-  ucnv_setDefaultName("UTF-8");
-  icu::UnicodeString ucontent(content.c_str());
-  icu::UnicodeString ureplacement(replacement.c_str());
-  auto matcher = buildMatcher(regex, ucontent);
-  if (matcher->find()) {
-    UErrorCode status = U_ZERO_ERROR;
-    ucontent.insert(matcher->start(status), ureplacement);
-    std::string tmp;
-    ucontent.toUTF8String(tmp);
-    return tmp;
-  }
-
-  return content;
-}

src/tools/regexTools.h

@@ -26,11 +26,5 @@ bool matchRegex(const std::string& content, const std::string& regex);
 std::string replaceRegex(const std::string& content,
                          const std::string& replacement,
                          const std::string& regex);
-std::string appendToFirstOccurence(const std::string& content,
-                                   const std::string& regex,
-                                   const std::string& replacement);
-std::string prependToFirstOccurence(const std::string& content,
-                                    const std::string& regex,
-                                    const std::string& replacement);
 
 #endif

src/tools/stringTools.cpp

@@ -49,6 +49,24 @@ void kiwix::loadICUExternalTables()
 #endif
 }
 
+kiwix::ICULanguageInfo::ICULanguageInfo(const std::string& langCode)
+  : locale(langCode.c_str())
+{}
+
+std::string kiwix::ICULanguageInfo::iso3Code() const
+{
+  return locale.getISO3Language();
+}
+
+std::string kiwix::ICULanguageInfo::selfName() const
+{
+  icu::UnicodeString langSelfNameICUString;
+  locale.getDisplayLanguage(locale, langSelfNameICUString);
+  std::string langSelfName;
+  langSelfNameICUString.toUTF8String(langSelfName);
+  return langSelfName;
+}
+
 std::string kiwix::removeAccents(const std::string& text)
 {
   loadICUExternalTables();
@@ -143,15 +161,14 @@ std::string kiwix::encodeDiples(const std::string& str)
   return result;
 }
 
-/* urlEncode() based on javascript encodeURI() &
-   encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
+namespace
+{
 
 bool isReservedUrlChar(char c)
 {
   switch (c) {
   case ';':
   case ',':
-  case '/':
   case '?':
   case ':':
   case '@':
@@ -159,22 +176,22 @@ bool isReservedUrlChar(char c)
   case '=':
   case '+':
   case '$':
+  case '#':
     return true;
   default:
     return false;
   }
 }
 
-bool needsEscape(char c, bool encodeReserved)
+bool isHarmlessUriChar(char c)
 {
   if (c >= 'a' && c <= 'z')
-    return false;
+    return true;
   if (c >= 'A' && c <= 'Z')
-    return false;
+    return true;
   if (c >= '0' && c <= '9')
-    return false;
-  if (isReservedUrlChar(c))
-    return encodeReserved;
+    return true;
+
   switch (c) {
   case '-':
   case '_':
@@ -185,8 +202,46 @@ bool needsEscape(char c, bool encodeReserved)
   case '\'':
   case '(':
   case ')':
-    return false;
+  case '/':
+    return true;
   }
+  return false;
+}
+
+bool mustBeUriEncodedFor(kiwix::URIComponentKind target, char c)
+{
+  if (isHarmlessUriChar(c))
+    return false;
+
+  switch (c) {
+  case '/': // There is no reason to encode the path separator in the general
+            // case. It must be encoded only in a path component when its
+            // semantics of a path separator has to be suppressed.
+            return false;
+
+  case '@': // In a relative URL of the form abc@def/xyz (with no / in abc)
+            // a non-encoded @ will make "abc" and "def" to be interpreted as
+            // username and host components, respectively
+            return target == kiwix::URIComponentKind::PATH;
+
+  case ':': // In a relative URL of the form abc:def/xyz (with no / in abc)
+            // a non-encoded : will make "abc" and "def" to be interpreted as
+            // host and port components, respectively
+            return target == kiwix::URIComponentKind::PATH;
+
+  case '?': // A non-encoded '?' acts as a separator between the path
+            // and query components
+            return target == kiwix::URIComponentKind::PATH;
+
+  case '&': return target == kiwix::URIComponentKind::QUERY;
+  case '=': return target == kiwix::URIComponentKind::QUERY;
+  case '+': return target == kiwix::URIComponentKind::QUERY;
+
+  case '#': // A non-encoded '#' in either path or query-component
+            // would mark the beginning of the fragment component
+            return true;
+  }
+
   return true;
 }
 
@@ -212,23 +267,43 @@ int hexToInt(char c) {
   }
 }
 
-std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
+} // unnamed namespace
+
+std::string kiwix::urlEncode(const std::string& value)
 {
   std::ostringstream os;
   os << std::hex << std::uppercase;
-  for (std::string::const_iterator it = value.begin();
-       it != value.end();
-       it++) {
-
-    if (!needsEscape(*it, encodeReserved)) {
-      os << *it;
+  for (const char c : value) {
+    if (isHarmlessUriChar(c)) {
+      os << c;
     } else {
-      os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
+      const unsigned int charVal = static_cast<unsigned char>(c);
+      os << '%' << std::setw(2) << std::setfill('0') << charVal;
     }
   }
   return os.str();
 }
 
+namespace kiwix
+{
+
+std::string uriEncode(URIComponentKind target, const std::string& value)
+{
+  std::ostringstream os;
+  os << std::hex << std::uppercase;
+  for (const char c : value) {
+    if ( mustBeUriEncodedFor(target, c)  ) {
+      const unsigned int charVal = static_cast<unsigned char>(c);
+      os << '%' << std::setw(2) << std::setfill('0') << charVal;
+    } else {
+      os << c;
+    }
+  }
+  return os.str();
+}
+
+} // namespace kiwix
+
 std::string kiwix::urlDecode(const std::string& value, bool component)
 {
   std::ostringstream os;
@@ -249,15 +324,15 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
       int iHi = hexToInt(hi);
       int iLo = hexToInt(lo);
       if (iHi < 0 || iLo < 0) {
-	// Invalid escape sequence
-	os << '%' << hi << lo;
-	continue;
+        // Invalid escape sequence
+        os << '%' << hi << lo;
+        continue;
       }
       char c = (char)(iHi << 4 | iLo);
       if (!component && isReservedUrlChar(c)) {
-	os << '%' << hi << lo;
+        os << '%' << hi << lo;
       } else {
-	os << c;
+        os << c;
       }
     } else {
       os << *it;

src/tools/stringTools.h

@@ -21,6 +21,7 @@
 #define KIWIX_STRINGTOOLS_H
 
 #include <unicode/unistr.h>
+#include <unicode/locid.h>
 
 #include <string>
 #include <vector>
@@ -41,9 +42,35 @@ std::string encodeDiples(const std::string& str);
 std::string removeAccents(const std::string& text);
 void loadICUExternalTables();
 
-std::string urlEncode(const std::string& value, bool encodeReserved = false);
+class ICULanguageInfo
+{
+public:
+  explicit ICULanguageInfo(const std::string& langCode);
+
+  std::string iso3Code() const;
+  std::string selfName() const;
+
+private:
+  const icu::Locale locale;
+};
+
+
+/* urlEncode() is the equivalent of JS encodeURIComponent(), with the only
+ * difference that the slash (/) symbol is NOT encoded. */
+std::string urlEncode(const std::string& value);
 std::string urlDecode(const std::string& value, bool component = false);
 
+// Only URI components that are of interest to libkiwix
+// are included in the below enumeration type
+enum class URIComponentKind
+{
+  PATH,
+  QUERY
+};
+
+// Encode 'value' for usage in a URI componenet specified by 'target'
+std::string uriEncode(URIComponentKind target, const std::string& value);
+
 std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);

static/i18n/ar.json

@@ -0,0 +1,33 @@
+{
+	"@metadata": {
+		"authors": [
+			"Asma",
+			"Ravan",
+			"محمد أحمد عبد الفتاح"
+		]
+	},
+	"name": "الإنجليزية",
+	"no-such-book": "لا يوجد مثل هذا الكتاب: {{BOOK_NAME}}",
+	"too-many-books": "طلب العديد من الكتب {{NB_BOOKS}} حيث الحد {{LIMIT}}",
+	"no-book-found": "لا يوجد كتاب يطابق معايير الاختيار",
+	"url-not-found": "لم يتم العثور على عنوان URL المطلوب \"{{url}}\" على هذا الخادم.",
+	"suggest-search": "قم بإجراء بحث عن النص الكامل لـ <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "مع الأسف! فشل اختيار مقال عشوائي :(",
+	"invalid-raw-data-type": "{{DATATYPE}} ليس طلبًا صالحًا للمحتوى الأولي.",
+	"no-value-for-arg": "لم يتم تقديم قيمة للوسيطة {{ARGUMENT}}",
+	"no-query": "لم يتم تقديم ملخص.",
+	"raw-entry-not-found": "لا يمكن العثور على إدخال {{DATATYPE}} {{ENTRY}}",
+	"400-page-title": "طلب غير صالح",
+	"400-page-heading": "طلب غير صالح",
+	"404-page-title": "المحتوى غير موجود",
+	"404-page-heading": "لم يتم العثور عليه",
+	"500-page-title": "خطأ في الخادم الداخلي",
+	"500-page-heading": "خطأ في الخادم الداخلي",
+	"fulltext-search-unavailable": "البحث عن النص الكامل غير متاح",
+	"no-search-results": "محرك البحث عن النص الكامل غير متاح لهذا المحتوى.",
+	"library-button-text": "اذهب لصفحة الترحيب",
+	"home-button-text": "انتقل إلى الصفحة الرئيسية لـ \"{{BOOK_TITLE}}\"",
+	"random-page-button-text": "اذهب إلى صفحة عشوائية",
+	"searchbox-tooltip": "بحث \"{{BOOK_TITLE}}\"",
+	"confusion-of-tongues": "قد يشارك في البحث كتابان أو أكثر بلغات مختلفة، مما قد يؤدي إلى نتائج محيرة."
+}

static/i18n/cs.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Spotter"
+		]
+	},
+	"name": "Čeština",
+	"suggest-full-text-search": "obsahující '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Žádná taková kniha: {{BOOK_NAME}}",
+	"too-many-books": "Bylo požadováno příliš mnoho knih ({{NB_BOOKS}}), kde je limit {{LIMIT}}",
+	"no-book-found": "Výběrovým kritériím nevyhovuje žádná kniha",
+	"url-not-found": "Požadovaná adresa URL \"{{url}}\" nebyla na tomto serveru nalezena.",
+	"suggest-search": "Proveďte fulltextové vyhledávání <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Jejda! Nepodařilo se vybrat náhodný článek :(",
+	"invalid-raw-data-type": "{{DATATYPE}} není platný požadavek na nezpracovaný obsah.",
+	"no-value-for-arg": "Pro argument {{ARGUMENT}} nebyla zadána žádná hodnota",
+	"no-query": "Nebyl poskytnut žádný dotaz.",
+	"raw-entry-not-found": "Nelze najít položku {{DATATYPE}} {{ENTRY}}",
+	"400-page-title": "Neplatný požadavek",
+	"400-page-heading": "Neplatný požadavek",
+	"404-page-title": "Obsah nenalezen",
+	"404-page-heading": "Nenalezeno",
+	"500-page-title": "Interní chyba serveru",
+	"500-page-heading": "Interní chyba serveru",
+	"fulltext-search-unavailable": "Fulltextové vyhledávání není k dispozici",
+	"no-search-results": "Fulltextový vyhledávač není pro tento obsah dostupný.",
+	"library-button-text": "Přejít na uvítací stránku",
+	"home-button-text": "Přejít na hlavní stránku '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Přejít na náhodně vybranou stránku",
+	"searchbox-tooltip": "Hledat '{{BOOK_TITLE}}'"
+}

static/i18n/de.json

@@ -0,0 +1,20 @@
+{
+	"@metadata": {
+		"authors": [
+			"Lucas Werkmeister",
+			"ThisCarthing"
+		]
+	},
+	"name": "Deutsch",
+	"random-article-failure": "Hoppla! Konnte keinen zufälligen Artikel auswählen :(",
+	"400-page-title": "Ungültige Anfrage",
+	"400-page-heading": "Ungültige Anfrage",
+	"404-page-title": "Inhalt nicht gefunden",
+	"404-page-heading": "Nicht gefunden",
+	"500-page-title": "Interner Server-Fehler",
+	"500-page-heading": "Interner Server-Fehler",
+	"library-button-text": "Zur Willkommensseite gehen",
+	"home-button-text": "Zur Hauptseite von '{{BOOK_TITLE}}' gehen",
+	"random-page-button-text": "Zu einer zufällig ausgewählten Seite gehen",
+	"searchbox-tooltip": "Nach '{{BOOK_TITLE}}' suchen"
+}

static/i18n/en.json

@@ -27,4 +27,5 @@
 	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
 	, "random-page-button-text": "Go to a randomly selected page"
 	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
+	, "confusion-of-tongues": "Two or more books in different languages would participate in search, which may lead to confusing results."
 }

static/i18n/fr.json

@@ -29,5 +29,6 @@
 	"library-button-text": "Aller à la page de bienvenue",
 	"home-button-text": "Aller à la page principale de « {{BOOK_TITLE}} »",
 	"random-page-button-text": "Aller à une page sélectionnée aléatoirement",
-	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »"
+	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »",
+	"confusion-of-tongues": "Deux livres ou plus dans des langues différentes participeraient à la recherche, ce qui pourrait conduire à des résultats confus."
 }

static/i18n/he.json

@@ -1,7 +1,8 @@
 {
 	"@metadata": {
 		"authors": [
-			"Amire80"
+			"Amire80",
+			"YaronSh"
 		]
 	},
 	"name": "עברית",
@@ -27,5 +28,6 @@
 	"library-button-text": "מעבר לדף הבית \"ברוך בואך\"",
 	"home-button-text": "מעבר לדף הראשי של \"{{BOOK_TITLE}}\"",
 	"random-page-button-text": "מעבר לדף שנבחר אקראית",
-	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\""
+	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\"",
+	"confusion-of-tongues": "שני ספרים או יותר בשפות שונות ישתתפו בחיפוש, מה שעלול להוביל לתוצאות מבלבלות."
 }

static/i18n/hy.json

@@ -1,12 +1,16 @@
 {
 	"@metadata": {
-		"authors": []
+		"authors": [
+			"Kareyac"
+		]
 	},
 	"name": "Հայերեն",
 	"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}",
 	"url-not-found": "Սխալ հասցե՝ {{url}}",
 	"suggest-search": "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"400-page-title": "Անվավեր հարցում",
+	"400-page-heading": "Անվավեր հարցում",
 	"404-page-title": "Սխալ հասցե",
 	"404-page-heading": "Սխալ հասցե",
 	"library-button-text": "Գրադարանի էջ",

static/i18n/it.json

@@ -1,15 +1,27 @@
 {
 	"@metadata": {
 		"authors": [
+			"Albano",
 			"Beta16"
 		]
 	},
 	"name": "italiano",
 	"suggest-full-text-search": "contenente '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Nessun libro del genere: {{BOOK_NAME}}",
+	"too-many-books": "Troppi libri richiesti ({{NB_BOOKS}}) dove il limite è {{LIMIT}}",
+	"no-book-found": "Nessun libro corrisponde ai criteri di selezione",
 	"url-not-found": "L'URL richiesto \"{{url}}\" non è stato trovato in questo server.",
+	"suggest-search": "Effettua una ricerca di testo completo per <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ops! Impossibile selezionare un articolo casuale :(",
+	"no-value-for-arg": "Nessun valore fornito per l'argomento {{ARGUMENT}}",
 	"400-page-title": "Richiesta non valida",
 	"400-page-heading": "Richiesta non valida",
 	"404-page-title": "Contenuto non trovato",
 	"404-page-heading": "Non trovato",
-	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'"
+	"500-page-title": "Errore interno del server",
+	"500-page-heading": "Errore interno del server",
+	"library-button-text": "Vai alla pagina di benvenuto",
+	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Vai a una pagina selezionata casualmente",
+	"searchbox-tooltip": "Cerca '{{BOOK_TITLE}}'"
 }

static/i18n/ku-latn.json

@@ -27,5 +27,6 @@
 	"library-button-text": "Here rûpela xêrhatinê",
 	"home-button-text": "Here rûpela destpêkê yê {{BOOK_TITLE}}",
 	"random-page-button-text": "Here rûpeleke ketober bijartî",
-	"searchbox-tooltip": "Li {{BOOK_TITLE}} bigere"
+	"searchbox-tooltip": "Li {{BOOK_TITLE}} bigere",
+	"confusion-of-tongues": "Du an zêdetir kitêbên bi zimanên cihê wê beşdarî lêgerînê bibin, ev jî dibe ku bibe sedema tevliheviya encaman."
 }

static/i18n/mk.json

@@ -27,5 +27,6 @@
 	"library-button-text": "Оди на воведната страница",
 	"home-button-text": "Оди на главната страница на „{{BOOK_TITLE}}“",
 	"random-page-button-text": "Оди на случајно избрана страница",
-	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“"
+	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“",
+	"confusion-of-tongues": "Во пребарувањето ќе учествуваат две или повеќе книги на различни јазици, што може да довете до збунувачки исход."
 }

static/i18n/nqo.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Lancine.kounfantoh.fofana"
+		]
+	},
+	"name": "ߒߞߏ",
+	"suggest-full-text-search": "ߞߣߐߞߍߣߍ߲߫ ߦߋ߫ '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "ߞߊ߬ߝߊ߫ ߛߎ߮ ߏ߬ ߕߴߦߋ߲߬: {{BOOK_NAME}}",
+	"too-many-books": "ߞߝߊ߬ ߛߌߦߊߡߊ߲߫ ߡߊߢߌ߬ߣߌ߲߬ߞߊ߬ߣߍ߲߫ ߞߏߖߎ߰ {{NB_BOOKS}} ߡߍ߲ ߞߐߘߊ߲ ߦߋ߫ {{LIMIT}} ߘߌ߫",
+	"no-book-found": "ߞߝߊ߬ ߛߌ߫ ߓߍ߬ߣߍ߲߬ ߕߍ߫ ߛߎߥߊ߲ߘߟߌ ߛߙߊߕߌ ߏ߬ ߡߊ߬",
+	"url-not-found": "ߍ߲߬ߤߍ߲߫ ''URL ''{{url}} ߡߊߢߌ߬ߣߌ߲߬ߞߊ߬ߣߍ߲ ߡߊ߫ ߛߐ߬ߘߐ߲߫ ߡߊ߬ߟߐ߬ߟߊ߲ ߣߌ߲߬ ߠߊ߫.",
+	"suggest-search": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲ ߞߍ߫ <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "ߏ߯߹ ߞߢߊ߬ ߓߘߊ߫ ߞߍ߫ ߓߍ߲߬ߛߋ߲߬ߡߊ߬ ߞߎߡߘߊ ߛߐ߬ߘߐ߲߬ ߠߊ߫:(",
+	"invalid-raw-data-type": "{{DATATYPE}} ߕߍ߫ ߡߞߊ߬ߣߌ߲߬ߣߍ߲߫ ߓߍ߲߬ߣߍ߲߫ ߘߌ߫ ߞߣߐߘߐ߫ ߘߝߊߣߍ߲ ߞߊ߲ߡߊ߬.",
+	"no-value-for-arg": "ߡߐ߬ߟߐ߲߬ ߡߊ߫ ߡߊߛߐ߫ ߘߊߘߐߡߌߣߊߞߎ߲ߧߊ {{ARGUMENT}} ߞߊ߲ߡߊ߬",
+	"no-query": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߛߌ߫ ߡߊ߫ ߡߊߛߐ߫.",
+	"raw-entry-not-found": "ߟߊ߬ߘߏ߲߬ߣߍ߲ {{ENTRY}} {{DATATYPE}} ߕߍ߫ ߣߊ߬ ߡߊߛߐ߬ߘߐ߲߫ ߠߊ߫",
+	"400-page-title": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߓߍ߲߬ߓߊߟߌ",
+	"400-page-heading": "ߡߞߊ߬ߣߌ߲߬ߠߌ߲߫ ߓߍ߲߬ߓߊߟߌ",
+	"404-page-title": "ߞߣߐߘߐ ߡߊ߫ ߛߐ߬ߘߐ߲߫",
+	"404-page-heading": "ߡߊ߫ ߛߐ߬ߘߐ߲߫",
+	"500-page-title": "ߞߣߐߟߊ ߡߊ߬ߛߐ߬ߟߊ߲ ߝߎ߬ߕߎ߲߬ߕߌ",
+	"500-page-heading": "ߞߣߐߟߊ ߡߊ߬ߛߐ߬ߟߊ߲ ߝߎ߬ߕߎ߲߬ߕߌ",
+	"fulltext-search-unavailable": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲ ߕߍ߫ ߦߋ߲߬",
+	"no-search-results": "ߛߓߍߟߌ߫ ߘߝߊߣߍ߲ ߢߌߣߌ߲ߠߌ߲߫ ߣߌߘߐ ߕߍ߫ ߦߋ߲߬ ߞߣߐߘߐ ߣߌ߲߬ ߢߍ߫.",
+	"library-button-text": "ߕߊ߯ ߟߊ߬ߛߣߍ߬ߟߌ߫ ߞߐߜߍ ߞߊ߲߬",
+	"home-button-text": "ߕߊ߯ {{BOOK_TITLE}} ߓߏ߬ߟߏ߲߬ߘߊ ߞߐߜߍ ߞߊ߲߬",
+	"random-page-button-text": "ߕߊ߯ ߓߍ߲߬ߛߋ߲߬ߡߊ߬ ߞߐߜߍ߫ ߛߎߥߊ߲ߘߌߣߍ߲ ߠߎ߬ ߞߊ߲߬",
+	"searchbox-tooltip": "ߕߌߙߌ߲ߠߌ߲ {{BOOK_TITLE}}"
+}

static/i18n/ru.json

@@ -3,15 +3,22 @@
 		"authors": [
 			"Fenixs-ru",
 			"Kareyac",
+			"Okras",
 			"Pacha Tchernof"
 		]
 	},
 	"name": "русский",
 	"suggest-full-text-search": "содержащее '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Такой книги нет: {{BOOK_NAME}}",
+	"too-many-books": "Запрошено слишком много книг ({{NB_BOOKS}}), максимальное количество — {{LIMIT}}.",
+	"no-book-found": "Ни одна книга не соответствует критериям отбора",
 	"url-not-found": "Запрошенный URL \"{{url}}\" не найден на этом сервере.",
 	"suggest-search": "Выполните полнотекстовый поиск для <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Ой! Не удалось выбрать случайную статью :(",
+	"invalid-raw-data-type": "{{DATATYPE}} не является допустимым запросом необработанного контента.",
+	"no-value-for-arg": "Не указано значение для аргумента {{ARGUMENT}}",
+	"no-query": "Не предоставлен запрос.",
+	"raw-entry-not-found": "Не удаётся найти запись {{ENTRY}} типа {{DATATYPE}}",
 	"400-page-title": "Недействительный запрос",
 	"400-page-heading": "Недействительный запрос",
 	"404-page-title": "Содержание не найдено",
@@ -23,5 +30,6 @@
 	"library-button-text": "Перейти на страницу-приветствие",
 	"home-button-text": "Перейти на главную страницу '{{BOOK_TITLE}}'",
 	"random-page-button-text": "Перейти на случайно выбранную страницу",
-	"searchbox-tooltip": "Искать '{{BOOK_TITLE}}'"
+	"searchbox-tooltip": "Искать '{{BOOK_TITLE}}'",
+	"confusion-of-tongues": "В поиске будут участвовать две или более книг на разных языках, что может привести к запутанным результатам."
 }

static/i18n/sc.json

@@ -7,10 +7,14 @@
 	"name": "Sardu",
 	"suggest-full-text-search": "chi cuntenet '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Perunu libru cun custu nùmene: {{BOOK_NAME}}",
+	"too-many-books": "Tropu libros pedidos, {{NB_BOOKS}} cando su lìmite est de {{LIMIT}}",
+	"no-book-found": "Perunu libru currispondet a sos critèrios de seletzione",
 	"url-not-found": "S'URL pedidu \"{{url}}\" non s'est atzapadu in custu serbidore.",
 	"suggest-search": "Faghe una chirca de testu intreu pro <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Oops! Sa seletzione de un'artìculu a casu est fallida :(",
 	"invalid-raw-data-type": "{{DATATYPE}} no est una rechesta vàlida pro cuntenutu puru.",
+	"no-value-for-arg": "Perunu valore frunidu pro s'argumentu {{ARGUMENT}}",
+	"no-query": "Peruna chirca frunida.",
 	"raw-entry-not-found": "Non faghet a atzapare s'elementu {{ENTRY}} de genia {{DATATYPE}}",
 	"400-page-title": "Rechesta non vàlida",
 	"400-page-heading": "Rechesta non vàlida",
@@ -23,5 +27,6 @@
 	"library-button-text": "Bae a sa pàgina de bene bènnidu",
 	"home-button-text": "Bae a sa pàgina printzipale de '{{BOOK_TITLE}}'",
 	"random-page-button-text": "Bae a una pàgina seletzionada a manera casuale",
-	"searchbox-tooltip": "Chirca '{{BOOK_TITLE}}'"
+	"searchbox-tooltip": "Chirca '{{BOOK_TITLE}}'",
+	"confusion-of-tongues": "Duos o prus libros in limbas diferentes diant pigare parte a sa chirca, cosa chi diat pòdere causare resurtados confusionosos."
 }

static/i18n/sk.json

@@ -7,10 +7,14 @@
 	"name": "slovenčina",
 	"suggest-full-text-search": "obsahuje '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Žiadna kniha ako: {{BOOK_NAME}}",
+	"too-many-books": "Príliš veľa požadovaných kníh ({{NB_BOOKS}}), limit je {{LIMIT}}",
+	"no-book-found": "Kritériám výberu nevyhovuje žiadna kniha",
 	"url-not-found": "Požadovaná adresa URL \"{{url}}\" na tomto serveri nebola nájdená.",
 	"suggest-search": "Spustite hľadanie celého textu <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Nepodarilo sa vybrať náhodný článok :(",
 	"invalid-raw-data-type": "{{DATATYPE}} nie je platná požiadavka pre surový obsah.",
+	"no-value-for-arg": "Pre argument {{ARGUMENT}} nebola poskytnutá žiadna hodnota",
+	"no-query": "Nebol poskytnutý žiadny dopyt.",
 	"raw-entry-not-found": "Nepodarilo sa nájsť {{DATATYPE}} položka {{ENTRY}}",
 	"400-page-title": "Neplatná požiadavka",
 	"400-page-heading": "Neplatná požiadavka",

static/i18n/sl.json

@@ -0,0 +1,32 @@
+{
+	"@metadata": {
+		"authors": [
+			"Eleassar"
+		]
+	},
+	"name": "slovenščina",
+	"suggest-full-text-search": "vsebuje »{{{SEARCH_TERMS}}}« ...",
+	"no-such-book": "Ni take knjige: {{BOOK_NAME}}",
+	"too-many-books": "Preveč zahtevanih knjig ({{NB_BOOKS}}), omejitev je {{LIMIT}}",
+	"no-book-found": "Izbirnim merilom ne ustreza nobena knjiga",
+	"url-not-found": "Zahtevanega URL-ja »{{url}}« v tem strežniku ni bilo mogoče najti.",
+	"suggest-search": "Preiščite celotno besedilo za <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ups! Ni bilo mogoče izbrati naključnega članka :(",
+	"invalid-raw-data-type": "{{DATATYPE}} ni veljaven zahtevek za neobdelano vsebino.",
+	"no-value-for-arg": "Argument {{ARGUMENT}} nima določene nobene vrednosti",
+	"no-query": "Poizvedba ni podana.",
+	"raw-entry-not-found": "Ni mogoče najti vnosa {{ENTRY}} vrste {{DATATYPE}}",
+	"400-page-title": "Neveljaven zahtevek",
+	"400-page-heading": "Neveljaven zahtevek",
+	"404-page-title": "Vsebine ni mogoče najti",
+	"404-page-heading": "Ni najdeno",
+	"500-page-title": "Notranja napaka strežnika",
+	"500-page-heading": "Notranja napaka strežnika",
+	"fulltext-search-unavailable": "Iskanje po celotnem besedilu ni na voljo",
+	"no-search-results": "Iskalnik po celotnem besedilu za to vsebino ni na voljo.",
+	"library-button-text": "Pojdite na pozdravno stran",
+	"home-button-text": "Pojdite na glavno stran »{{BOOK_TITLE}}«",
+	"random-page-button-text": "Pojdite na naključno izbrano stran",
+	"searchbox-tooltip": "Poiščite »{{BOOK_TITLE}}«",
+	"confusion-of-tongues": "V iskanju bi bili uporabljeni dve ali več knjig v različnih jezikih, kar lahko pripelje do nejasnih zadetkov."
+}

static/i18n/sv.json

@@ -1,15 +1,22 @@
 {
 	"@metadata": {
 		"authors": [
-			"Sabelöga"
+			"Jopparn",
+			"Sabelöga",
+			"WikiPhoenix"
 		]
 	},
+	"name": "Svenska",
 	"suggest-full-text-search": "innehåller '{{{SEARCH_TERMS}}}'...",
 	"no-such-book": "Ingen sådan bok: {{BOOK_NAME}}",
+	"too-many-books": "För många böcker begärda ({{NB_BOOKS}}) där gränsen är {{LIMIT}}",
+	"no-book-found": "Ingen bok matchar urvalskriterierna",
 	"url-not-found": "Den begärda webbadressen \"{{url}}\" hittades inte på denna server.",
 	"suggest-search": "Utför en fulltextsökning för <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
 	"random-article-failure": "Hoppsan! Kunde inte välja en slumpartikel :(",
 	"invalid-raw-data-type": "{{DATATYPE}} är ingen giltig begäran för oformaterat innehåll.",
+	"no-value-for-arg": "Inget värde angett för argumentet {{ARGUMENT}}",
+	"no-query": "Ingen fråga tillhandahålls.",
 	"raw-entry-not-found": "Kunde inte hitta {{DATATYPE}}-inlägget {{ENTRY}}",
 	"400-page-title": "Ogiltig begäran",
 	"400-page-heading": "Ogiltig begäran",
@@ -22,5 +29,6 @@
 	"library-button-text": "Gå till hemsidan",
 	"home-button-text": "Gå till huvudsidan för \"{{BOOK_TITLE}}\"",
 	"random-page-button-text": "Gå till en slumpmässigt utvald sida",
-	"searchbox-tooltip": "Sök efter \"{{BOOK_TITLE}}\""
+	"searchbox-tooltip": "Sök efter \"{{BOOK_TITLE}}\"",
+	"confusion-of-tongues": "Två eller fler böcker på olika språk skulle delta i sökningen, vilket kan ge förvirrande resultat."
 }

static/i18n/test.json

@@ -0,0 +1,20 @@
+{
+	"@metadata": {
+		"authors": [
+			"Kareyac"
+		]
+	},
+	"name": "Fake language for i18n testing"
+	, "suggest-full-text-search": "[I18N TESTING] cOnTaInInG '{{{SEARCH_TERMS}}}'..."
+	, "no-such-book": "[I18N TESTING] No such book: {{BOOK_NAME}}. Sorry."
+	, "url-not-found": "[I18N TESTING] URL not found: {{url}}"
+	, "suggest-search": "[I18N TESTING] Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
+	, "400-page-title": "[I18N TESTING] Invalid request ($400 fine must be paid)"
+	, "400-page-heading": "[I18N TESTING] -400 karma for an invalid request"
+	, "404-page-title": "[I18N TESTING] Not Found - Try Again"
+	, "404-page-heading": "[I18N TESTING] Content not found, but at least the server is alive"
+	, "library-button-text": "[I18N TESTING] Navigate to the welcome page"
+	, "home-button-text": "[I18N TESTING] Jump to the main page of '{{BOOK_TITLE}}'"
+	, "random-page-button-text": "[I18N TESTING] I am tired of determinism"
+	, "searchbox-tooltip": "[I18N TESTING] Let's search in '{{BOOK_TITLE}}'"
+}

static/i18n/tr.json

@@ -7,10 +7,14 @@
 	"name": "Türkçe",
 	"suggest-full-text-search": "'{{{SEARCH_TERMS}}}' içeriyor...",
 	"no-such-book": "Böyle bir kitap yok: {{BOOK_NAME}}",
+	"too-many-books": "Sınır {{LIMIT}} olduğunda çok fazla ({{NB_BOOKS}}) kitap istendi",
+	"no-book-found": "Seçim kriterleriyle eşleşen kitap yok",
 	"url-not-found": "İstenen \"{{url}}\" URL'si bu sunucuda bulunamadı.",
 	"suggest-search": "<a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> için tam metin araması yapın",
 	"random-article-failure": "Hata! Rastgele bir madde seçilemedi :(",
 	"invalid-raw-data-type": "{{DATATYPE}}, ham içerik için geçerli bir istek değil.",
+	"no-value-for-arg": "{{ARGUMENT}} bağımsız değişkeni için değer sağlanmadı",
+	"no-query": "Sorgu sağlanmadı.",
 	"raw-entry-not-found": "{{DATATYPE}} {{ENTRY}} girişi bulunamadı",
 	"400-page-title": "Geçersiz istek",
 	"400-page-heading": "Geçersiz istek",

static/i18n/zh-hans.json

@@ -0,0 +1,16 @@
+{
+	"@metadata": {
+		"authors": [
+			"GuoPC",
+			"StarrySky"
+		]
+	},
+	"name": "英语",
+	"no-query": "未提供查询。",
+	"400-page-title": "无效请求",
+	"400-page-heading": "无效请求",
+	"404-page-heading": "未找到",
+	"500-page-title": "内部服务器错误",
+	"500-page-heading": "内部服务器错误",
+	"library-button-text": "前往欢迎页面"
+}

static/i18n/zh-hant.json

@@ -1,15 +1,33 @@
 {
 	"@metadata": {
 		"authors": [
+			"Kly",
 			"Winston Sung"
 		]
 	},
 	"name": "繁體中文",
+	"suggest-full-text-search": "正在包含「{{{SEARCH_TERMS}}}」…",
+	"no-such-book": "沒有這樣的書籍：{{BOOK_NAME}}",
+	"too-many-books": "請求太多個書籍（{{NB_BOOKS}}），上限是 {{LIMIT}} 個",
+	"no-book-found": "沒有書籍符合選擇標準",
+	"url-not-found": "在此伺服器上找不到請求的 URL「{{url}}」。",
+	"suggest-search": "建立 <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> 使用的全文搜尋",
+	"random-article-failure": "哎呀！隨機挑選條目失敗 :(",
+	"invalid-raw-data-type": "{{DATATYPE}}不是原始內容的有效請求。",
+	"no-value-for-arg": "沒有為引數 {{ARGUMENT}} 提供內容",
+	"no-query": "未提供查詢。",
+	"raw-entry-not-found": "找不到{{DATATYPE}}項目{{ENTRY}}",
 	"400-page-title": "無效請求",
 	"400-page-heading": "無效請求",
 	"404-page-title": "查無內容",
 	"404-page-heading": "查無頁面",
 	"500-page-title": "內部伺服器錯誤",
 	"500-page-heading": "內部伺服器錯誤",
-	"searchbox-tooltip": "在{{BOOK_TITLE}}搜尋"
+	"fulltext-search-unavailable": "全文搜尋無效",
+	"no-search-results": "全文搜尋引擎不適用此內容。",
+	"library-button-text": "前往歡迎首頁",
+	"home-button-text": "前往「{{BOOK_TITLE}}」的首頁",
+	"random-page-button-text": "前往隨機選取頁面",
+	"searchbox-tooltip": "在{{BOOK_TITLE}}搜尋",
+	"confusion-of-tongues": "搜索裡有加入兩本或更多不同語言的書籍，這可能會導致混淆結果。"
 }

static/i18n_resources_list.txt

@@ -1,4 +1,7 @@
+i18n/ar.json
 i18n/bn.json
+i18n/cs.json
+i18n/de.json
 i18n/en.json
 i18n/fr.json
 i18n/he.json
@@ -8,10 +11,14 @@ i18n/ja.json
 i18n/ko.json
 i18n/ku-latn.json
 i18n/mk.json
+i18n/nqo.json
 i18n/pl.json
 i18n/ru.json
 i18n/sc.json
 i18n/sk.json
+i18n/sl.json
 i18n/sv.json
+i18n/test.json
 i18n/tr.json
+i18n/zh-hans.json
 i18n/zh-hant.json

static/meson.build

@@ -1,6 +1,7 @@
 resource_files = run_command(res_manager,
                     '--list-all',
-                    files('resources_list.txt')
+                    files('resources_list.txt'),
+                    check: true
                  ).stdout().strip().split('\n')
 
 preprocessed_resources = custom_target('preprocessed_resource_files',
@@ -15,7 +16,7 @@ preprocessed_resources = custom_target('preprocessed_resource_files',
 
 lib_resources = custom_target('resources',
     input: preprocessed_resources,
-    output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
+    output: ['libkiwix-resources.cpp', 'libkiwix-resources.h'],
     command:[res_compiler,
              '--cxxfile', '@OUTPUT0@',
              '--hfile', '@OUTPUT1@',
@@ -33,7 +34,8 @@ lib_resources = custom_target('resources',
 i18n_resource_files = run_command(find_program('python3'),
                                     '-c',
                                     'import sys; f=open(sys.argv[1]); print(f.read())',
-                                    files('i18n_resources_list.txt')
+                                    files('i18n_resources_list.txt'),
+                                    check: true
                                  ).stdout().strip().split('\n')
 
 i18n_resources = custom_target('i18n_resources',

static/resources_list.txt

@@ -1,55 +1,52 @@
-skin/jquery-ui/jquery-ui.structure.min.css
-skin/jquery-ui/jquery-ui.min.js
-skin/jquery-ui/external/jquery/jquery.js
-skin/jquery-ui/images/ui-bg_flat_0_aaaaaa_40x100.png
-skin/jquery-ui/images/ui-bg_flat_75_ffffff_40x100.png
-skin/jquery-ui/images/ui-icons_222222_256x240.png
-skin/jquery-ui/images/ui-bg_glass_55_fbf9ee_1x400.png
-skin/jquery-ui/images/ui-bg_highlight-soft_75_cccccc_1x100.png
-skin/jquery-ui/images/ui-bg_glass_65_ffffff_1x400.png
-skin/jquery-ui/images/ui-icons_2e83ff_256x240.png
-skin/jquery-ui/images/ui-icons_cd0a0a_256x240.png
-skin/jquery-ui/images/ui-icons_888888_256x240.png
-skin/jquery-ui/images/ui-bg_glass_75_e6e6e6_1x400.png
-skin/jquery-ui/images/animated-overlay.gif
-skin/jquery-ui/images/ui-bg_glass_75_dadada_1x400.png
-skin/jquery-ui/images/ui-icons_454545_256x240.png
-skin/jquery-ui/images/ui-bg_glass_95_fef1ec_1x400.png
-skin/jquery-ui/jquery-ui.theme.min.css
-skin/jquery-ui/jquery-ui.min.css
 skin/caret.png
 skin/bittorrent.png
 skin/magnet.png
 skin/download.png
 skin/hash.png
 skin/search-icon.svg
-skin/taskbar.js
 skin/iso6391To3.js
 skin/isotope.pkgd.min.js
 skin/index.js
+skin/autoComplete.min.js
 skin/taskbar.css
 skin/index.css
 skin/fonts/Poppins.ttf
 skin/fonts/Roboto.ttf
-skin/block_external.js
 skin/search_results.css
+skin/blank.html
+skin/viewer.js
+viewer.html
 templates/search_result.html
 templates/search_result.xml
 templates/error.html
 templates/error.xml
 templates/index.html
 templates/suggestion.json
-templates/head_taskbar.html
-templates/taskbar_part.html
-templates/external_blocker_part.html
 templates/captured_external.html
 templates/catalog_entries.xml
 templates/catalog_v2_root.xml
 templates/catalog_v2_entries.xml
 templates/catalog_v2_entry.xml
+templates/catalog_v2_partial_entry.xml
 templates/catalog_v2_categories.xml
 templates/catalog_v2_languages.xml
 templates/url_of_search_results_css
+templates/viewer_settings.js
 opensearchdescription.xml
 ft_opensearchdescription.xml
 catalog_v2_searchdescription.xml
+skin/css/autoComplete.css
+skin/favicon/android-chrome-192x192.png
+skin/favicon/android-chrome-512x512.png
+skin/favicon/apple-touch-icon.png
+skin/favicon/browserconfig.xml
+skin/favicon/favicon-16x16.png
+skin/favicon/favicon-32x32.png
+skin/favicon/favicon.ico
+skin/favicon/mstile-70x70.png
+skin/favicon/mstile-144x144.png
+skin/favicon/mstile-150x150.png
+skin/favicon/mstile-310x150.png
+skin/favicon/mstile-310x310.png
+skin/favicon/safari-pinned-tab.svg
+skin/favicon/site.webmanifest

static/skin/blank.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <title>Blank page</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  </head>
+
+  <body>
+  </body>
+</html>

static/skin/block_external.js

@@ -1,74 +0,0 @@
-const root = document.querySelector( `link[type='root']` ).getAttribute("href");
-// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
-var block_path = `${root}/catch/external`;
-// called only on external links
-function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
-
-// called on all link clicks. filters external and call capture_event
-function on_click_event(e) {
-  var target = findParent("a", e.target);
-  if (target !== null && "href" in target) {
-    var href = target.href;
-    if (window.location.pathname.indexOf(block_path) == 0) // already in catch page
-      return;
-    if (href.indexOf(window.location.origin) == 0)
-      return;
-    if (href.substr(0, 2) == "//")
-      return capture_event(e, target);
-    if (href.substr(0, 5) == "http:")
-      return capture_event(e, target);
-    if (href.substr(0, 6) == "https:")
-      return capture_event(e, target);
-    return;
-  }
-}
-
-// script entrypoint (called on document ready)
-function run() { live('a', 'click', on_click_event); }
-
-// find first parent with tagname
-function findParent(tagname, el) {
-  while (el) {
-    if ((el.nodeName || el.tagName).toLowerCase() === tagname.toLowerCase()) {
-      return el;
-    }
-    el = el.parentNode;
-  }
-  return null;
-}
-
-// matches polyfill
-this.Element && function(ElementPrototype) {
-    ElementPrototype.matches = ElementPrototype.matches ||
-    ElementPrototype.matchesSelector ||
-    ElementPrototype.webkitMatchesSelector ||
-    ElementPrototype.msMatchesSelector ||
-    function(selector) {
-        var node = this, nodes = (node.parentNode || node.document).querySelectorAll(selector), i = -1;
-        while (nodes[++i] && nodes[i] != node);
-        return !!nodes[i];
-    }
-}(Element.prototype);
-
-// helper for enabling IE 8 event bindings
-function addEvent(el, type, handler) {
-    if (el.attachEvent) el.attachEvent('on'+type, handler); else el.addEventListener(type, handler);
-}
-
-// live binding helper using matchesSelector
-function live(selector, event, callback, context) {
-    addEvent(context || document, event, function(e) {
-        var found, el = e.target || e.srcElement;
-        while (el && el.matches && el !== context && !(found = el.matches(selector))) el = el.parentElement;
-        if (found) callback.call(el, e);
-    });
-}
-
-// in case the document is already rendered
-if (document.readyState!='loading') run();
-// modern browsers
-else if (document.addEventListener) document.addEventListener('DOMContentLoaded', run);
-// IE <= 8
-else document.attachEvent('onreadystatechange', function(){
-    if (document.readyState=='complete') run();
-});

static/skin/css/autoComplete.css

@@ -0,0 +1,91 @@
+.autoComplete_wrapper {
+  display: inline-block;
+  position: relative;
+}
+
+.autoComplete_wrapper > input {
+  width: 370px;
+  height: 40px;
+  padding-left: 20px;
+  font-size: 1rem;
+  color: rgba(123, 123, 123, 1);
+  border-radius: 8px;
+  border: 0;
+  outline: none;
+  background-color: #f1f3f4;
+}
+
+.autoComplete_wrapper > input::placeholder {
+  color: rgba(123, 123, 123, 0.5);
+  transition: all 0.3s ease;
+}
+
+.autoComplete_wrapper > ul {
+  position: absolute;
+  max-height: 226px;
+  overflow-y: scroll;
+  top: 100%;
+  left: 0;
+  right: 0;
+  padding: 0;
+  margin: 0.5rem 0 0;
+  border-radius: 0.6rem;
+  background-color: #fff;
+  box-shadow: 0 3px 6px rgba(149, 157, 165, 0.15);
+  border: 1px solid rgba(33, 33, 33, 0.07);
+  z-index: 1000;
+  outline: none;
+}
+
+.autoComplete_wrapper > ul[hidden],
+.autoComplete_wrapper > ul:empty {
+  display: block;
+  opacity: 0;
+  transform: scale(0);
+}
+
+.autoComplete_wrapper > ul > li {
+  margin: 0.3rem;
+  padding: 0.3rem 0.5rem;
+  list-style: none;
+  text-align: left;
+  font-size: 1rem;
+  color: #212121;
+  border-radius: 0.35rem;
+  background-color: rgba(255, 255, 255, 1);
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  transition: all 0.2s ease;
+}
+
+.autoComplete_wrapper > ul > li::selection {
+  color: rgba(#ffffff, 0);
+  background-color: rgba(#ffffff, 0);
+}
+
+.autoComplete_wrapper > ul > li:hover {
+  cursor: pointer;
+  background-color: rgba(123, 123, 123, 0.1);
+}
+
+.autoComplete_wrapper > ul > li mark {
+  background-color: transparent;
+  color: rgba(255, 122, 122, 1);
+  font-weight: bold;
+}
+
+.autoComplete_wrapper > ul > li mark::selection {
+  color: rgba(#ffffff, 0);
+  background-color: rgba(#ffffff, 0);
+}
+
+.autoComplete_wrapper > ul > li[aria-selected="true"] {
+  background-color: rgba(123, 123, 123, 0.1);
+}
+
+@media only screen and (max-width: 600px) {
+  .autoComplete_wrapper > input {
+    width: 18rem;
+  }
+}

static/skin/favicon/browserconfig.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square70x70logo src="skin/favicon/mstile-70x70.png"/>
+            <square144x144logo src="skin/favicon/mstile-144x144.png"/>
+            <square150x150logo src="skin/favicon/mstile-150x150.png"/>
+            <square310x150logo src="skin/favicon/mstile-310x150.png"/>
+            <square310x310logo src="skin/favicon/mstile-310x310.png"/>
+            <TileColor>#da532c</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

static/skin/favicon/safari-pinned-tab.svg

@@ -0,0 +1,47 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
+ "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
+ width="700.000000pt" height="700.000000pt" viewBox="0 0 700.000000 700.000000"
+ preserveAspectRatio="xMidYMid meet">
+<metadata>
+Created by potrace 1.14, written by Peter Selinger 2001-2017
+</metadata>
+<g transform="translate(0.000000,700.000000) scale(0.100000,-0.100000)"
+fill="#000000" stroke="none">
+<path d="M2130 5897 c-3 -3 -38 -7 -78 -11 -41 -3 -77 -8 -80 -10 -4 -2 -27
+-7 -52 -10 -25 -3 -73 -13 -105 -22 -33 -8 -67 -17 -75 -19 -51 -11 -211 -75
+-305 -120 -93 -45 -289 -163 -305 -184 -3 -3 -27 -24 -55 -46 -108 -86 -238
+-229 -323 -357 -48 -71 -142 -245 -142 -263 0 -4 -6 -21 -14 -38 -21 -46 -51
+-149 -72 -247 -19 -89 -26 -391 -11 -475 3 -22 9 -56 12 -75 46 -278 218 -619
+460 -909 97 -117 141 -182 172 -256 19 -45 85 -251 118 -365 7 -25 35 -119 63
+-210 144 -469 182 -592 184 -607 3 -14 -10 -16 -77 -18 -93 -2 -140 -20 -185
+-71 -28 -33 -57 -110 -46 -122 4 -3 109 -6 234 -7 l227 -1 105 -51 c154 -74
+310 -148 440 -209 63 -30 133 -63 155 -74 25 -12 43 -16 49 -10 6 6 10 39 10
+74 1 56 -3 70 -29 108 -25 36 -49 52 -142 98 -61 30 -114 57 -116 60 -3 3 120
+5 273 5 l278 0 -3 25 c-9 69 -45 119 -110 153 -38 19 -58 20 -345 22 -272 1
+-309 3 -343 19 -48 24 -102 86 -115 134 -5 20 -42 143 -82 272 -40 129 -88
+287 -107 350 -19 63 -46 153 -60 200 -48 157 -30 273 54 357 65 65 153 95 237
+79 45 -9 96 -60 130 -132 20 -41 286 -885 286 -905 0 -2 -37 -5 -83 -5 -130
+-2 -196 -47 -227 -155 -7 -23 -5 -31 8 -37 9 -5 112 -8 227 -7 l210 1 110 -52
+c61 -28 167 -79 237 -112 70 -34 147 -70 172 -82 25 -11 87 -40 138 -65 54
+-26 96 -41 101 -36 19 19 20 100 3 146 -23 59 -63 91 -178 145 -54 25 -98 47
+-98 50 0 3 88 6 196 5 l196 0 60 -80 c208 -276 501 -419 838 -410 381 10 721
+235 881 582 80 174 109 431 69 614 -102 469 -504 798 -976 797 -446 0 -828
+-283 -958 -708 -35 -114 -45 -194 -41 -325 4 -93 10 -141 37 -255 5 -22 -465
+-24 -532 -3 -69 23 -114 78 -144 176 -210 687 -376 1234 -376 1241 0 9 88 51
+155 74 107 37 218 62 350 79 117 15 593 8 705 -11 14 -2 61 -7 105 -11 44 -3
+94 -8 111 -10 160 -20 504 -27 624 -11 240 30 439 120 611 273 102 91 142 113
+207 113 75 0 142 -25 252 -94 334 -208 677 -541 882 -856 46 -71 158 -266 158
+-275 0 -2 15 -36 34 -77 18 -40 41 -95 50 -121 l18 -48 31 19 c50 29 87 90 94
+153 10 95 -33 204 -168 434 -82 140 -272 379 -429 540 -214 220 -385 364 -669
+564 -77 54 -138 100 -136 103 2 2 7 42 11 89 26 336 -200 696 -540 858 -131
+63 -205 82 -346 88 -128 5 -216 -9 -332 -53 l-68 -26 -43 38 c-23 21 -44 40
+-47 44 -22 27 -248 190 -348 252 -133 81 -408 208 -452 208 -9 0 -20 4 -26 9
+-9 9 -189 56 -254 66 -19 3 -60 10 -90 16 -54 10 -421 24 -430 16z m2316
+-1050 c97 -49 163 -150 165 -253 2 -87 -33 -199 -39 -126 -6 72 -105 152 -187
+152 -95 0 -185 -84 -190 -177 -3 -50 14 -104 43 -137 16 -18 16 -19 -13 -12
+-16 4 -51 21 -76 37 -119 76 -167 241 -107 369 27 58 48 84 100 123 81 61 211
+71 304 24z"/>
+</g>
+</svg>

static/skin/favicon/site.webmanifest

@@ -0,0 +1,19 @@
+{
+    "name": "",
+    "short_name": "",
+    "icons": [
+        {
+            "src": "android-chrome-192x192.png",
+            "sizes": "192x192",
+            "type": "image/png"
+        },
+        {
+            "src": "android-chrome-512x512.png",
+            "sizes": "512x512",
+            "type": "image/png"
+        }
+    ],
+    "theme_color": "#ffffff",
+    "background_color": "#ffffff",
+    "display": "standalone"
+}

static/skin/index.js

@@ -1,5 +1,5 @@
 (function() {
-    const root = $(`link[type='root']`).attr('href');
+    const root = document.querySelector(`link[type='root']`).getAttribute('href');
     const incrementalLoadingParams = {
         start: 0,
         count: viewPortToCount()
@@ -77,6 +77,13 @@
         return queryNode != null ? queryNode.innerHTML : "";
     }
 
+    function generateTagLink(tagValue) {
+        tagValue = tagValue.toLowerCase();
+        const humanFriendlyTagValue = humanFriendlyTitle(tagValue);
+        const tagMessage = `Filter by tag "${humanFriendlyTagValue}"`;
+        return `<span class='tag__link' aria-label='${tagMessage}' title='${tagMessage}' data-tag=${tagValue}>${humanFriendlyTagValue}</span>`
+    }
+
     function generateBookHtml(book, sort = false) {
         const link = book.querySelector('link[type="text/html"]').getAttribute('href');
         let iconUrl;
@@ -91,9 +98,9 @@
         const langCode = getInnerHtml(book, 'language');
         const language = languages[langCode];
         const tags = getInnerHtml(book, 'tags');
-        let tagHtml = tags.split(';').filter(tag => {return !(tag.split(':')[0].startsWith('_'))})
-            .map((tag) => {return tag.charAt(0).toUpperCase() + tag.slice(1)})
-            .join(' | ').replace(/_/g, ' ');
+        const tagList = tags.split(';').filter(tag => {return !(tag.startsWith('_'))});
+        const tagFilterLinks = tagList.map((tagValue) => generateTagLink(tagValue));
+        const tagHtml = tagFilterLinks.join(' | ');
         let downloadLink;
         let zimSize = 0;
         try {
@@ -103,6 +110,9 @@
         } catch {
             downloadLink = '';
         }
+        const bookName = link.split('/').pop();
+        const viewerLink = `${root}/viewer#${bookName}`;
+
         const humanFriendlyZimSize = humanFriendlySize(zimSize);
 
         const divTag = document.createElement('div');
@@ -113,17 +123,21 @@
         }
         const faviconAttr = iconUrl != undefined ? `style="background-image: url('${iconUrl}')"` : '';
         const languageAttr = langCode != '' ? `title="${language}" aria-label="${language}"` : 'style="background-color: transparent"';
-        divTag.innerHTML = `<a class="book__link" href="${link}" data-hover="Preview">
+        divTag.innerHTML = `
             <div class="book__wrapper">
+            <a class="book__link" href="${viewerLink}" data-hover="Preview">
+            <div class="book__link__wrapper">
             <div class="book__icon" ${faviconAttr}></div>
             <div class="book__header">
                 <div id="book__title">${title}</div>
                 ${downloadLink ? `<div class="book__download"><span data-link="${downloadLink}">Download ${humanFriendlyZimSize ? ` - ${humanFriendlyZimSize}</span></div>`: ''}` : ''}
             </div>
             <div class="book__description" title="${description}">${description}</div>
+            </div>
+            </a>
             <div class="book__languageTag" ${languageAttr}>${getLanguageCodeToDisplay(langCode)}</div>
             <div class="book__tags"><div class="book__tags--wrapper">${tagHtml}</div></div>
-            </div></div></a>`;
+            </div></div>`;
         return divTag;
     }
 
@@ -168,12 +182,12 @@
                         <div onclick="closeModal()" class="modal-close-button">
                             <div>
                                 <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 14 14" fill="none">
-                                    <path fill-rule="evenodd" clip-rule="evenodd" d="M13.7071 1.70711C14.0976 1.31658 14.0976 
+                                    <path fill-rule="evenodd" clip-rule="evenodd" d="M13.7071 1.70711C14.0976 1.31658 14.0976
                                     0.683417 13.7071 0.292893C13.3166 -0.0976311 12.6834 -0.0976311 12.2929 0.292893L7 5.58579L1.70711
                                     0.292893C1.31658 -0.0976311 0.683417 -0.0976311 0.292893 0.292893C-0.0976311 0.683417
                                     -0.0976311 1.31658 0.292893 1.70711L5.58579 7L0.292893 12.2929C-0.0976311 12.6834
                                     -0.0976311 13.3166 0.292893 13.7071C0.683417 14.0976 1.31658 14.0976 1.70711 13.7071L7
-                                    8.41421L12.2929 13.7071C12.6834 14.0976 13.3166 14.0976 13.7071 13.7071C14.0976 13.3166 
+                                    8.41421L12.2929 13.7071C12.6834 14.0976 13.3166 14.0976 13.7071 13.7071C14.0976 13.3166
                                     14.0976 12.6834 13.7071 12.2929L8.41421 7L13.7071 1.70711Z" fill="black" />
                                 </svg>
                             </div>
@@ -192,7 +206,7 @@
                                 <div>Sha256 hash</div>
                             </a>
                         </div>
-                        ${magnetLink ? 
+                        ${magnetLink ?
                         `<div class="modal-regular-download">
                             <a href="${magnetLink}" target="_blank">
                                 <img src="../skin/magnet.png?KIWIXCACHEID" alt="download magnet" />
@@ -267,6 +281,16 @@
         });
     }
 
+    function setNoResultsContent() {
+        const kiwixHomeBody = document.querySelector('.kiwixHomeBody');
+        const divTag = document.createElement('div');
+        divTag.setAttribute('class', 'noResults');
+        divTag.innerHTML = `No result. Would you like to <a href="?lang=">reset filter</a>?`;
+        kiwixHomeBody.append(divTag);
+        kiwixHomeBody.setAttribute('style', 'display: flex; justify-content: center; align-items: center');
+        loader.setAttribute('style', 'position: absolute; top: 50%');
+    }
+
     function checkAndInjectEmptyMessage() {
         const kiwixHomeBody = document.querySelector('.kiwixHomeBody');
         if (!bookOrderMap.size) {
@@ -274,28 +298,7 @@
                 noResultInjected = true;
                 iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('div'));
                 iso.layout();
-                setTimeout(() => {
-                    const divTag = document.createElement('div');
-                    divTag.setAttribute('class', 'noResults');
-                    divTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter</a>?`;
-                    kiwixHomeBody.append(divTag);
-                    kiwixHomeBody.setAttribute('style', 'display: flex; justify-content: center; align-items: center');
-                    divTag.getElementsByTagName('a')[0].onclick = (event) => {
-                        event.preventDefault();
-                        window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
-                        setCookie(filterCookieName, 'lang=');
-                        resetAndFilter();
-                        document.querySelectorAll('.filter').forEach(filter => {
-                            filter.value = params.get(filter.name) || '';
-                            if (filter.value) {
-                                filter.style = 'background-color: #858585; color: #fff';
-                            } else {
-                                filter.style = 'background-color: #ffffff; color: black';
-                            }
-                        })
-                    };
-                    loader.setAttribute('style', 'position: absolute; top: 50%');
-                }, 300);
+                setTimeout(setNoResultsContent, 300);
             }
             return true;
         } else if (noResultInjected) {
@@ -320,7 +323,7 @@
         const booksToFilter = new Set();
         const booksToDelete = new Set();
         iso.arrange({
-            filter: function (idx, elem) {
+            filter: function (elem) {
                 const id = elem.getAttribute('data-id');
                 const retVal = bookOrderMap.has(id);
                 if (retVal) {
@@ -344,6 +347,7 @@
                 insertModal(downloadButton);
             }
         });
+        refreshTagLinks();
     }
 
     async function resetAndFilter(filterType = '', filterValue = '') {
@@ -355,22 +359,16 @@
         params = new URLSearchParams(window.location.search);
         if (filterType) {
             params.set(filterType, filterValue);
-            window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
+            window.history.pushState({}, null, `?${params.toString()}`);
             setCookie(filterCookieName, params.toString());
         }
-        document.querySelectorAll('.filter').forEach(filter => {
-            if (filter.value) {
-                filter.style = 'background-color: #858585; color: #fff';
-            } else {
-                filter.style = 'background-color: #ffffff; color: black';
-            }
-        });
+        updateFilterColors();
         await loadAndDisplayBooks(true);
     }
 
     window.addEventListener('popstate', async () => {
         await resetAndFilter();
-        document.querySelectorAll('.filter').forEach(filter => {filter.value = params.get(filter.name) || ''});
+        updateVisibleParams();
     });
 
     async function loadSubset() {
@@ -384,6 +382,56 @@
         }
     }
 
+    function updateFilterColors() {
+        document.querySelectorAll('.filter').forEach(filter => {
+            if (filter.value) {
+                filter.style = 'background-color: #858585; color: #fff';
+            } else {
+                filter.style = 'background-color: #ffffff; color: black';
+            }
+        });
+    }
+
+    function addTagElement(tagValue, resetFilter) {
+        const tagElement = document.getElementsByClassName('tagFilterLabel')[0];
+        tagElement.style.display = 'inline-block';
+        const humanFriendlyTagValue = humanFriendlyTitle(tagValue);
+        tagElement.innerHTML = `${humanFriendlyTagValue}`;
+        const tagMessage = `Stop filtering by tag "${humanFriendlyTagValue}"`;
+        tagElement.setAttribute('aria-label', tagMessage);
+        tagElement.setAttribute('title', tagMessage);
+        if (resetFilter)
+            resetAndFilter('tag', tagValue);
+    }
+
+    function refreshTagLinks() {
+        const tagLinks = document.getElementsByClassName('tag__link');
+        [...tagLinks].forEach(elem => {
+            if (!elem.getAttribute('click-listener')) {
+                elem.addEventListener('click', () => addTagElement(elem.dataset.tag, true));
+                elem.setAttribute('click-listener', 'true');
+            }
+        });
+    }
+
+    function removeTagElement(resetFilter) {
+        const tagElement = document.getElementsByClassName('tagFilterLabel')[0];
+        tagElement.style.display = 'none';
+        if (resetFilter)
+            resetAndFilter('tag', '');
+    }
+
+    function updateVisibleParams() {
+        document.querySelectorAll('.filter').forEach(filter => {filter.value = params.get(filter.name) || ''});
+        updateFilterColors();
+        const tagKey = params.get('tag');
+        if (tagKey !== null && tagKey.trim() !== '') {
+            addTagElement(tagKey, false);
+        } else {
+            removeTagElement(false);
+        }
+    }
+
     window.addEventListener('resize', (event) => {
         if (timer) {clearTimeout(timer)}
         timer = setTimeout(() => {
@@ -394,6 +442,12 @@
 
     window.addEventListener('scroll', loadSubset);
 
+    window.addEventListener('keydown', function (event) {
+        if (event.key === "Escape" ) {
+            closeModal();
+        }
+    });
+
     window.onload = async () => {
         iso = new Isotope( '.book__list', {
             itemSelector: '.book',
@@ -404,10 +458,9 @@
                 }
             },
             sortBy: 'weight',
-            layoutMode: 'cellsByRow',
-            cellsByRow: {
-                columnWidth: '.book',
-                rowHeight: '.book'
+            layoutMode: 'masonry',
+            masonry: {
+                fitWidth: true
             }
         });
         footer = document.getElementById('kiwixfooter');
@@ -419,15 +472,16 @@
         document.querySelectorAll('.filter').forEach(filter => {
             filter.addEventListener('change', () => {resetAndFilter(filter.name, filter.value)});
         });
+        const tagElement = document.getElementsByClassName('tagFilterLabel')[0];
+        tagElement.addEventListener('click', () => removeTagElement(true));
         if (filters) {
-            window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
-        }
-        params.forEach((value, key) => {
-            const selectBox = document.getElementsByName(key)[0];
-            if (selectBox) {
-                selectBox.value = value
+            const currentLink = window.location.search;
+            const newLink = `?${params.toString()}`;
+            if (currentLink != newLink) {
+                window.history.pushState({}, null, newLink);
             }
-        });
+        }
+        updateVisibleParams();
         document.getElementById('kiwixSearchForm').onsubmit = (event) => {event.preventDefault()};
         if (!window.location.search) {
             const browserLang = navigator.language.split('-')[0];
@@ -438,13 +492,7 @@
                 langFilter.dispatchEvent(new Event('change'));
             }
         }
-        document.querySelectorAll('.filter').forEach(filter => {
-            if (filter.value) {
-                filter.style = 'background-color: #858585; color: #fff';
-            } else {
-                filter.style = 'background-color: #ffffff; color: black';
-            }
-        });
         setCookie(filterCookieName, params.toString());
     }
 })();
+