Merge pull request #784 from kiwix/version_11.0.0

Included the word "Html" in the names of those functions and variables
which will get Xml siblings soon.

.github/workflows/ci.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v1
-      - name: Setup python 3.5
-        uses: actions/setup-python@v1
+      - name: Setup python 3.10
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.5'
+          python-version: '3.10'
       - name: Install packages
         run: |
           brew update
@@ -37,17 +37,8 @@ jobs:
           export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
           cd build
           meson test --verbose
-          ninja coverage
         env:
           SKIP_BIG_MEMORY_TEST: 1
-      - name: Publish coverage
-        shell: bash
-        run: |
-          curl https://codecov.io/bash -o codecov.sh
-          bash codecov.sh -n osx_native_dyn -Z
-          rm codecov.sh
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   Linux:
     strategy:
@@ -56,7 +47,6 @@ jobs:
         name:
           - native_static
           - native_dyn
-          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
@@ -64,31 +54,27 @@ jobs:
         include:
           - name: native_static
             target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: native_dyn
-            target: native_dyn
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - name: native_dyn_bionic
             target: native_dyn
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - name: android_arm
             target: android_arm
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/arm-linux-androideabi'
           - name: android_arm64
             target: android_arm64
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
+            image_variant: bionic
+            lib_postfix: '/aarch64-linux-android'
           - name: win32_static
             target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - name: win32_dyn
             target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
     env:
       HOME: /home/runner
@@ -132,7 +118,7 @@ jobs:
           MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
         fi
         if [[ "${{matrix.target}}" =~ android_.* ]]; then
-          MESON_OPTION="$MESON_OPTION -Dandroid=true"
+          MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
         fi
         cd $HOME/libkiwix
         meson . build ${MESON_OPTION}
@@ -158,6 +144,6 @@ jobs:
         curl https://codecov.io/bash -o codecov.sh
         bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
         rm codecov.sh
-      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
+      if: startsWith(matrix.target, 'native_')
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -10,7 +10,6 @@ jobs:
         distro:
           - ubuntu-jammy
           - ubuntu-impish
-          - ubuntu-hirsute
           - ubuntu-focal
           - ubuntu-bionic
     steps:
@@ -51,14 +50,6 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
       - uses: legoktm/gh-action-build-deb@ubuntu-focal
         if: matrix.distro == 'ubuntu-focal'
         name: Build package for ubuntu-focal

ChangeLog

@@ -1,3 +1,62 @@
+libkiwix 11.0.0
+===============
+
+ * [server] Add support for internationalization (@veloman-yunkan #679)
+ * [server] Use gzip compression instead of deflat (mgautierfr #757)
+ * [server] Version the static resources. This allow better invalidating
+   browser cache when resources are changed (@veloman-yunkan #712)
+ * [server|front] Use integer to query the host for page length (@juuz #772)
+ * [server] Improve multizim search API:
+   - Improvement of the cache system
+   - Better API to select on which books to search in.
+   - SysAdmin is now able to limit the number of book we search in for a multizim search
+ * [server] Introduce a opensearch API for multizim fulltext search
+ * [wrapper] Remove java wrapper
+ * Testing:
+   - Testing of search result pages content (@veloman-yunkan #765)
+   - Better testing structure of xml search result (@veloman-yunkan #780)
+
+libkiwix 10.1.1
+===============
+
+ * Correctly detect the number of article for older zims (<=6) (@mgautier #743)
+ * [server] Fix fulltext search (@mgautierfr #724)
+ * [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
+ * Fix CI (@mgautierfr #736)
+
+libkiwix 10.1.0
+===============
+
+ This release is an important one as it fixes a Xss vulnerability introduced
+ in libkiwix 10.0.0
+
+ * [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
+ * [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
+ * [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
+ * [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
+ * Add `dc:issued` to opds output stream (@veloman-yunkan #715)
+ * Add handling of several languages not provided by ICU (@juuz0 #701)
+ * [server] Add a caching system for search and suggestion (@maneeshpm #620)
+ * Fix cross-compilation (@kelson42 #703)
+ * Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
+ * Better testing system of html response (@veloman-yunkan #725)
+
+libkiwix 10.0.1
+===============
+
+ * [server] The catalog search interpret `count=0` as no limit.
+   This was the case for a long time. This was changed unintentionally
+   (@veloman-yunkan #686)
+ * [server] Correctly generere a human friendly title in the server frontend.
+   (@juuz0 #687, @kelson42 #689)
+ * [server] Fix download button if there is no url do download from.
+   (@juuz0 #691)
+ * Add non-minified isotope.pkdg.js
+   Needed for debian packaging as we need the source and minified version is
+   not the source (@legoktm #693)
+ * [server] Add a tooltip with the full language for the lang tag.
+ * CI fixes (@kelson42 @legoktm)
+
 libkiwix 10.0.0
 ===============
 

README.md

@@ -5,8 +5,10 @@ The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
 core. It contains the code shared by all Kiwix ports (Windows,
 GNU/Linux, macOS, Android, iOS, ...).
 
+[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
 [![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
 [![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
+[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
 [![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/libkiwix'

debian/control

@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
  meson,
  pkg-config,
- libzim-dev (>= 7.2.0),
+ libzim-dev (>= 7.2.0~),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
@@ -23,7 +23,7 @@ Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
- libzim-dev (>= 7.2.0),
+ libzim-dev (>= 7.2.0~),
  libicu-dev,
  libpugixml-dev,
  libcurl4-gnutls-dev,

debian/libkiwix-dev.manpages

@@ -1 +1,2 @@
 usr/share/man/man1/kiwix-compile-resources.1*
+usr/share/man/man1/kiwix-compile-i18n.1*

docs/requirements.txt

@@ -1,3 +1,2 @@
 breathe
 exhale
-sphinx<4

include/library.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <mutex>
 #include <zim/archive.h>
+#include <zim/search.h>
 
 #include "book.h"
 #include "bookmark.h"
@@ -140,53 +141,35 @@ private: // functions
     bool accept(const Book& book) const;
 };
 
-/**
- * This class is not part of the libkiwix API. Its only purpose is
- * to simplify the implementation of the Library's move operations
- * and avoid bugs should new data members be added to Library.
- */
-class LibraryBase
+
+class ZimSearcher : public zim::Searcher
 {
-protected: // types
-  typedef uint64_t LibraryRevision;
+  public:
+    explicit ZimSearcher(zim::Searcher&& searcher)
+      : zim::Searcher(searcher)
+    {}
 
-  struct Entry : Book
-  {
-    LibraryRevision lastUpdatedRevision = 0;
-
-    // May also keep the Archive and Reader pointers here and get
-    // rid of the m_readers and m_archives data members in Library
-  };
-
-protected: // data
-  LibraryRevision m_revision;
-  std::map<std::string, Entry> m_books;
-  std::map<std::string, std::shared_ptr<Reader>> m_readers;
-  std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
-  std::vector<kiwix::Bookmark> m_bookmarks;
-  class BookDB;
-  std::unique_ptr<BookDB> m_bookDB;
-
-protected: // functions
-  LibraryBase();
-  ~LibraryBase();
-
-  LibraryBase(LibraryBase&& );
-  LibraryBase& operator=(LibraryBase&& );
+    std::unique_lock<std::mutex> getLock() {
+      return std::unique_lock<std::mutex>(m_mutex);
+    }
+    virtual ~ZimSearcher() = default;
+  private:
+    std::mutex m_mutex;
 };
 
 /**
  * A Library store several books.
  */
-class Library : private LibraryBase
+class Library
 {
   // all data fields must be added in LibraryBase
   mutable std::mutex m_mutex;
 
  public:
-  typedef LibraryRevision Revision;
+  typedef uint64_t Revision;
   typedef std::vector<std::string> BookIdCollection;
   typedef std::map<std::string, int> AttributeCounts;
+  typedef std::set<std::string> BookIdSet;
 
  public:
   Library();
@@ -242,6 +225,10 @@ class Library : private LibraryBase
 
   DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
   std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
+  std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
+    return getSearcherByIds(BookIdSet{id});
+  }
+  std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
 
   /**
    * Remove a book from the library.
@@ -351,7 +338,7 @@ class Library : private LibraryBase
    *
    * @return Current revision of the library.
    */
-  LibraryRevision getRevision() const;
+  Revision getRevision() const;
 
   /**
    * Remove books that have not been updated since the specified revision.
@@ -359,20 +346,24 @@ class Library : private LibraryBase
    * @param  rev the library revision to use
    * @return Count of books that were removed by this operation.
    */
-  uint32_t removeBooksNotUpdatedSince(LibraryRevision rev);
+  uint32_t removeBooksNotUpdatedSince(Revision rev);
 
   friend class OPDSDumper;
   friend class libXMLDumper;
 
 private: // types
   typedef const std::string& (Book::*BookStrPropMemFn)() const;
+  struct Impl;
 
 private: // functions
   AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
   std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
   BookIdCollection filterViaBookDB(const Filter& filter) const;
   void updateBookDB(const Book& book);
-  void dropReader(const std::string& bookId);
+  void dropCache(const std::string& bookId);
+
+private: //data
+  std::unique_ptr<Impl> mp_impl;
 };
 
 }

include/search_renderer.h

@@ -22,6 +22,7 @@
 
 #include <string>
 #include <zim/search.h>
+#include "library.h"
 
 namespace kiwix
 {
@@ -48,6 +49,10 @@ class SearchRenderer
   /**
    * Construct a SearchRenderer from a SearchResultSet.
    *
+   * The constructed version of the SearchRenderer will not introduce
+   * the book name for each result. It is better to use the other constructor
+   * with a Library pointer to have a better html page.
+   *
    * @param srs The `SearchResultSet` to render.
    * @param mapper The `NameMapper` to use to do the rendering.
    * @param start The start offset used for the srs.
@@ -56,14 +61,29 @@ class SearchRenderer
   SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                  unsigned int start, unsigned int estimatedResultCount);
 
+  /**
+   * Construct a SearchRenderer from a SearchResultSet.
+   *
+   * @param srs The `SearchResultSet` to render.
+   * @param mapper The `NameMapper` to use to do the rendering.
+   * @param library The `Library` to use to look up book details for search results.
+   * @param start The start offset used for the srs.
+   * @param estimatedResultCount The estimatedResultCount of the whole search
+   */
+  SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                 unsigned int start, unsigned int estimatedResultCount);
+
   ~SearchRenderer();
 
+  /**
+   * Set the search pattern used to do the search
+   */
   void setSearchPattern(const std::string& pattern);
 
   /**
-   * Set the search content id.
+   * Set the querystring used to select books
    */
-  void setSearchContent(const std::string& name);
+  void setSearchBookQuery(const std::string& bookQuery);
 
   /**
    * Set protocol prefix.
@@ -82,16 +102,25 @@ class SearchRenderer
     this->pageLength  = pageLength;
   }
 
+  std::string renderTemplate(const std::string& tmpl_str);
+
   /**
    * Generate the html page with the resutls of the search.
    */
   std::string getHtml();
 
+    /**
+   * Generate the xml page with the resutls of the search.
+   */
+  std::string getXml();
+
+
  protected:
   std::string beautifyInteger(const unsigned int number);
   zim::SearchResultSet m_srs;
   NameMapper* mp_nameMapper;
-  std::string searchContent;
+  Library* mp_library;
+  std::string searchBookQuery;
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;

include/searcher.h

@@ -76,10 +76,10 @@ class Searcher
    * @return true if the reader has been added.
    *         false if the reader cannot be added (no embedded fulltext index present)
    */
-  bool add_reader(Reader* reader);
+  bool add_reader(std::shared_ptr<Reader> reader);
 
 
-  Reader* get_reader(int index);
+  std::shared_ptr<Reader> get_reader(int index);
 
   /**
    * Start a search on the zim associated to the Searcher.
@@ -161,7 +161,7 @@ class Searcher
                      const unsigned int maxResultCount,
                      const bool verbose = false);
 
-  std::vector<Reader*> readers;
+  std::vector<std::shared_ptr<Reader>> readers;
   std::unique_ptr<SearcherInternal> internal;
   std::unique_ptr<SuggestionInternal> suggestionInternal;
   std::string searchPattern;

include/server.h

@@ -54,6 +54,8 @@ namespace kiwix
        void setAddress(const std::string& addr) { m_addr = addr; }
        void setPort(int port) { m_port = port; }
        void setNbThreads(int threads) { m_nbThreads = threads; }
+       void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
+       void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
@@ -62,7 +64,7 @@ namespace kiwix
         { m_blockExternalLinks = blockExternalLinks; }
        int getPort();
        std::string getAddress();
-       
+
      protected:
        Library* mp_library;
        NameMapper* mp_nameMapper;
@@ -71,10 +73,12 @@ namespace kiwix
        std::string m_indexTemplateString = "";
        int m_port = 80;
        int m_nbThreads = 1;
+       unsigned int m_multizimSearchLimit = 0;
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
        bool m_blockExternalLinks = false;
+       int m_ipConnectionLimit = 0;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

meson.build

@@ -1,29 +1,20 @@
 project('libkiwix', 'cpp',
-  version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '11.0.0',
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
-wrapper = get_option('wrapper')
+static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
 
-static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
-if wrapper.contains('android')
-  extra_libs = ['-llog']
+# See https://github.com/kiwix/libkiwix/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
+  extra_libs = ['-latomic']
 else
   extra_libs = []
 endif
 
-if wrapper.contains('java')
-  add_languages('java')
-endif
-
-# See https://github.com/kiwix/libkiwix/issues/371
-if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
-  extra_libs += '-latomic'
-endif
-
-if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
+if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
   # C++ std::thread is implemented using pthread on linux by gcc
   thread_dep = dependency('threads')
 else
@@ -51,12 +42,12 @@ endif
 
 
 extra_cflags = ''
-if target_machine.system() == 'windows' and static_deps
+if host_machine.system() == 'windows' and static_deps
   add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-if target_machine.system() == 'windows'
+if host_machine.system() == 'windows'
   add_project_arguments('-DNOMINMAX', language: 'cpp')
 endif
 

meson_options.txt

@@ -1,4 +1,4 @@
-option('wrapper', type:'array', choices:['java', 'android'], value:[],
-  description: 'The wrapper to generate.')
+option('static-linkage', type : 'boolean', value : false,
+  description : 'Link statically with the dependencies.')
 option('doc', type : 'boolean', value : false,
   description : 'Build the documentations.')

scripts/kiwix-compile-i18n

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import os.path
+import re
+import json
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+def lang_code(filename):
+    filename = os.path.basename(filename)
+    lang = to_identifier(os.path.splitext(filename)[0])
+    print(filename, '->', lang)
+    return lang
+
+from string import Template
+
+def expand_cxx_template(t, **kwargs):
+    return Template(t).substitute(**kwargs)
+
+def cxx_string_literal(s):
+    # Taking advantage of the fact the JSON string escape rules match
+    # those of C++
+    return 'u8' + json.dumps(s)
+
+string_table_cxx_template = '''
+const I18nString $TABLE_NAME[] = {
+  $TABLE_ENTRIES
+};
+'''
+
+lang_table_entry_cxx_template = '''
+    {
+        $LANG_STRING_LITERAL,
+        ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
+        $STRING_TABLE_NAME
+    }'''
+
+cxxfile_template = '''// This file is automatically generated. Do not modify it.
+
+#include "server/i18n.h"
+
+namespace kiwix {
+namespace i18n {
+
+namespace
+{
+
+$STRING_DATA
+
+} // unnamed namespace
+
+#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
+
+extern const I18nStringTable stringTables[] = {
+  $LANG_TABLE
+};
+
+extern const size_t langCount = $LANG_COUNT;
+
+} // namespace i18n
+} // namespace kiwix
+'''
+
+class Resource:
+    def __init__(self, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.lang_code = lang_code(filename)
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.data = f.read()
+
+    def get_string_table_name(self):
+        return "string_table_for_" + self.lang_code
+
+    def get_string_table(self):
+        table_entries = ",\n  ".join(self.get_string_table_entries())
+        return expand_cxx_template(string_table_cxx_template,
+                    TABLE_NAME=self.get_string_table_name(),
+                    TABLE_ENTRIES=table_entries)
+
+    def get_string_table_entries(self):
+        d = json.loads(self.data)
+        for k in sorted(d.keys()):
+            if k != "@metadata":
+                key_string = cxx_string_literal(k)
+                value_string = cxx_string_literal(d[k])
+                yield '{ ' + key_string + ', ' + value_string + ' }'
+
+    def get_lang_table_entry(self):
+        return expand_cxx_template(lang_table_entry_cxx_template,
+                LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
+                STRING_TABLE_NAME=self.get_string_table_name())
+
+
+
+def gen_c_file(resources):
+    string_data = []
+    lang_table = []
+    for r in resources:
+        string_data.append(r.get_string_table())
+        lang_table.append(r.get_lang_table_entry())
+
+    return expand_cxx_template(cxxfile_template,
+                STRING_DATA="\n".join(string_data),
+                LANG_TABLE=",\n  ".join(lang_table),
+                LANG_COUNT=len(resources)
+           )
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        required=True,
+                        help='The Cpp file name to generate')
+    parser.add_argument('i18n_resource_files', nargs='+',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    resources = [Resource(filename) for filename in args.i18n_resource_files]
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources))
+

scripts/kiwix-compile-i18n.1

@@ -0,0 +1,18 @@
+.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
+.SH SYNOPSIS
+\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
+.SH DESCRIPTION
+.TP
+i18n_resource_files ...
+The list of i18n resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+.SH AUTHOR
+Veloman Yunkan <veloman.yunkan@gmail.com>

scripts/kiwix-compile-resources

@@ -102,7 +102,7 @@ class Resource:
 
 
 
-master_c_template = """//This file is automaically generated. Do not modify it.
+master_c_template = """//This file is automatically generated. Do not modify it.
 
 #include <stdlib.h>
 #include <fstream>

scripts/kiwix-resources

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_file(src_path, dst_path):
+    with open(src_path, 'rb') as src:
+        with open(dst_path, 'wb') as dst:
+            dst.write(src.read())
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        preprocess_resource(resource)
+    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,6 +1,13 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
 
 install_man('kiwix-compile-resources.1')
+
+i18n_compiler = find_program('kiwix-compile-i18n')
+
+install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-i18n.1')

src/book.cpp

@@ -83,7 +83,7 @@ void Book::update(const zim::Archive& archive) {
   m_flavour = getMetaFlavour(archive);
   m_tags = getMetaTags(archive);
   m_category = getCategoryFromTags();
-  m_articleCount = archive.getArticleCount();
+  m_articleCount = getArchiveArticleCount(archive);
   m_mediaCount = getArchiveMediaCount(archive);
   m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
 
@@ -161,7 +161,9 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_language = VALUE("language");
   m_creator = node.child("author").child("name").child_value();
   m_publisher = node.child("publisher").child("name").child_value();
-  m_date = fromOpdsDate(VALUE("updated"));
+  const std::string dcIssuedDate = VALUE("dc:issued");
+  m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
+  m_date = fromOpdsDate(m_date);
   m_name = VALUE("name");
   m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");

src/library.cpp

@@ -27,10 +27,13 @@
 #include "tools/regexTools.h"
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
 
 #include <pugixml.hpp>
 #include <algorithm>
 #include <set>
+#include <cmath>
 #include <unicode/locid.h>
 #include <xapian.h>
 
@@ -56,68 +59,129 @@ bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
       && book1.getPath() == book2.getPath();
 }
 
-} // unnamed namespace
-
-class LibraryBase::BookDB : public Xapian::WritableDatabase
+template<typename Key, typename Value>
+class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
 {
-public:
-  BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
+  public:
+    explicit MultiKeyCache(size_t maxEntries)
+      : ConcurrentCache<std::set<Key>, Value>(maxEntries)
+    {}
+
+    bool drop(const Key& key)
+    {
+      std::unique_lock<std::mutex> l(this->lock_);
+      bool removed = false;
+      for(auto& cache_key: this->impl_.keys()) {
+        if(cache_key.find(key)!=cache_key.end()) {
+          removed |= this->impl_.drop(cache_key);
+        }
+      }
+      return removed;
+    }
 };
 
-LibraryBase::LibraryBase()
-  : m_bookDB(new BookDB)
+} // unnamed namespace
+
+struct Library::Impl
+{
+  struct Entry : Book
+  {
+    Library::Revision lastUpdatedRevision = 0;
+  };
+
+  Library::Revision m_revision;
+  std::map<std::string, Entry> m_books;
+  using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
+  std::unique_ptr<ArchiveCache> mp_archiveCache;
+  using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
+  std::unique_ptr<SearcherCache> mp_searcherCache;
+  std::vector<kiwix::Bookmark> m_bookmarks;
+  Xapian::WritableDatabase m_bookDB;
+
+  unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
+
+  Impl();
+  ~Impl();
+
+  Impl(Impl&& );
+  Impl& operator=(Impl&& );
+};
+
+Library::Impl::Impl()
+  : mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+    mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+    m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
 {
 }
 
-LibraryBase::~LibraryBase()
+Library::Impl::~Impl()
 {
 }
 
-LibraryBase::LibraryBase(LibraryBase&& ) = default;
-LibraryBase& LibraryBase::operator=(LibraryBase&& ) = default;
+Library::Impl::Impl(Library::Impl&& ) = default;
+Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
+
+unsigned int
+Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
+{
+  unsigned int result = 0;
+  for (auto& pair: m_books) {
+    auto& book = pair.second;
+    if ((!book.getPath().empty() && localBooks)
+        || (!book.getUrl().empty() && remoteBooks)) {
+      result++;
+    }
+  }
+  return result;
+}
 
 /* Constructor */
 Library::Library()
+  : mp_impl(new Library::Impl)
 {
 }
 
 Library::Library(Library&& other)
-  : LibraryBase(std::move(other))
+  : mp_impl(std::move(other.mp_impl))
 {
 }
 
 Library& Library::operator=(Library&& other)
 {
-  LibraryBase::operator=(std::move(other));
+  mp_impl = std::move(other.mp_impl);
   return *this;
 }
 
 /* Destructor */
-Library::~Library()
-{
-}
-
+Library::~Library() = default;
 
 bool Library::addBook(const Book& book)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  ++m_revision;
+  ++mp_impl->m_revision;
   /* Try to find it */
   updateBookDB(book);
   try {
-    auto& oldbook = m_books.at(book.getId());
+    auto& oldbook = mp_impl->m_books.at(book.getId());
     if ( ! booksReferToTheSameArchive(oldbook, book) ) {
-      dropReader(book.getId());
+      dropCache(book.getId());
     }
     oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
                           // XXX: Then m_bookDB will become out-of-sync with
                           // XXX: the real contents of the library.
-    oldbook.lastUpdatedRevision = m_revision;
+    oldbook.lastUpdatedRevision = mp_impl->m_revision;
     return false;
   } catch (std::out_of_range&) {
-    Entry& newEntry = m_books[book.getId()];
+    auto& newEntry = mp_impl->m_books[book.getId()];
     static_cast<Book&>(newEntry) = book;
-    newEntry.lastUpdatedRevision = m_revision;
+    newEntry.lastUpdatedRevision = mp_impl->m_revision;
+    size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
+    if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+    }
+    if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+      mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+    }
     return true;
   }
 }
@@ -125,15 +189,15 @@ bool Library::addBook(const Book& book)
 void Library::addBookmark(const Bookmark& bookmark)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  m_bookmarks.push_back(bookmark);
+  mp_impl->m_bookmarks.push_back(bookmark);
 }
 
 bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
+  for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
     if (it->getBookId() == zimId && it->getUrl() == url) {
-      m_bookmarks.erase(it);
+      mp_impl->m_bookmarks.erase(it);
       return true;
     }
   }
@@ -141,32 +205,38 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
 }
 
 
-void Library::dropReader(const std::string& id)
+void Library::dropCache(const std::string& id)
 {
-  m_readers.erase(id);
-  m_archives.erase(id);
+  mp_impl->mp_archiveCache->drop(id);
+  mp_impl->mp_searcherCache->drop(id);
 }
 
 bool Library::removeBookById(const std::string& id)
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  m_bookDB->delete_document("Q" + id);
-  dropReader(id);
-  return m_books.erase(id) == 1;
+  mp_impl->m_bookDB.delete_document("Q" + id);
+  dropCache(id);
+  // We do not change the cache size here
+  // Most of the time, the book is remove in case of library refresh, it is
+  // often associated with addBook calls (which will properly set the cache size)
+  // Having a too big cache is not a problem here (or it would have been before)
+  // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+  //  will be removed in put or getOrPut).
+  return mp_impl->m_books.erase(id) == 1;
 }
 
 Library::Revision Library::getRevision() const
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  return m_revision;
+  return mp_impl->m_revision;
 }
 
-uint32_t Library::removeBooksNotUpdatedSince(LibraryRevision libraryRevision)
+uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
 {
   BookIdCollection booksToRemove;
   {
     std::lock_guard<std::mutex> lock(m_mutex);
-    for ( const auto& entry : m_books) {
+    for ( const auto& entry : mp_impl->m_books) {
       if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
         booksToRemove.push_back(entry.first);
       }
@@ -185,7 +255,7 @@ const Book& Library::getBookById(const std::string& id) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  return m_books.at(id);
+  return mp_impl->m_books.at(id);
 }
 
 Book Library::getBookByIdThreadSafe(const std::string& id) const
@@ -198,7 +268,7 @@ const Book& Library::getBookByPath(const std::string& path) const
 {
   // XXX: Doesn't make sense to lock this operation since it cannot
   // XXX: guarantee thread-safety because of its return type
-  for(auto& it: m_books) {
+  for(auto& it: mp_impl->m_books) {
     auto& book = it.second;
     if (book.getPath() == path)
       return book;
@@ -210,50 +280,56 @@ const Book& Library::getBookByPath(const std::string& path) const
 
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
-  try {
-    std::lock_guard<std::mutex> lock(m_mutex);
-    return m_readers.at(id);
-  } catch (std::out_of_range& e) {}
-
-  const auto archive = getArchiveById(id);
-  if ( !archive )
+  auto archive = getArchiveById(id);
+  if(archive) {
+    return std::shared_ptr<Reader>(new Reader(archive, true));
+  } else {
     return nullptr;
-
-  const shared_ptr<Reader> reader(new Reader(archive, true));
-  std::lock_guard<std::mutex> lock(m_mutex);
-  m_readers[id] = reader;
-  return reader;
+  }
 }
 
 std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
 {
-  std::lock_guard<std::mutex> lock(m_mutex);
   try {
-    return m_archives.at(id);
-  } catch (std::out_of_range& e) {}
-
-  auto book = getBookById(id);
-  if (!book.isPathValid())
+    return mp_impl->mp_archiveCache->getOrPut(id,
+    [&](){
+      auto book = getBookById(id);
+      if (!book.isPathValid()) {
+        throw std::invalid_argument("");
+      }
+      return std::make_shared<zim::Archive>(book.getPath());
+    });
+  } catch (std::invalid_argument&) {
     return nullptr;
+  }
+}
 
-  auto sptr = make_shared<zim::Archive>(book.getPath());
-  m_archives[id] = sptr;
-  return sptr;
+std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
+{
+  assert(!ids.empty());
+  try {
+    return mp_impl->mp_searcherCache->getOrPut(ids,
+    [&](){
+      std::vector<zim::Archive> archives;
+      for(auto& id:ids) {
+        auto archive = getArchiveById(id);
+        if(!archive) {
+          throw std::invalid_argument("");
+        }
+        archives.push_back(*archive);
+      }
+      return std::make_shared<ZimSearcher>(zim::Searcher(archives));
+    });
+  } catch (std::invalid_argument&) {
+    return nullptr;
+  }
 }
 
 unsigned int Library::getBookCount(const bool localBooks,
                                    const bool remoteBooks) const
 {
   std::lock_guard<std::mutex> lock(m_mutex);
-  unsigned int result = 0;
-  for (auto& pair: m_books) {
-    auto& book = pair.second;
-    if ((!book.getPath().empty() && localBooks)
-        || (book.getPath().empty() && remoteBooks)) {
-      result++;
-    }
-  }
-  return result;
+  return mp_impl->getBookCount(localBooks, remoteBooks);
 }
 
 bool Library::writeToFile(const std::string& path) const
@@ -284,7 +360,7 @@ Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) con
   std::lock_guard<std::mutex> lock(m_mutex);
   AttributeCounts propValueCounts;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     if (book.getOrigId().empty()) {
       propValueCounts[(book.*p)()] += 1;
@@ -317,7 +393,7 @@ std::vector<std::string> Library::getBooksCategories() const
   std::lock_guard<std::mutex> lock(m_mutex);
   std::set<std::string> categories;
 
-  for (const auto& pair: m_books) {
+  for (const auto& pair: mp_impl->m_books) {
     const auto& book = pair.second;
     const auto& c = book.getCategory();
     if ( !c.empty() ) {
@@ -341,12 +417,12 @@ std::vector<std::string> Library::getBooksPublishers() const
 const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
 {
   if (!onlyValidBookmarks) {
-    return m_bookmarks;
+    return mp_impl->m_bookmarks;
   }
   std::vector<kiwix::Bookmark> validBookmarks;
   auto booksId = getBooksIds();
   std::lock_guard<std::mutex> lock(m_mutex);
-  for(auto& bookmark:m_bookmarks) {
+  for(auto& bookmark:mp_impl->m_bookmarks) {
     if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
       validBookmarks.push_back(bookmark);
     }
@@ -359,7 +435,7 @@ Library::BookIdCollection Library::getBooksIds() const
   std::lock_guard<std::mutex> lock(m_mutex);
   BookIdCollection bookIds;
 
-  for (auto& pair: m_books) {
+  for (auto& pair: mp_impl->m_books) {
     bookIds.push_back(pair.first);
   }
 
@@ -405,7 +481,7 @@ void Library::updateBookDB(const Book& book)
 
   doc.set_data(book.getId());
 
-  m_bookDB->replace_document(idterm, doc);
+  mp_impl->m_bookDB.replace_document(idterm, doc);
 }
 
 namespace
@@ -538,9 +614,9 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
   BookIdCollection bookIds;
 
   std::lock_guard<std::mutex> lock(m_mutex);
-  Xapian::Enquire enquire(*m_bookDB);
+  Xapian::Enquire enquire(mp_impl->m_bookDB);
   enquire.set_query(query);
-  const auto results = enquire.get_mset(0, m_books.size());
+  const auto results = enquire.get_mset(0, mp_impl->m_books.size());
   for ( auto it = results.begin(); it != results.end(); ++it  ) {
     bookIds.push_back(it.get_document().get_data());
   }
@@ -554,7 +630,7 @@ Library::BookIdCollection Library::filter(const Filter& filter) const
   const auto preliminaryResult = filterViaBookDB(filter);
   std::lock_guard<std::mutex> lock(m_mutex);
   for(auto id : preliminaryResult) {
-    if(filter.accept(m_books.at(id))) {
+    if(filter.accept(mp_impl->m_books.at(id))) {
       result.push_back(id);
     }
   }

src/meson.build

@@ -28,10 +28,12 @@ kiwix_sources = [
   'server/response.cpp',
   'server/internalServer.cpp',
   'server/internalServer_catalog_v2.cpp',
+  'server/i18n.cpp',
   'opds_catalog.cpp',
   'version.cpp'
 ]
 kiwix_sources += lib_resources
+kiwix_sources += i18n_resources
 
 if host_machine.system() == 'windows'
   kiwix_sources += 'subprocess_windows.cpp'
@@ -39,15 +41,7 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if wrapper.contains('android')
-  install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
-else
-  install_dir = get_option('libdir')
-endif
-
-if wrapper.contains('android') or wrapper.contains('java')
-  subdir('wrapper/java')
-endif
+install_dir = get_option('libdir')
 
 config_h = configure_file(output : 'kiwix_config.h',
                configuration : conf,

src/opds_dumper.cpp

@@ -72,9 +72,7 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
 
 kainjow::mustache::object getSingleBookData(const Book& book)
 {
-    const MustacheData bookUrl = book.getUrl().empty()
-                               ? MustacheData(false)
-                               : MustacheData(book.getUrl());
+    const auto bookDate = book.getDate() + "T00:00:00Z";
     return kainjow::mustache::object{
       {"id", book.getId()},
       {"name", book.getName()},
@@ -82,7 +80,8 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"description", book.getDescription()},
       {"language", book.getLanguage()},
       {"content_id",  urlEncode(book.getHumanReadableIdFromPath(), true)},
-      {"updated", book.getDate() + "T00:00:00Z"},
+      {"updated", bookDate}, // XXX: this should be the entry update datetime
+      {"book_date", bookDate},
       {"category", book.getCategory()},
       {"flavour", book.getFlavour()},
       {"tags", book.getTags()},
@@ -90,7 +89,7 @@ kainjow::mustache::object getSingleBookData(const Book& book)
       {"media_count", to_string(book.getMediaCount())},
       {"author_name", book.getCreator()},
       {"publisher_name", book.getPublisher()},
-      {"url", bookUrl},
+      {"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
       {"size", to_string(book.getSize())},
       {"icons", getBookIllustrationInfo(book)},
     };
@@ -124,13 +123,63 @@ BooksData getBooksData(const Library* library, const std::vector<std::string>& b
   return booksData;
 }
 
+std::map<std::string, std::string> iso639_3 = {
+  {"atj", "atikamekw"},
+  {"azb", "آذربایجان دیلی"},
+  {"bcl", "central bikol"},
+  {"bgs", "tagabawa"},
+  {"bxr", "буряад хэлэн"},
+  {"cbk", "chavacano"},
+  {"cdo", "閩東語"},
+  {"dag", "Dagbani"},
+  {"diq", "dimli"},
+  {"dty", "डोटेली"},
+  {"eml", "emiliân-rumagnōl"},
+  {"fbs", "српскохрватски"},
+  {"ido", "ido"},
+  {"kbp", "kabɩyɛ"},
+  {"kld", "Gamilaraay"},
+  {"lbe", "лакку маз"},
+  {"lbj", "ལ་དྭགས་སྐད་"},
+  {"map", "Austronesian"},
+  {"mhr", "марий йылме"},
+  {"mnw", "ဘာသာမန်"},
+  {"myn", "mayan"},
+  {"nah", "nahuatl"},
+  {"nai", "north American Indian"},
+  {"nds", "plattdütsch"},
+  {"nrm", "bhasa narom"},
+  {"olo", "livvi"},
+  {"pih", "Pitcairn-Norfolk"},
+  {"pnb", "Western Panjabi"},
+  {"rmr", "Caló"},
+  {"rmy", "romani shib"},
+  {"roa", "romance languages"},
+  {"twi", "twi"}
+};
+
+std::once_flag fillLanguagesFlag;
+
+void fillLanguagesMap()
+{
+  for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
+    auto lang = *icuLangPtr;
+    const icu::Locale locale(lang);
+    icu::UnicodeString ustring;
+    locale.getDisplayLanguage(locale, ustring);
+    std::string displayLanguage;
+    ustring.toUTF8String(displayLanguage);
+    std::string iso3LangCode = locale.getISO3Language();
+    iso639_3.insert({iso3LangCode, displayLanguage});
+  }
+}
+
 std::string getLanguageSelfName(const std::string& lang) {
-  const icu::Locale locale(lang.c_str());
-  icu::UnicodeString ustring;
-  locale.getDisplayLanguage(locale, ustring);
-  std::string result;
-  ustring.toUTF8String(result);
-  return result;
+  const auto itr = iso639_3.find(lang);
+  if (itr != iso639_3.end()) {
+    return itr->second;
+  }
+  return lang;
 };
 
 } // unnamed namespace
@@ -142,7 +191,7 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
      {"date", gen_date_str()},
      {"root", rootLocation},
      {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
      {"itemsPerPage", to_string(m_count)},
@@ -162,7 +211,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
      {"date", gen_date_str()},
      {"endpoint_root", endpointRoot},
      {"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
-     {"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
+     {"filter", onlyAsNonEmptyMustacheValue(query)},
      {"query", query.empty() ? "" : "?" + urlEncode(query)},
      {"totalResults", to_string(m_totalResults)},
      {"startIndex", to_string(m_startIndex)},
@@ -208,6 +257,7 @@ std::string OPDSDumper::languagesOPDSFeed() const
 {
   const auto now = gen_date_str();
   kainjow::mustache::list languageData;
+  std::call_once(fillLanguagesFlag, fillLanguagesMap);
   for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
     const std::string languageCode = langAndBookCount.first;
     const int bookCount = langAndBookCount.second;

src/search_renderer.cpp

@@ -26,6 +26,8 @@
 #include "library.h"
 #include "name_mapper.h"
 
+#include "tools/archiveTools.h"
+
 #include <zim/search.h>
 
 #include <mustache.hpp>
@@ -37,20 +39,27 @@ namespace kiwix
 
 /* Constructor */
 SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
-    : m_srs(searcher->getSearchResultSet()),
-      mp_nameMapper(mapper),
-      protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
-      estimatedResultCount(searcher->getEstimatedResultCount()),
-      resultStart(searcher->getResultStart())
+    : SearchRenderer(
+        /* srs */ searcher->getSearchResultSet(),
+        /* mapper */ mapper,
+        /* library */ nullptr,
+        /* start */ searcher->getResultStart(),
+        /* estimatedResultCount */ searcher->getEstimatedResultCount()
+        )
 {}
 
 SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
                       unsigned int start, unsigned int estimatedResultCount)
+    : SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
+{}
+
+SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
+                      unsigned int start, unsigned int estimatedResultCount)
     : m_srs(srs),
       mp_nameMapper(mapper),
+      mp_library(library),
       protocolPrefix("zim://"),
-      searchProtocolPrefix("search://?"),
+      searchProtocolPrefix("search://"),
       estimatedResultCount(estimatedResultCount),
       resultStart(start)
 {}
@@ -60,12 +69,12 @@ SearchRenderer::~SearchRenderer() = default;
 
 void SearchRenderer::setSearchPattern(const std::string& pattern)
 {
-  this->searchPattern = pattern;
+  searchPattern = pattern;
 }
 
-void SearchRenderer::setSearchContent(const std::string& name)
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
 {
-  this->searchContent = name;
+  searchBookQuery = bookQuery;
 }
 
 void SearchRenderer::setProtocolPrefix(const std::string& prefix)
@@ -78,81 +87,160 @@ void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
   this->searchProtocolPrefix = prefix;
 }
 
-std::string SearchRenderer::getHtml()
-{
-  kainjow::mustache::data results{kainjow::mustache::data::type::list};
+std::string extractValueFromQuery(const std::string& query, const std::string& key) {
+  const std::string p = key + "=";
+  const size_t i = query.find(p);
+  if (i == std::string::npos) {
+    return "";
+  }
+  std::string r = query.substr(i + p.size());
+  return r.substr(0, r.find("&"));
+}
 
+kainjow::mustache::data buildQueryData
+(
+  const std::string& searchProtocolPrefix,
+  const std::string& pattern,
+  const std::string& bookQuery
+) {
+  kainjow::mustache::data query;
+  query.set("pattern", kiwix::encodeDiples(pattern));
+  std::ostringstream ss;
+  ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
+  ss << "&" << bookQuery;
+  query.set("unpaginatedQuery", ss.str());
+  auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
+  if(!lang.empty()) {
+    query.set("lang", lang);
+  }
+  return query;
+}
+
+kainjow::mustache::data buildPagination(
+  unsigned int pageLength,
+  unsigned int resultsCount,
+  unsigned int resultsStart
+)
+{
+  assert(pageLength!=0);
+  kainjow::mustache::data pagination;
+  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+
+  if (resultsCount == 0) {
+    // Easy case
+    pagination.set("itemsPerPage", to_string(pageLength));
+    pagination.set("hasPages", false);
+    pagination.set("pages", pages);
+    return pagination;
+  }
+
+  // First we want to display pages starting at a multiple of `pageLength`
+  // so, let's calculate the start index of the current page.
+  auto currentPage = resultsStart/pageLength;
+  auto lastPage = ((resultsCount-1)/pageLength);
+  auto lastPageStart = lastPage*pageLength;
+  auto nbPages = lastPage + 1;
+
+  auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
+  auto lastPageGenerated = min(currentPage+4, lastPage);
+
+  if (nbPages != 1) {
+    if (firstPageGenerated!=0) {
+      kainjow::mustache::data page;
+      page.set("label", "◀");
+      page.set("start", to_string(0));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+
+    for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
+      kainjow::mustache::data page;
+      page.set("label", to_string(i+1));
+      page.set("start", to_string(i*pageLength));
+      page.set("current", bool(i == currentPage));
+      pages.push_back(page);
+    }
+
+    if (lastPageGenerated!=lastPage) {
+      kainjow::mustache::data page;
+      page.set("label", "▶");
+      page.set("start", to_string(lastPageStart));
+      page.set("current", false);
+      pages.push_back(page);
+    }
+  }
+
+  pagination.set("itemsPerPage", to_string(pageLength));
+  pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
+  pagination.set("pages", pages);
+  return pagination;
+}
+
+std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
+{
+  // Build the results list
+  kainjow::mustache::data items{kainjow::mustache::data::type::list};
   for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
     kainjow::mustache::data result;
+    std::string zim_id(it.getZimId());
     result.set("title", it.getTitle());
-    result.set("url", it.getPath());
+    result.set("absolutePath", protocolPrefix + urlEncode(mp_nameMapper->getNameForId(zim_id), true) + "/" + urlEncode(it.getPath()));
     result.set("snippet", it.getSnippet());
-    std::ostringstream s;
-    s << it.getZimId();
-    result.set("resultContentId", mp_nameMapper->getNameForId(s.str()));
-
+    if (mp_library) {
+      result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
+    }
     if (it.getWordCount() >= 0) {
       result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
     }
 
-    results.push_back(result);
+    items.push_back(result);
   }
+  kainjow::mustache::data results;
+  results.set("items", items);
+  results.set("count", kiwix::beautifyInteger(estimatedResultCount));
+  results.set("hasResults", estimatedResultCount != 0);
+  results.set("start", kiwix::beautifyInteger(resultStart));
+  results.set("end", kiwix::beautifyInteger(min(resultStart+pageLength-1, estimatedResultCount)));
 
-  // pages
-  kainjow::mustache::data pages{kainjow::mustache::data::type::list};
+  // pagination
+  auto pagination = buildPagination(
+    pageLength,
+    estimatedResultCount,
+    resultStart
+  );
 
-  auto resultEnd = 0U;
-  auto currentPage = 0U;
-  auto pageStart = 0U;
-  auto pageEnd = 0U;
-  auto lastPageStart = 0U;
-  if (pageLength) {
-    currentPage = resultStart/pageLength;
-    pageStart = currentPage > 4 ? currentPage-4 : 0;
-    pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / pageLength) {
-      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
-    }
-    if (estimatedResultCount > pageLength) {
-      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
-    }
-  }
+  kainjow::mustache::data query = buildQueryData(
+    searchProtocolPrefix,
+    searchPattern,
+    searchBookQuery
+  );
 
-  resultEnd = resultStart+pageLength; //setting result end
-
-  for (unsigned int i = pageStart; i < pageEnd; i++) {
-    kainjow::mustache::data page;
-    page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * pageLength));
-
-    if (i == currentPage) {
-      page.set("selected", true);
-    }
-    pages.push_back(page);
-  }
-
-  std::string template_str = RESOURCE::templates::search_result_html;
-  kainjow::mustache::mustache tmpl(template_str);
 
   kainjow::mustache::data allData;
+  allData.set("protocolPrefix", protocolPrefix);
   allData.set("results", results);
-  allData.set("pages", pages);
-  allData.set("hasResults", estimatedResultCount != 0);
-  allData.set("hasPages", pageStart != pageEnd);
-  allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
-  allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
-  allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
-  allData.set("resultStart", to_string(resultStart + 1));
-  allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("pageLength", to_string(pageLength));
-  allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("protocolPrefix", this->protocolPrefix);
-  allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
-  allData.set("contentId", this->searchContent);
+  allData.set("pagination", pagination);
+  allData.set("query", query);
+
+  kainjow::mustache::mustache tmpl(tmpl_str);
 
   std::stringstream ss;
   tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
+  if (!tmpl.is_valid()) {
+    throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
+  }
   return ss.str();
 }
 
+std::string SearchRenderer::getHtml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_html);
+}
+
+std::string SearchRenderer::getXml()
+{
+  return renderTemplate(RESOURCE::templates::search_result_xml);
+}
+
+
 }

src/searcher.cpp

@@ -89,13 +89,13 @@ Searcher::~Searcher()
 {
 }
 
-bool Searcher::add_reader(Reader* reader)
+bool Searcher::add_reader(std::shared_ptr<Reader> reader)
 {
   if (!reader->hasFulltextIndex()) {
       return false;
   }
 
-  for ( const Reader* const existing_reader : readers ) {
+  for ( auto existing_reader : readers ) {
     if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
       return false;
   }
@@ -105,7 +105,7 @@ bool Searcher::add_reader(Reader* reader)
 }
 
 
-Reader* Searcher::get_reader(int readerIndex)
+std::shared_ptr<Reader> Searcher::get_reader(int readerIndex)
 {
   return readers.at(readerIndex);
 }

src/server.cpp

@@ -45,11 +45,13 @@ bool Server::start() {
     m_port,
     m_root,
     m_nbThreads,
+    m_multizimSearchLimit,
     m_verbose,
     m_withTaskbar,
     m_withLibraryButton,
     m_blockExternalLinks,
-    m_indexTemplateString));
+    m_indexTemplateString,
+    m_ipConnectionLimit));
   return mp_server->start();
 }
 

src/server/i18n.cpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "i18n.h"
+
+#include "tools/otherTools.h"
+
+#include <algorithm>
+#include <map>
+
+namespace kiwix
+{
+
+const char* I18nStringTable::get(const std::string& key) const
+{
+  const I18nString* const begin = entries;
+  const I18nString* const end = begin + entryCount;
+  const I18nString* found = std::lower_bound(begin, end, key,
+      [](const I18nString& a, const std::string& k) {
+        return a.key < k;
+  });
+  return (found == end || found->key != key) ? nullptr : found->value;
+}
+
+namespace i18n
+{
+// this data is generated by the i18n resource compiler
+extern const I18nStringTable stringTables[];
+extern const size_t langCount;
+}
+
+namespace
+{
+
+class I18nStringDB
+{
+public: // functions
+  I18nStringDB() {
+    for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
+      const auto& t = kiwix::i18n::stringTables[i];
+      lang2TableMap[t.lang] = &t;
+    }
+    enStrings = lang2TableMap.at("en");
+  };
+
+  std::string get(const std::string& lang, const std::string& key) const {
+    const char* s = getStringsFor(lang)->get(key);
+    if ( s == nullptr ) {
+      s = enStrings->get(key);
+      if ( s == nullptr ) {
+        throw std::runtime_error("Invalid message id");
+      }
+    }
+    return s;
+  }
+
+private: // functions
+  const I18nStringTable* getStringsFor(const std::string& lang) const {
+    try {
+      return lang2TableMap.at(lang);
+    } catch(const std::out_of_range&) {
+      return enStrings;
+    }
+  }
+
+private: // data
+  std::map<std::string, const I18nStringTable*> lang2TableMap;
+  const I18nStringTable* enStrings;
+};
+
+} // unnamed namespace
+
+std::string getTranslatedString(const std::string& lang, const std::string& key)
+{
+  static const I18nStringDB stringDb;
+
+  return stringDb.get(lang, key);
+}
+
+namespace i18n
+{
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params)
+{
+  const std::string tmpl = getTranslatedString(lang, key);
+  return render_template(tmpl, params);
+}
+
+} // namespace i18n
+
+std::string ParameterizedMessage::getText(const std::string& lang) const
+{
+  return i18n::expandParameterizedString(lang, msgId, params);
+}
+
+} // namespace kiwix

src/server/i18n.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIX_SERVER_I18N
+#define KIWIX_SERVER_I18N
+
+#include <string>
+#include <mustache.hpp>
+
+namespace kiwix
+{
+
+struct I18nString {
+  const char* const key;
+  const char* const value;
+};
+
+struct I18nStringTable {
+  const char* const lang;
+  const size_t entryCount;
+  const I18nString* const entries;
+
+  const char* get(const std::string& key) const;
+};
+
+std::string getTranslatedString(const std::string& lang, const std::string& key);
+
+namespace i18n
+{
+
+typedef kainjow::mustache::object Parameters;
+
+std::string expandParameterizedString(const std::string& lang,
+                                      const std::string& key,
+                                      const Parameters& params);
+
+class GetTranslatedString
+{
+public:
+  explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
+
+  std::string operator()(const std::string& key) const
+  {
+    return getTranslatedString(m_lang, key);
+  }
+
+  std::string operator()(const std::string& key, const Parameters& params) const
+  {
+    return expandParameterizedString(m_lang, key, params);
+  }
+
+private:
+  const std::string m_lang;
+};
+
+} // namespace i18n
+
+struct ParameterizedMessage
+{
+public: // types
+  typedef kainjow::mustache::object Parameters;
+
+public: // functions
+  ParameterizedMessage(const std::string& msgId, const Parameters& params)
+    : msgId(msgId)
+    , params(params)
+  {}
+
+  std::string getText(const std::string& lang) const;
+
+private: // data
+  const std::string msgId;
+  const Parameters  params;
+};
+
+} // namespace kiwix
+
+#endif // KIWIX_SERVER_I18N

src/server/internalServer.cpp

@@ -55,11 +55,10 @@ extern "C" {
 #include "searcher.h"
 #include "search_renderer.h"
 #include "opds_dumper.h"
+#include "i18n.h"
 
 #include <zim/uuid.h>
 #include <zim/error.h>
-#include <zim/search.h>
-#include <zim/suggestion.h>
 #include <zim/entry.h>
 #include <zim/item.h>
 
@@ -79,7 +78,7 @@ extern "C" {
 #include "response.h"
 
 #define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#define DEFAULT_CACHE_SIZE 2
 
 namespace kiwix {
 
@@ -96,8 +95,239 @@ inline std::string normalizeRootUrl(std::string rootUrl)
   return rootUrl.empty() ? rootUrl : "/" + rootUrl;
 }
 
+Filter get_search_filter(const RequestContext& request, const std::string& prefix="")
+{
+    auto filter = kiwix::Filter().valid(true).local(true);
+    try {
+      filter.query(request.get_argument(prefix+"q"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(request.get_argument<unsigned long>(prefix+"maxsize"));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument(prefix+"name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.category(request.get_argument(prefix+"category"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument(prefix+"lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument(prefix+"tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument(prefix+"notag"), ";"));
+    } catch (...) {}
+    return filter;
+}
+
+template<class T>
+std::vector<T> subrange(const std::vector<T>& v, size_t s, size_t n)
+{
+  const size_t e = std::min(v.size(), s+n);
+  return std::vector<T>(v.begin()+std::min(v.size(), s), v.begin()+e);
+}
+
+std::string renderUrl(const std::string& root, const std::string& urlTemplate)
+{
+  MustacheData data;
+  data.set("root", root);
+  auto url = kainjow::mustache::mustache(urlTemplate).render(data);
+  if ( url.back() == '\n' )
+    url.pop_back();
+  return url;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
+{
+  return i18n::expandParameterizedString(lang, "suggest-full-text-search",
+               {
+                  {"SEARCH_TERMS", queryString}
+               }
+         );
+}
+
+ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
+{
+  return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
+}
+
+ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
+{
+  return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
+}
+
+ParameterizedMessage noValueForArgMsg(const std::string& argument)
+{
+  return ParameterizedMessage("no-value-for-arg", { {"ARGUMENT", argument} });
+}
+
+ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
+{
+  return ParameterizedMessage("raw-entry-not-found",
+                              {
+                                {"DATATYPE", dt},
+                                {"ENTRY", entry},
+                              }
+  );
+}
+
+ParameterizedMessage tooManyBooksMsg(size_t nbBooks, size_t limit)
+{
+  return ParameterizedMessage("too-many-books",
+                              {
+                                {"NB_BOOKS", beautifyInteger(nbBooks)},
+                                {"LIMIT", beautifyInteger(limit)},
+                              }
+  );
+}
+
+ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
+{
+  const ParameterizedMessage::Parameters noParams;
+  return ParameterizedMessage(msgId, noParams);
+}
+
+struct Error : public std::runtime_error {
+  explicit Error(const ParameterizedMessage& message)
+    : std::runtime_error("Error while handling request"),
+      _message(message)
+  {}
+
+  const ParameterizedMessage& message() const
+  {
+    return _message;
+  }
+
+  const ParameterizedMessage _message;
+};
+
+void checkBookNumber(const Library::BookIdSet& bookIds, size_t limit) {
+  if (bookIds.empty()) {
+    throw Error(nonParameterizedMessage("no-book-found"));
+  }
+  if (limit > 0 && bookIds.size() > limit) {
+    throw Error(tooManyBooksMsg(bookIds.size(), limit));
+  }
+}
+
 } // unnamed namespace
 
+std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
+{
+  // Try old API
+  try {
+    auto bookName = request.get_argument("content");
+    try {
+      const auto bookIds = Library::BookIdSet{mp_nameMapper->getIdForName(bookName)};
+      const auto queryString = request.get_query([&](const std::string& key){return key == "content";}, true);
+      return {queryString, bookIds};
+    } catch (const std::out_of_range&) {
+      throw Error(noSuchBookErrorMsg(bookName));
+    }
+  } catch(const std::out_of_range&) {
+    // We've catch the out_of_range of get_argument
+    // continue
+  }
+
+  // Does user directly gives us ids ?
+  try {
+    auto id_vec = request.get_arguments("books.id");
+    if (id_vec.empty()) {
+      throw Error(noValueForArgMsg("books.id"));
+    }
+    for(const auto& bookId: id_vec) {
+      try {
+        // This is a silly way to check that bookId exists
+        mp_nameMapper->getNameForId(bookId);
+      } catch (const std::out_of_range&) {
+        throw Error(noSuchBookErrorMsg(bookId));
+      }
+    }
+    const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.id";}, true);
+    return {queryString, bookIds};
+  } catch(const std::out_of_range&) {}
+
+  // Use the names
+  try {
+    auto name_vec = request.get_arguments("books.name");
+    if (name_vec.empty()) {
+      throw Error(noValueForArgMsg("books.name"));
+    }
+    Library::BookIdSet bookIds;
+    for(const auto& bookName: name_vec) {
+      try {
+        bookIds.insert(mp_nameMapper->getIdForName(bookName));
+      } catch(const std::out_of_range&) {
+        throw Error(noSuchBookErrorMsg(bookName));
+      }
+    }
+    const auto queryString = request.get_query([&](const std::string& key){return key == "books.name";}, true);
+    return {queryString, bookIds};
+  } catch(const std::out_of_range&) {}
+
+  // Check for filtering
+  Filter filter = get_search_filter(request, "books.filter.");
+  auto id_vec = mp_library->filter(filter);
+  if (id_vec.empty()) {
+    throw Error(nonParameterizedMessage("no-book-found"));
+  }
+  const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+  const auto queryString = request.get_query([&](const std::string& key){return startsWith(key, "books.filter.");}, true);
+  return {queryString, bookIds};
+}
+
+SearchInfo InternalServer::getSearchInfo(const RequestContext& request) const
+{
+  auto bookIds = selectBooks(request);
+  checkBookNumber(bookIds.second, m_multizimSearchLimit);
+  auto pattern = request.get_optional_param<std::string>("pattern", "");
+  GeoQuery geoQuery;
+
+  /* Retrive geo search */
+  try {
+    auto latitude = request.get_argument<float>("latitude");
+    auto longitude = request.get_argument<float>("longitude");
+    auto distance = request.get_argument<float>("distance");
+    geoQuery = GeoQuery(latitude, longitude, distance);
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  if (!geoQuery && pattern.empty()) {
+    throw Error(nonParameterizedMessage("no-query"));
+  }
+
+  return SearchInfo(pattern, geoQuery, bookIds.second, bookIds.first);
+}
+
+SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterQuery)
+  : pattern(pattern),
+    geoQuery(geoQuery),
+    bookIds(bookIds),
+    bookFilterQuery(bookFilterQuery)
+{}
+
+zim::Query SearchInfo::getZimQuery(bool verbose) const {
+  zim::Query query;
+  if (verbose) {
+    std::cout << "Performing query '" << pattern<< "'";
+  }
+  query.setQuery(pattern);
+  if (geoQuery) {
+    if (verbose) {
+      std::cout << " with geo query '" << geoQuery.distance << "&(" << geoQuery.latitude << ";" << geoQuery.longitude << ")'";
+    }
+    query.setGeorange(geoQuery.latitude, geoQuery.longitude, geoQuery.distance);
+  }
+  if (verbose) {
+    std::cout << std::endl;
+  }
+  return query;
+}
+
+
 static IdNameMapper defaultNameMapper;
 
 static MHD_Result staticHandlerCallback(void* cls,
@@ -116,23 +346,29 @@ InternalServer::InternalServer(Library* library,
                                int port,
                                std::string root,
                                int nbThreads,
+                               unsigned int multizimSearchLimit,
                                bool verbose,
                                bool withTaskbar,
                                bool withLibraryButton,
                                bool blockExternalLinks,
-                               std::string indexTemplateString) :
+                               std::string indexTemplateString,
+                               int ipConnectionLimit) :
   m_addr(addr),
   m_port(port),
   m_root(normalizeRootUrl(root)),
   m_nbThreads(nbThreads),
+  m_multizimSearchLimit(multizimSearchLimit),
   m_verbose(verbose),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
   m_blockExternalLinks(blockExternalLinks),
   m_indexTemplateString(indexTemplateString.empty() ? RESOURCE::templates::index_html : indexTemplateString),
+  m_ipConnectionLimit(ipConnectionLimit),
   mp_daemon(nullptr),
   mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
+  searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
+  suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
 {}
 
 bool InternalServer::start() {
@@ -144,7 +380,6 @@ bool InternalServer::start() {
   if (m_verbose.load())
     flags |= MHD_USE_DEBUG;
 
-
   struct sockaddr_in sockAddr;
   memset(&sockAddr, 0, sizeof(sockAddr));
   sockAddr.sin_family = AF_INET;
@@ -168,6 +403,7 @@ bool InternalServer::start() {
                             this,
                             MHD_OPTION_SOCK_ADDR, &sockAddr,
                             MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_PER_IP_CONNECTION_LIMIT, m_ipConnectionLimit,
                             MHD_OPTION_END);
   if (mp_daemon == nullptr) {
     std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
@@ -262,8 +498,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
 std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
 {
   try {
-    if (! request.is_valid_url())
-      return Response::build_404(*this, request.get_full_url(), "", "");
+    if (! request.is_valid_url()) {
+      return HTTP404Response(*this, request)
+             + urlNotFoundMsg;
+    }
 
     const ETag etag = get_matching_if_none_match_etag(request);
     if ( etag )
@@ -281,6 +519,14 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     if (request.get_url() == "/search")
       return handle_search(request);
 
+    if (request.get_url() == "/search/searchdescription.xml") {
+      return ContentResponse::build(
+        *this,
+        RESOURCE::ft_opensearchdescription_xml,
+        get_default_data(),
+        "application/opensearchdescription+xml");
+    }
+
     if (request.get_url() == "/suggest")
      return handle_suggest(request);
 
@@ -293,10 +539,12 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
     return handle_content(request);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return Response::build_500(*this, e.what());
+    return HTTP500Response(*this, request)
+         + e.what();
   } catch (...) {
     fprintf(stderr, "===== Unhandled unknown error\n");
-    return Response::build_500(*this, "Unknown error");
+    return HTTP500Response(*this, request)
+         + "Unknown error";
   }
 }
 
@@ -337,14 +585,15 @@ std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& r
  * Archive and Zim handlers begin
  **/
 
-// TODO: retrieve searcher from caching mechanism
-SuggestionsList_t getSuggestions(const zim::Archive* const archive,
-                  const std::string& queryString, int start, int suggestionCount)
+SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Archive* const archive,
+                  const std::string& bookId, const std::string& queryString, int start, int suggestionCount)
 {
   SuggestionsList_t suggestions;
-  auto searcher = zim::SuggestionSearcher(*archive);
+  std::shared_ptr<zim::SuggestionSearcher> searcher;
+  searcher = cache.getOrPut(bookId, [=](){ return make_shared<zim::SuggestionSearcher>(*archive); });
+
   if (archive->hasTitleIndex()) {
-    auto search = searcher.suggest(queryString);
+    auto search = searcher->suggest(queryString);
     auto srs = search.getResults(start, suggestionCount);
 
     for (auto it : srs) {
@@ -357,7 +606,7 @@ SuggestionsList_t getSuggestions(const zim::Archive* const archive,
     std::vector<std::string> variants = getTitleVariants(queryString);
     int currCount = 0;
     for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
-      auto search = searcher.suggest(queryString);
+      auto search = searcher->suggest(queryString);
       auto srs = search.getResults(0, suggestionCount);
       for (auto it : srs) {
         SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
@@ -370,26 +619,26 @@ SuggestionsList_t getSuggestions(const zim::Archive* const archive,
   return suggestions;
 }
 
-
 std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
 {
   if (m_verbose.load()) {
     printf("** running handle_suggest\n");
   }
 
-  std::string bookName;
+  std::string bookName, bookId;
   std::shared_ptr<zim::Archive> archive;
   try {
     bookName = request.get_argument("content");
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    bookId = mp_nameMapper->getIdForName(bookName);
     archive = mp_library->getArchiveById(bookId);
   } catch (const std::out_of_range&) {
     // error handled by the archive == nullptr check below
   }
 
   if (archive == nullptr) {
-    const std::string error_details = "No such book: " + bookName;
-    return Response::build_404(*this, "", bookName, "", error_details);
+    return HTTP404Response(*this, request)
+           + noSuchBookErrorMsg(bookName)
+           + TaskbarInfo(bookName);
   }
 
   const auto queryString = request.get_optional_param("term", std::string());
@@ -408,7 +657,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
   bool first = true;
 
   /* Get the suggestions */
-  SuggestionsList_t suggestions = getSuggestions(archive.get(), queryString, start, count);
+  SuggestionsList_t suggestions = getSuggestions(suggestionSearcherCache, archive.get(),
+                                                  bookId, queryString, start, count);
   for(auto& suggestion:suggestions) {
     MustacheData result;
     result.set("label", suggestion.getTitle());
@@ -429,7 +679,8 @@ std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& r
   /* Propose the fulltext search if possible */
   if (archive->hasFulltextIndex()) {
     MustacheData result;
-    result.set("label", "containing '" + queryString + "'...");
+    const auto lang = request.get_user_language();
+    result.set("label", makeFulltextSearchSuggestion(lang, queryString));
     result.set("value", queryString + " ");
     result.set("kind", "pattern");
     result.set("first", first);
@@ -458,7 +709,8 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
     response->set_cacheable();
     return std::move(response);
   } catch (const ResourceNotFound& e) {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 }
 
@@ -468,111 +720,80 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     printf("** running handle_search\n");
   }
 
-  std::string patternString;
   try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
+    auto searchInfo = getSearchInfo(request);
+    auto bookIds = searchInfo.getBookIds();
 
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
+    /* Make the search */
+    // Try to get a search from the searchInfo, else build it
+    auto searcher = mp_library->getSearcherByIds(bookIds);
+    auto lock(searcher->getLock());
 
-  std::string bookName;
-  std::shared_ptr<zim::Archive> archive;
-  try {
-    bookName = request.get_argument("content");
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    archive = mp_library->getArchiveById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Make the search */
-  if ( (!archive && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
-    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
-    response->set_code(MHD_HTTP_NOT_FOUND);
-    return std::move(response);
-  }
-
-  std::shared_ptr<zim::Searcher> searcher;
-  if (archive) {
-    searcher = std::make_shared<zim::Searcher>(*archive);
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentArchive = mp_library->getArchiveById(bookId);
-      if (currentArchive) {
-        if (! searcher) {
-          searcher = std::make_shared<zim::Searcher>(*currentArchive);
-        } else {
-          searcher->addArchive(*currentArchive);
+    std::shared_ptr<zim::Search> search;
+    try {
+      search = searchCache.getOrPut(searchInfo,
+        [=](){
+          return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
         }
+      );
+    } catch(std::runtime_error& e) {
+      // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
+      // (in case of zim file not containing a index)
+      const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
+      HTTPErrorResponse response(*this, request, MHD_HTTP_NOT_FOUND,
+                                 "fulltext-search-unavailable",
+                                 "404-page-heading",
+                                 cssUrl);
+      response += nonParameterizedMessage("no-search-results");
+      if(bookIds.size() == 1) {
+        auto bookId = *bookIds.begin();
+        auto bookName = mp_nameMapper->getNameForId(bookId);
+        response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
       }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-
-  auto pageLength = 25;
-  try {
-    pageLength = request.get_argument<unsigned int>("pageLength");
-  } catch (const std::exception&) {}
-  if (pageLength > MAX_SEARCH_LEN) {
-    pageLength = MAX_SEARCH_LEN;
-  }
-  if (pageLength == 0) {
-    pageLength = 25;
-  }
-
-  /* Get the results */
-  try {
-    zim::Query query;
-    if (patternString.empty()) {
-      // Execute geo-search
-      if (m_verbose.load()) {
-        cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
-      }
-
-      query.setQuery("");
-      query.setGeorange(latitude, longitude, distance);
-    } else {
-      // Execute Ft search
-      if (m_verbose.load()) {
-          cout << "Performing query `" << patternString << "'" << endl;
-      }
-
-      std::string queryString = removeAccents(patternString);
-      query.setQuery(queryString);
+      return response;
     }
 
-    zim::Search search = searcher->search(query);
-    SearchRenderer renderer(search.getResults(start, pageLength), mp_nameMapper, start,
-                            search.getEstimatedMatches());
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
+    auto start = 1;
+    try {
+      start = request.get_argument<unsigned int>("start");
+    } catch (const std::exception&) {}
+    start = max(1, start);
+
+    auto pageLength = 25;
+    try {
+      pageLength = request.get_argument<unsigned int>("pageLength");
+    } catch (const std::exception&) {}
+    if (pageLength > MAX_SEARCH_LEN) {
+      pageLength = MAX_SEARCH_LEN;
+    }
+    if (pageLength == 0) {
+      pageLength = 25;
+    }
+
+    /* Get the results */
+    SearchRenderer renderer(search->getResults(start-1, pageLength), mp_nameMapper, mp_library, start,
+                            search->getEstimatedMatches());
+    renderer.setSearchPattern(searchInfo.pattern);
+    renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
     renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setSearchProtocolPrefix(m_root + "/search");
     renderer.setPageLength(pageLength);
+    if (request.get_requested_format() == "xml") {
+      return ContentResponse::build(*this, renderer.getXml(), "application/rss+xml; charset=utf-8",
+      /*isHomePage =*/false,
+      /*raw =*/true);
+    }
     auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
-    response->set_taskbar(bookName, archive ? getArchiveTitle(*archive) : "");
-
+    if(bookIds.size() == 1) {
+      auto bookId = *bookIds.begin();
+      auto bookName = mp_nameMapper->getNameForId(bookId);
+      response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
+    }
     return std::move(response);
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-    return Response::build_500(*this, e.what());
+  } catch (const Error& e) {
+    return HTTP400Response(*this, request)
+      + invalidUrlMsg
+      + e.message();
   }
 }
 
@@ -593,16 +814,18 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
   }
 
   if (archive == nullptr) {
-    const std::string error_details = "No such book: " + bookName;
-    return Response::build_404(*this, "", bookName, "", error_details);
+    return HTTP404Response(*this, request)
+           + noSuchBookErrorMsg(bookName)
+           + TaskbarInfo(bookName);
   }
 
   try {
     auto entry = archive->getRandomEntry();
     return build_redirect(bookName, getFinalItem(*archive, entry));
   } catch(zim::EntryNotFound& e) {
-    const std::string error_details = "Oops! Failed to pick a random article :(";
-    return Response::build_404(*this, "", bookName, getArchiveTitle(*archive), error_details);
+    return HTTP404Response(*this, request)
+           + nonParameterizedMessage("random-article-failure")
+           + TaskbarInfo(bookName, archive.get());
   }
 }
 
@@ -613,8 +836,10 @@ std::unique_ptr<Response> InternalServer::handle_captured_external(const Request
     source = kiwix::urlDecode(request.get_argument("source"));
   } catch (const std::out_of_range& e) {}
 
-  if (source.empty())
-    return Response::build_404(*this, request.get_full_url(), "", "");
+  if (source.empty()) {
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
+  }
 
   auto data = get_default_data();
   data.set("source", source);
@@ -633,7 +858,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
     host = request.get_header("Host");
     url  = request.get_url_part(1);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "v2") {
@@ -641,7 +867,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   }
 
   if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "searchdescription.xml") {
@@ -669,45 +896,6 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   return std::move(response);
 }
 
-namespace
-{
-
-Filter get_search_filter(const RequestContext& request)
-{
-    auto filter = kiwix::Filter().valid(true).local(true);
-    try {
-      filter.query(request.get_argument("q"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(request.get_argument<unsigned long>("maxsize"));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.category(request.get_argument("category"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    return filter;
-}
-
-template<class T>
-std::vector<T> subrange(const std::vector<T>& v, size_t s, size_t n)
-{
-  const size_t e = std::min(v.size(), s+n);
-  return std::vector<T>(v.begin()+std::min(v.size(), s), v.begin()+e);
-}
-
-} // unnamed namespace
-
 std::vector<std::string>
 InternalServer::search_catalog(const RequestContext& request,
                                kiwix::OPDSDumper& opdsDumper)
@@ -720,7 +908,8 @@ InternalServer::search_catalog(const RequestContext& request,
     const auto totalResults = bookIdsToDump.size();
     const size_t count = request.get_optional_param("count", 10UL);
     const size_t startIndex = request.get_optional_param("start", 0UL);
-    bookIdsToDump = subrange(bookIdsToDump, startIndex, count);
+    const size_t intendedCount = count > 0 ? count : bookIdsToDump.size();
+    bookIdsToDump = subrange(bookIdsToDump, startIndex, intendedCount);
     opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
     return bookIdsToDump;
 }
@@ -737,13 +926,13 @@ std::string get_book_name(const RequestContext& request)
   }
 }
 
-std::string searchSuggestionHTML(const std::string& searchURL, const std::string& pattern)
+ParameterizedMessage suggestSearchMsg(const std::string& searchURL, const std::string& pattern)
 {
-  kainjow::mustache::mustache tmpl("Make a full text search for <a href=\"{{{searchURL}}}\">{{pattern}}</a>");
-  MustacheData data;
-  data.set("pattern", pattern);
-  data.set("searchURL", searchURL);
-  return (tmpl.render(data));
+  return ParameterizedMessage("suggest-search",
+                              {
+                                { "PATTERN",    pattern   },
+                                { "SEARCH_URL", searchURL }
+                              });
 }
 
 } // unnamed namespace
@@ -774,10 +963,11 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
   } catch (const std::out_of_range& e) {}
 
   if (archive == nullptr) {
-    std::string searchURL = m_root+"/search?pattern="+pattern; // Make a full search on the entire library.
-    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
-
-    return Response::build_404(*this, request.get_full_url(), bookName, "", details);
+    const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
+           + TaskbarInfo(bookName);
   }
 
   auto urlStr = request.get_url().substr(bookName.size()+1);
@@ -794,7 +984,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     }
     auto response = ItemResponse::build(*this, request, entry.getItem());
     try {
-      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, getArchiveTitle(*archive));
+      dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, archive.get());
     } catch (std::bad_cast& e) {}
 
     if (m_verbose.load()) {
@@ -807,10 +997,11 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
     if (m_verbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
 
-    std::string searchURL = m_root+"/search?content="+bookName+"&pattern="+pattern; // Make a search on this specific book only.
-    const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
-
-    return Response::build_404(*this, request.get_full_url(), bookName, getArchiveTitle(*archive), details);
+    std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg
+           + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern))
+           + TaskbarInfo(bookName, archive.get());
   }
 }
 
@@ -827,12 +1018,14 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
      bookName = request.get_url_part(1);
      kind = request.get_url_part(2);
   } catch (const std::out_of_range& e) {
-     return Response::build_404(*this, request.get_full_url(), bookName, "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (kind != "meta" && kind!= "content") {
-    const std::string error_details = kind + " is not a valid request for raw content.";
-    return Response::build_404(*this, request.get_full_url(), bookName, "", error_details);
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg
+           + invalidRawAccessMsg(kind);
   }
 
   std::shared_ptr<zim::Archive> archive;
@@ -842,8 +1035,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   } catch (const std::out_of_range& e) {}
 
   if (archive == nullptr) {
-    const std::string error_details = "No such book: " + bookName;
-    return Response::build_404(*this, request.get_full_url(), bookName, "", error_details);
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg
+           + noSuchBookErrorMsg(bookName);
   }
 
   // Remove the beggining of the path:
@@ -867,8 +1061,9 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
     if (m_verbose.load()) {
       printf("Failed to find %s\n", itemPath.c_str());
     }
-    const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
-    return Response::build_404(*this, request.get_full_url(), bookName, getArchiveTitle(*archive), error_details);
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg
+           + rawEntryNotFoundMsg(kind, itemPath);
   }
 }
 

src/server/internalServer.h

@@ -28,6 +28,9 @@ extern "C" {
 #include "library.h"
 #include "name_mapper.h"
 
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
 #include <mustache.hpp>
 
 #include <atomic>
@@ -36,9 +39,57 @@ extern "C" {
 #include "server/request_context.h"
 #include "server/response.h"
 
+#include "tools/concurrent_cache.h"
+
 namespace kiwix {
 
+struct GeoQuery {
+  GeoQuery()
+    : GeoQuery(0, 0, -1)
+  {}
+
+  GeoQuery(float latitude, float longitude, float distance)
+    : latitude(latitude), longitude(longitude), distance(distance)
+  {}
+  float latitude;
+  float longitude;
+  float distance;
+
+  explicit operator bool() const {
+    return distance >= 0;
+  }
+
+  friend bool operator<(const GeoQuery& l, const GeoQuery& r)
+  {
+      return std::tie(l.latitude, l.longitude, l.distance)
+           < std::tie(r.latitude, r.longitude, r.distance); // keep the same order
+  }
+};
+
+class SearchInfo {
+  public:
+    SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
+
+    zim::Query getZimQuery(bool verbose) const;
+    const Library::BookIdSet& getBookIds() const { return bookIds; }
+
+    friend bool operator<(const SearchInfo& l, const SearchInfo& r)
+    {
+        return std::tie(l.bookIds, l.pattern, l.geoQuery)
+             < std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
+    }
+
+  public: //data
+    std::string pattern;
+    GeoQuery geoQuery;
+    Library::BookIdSet bookIds;
+    std::string bookFilterQuery;
+};
+
+
 typedef kainjow::mustache::data MustacheData;
+typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
+typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
 
 class Entry;
 class OPDSDumper;
@@ -51,11 +102,13 @@ class InternalServer {
                    int port,
                    std::string root,
                    int nbThreads,
+                   unsigned int multizimSearchLimit,
                    bool verbose,
                    bool withTaskbar,
                    bool withLibraryButton,
                    bool blockExternalLinks,
-                   std::string indexTemplateString);
+                   std::string indexTemplateString,
+                   int ipConnectionLimit);
     virtual ~InternalServer() = default;
 
     MHD_Result handlerCallback(struct MHD_Connection* connection,
@@ -68,7 +121,7 @@ class InternalServer {
     bool start();
     void stop();
     std::string getAddress() { return m_addr; }
-    int getPort() { return m_port; } 
+    int getPort() { return m_port; }
 
   private: // functions
     std::unique_ptr<Response> handle_request(const RequestContext& request);
@@ -97,30 +150,35 @@ class InternalServer {
 
     bool etag_not_needed(const RequestContext& r) const;
     ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+    std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
+    SearchInfo getSearchInfo(const RequestContext& r) const;
 
   private: // data
     std::string m_addr;
     int m_port;
     std::string m_root;
     int m_nbThreads;
+    unsigned int m_multizimSearchLimit;
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
     std::string m_indexTemplateString;
+    int m_ipConnectionLimit;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
     NameMapper* mp_nameMapper;
 
+    SearchCache searchCache;
+    SuggestionSearcherCache suggestionSearcherCache;
+
     std::string m_server_id;
     std::string m_library_id;
 
     friend std::unique_ptr<Response> Response::build(const InternalServer& server);
     friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
     friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
-    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
-
 };
 
 }

src/server/internalServer_catalog_v2.cpp

@@ -43,7 +43,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   try {
     url  = request.get_url_part(2);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   if (url == "root.xml") {
@@ -69,7 +70,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
   } else if (url == "illustration") {
     return handle_catalog_v2_illustration(request);
   } else {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 }
 
@@ -110,7 +112,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
   try {
     mp_library->getBookById(entryId);
   } catch (const std::out_of_range&) {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 
   OPDSDumper opdsDumper(mp_library);
@@ -158,7 +161,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const R
     auto illustration = book.getIllustration(size);
     return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
   } catch(...) {
-    return Response::build_404(*this, request.get_full_url(), "", "");
+    return HTTP404Response(*this, request)
+           + urlNotFoundMsg;
   }
 }
 

src/server/request_context.cpp

@@ -75,15 +75,15 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   method(str2RequestMethod(_method)),
   version(version),
   requestIndex(s_requestIndex++),
-  acceptEncodingDeflate(false),
+  acceptEncodingGzip(false),
   byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
 
   try {
-    acceptEncodingDeflate =
-        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
+    acceptEncodingGzip =
+        (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("gzip") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
   try {
@@ -106,7 +106,7 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
                                          const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->arguments[key] = value == nullptr ? "" : value;
+  _this->arguments[key].push_back(value == nullptr ? "" : value);
   return MHD_YES;
 }
 
@@ -121,13 +121,19 @@ void RequestContext::print_debug_info() const {
     printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
   }
   printf("arguments :\n");
-  for (auto it=arguments.begin(); it!=arguments.end(); it++) {
-    printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+  for (auto& pair:arguments) {
+    printf(" - %s :", pair.first.c_str());
+    bool first = true;
+    for (auto& v: pair.second) {
+      printf("%s %s", first?"":",", v.c_str());
+      first = false;
+    }
+    printf("\n");
   }
   printf("Parsed : \n");
   printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
-  printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
+  printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
   printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
@@ -176,21 +182,29 @@ ByteRange RequestContext::get_range() const {
 
 template<>
 std::string RequestContext::get_argument(const std::string& name) const {
-  return arguments.at(name);
+  return arguments.at(name)[0];
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
   return headers.at(lcAll(name));
 }
 
-std::string RequestContext::get_query() const {
-  std::string q;
-  const char* sep = "";
-  for ( const auto& a : arguments ) {
-    q += sep + a.first + '=' + a.second;
-    sep = "&";
-  }
-  return q;
+std::string RequestContext::get_user_language() const
+{
+  try {
+    return get_argument("userlang");
+  } catch(const std::out_of_range&) {}
+
+  try {
+    return get_header("Accept-Language");
+  } catch(const std::out_of_range&) {}
+
+  return "en";
+}
+
+std::string RequestContext::get_requested_format() const
+{
+  return get_optional_param<std::string>("format", "html");
 }
 
 }

src/server/request_context.h

@@ -25,9 +25,11 @@
 #include <string>
 #include <sstream>
 #include <map>
+#include <vector>
 #include <stdexcept>
 
 #include "byte_range.h"
+#include "tools/stringTools.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
@@ -68,10 +70,11 @@ class RequestContext {
     std::string get_header(const std::string& name) const;
     template<typename T=std::string>
     T get_argument(const std::string& name) const {
-        std::istringstream stream(arguments.at(name));
-        T v;
-        stream >> v;
-        return v;
+        return extractFromString<T>(get_argument(name));
+    }
+
+    std::vector<std::string> get_arguments(const std::string& name) const {
+      return arguments.at(name);
     }
 
     template<class T>
@@ -88,11 +91,34 @@ class RequestContext {
     std::string get_url() const;
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
-    std::string get_query() const;
+
+    std::string get_query(bool mustEncode = false) const {
+      return get_query([](const std::string& key) {return true;}, mustEncode);
+    }
+
+    template<class F>
+    std::string get_query(F filter, bool mustEncode) const {
+      std::string q;
+      const char* sep = "";
+      auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value, true):value; };
+      for ( const auto& a : arguments ) {
+        if (!filter(a.first)) {
+          continue;
+        }
+        for (const auto& v: a.second) {
+          q += sep + encode(a.first) + '=' + encode(v);
+          sep = "&";
+        }
+      }
+      return q;
+    }
 
     ByteRange get_range() const;
 
-    bool can_compress() const { return acceptEncodingDeflate; }
+    bool can_compress() const { return acceptEncodingGzip; }
+
+    std::string get_user_language() const;
+    std::string get_requested_format() const;
 
   private: // data
     std::string full_url;
@@ -101,11 +127,11 @@ class RequestContext {
     std::string version;
     unsigned long long requestIndex;
 
-    bool acceptEncodingDeflate;
+    bool acceptEncodingGzip;
 
     ByteRange byteRange_;
     std::map<std::string, std::string> headers;
-    std::map<std::string, std::string> arguments;
+    std::map<std::string, std::vector<std::string>> arguments;
 
   private: // functions
     static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);

src/server/response.cpp

@@ -25,13 +25,23 @@
 #include "tools/regexTools.h"
 #include "tools/stringTools.h"
 #include "tools/otherTools.h"
+#include "tools/archiveTools.h"
 
 #include "string.h"
 #include <mustache.hpp>
 #include <zlib.h>
 
+#include <array>
 
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+// This is somehow a magic value.
+// If this value is too small, we will compress (and lost cpu time) too much
+// content.
+// If this value is too big, we will not compress enough content and send too
+// much data.
+// If we assume that MTU is 1500 Bytes it is useless to compress
+// content smaller as the content will be sent in one packet anyway.
+// 1400 Bytes seems to be a common accepted limit.
+#define KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS 1400
 
 namespace kiwix {
 
@@ -57,6 +67,41 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/json") != string::npos;
 }
 
+bool compress(std::string &content) {
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+
+  auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
+                          Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK) { return false; }
+
+  strm.avail_in = static_cast<decltype(strm.avail_in)>(content.size());
+  strm.next_in =
+      const_cast<Bytef *>(reinterpret_cast<const Bytef *>(content.data()));
+
+  std::string compressed;
+
+  std::array<char, 16384> buff{};
+  do {
+    strm.avail_out = buff.size();
+    strm.next_out = reinterpret_cast<Bytef *>(buff.data());
+    ret = deflate(&strm, Z_FINISH);
+    assert(ret != Z_STREAM_ERROR);
+    compressed.append(buff.data(), buff.size() - strm.avail_out);
+  } while (strm.avail_out == 0);
+
+  assert(ret == Z_STREAM_END);
+  assert(strm.avail_in == 0);
+
+  content.swap(compressed);
+
+  deflateEnd(&strm);
+  return true;
+}
+
+
 
 } // unnamed namespace
 
@@ -83,21 +128,137 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   return response;
 }
 
-std::unique_ptr<Response> Response::build_404(const InternalServer& server, const std::string& url, const std::string& bookName, const std::string& bookTitle, const std::string& details)
+const UrlNotFoundMsg urlNotFoundMsg;
+const InvalidUrlMsg invalidUrlMsg;
+
+std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
 {
-  MustacheData results;
-  if ( !url.empty() ) {
-    results.set("url", url);
-  }
-  results.set("details", details);
-
-  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
-  response->set_code(MHD_HTTP_NOT_FOUND);
-  response->set_taskbar(bookName, bookTitle);
-
-  return std::move(response);
+  return getTranslatedString(m_request.get_user_language(), msgId);
 }
 
+std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
+{
+  auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
+  r->set_code(m_httpStatusCode);
+  if ( m_taskbarInfo ) {
+    r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
+  }
+  return r;
+}
+
+HTTPErrorResponse::HTTPErrorResponse(const InternalServer& server,
+                                     const RequestContext& request,
+                                     int httpStatusCode,
+                                     const std::string& pageTitleMsgId,
+                                     const std::string& headingMsgId,
+                                     const std::string& cssUrl)
+  : ContentResponseBlueprint(&server,
+                             &request,
+                             httpStatusCode,
+                             request.get_requested_format() == "html" ? "text/html; charset=utf-8" : "application/xml; charset=utf-8",
+                             request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml)
+{
+  kainjow::mustache::list emptyList;
+  this->m_data = kainjow::mustache::object{
+                    {"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
+                    {"PAGE_TITLE",   getMessage(pageTitleMsgId)},
+                    {"PAGE_HEADING", getMessage(headingMsgId)},
+                    {"details", emptyList}
+  };
+}
+
+HTTP404Response::HTTP404Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_NOT_FOUND,
+                      "404-page-title",
+                      "404-page-heading")
+{
+}
+
+HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
+{
+  const std::string requestUrl = m_request.get_full_url();
+  return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+(const std::string& msg)
+{
+  m_data["details"].push_back({"p", msg});
+  return *this;
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+(const ParameterizedMessage& details)
+{
+  return *this + details.getText(m_request.get_user_language());
+}
+
+HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& details)
+{
+  // operator+() is already a state-modifying operator (akin to operator+=)
+  return *this + details;
+}
+
+
+HTTP400Response::HTTP400Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_BAD_REQUEST,
+                      "400-page-title",
+                      "400-page-heading")
+{
+}
+
+HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
+{
+  std::string requestUrl = m_request.get_full_url();
+  const auto query = m_request.get_query();
+  if (!query.empty()) {
+    requestUrl += "?" + encodeDiples(query);
+  }
+  kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
+  return *this + msgTmpl.render({"url", requestUrl});
+}
+
+HTTP500Response::HTTP500Response(const InternalServer& server,
+                                 const RequestContext& request)
+  : HTTPErrorResponse(server,
+                      request,
+                      MHD_HTTP_INTERNAL_SERVER_ERROR,
+                      "500-page-title",
+                      "500-page-heading")
+{
+  // operator+() is a state-modifying operator (akin to operator+=)
+  *this + "An internal server error occured. We are sorry about that :/";
+}
+
+std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
+{
+  // We want a 500 response to be a minimalistic one (so that the server doesn't
+  // have to provide additional resources required for its proper rendering)
+  // ";raw=true" in the MIME-type below disables response decoration
+  // (see ContentResponse::contentDecorationAllowed())
+  const std::string mimeType = "text/html;charset=utf-8;raw=true";
+  auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
+  r->set_code(m_httpStatusCode);
+  return r;
+}
+
+ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
+{
+  this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
+  return *this;
+}
+
+ContentResponseBlueprint& ContentResponseBlueprint::operator+=(const TaskbarInfo& taskbarInfo)
+{
+  // operator+() is already a state-modifying operator (akin to operator+=)
+  return *this + taskbarInfo;
+}
+
+
 std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
 {
   auto response = Response::build(server);
@@ -111,26 +272,6 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
   return response;
 }
 
-std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  auto content = render_template(RESOURCE::templates::_500_html, data);
-  std::unique_ptr<Response> response (
-      new ContentResponse(
-        server.m_root, //root
-        true, //verbose
-        true, //raw
-        false, //withTaskbar
-        false, //withLibraryButton
-        false, //blockExternalLinks
-        content, //content
-        "text/html" //mimetype
-  ));
-  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
 
 std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
 {
@@ -196,14 +337,20 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-void ContentResponse::introduce_taskbar()
+void ContentResponse::introduce_taskbar(const std::string& lang)
 {
-  kainjow::mustache::data data;
-  data.set("root", m_root);
-  data.set("content", m_bookName);
-  data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
-  data.set("title", m_bookTitle);
-  data.set("withlibrarybutton", m_withLibraryButton);
+  i18n::GetTranslatedString t(lang);
+  kainjow::mustache::object data{
+    {"root", m_root},
+    {"content", m_bookName},
+    {"hascontent", (!m_bookName.empty() && !m_bookTitle.empty())},
+    {"title", m_bookTitle},
+    {"withlibrarybutton", m_withLibraryButton},
+    {"LIBRARY_BUTTON_TEXT", t("library-button-text")},
+    {"HOME_BUTTON_TEXT", t("home-button-text", {{"BOOK_TITLE", m_bookTitle}}) },
+    {"RANDOM_PAGE_BUTTON_TEXT", t("random-page-button-text") },
+    {"SEARCHBOX_TOOLTIP", t("searchbox-tooltip", {{"BOOK_TITLE", m_bookTitle}}) },
+  };
   auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
   m_content = prependToFirstOccurence(
     m_content,
@@ -241,7 +388,7 @@ ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
-      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
+      && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
 }
 
 bool
@@ -268,42 +415,24 @@ ContentResponse::create_mhd_response(const RequestContext& request)
     inject_root_link();
 
     if (m_withTaskbar) {
-      introduce_taskbar();
+      introduce_taskbar(request.get_user_language());
     }
     if (m_blockExternalLinks) {
       inject_externallinks_blocker();
     }
   }
 
-  bool shouldCompress = can_compress(request);
-  if (shouldCompress) {
-    std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
-    uLongf comprLen = compr_buffer.capacity();
-    int err = compress(&compr_buffer[0],
-                       &comprLen,
-                       (const Bytef*)(m_content.data()),
-                       m_content.size());
-    if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
-      /* /!\ Internet Explorer has a bug with deflate compression.
-         It can not handle the first two bytes (compression headers)
-         We need to chunk them off (move the content 2bytes)
-         It has no incidence on other browsers
-         See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
-      m_content = string((char*)&compr_buffer[2], comprLen - 2);
-      m_etag.set_option(ETag::COMPRESSED_CONTENT);
-    } else {
-      shouldCompress = false;
-    }
-  }
+  const bool isCompressed = can_compress(request) && compress(m_content);
 
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  if (shouldCompress) {
+  if (isCompressed) {
+    m_etag.set_option(ETag::COMPRESSED_CONTENT);
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
     MHD_add_response_header(
-        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
+        response, MHD_HTTP_HEADER_CONTENT_ENCODING, "gzip");
   }
   return response;
 }
@@ -332,10 +461,10 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   return ret;
 }
 
-void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
 {
   m_bookName = bookName;
-  m_bookTitle = bookTitle;
+  m_bookTitle = archive ? getArchiveTitle(*archive) : "";
 }
 
 

src/server/response.h

@@ -28,18 +28,21 @@
 #include "byte_range.h"
 #include "entry.h"
 #include "etag.h"
+#include "i18n.h"
 
 extern "C" {
 #include "microhttpd_wrapper.h"
 }
 
+namespace zim {
+class Archive;
+} // namespace zim
+
 namespace kiwix {
 
 class InternalServer;
 class RequestContext;
 
-class EntryResponse;
-
 class Response {
   public:
     Response(bool verbose);
@@ -47,9 +50,7 @@ class Response {
 
     static std::unique_ptr<Response> build(const InternalServer& server);
     static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
-    static std::unique_ptr<Response> build_404(const InternalServer& server, const std::string& url, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
     static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
-    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
     static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
@@ -100,12 +101,12 @@ class ContentResponse : public Response {
       const std::string& mimetype,
       bool isHomePage = false);
 
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+    void set_taskbar(const std::string& bookName, const zim::Archive* archive);
 
   private:
     MHD_Response* create_mhd_response(const RequestContext& request);
 
-    void introduce_taskbar();
+    void introduce_taskbar(const std::string& lang);
     void inject_externallinks_blocker();
     void inject_root_link();
     bool can_compress(const RequestContext& request) const;
@@ -124,6 +125,115 @@ class ContentResponse : public Response {
     std::string m_bookTitle;
  };
 
+struct TaskbarInfo
+{
+  const std::string bookName;
+  const zim::Archive* const archive;
+
+  TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
+    : bookName(bookName)
+    , archive(a)
+  {}
+};
+
+class ContentResponseBlueprint
+{
+public: // functions
+  ContentResponseBlueprint(const InternalServer* server,
+                           const RequestContext* request,
+                           int httpStatusCode,
+                           const std::string& mimeType,
+                           const std::string& templateStr)
+    : m_server(*server)
+    , m_request(*request)
+    , m_httpStatusCode(httpStatusCode)
+    , m_mimeType(mimeType)
+    , m_template(templateStr)
+  {}
+
+  virtual ~ContentResponseBlueprint() = default;
+
+  operator std::unique_ptr<ContentResponse>() const
+  {
+    return generateResponseObject();
+  }
+
+  operator std::unique_ptr<Response>() const
+  {
+    return operator std::unique_ptr<ContentResponse>();
+  }
+
+
+  ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
+  ContentResponseBlueprint& operator+=(const TaskbarInfo& taskbarInfo);
+
+protected: // functions
+  std::string getMessage(const std::string& msgId) const;
+  virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
+
+public: //data
+  const InternalServer& m_server;
+  const RequestContext& m_request;
+  const int m_httpStatusCode;
+  const std::string m_mimeType;
+  const std::string m_template;
+  kainjow::mustache::data m_data;
+  std::unique_ptr<TaskbarInfo> m_taskbarInfo;
+};
+
+struct HTTPErrorResponse : ContentResponseBlueprint
+{
+  HTTPErrorResponse(const InternalServer& server,
+                    const RequestContext& request,
+                    int httpStatusCode,
+                    const std::string& pageTitleMsgId,
+                    const std::string& headingMsgId,
+                    const std::string& cssUrl = "");
+
+  using ContentResponseBlueprint::operator+;
+  using ContentResponseBlueprint::operator+=;
+  HTTPErrorResponse& operator+(const std::string& msg);
+  HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
+  HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
+};
+
+class UrlNotFoundMsg {};
+
+extern const UrlNotFoundMsg urlNotFoundMsg;
+
+struct HTTP404Response : HTTPErrorResponse
+{
+  HTTP404Response(const InternalServer& server,
+                  const RequestContext& request);
+
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(UrlNotFoundMsg /*unused*/);
+};
+
+class InvalidUrlMsg {};
+
+extern const InvalidUrlMsg invalidUrlMsg;
+
+struct HTTP400Response : HTTPErrorResponse
+{
+  HTTP400Response(const InternalServer& server,
+                  const RequestContext& request);
+
+  using HTTPErrorResponse::operator+;
+  HTTPErrorResponse& operator+(InvalidUrlMsg /*unused*/);
+};
+
+struct HTTP500Response : HTTPErrorResponse
+{
+  HTTP500Response(const InternalServer& server,
+                  const RequestContext& request);
+
+private: // overrides
+  // generateResponseObject() is overriden in order to produce a minimal
+  // response without any need for additional resources from the server
+  std::unique_ptr<ContentResponse> generateResponseObject() const override;
+};
+
 class ItemResponse : public Response {
   public:
     ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);

src/tools/archiveTools.cpp

@@ -125,6 +125,30 @@ unsigned int getArchiveMediaCount(const zim::Archive& archive) {
   return counter;
 }
 
+unsigned int getArchiveArticleCount(const zim::Archive& archive) {
+  // [HACK]
+  // getArticleCount() returns different things depending of the "version" of the zim.
+  // On old zim (<=6), it returns the number of entry in `A` namespace
+  // On recent zim (>=7), it returns:
+  //  - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
+  //  - the number of front article if a frontArticleIndex is present
+  // The use case >=7 without frontArticleIndex is pretty rare so we don't care
+  // We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
+  if (archive.hasNewNamespaceScheme()) {
+    //The articleCount is "good"
+    return archive.getArticleCount();
+  } else {
+    // We have to parse the `M/Counter` metadata
+    unsigned int counter = 0;
+    for(const auto& pair:parseArchiveCounter(archive)) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
+    }
+    return counter;
+  }
+}
+
 unsigned int getArchiveFileSize(const zim::Archive& archive) {
   return archive.getFilesize() / 1024;
 }

src/tools/archiveTools.h

@@ -46,6 +46,7 @@ namespace kiwix
                            std::string& content, std::string& mimeType);
 
     unsigned int getArchiveMediaCount(const zim::Archive& archive);
+    unsigned int getArchiveArticleCount(const zim::Archive& archive);
     unsigned int getArchiveFileSize(const zim::Archive& archive);
 
     zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);

src/tools/concurrent_cache.h

@@ -0,0 +1,208 @@
+
+/*
+ * Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (C) 2020 Veloman Yunkan
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
+ * NON-INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifndef ZIM_CONCURRENT_CACHE_H
+#define ZIM_CONCURRENT_CACHE_H
+
+#include "lrucache.h"
+
+#include <future>
+#include <mutex>
+
+namespace kiwix
+{
+
+/**
+   ConcurrentCache implements a concurrent thread-safe cache
+
+   Compared to kiwix::lru_cache, each access operation is slightly more expensive.
+   However, different slots of the cache can be safely accessed concurrently
+   with minimal blocking. Concurrent access to the same element is also
+   safe, and, in case of a cache miss, will block until that element becomes
+   available.
+ */
+template <typename Key, typename Value>
+class ConcurrentCache
+{
+private: // types
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      try {
+        valuePromise.set_value(f());
+      } catch (std::exception& e) {
+        drop(key);
+        throw;
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  Impl impl_;
+  std::mutex lock_;
+};
+
+
+/**
+  WeakStore represent a thread safe store (map) of weak ptr.
+  It allows to store weak_ptr from shared_ptr and retrieve shared_ptr from
+  potential non expired weak_ptr.
+  It is not limited in size.
+*/
+template<typename Key, typename Value>
+class WeakStore {
+  private: // types
+    typedef std::weak_ptr<Value> WeakValue;
+
+  public:
+    explicit WeakStore() = default;
+
+    std::shared_ptr<Value> get(const Key& key)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      auto it = m_weakMap.find(key);
+      if (it != m_weakMap.end()) {
+        auto shared = it->second.lock();
+        if (shared) {
+          return shared;
+        } else {
+          m_weakMap.erase(it);
+        }
+      }
+      throw std::runtime_error("No weak ptr");
+    }
+
+    void add(const Key& key, std::shared_ptr<Value> shared)
+    {
+      std::lock_guard<std::mutex> l(m_lock);
+      m_weakMap[key] = WeakValue(shared);
+    }
+
+
+  private: //data
+    std::map<Key, WeakValue> m_weakMap;
+    std::mutex m_lock;
+};
+
+template <typename Key, typename RawValue>
+class ConcurrentCache<Key, std::shared_ptr<RawValue>>
+{
+private: // types
+  typedef std::shared_ptr<RawValue> Value;
+  typedef std::shared_future<Value> ValuePlaceholder;
+  typedef lru_cache<Key, ValuePlaceholder> Impl;
+
+public: // types
+  explicit ConcurrentCache(size_t maxEntries)
+    : impl_(maxEntries)
+  {}
+
+  // Gets the entry corresponding to the given key. If the entry is not in the
+  // cache, it is obtained by calling f() (without any arguments) and the
+  // result is put into the cache.
+  //
+  // The cache as a whole is locked only for the duration of accessing
+  // the respective slot. If, in the case of the a cache miss, the generation
+  // of the missing element takes a long time, only attempts to access that
+  // element will block - the rest of the cache remains open to concurrent
+  // access.
+  template<class F>
+  Value getOrPut(const Key& key, F f)
+  {
+    std::promise<Value> valuePromise;
+    std::unique_lock<std::mutex> l(lock_);
+    const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
+    l.unlock();
+    if ( x.miss() ) {
+      // Try to get back the shared_ptr from the weak_ptr first.
+      try {
+        valuePromise.set_value(m_weakStore.get(key));
+      } catch(const std::runtime_error& e) {
+        try {
+          const auto value = f();
+          valuePromise.set_value(value);
+          m_weakStore.add(key, value);
+        } catch (std::exception& e) {
+          drop(key);
+          throw;
+        }
+      }
+    }
+
+    return x.value().get();
+  }
+
+  bool drop(const Key& key)
+  {
+    std::unique_lock<std::mutex> l(lock_);
+    return impl_.drop(key);
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    std::unique_lock<std::mutex> l(lock_);
+    return  impl_.setMaxSize(new_size);
+  }
+
+protected: // data
+  std::mutex lock_;
+  Impl impl_;
+  WeakStore<Key, RawValue> m_weakStore;
+};
+
+} // namespace kiwix
+
+#endif // ZIM_CONCURRENT_CACHE_H
+

src/tools/lrucache.h

@@ -0,0 +1,175 @@
+/*
+ * Copyrigth (c) 2021, Matthieu Gautier <mgautier@kymeria.fr>
+ * Copyright (c) 2020, Veloman Yunkan
+ * Copyright (c) 2014, lamerman
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of lamerman nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * File:   lrucache.hpp
+ * Author: Alexander Ponomarev
+ *
+ * Created on June 20, 2013, 5:09 PM
+ */
+
+#ifndef _LRUCACHE_HPP_INCLUDED_
+#define _LRUCACHE_HPP_INCLUDED_
+
+#include <map>
+#include <list>
+#include <set>
+#include <cstddef>
+#include <stdexcept>
+#include <cassert>
+
+namespace kiwix {
+
+template<typename key_t, typename value_t>
+class lru_cache {
+public: // types
+  typedef typename std::pair<key_t, value_t> key_value_pair_t;
+  typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+
+  enum AccessStatus {
+    HIT, // key was found in the cache
+    PUT, // key was not in the cache but was created by the getOrPut() access
+    MISS // key was not in the cache; get() access failed
+  };
+
+  class AccessResult
+  {
+    const AccessStatus status_;
+    const value_t val_;
+  public:
+    AccessResult(const value_t& val, AccessStatus status)
+      : status_(status), val_(val)
+    {}
+    AccessResult() : status_(MISS), val_() {}
+
+    bool hit() const { return status_ == HIT; }
+    bool miss() const { return !hit(); }
+    const value_t& value() const
+    {
+      if ( status_ == MISS )
+        throw std::range_error("There is no such key in cache");
+      return val_;
+    }
+
+    operator const value_t& () const { return value(); }
+  };
+
+public: // functions
+  explicit lru_cache(size_t max_size) :
+    _max_size(max_size) {
+  }
+
+  // If 'key' is present in the cache, returns the associated value,
+  // otherwise puts the given value into the cache (and returns it with
+  // a status of a cache miss).
+  AccessResult getOrPut(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    } else {
+      putMissing(key, value);
+      return AccessResult(value, PUT);
+    }
+  }
+
+  void put(const key_t& key, const value_t& value) {
+    auto it = _cache_items_map.find(key);
+    if (it != _cache_items_map.end()) {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      it->second->second = value;
+    } else {
+      putMissing(key, value);
+    }
+  }
+
+  AccessResult get(const key_t& key) {
+    auto it = _cache_items_map.find(key);
+    if (it == _cache_items_map.end()) {
+      return AccessResult();
+    } else {
+      _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+      return AccessResult(it->second->second, HIT);
+    }
+  }
+
+  bool drop(const key_t& key) {
+    try {
+      auto list_it = _cache_items_map.at(key);
+      _cache_items_list.erase(list_it);
+      _cache_items_map.erase(key);
+      return true;
+    } catch (std::out_of_range& e) {
+      return false;
+    }
+  }
+
+  bool exists(const key_t& key) const {
+    return _cache_items_map.find(key) != _cache_items_map.end();
+  }
+
+  size_t size() const {
+    return _cache_items_map.size();
+  }
+
+  size_t setMaxSize(size_t new_size) {
+    size_t previous = _max_size;
+    _max_size = new_size;
+    return previous;
+  }
+
+  std::set<key_t> keys() const  {
+    std::set<key_t> keys;
+    for(auto& item:_cache_items_map) {
+      keys.insert(item.first);
+    }
+    return keys;
+  }
+
+private: // functions
+  void putMissing(const key_t& key, const value_t& value) {
+    assert(_cache_items_map.find(key) == _cache_items_map.end());
+    _cache_items_list.push_front(key_value_pair_t(key, value));
+    _cache_items_map[key] = _cache_items_list.begin();
+    while (_cache_items_map.size() > _max_size) {
+      _cache_items_map.erase(_cache_items_list.back().first);
+      _cache_items_list.pop_back();
+    }
+  }
+
+private: // data
+  std::list<key_value_pair_t> _cache_items_list;
+  std::map<key_t, list_iterator_t> _cache_items_map;
+  size_t _max_size;
+};
+
+} // namespace kiwix
+
+#endif  /* _LRUCACHE_HPP_INCLUDED_ */

src/tools/otherTools.cpp

@@ -370,6 +370,13 @@ std::string kiwix::gen_uuid(const std::string& s)
   return kiwix::to_string(zim::Uuid::generate(s));
 }
 
+kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
+{
+  return s.empty()
+       ? kainjow::mustache::data(false)
+       : kainjow::mustache::data(s);
+}
+
 std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
 {
   kainjow::mustache::mustache tmpl(template_str);

src/tools/otherTools.h

@@ -23,9 +23,12 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <cstdlib>
 #include <zim/zim.h>
 #include <mustache.hpp>
 
+#include "stringTools.h"
+
 namespace pugi {
   class xml_node;
 }
@@ -48,7 +51,25 @@ namespace kiwix
   std::string gen_date_str();
   std::string gen_uuid(const std::string& s);
 
+  // if s is empty then returns kainjow::mustache::data(false)
+  // otherwise kainjow::mustache::data(value)
+  kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
+
   std::string render_template(const std::string& template_str, kainjow::mustache::data data);
+
+  template<typename T>
+  T getEnvVar(const char* name, const T& defaultValue)
+  {
+    try {
+      const char* envString = std::getenv(name);
+      if (envString == nullptr) {
+        throw std::runtime_error("Environment variable not set");
+      }
+      return extractFromString<T>(envString);
+    } catch (...) {}
+
+    return defaultValue;
+  }
 }
 
 #endif

src/tools/stringTools.cpp

@@ -405,3 +405,8 @@ std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
   variants.push_back(kiwix::toTitle(title));
   return variants;
 }
+
+template<>
+std::string kiwix::extractFromString(const std::string& str) {
+  return str;
+}

src/tools/stringTools.h

@@ -25,6 +25,7 @@
 #include <string>
 #include <vector>
 #include <sstream>
+#include <stdexcept>
 
 namespace kiwix
 {
@@ -65,9 +66,15 @@ T extractFromString(const std::string& str) {
     std::istringstream iss(str);
     T ret;
     iss >> ret;
+    if(iss.fail() || !iss.eof()) {
+        throw std::invalid_argument("no conversion");
+    }
     return ret;
 }
 
+template<>
+std::string extractFromString(const std::string& str);
+
 bool startsWith(const std::string& base, const std::string& start);
 
 std::vector<std::string> getTitleVariants(const std::string& title);

src/wrapper/java/AndroidManifest.xml

@@ -1,13 +0,0 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-
-  package="kiwix.org.kiwixlib"
->
-
-  <application android:allowBackup="true"
-    android:label="@string/app_name"
-    android:supportsRtl="true"
-  >
-
-  </application>
-
-</manifest>

src/wrapper/java/book.cpp

@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Book.h"
-
-#include "utils.h"
-#include "book.h"
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_allocate(
-  JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Book>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Book>(env, thisObj);
-}
-
-#define BOOK (getPtr<kiwix::Book>(env, thisObj))
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
-{
-  BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
-}
-
-METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
-{
-  BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
-}
-
-#define GETTER(retType, name) JNIEXPORT retType JNICALL \
-Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
-{ \
-  auto cRet = BOOK->name(); \
-  retType ret = c2jni(cRet, env); \
-  return ret; \
-}
-
-GETTER(jstring, getId)
-GETTER(jstring, getPath)
-GETTER(jboolean, isPathValid)
-GETTER(jstring, getTitle)
-GETTER(jstring, getDescription)
-GETTER(jstring, getLanguage)
-GETTER(jstring, getCreator)
-GETTER(jstring, getPublisher)
-GETTER(jstring, getDate)
-GETTER(jstring, getUrl)
-GETTER(jstring, getName)
-GETTER(jstring, getFlavour)
-GETTER(jstring, getCategory)
-GETTER(jstring, getTags)
-GETTER(jlong, getArticleCount)
-GETTER(jlong, getMediaCount)
-GETTER(jlong, getSize)
-GETTER(jstring, getFavicon)
-GETTER(jstring, getFaviconUrl)
-GETTER(jstring, getFaviconMimeType)
-
-METHOD(jstring, Book, getTagStr, jstring tagName) try {
-  auto cRet = BOOK->getTagStr(jni2c(tagName, env));
-  return c2jni(cRet, env);
-} catch(...) {
-  return c2jni<std::string>("", env);
-}
-
-#undef GETTER

src/wrapper/java/filter.cpp

@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Filter.h"
-
-#include "library.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-METHOD0(void, Filter, allocate) {
-  allocate<kiwix::Filter>(env, thisObj);
-}
-
-METHOD0(void, Filter, dispose) {
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define FILTER (getPtr<kiwix::Filter>(env, thisObj))
-
-#define FORWARD(name, args_type) \
-METHOD(jobject, Filter, name, args_type value) { \
-  FILTER->name(jni2c(value, env)); \
-  return thisObj; \
-}
-
-#define FORWARDA(name, args_type) \
-METHOD(jobject, Filter, name, jobjectArray value) { \
-  FILTER->name(jni2c<args_type>(value, env)); \
-  return thisObj; \
-}
-
-
-
-FORWARD(local, jboolean)
-FORWARD(remote, jboolean)
-FORWARD(valid, jboolean)
-FORWARDA(acceptTags, jstring)
-FORWARDA(rejectTags, jstring)
-FORWARD(lang, jstring)
-FORWARD(publisher, jstring)
-FORWARD(creator, jstring)
-FORWARD(maxSize, jlong)
-FORWARD(query, jstring)
-
-

src/wrapper/java/kiwixicu.cpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIICU.h"
-
-#include <iostream>
-#include <string>
-
-#include "unicode/putil.h"
-
-#include "utils.h"
-
-std::mutex globalLock;
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIICU_setDataDirectory(
-    JNIEnv* env, jclass kclass, jstring dirStr)
-{
-  std::string cPath = jni2c(dirStr, env);
-
-  Lock l;
-  try {
-    u_setDataDirectory(cPath.c_str());
-  } catch (...) {
-    std::cerr << "Unable to set data directory " << cPath << std::endl;
-  }
-}

src/wrapper/java/kiwixreader.cpp

@@ -1,561 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include <exception>
-#include "org_kiwix_kiwixlib_JNIKiwixReader.h"
-
-#include "tools/base64.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader(
-    JNIEnv* env, jobject obj, jstring filename)
-{
-  std::string cPath = jni2c(filename, env);
-
-  LOG("Attempting to create reader with: %s", cPath.c_str());
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(cPath);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-namespace
-{
-
-int jni2fd(const jobject& fdObj, JNIEnv* env)
-{
-  jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
-  jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I");
-  if ( field_fd == NULL )
-  {
-    env->ExceptionClear();
-    // Under Android the (private) 'fd' field of java.io.FileDescriptor has been
-    // renamed to 'descriptor'. See, for example,
-    // https://android.googlesource.com/platform/libcore/+/refs/tags/android-8.1.0_r1/ojluni/src/main/java/java/io/FileDescriptor.java#55
-    field_fd = env->GetFieldID(class_fdesc, "descriptor", "I");
-  }
-  return env->GetIntField(fdObj, field_fd);
-}
-
-} // unnamed namespace
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderByFD(
-    JNIEnv* env, jobject obj, jobject fdObj)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderByFD() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderEmbedded(
-    JNIEnv* env, jobject obj, jobject fdObj, jlong offset, jlong size)
-{
-#ifndef _WIN32
-  int fd = jni2fd(fdObj, env);
-
-  LOG("Attempting to create reader with fd: %d", fd);
-  Lock l;
-  try {
-    kiwix::Reader* reader = new kiwix::Reader(fd, offset, size);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Reader>(reader));
-  } catch (std::exception& e) {
-    LOG("Error opening ZIM file");
-    LOG(e.what());
-    return 0;
-  }
-#else
-  jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
-  env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderEmbedded() is not supported under Windows");
-  return 0;
-#endif
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Reader>::dispose(env, obj);
-}
-
-#define READER (Handle<kiwix::Reader>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMainPage(JNIEnv* env, jobject obj)
-{
-  jstring url;
-
-  try {
-    std::string cUrl = READER->getMainPage().getPath();
-    url = c2jni(cUrl, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM main page");
-    LOG(e.what());
-    url = NULL;
-  }
-  return url;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getId(JNIEnv* env, jobject obj)
-{
-  jstring id;
-
-  try {
-    std::string cId = READER->getId();
-    id = c2jni(cId, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-    id = NULL;
-  }
-
-  return id;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFileSize(JNIEnv* env, jobject obj)
-{
-  jint size = 0;
-
-  try {
-    int cSize = READER->getFileSize();
-    size = c2jni(cSize, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return size;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getCreator(JNIEnv* env, jobject obj)
-{
-  jstring creator;
-
-  try {
-    std::string cCreator = READER->getCreator();
-    creator = c2jni(cCreator, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM creator");
-    LOG(e.what());
-    creator = NULL;
-  }
-
-  return creator;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPublisher(JNIEnv* env, jobject obj)
-{
-  jstring publisher;
-
-  try {
-    std::string cPublisher = READER->getPublisher();
-    publisher = c2jni(cPublisher, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM publish");
-    LOG(e.what());
-    publisher = NULL;
-  }
-  return publisher;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getName(JNIEnv* env, jobject obj)
-{
-  jstring name;
-
-  try {
-    std::string cName = READER->getName();
-    name = c2jni(cName, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM name");
-    LOG(e.what());
-    name = NULL;
-  }
-  return name;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getFavicon(JNIEnv* env, jobject obj)
-{
-  jstring favicon;
-
-  try {
-    std::string cContent;
-    std::string cMime;
-    READER->getFavicon(cContent, cMime);
-    favicon = c2jni(
-        base64_encode(cContent),
-        env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM favicon");
-    LOG(e.what());
-    favicon = NULL;
-  }
-  return favicon;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDate(JNIEnv* env, jobject obj)
-{
-  jstring date;
-
-  try {
-    std::string cDate = READER->getDate();
-    date = c2jni(cDate, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM date");
-    LOG(e.what());
-    date = NULL;
-  }
-  return date;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getLanguage(JNIEnv* env, jobject obj)
-{
-  jstring language;
-
-  try {
-    std::string cLanguage = READER->getLanguage();
-    language = c2jni(cLanguage, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM language");
-    LOG(e.what());
-    language = NULL;
-  }
-
-  return language;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getMimeType(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  jstring mimeType;
-
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    auto cMimeType = entry.getMimetype();
-    mimeType = c2jni(cMimeType, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get mime-type for url: %s", cUrl.c_str());
-    LOG(e.what());
-    mimeType = NULL;
-  }
-  return mimeType;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_checkUrl(
-     JNIEnv* env, jobject obj, jstring url)
-{
-  jstring finalUrl;
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    finalUrl = c2jni(entry.getPath(), env);
-  } catch (std::exception& e) {
-    finalUrl = c2jni(std::string(), env);
-  }
-  return finalUrl;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContent(
-    JNIEnv* env, jobject obj, jobject url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
-{
-  /* Default values */
-  setStringObjValue("", titleObj, env);
-  setStringObjValue("", mimeTypeObj, env);
-  setIntObjValue(0, sizeObj, env);
-  jbyteArray data = env->NewByteArray(0);
-
-  /* Retrieve the content */
-  std::string cUrl = getStringObjValue(url, env);
-  unsigned int cSize = 0;
-
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    bool isRedirect = entry.isRedirect();
-    entry = entry.getFinalEntry();
-    cSize = entry.getSize();
-    setIntObjValue(cSize, sizeObj, env);
-    setStringObjValue(entry.getMimetype(), mimeTypeObj, env);
-    setStringObjValue(entry.getTitle(), titleObj, env);
-    if (isRedirect) {
-      setStringObjValue(entry.getPath(), url, env);
-    } else {
-      data = env->NewByteArray(cSize);
-      env->SetByteArrayRegion(
-          data, 0, cSize, reinterpret_cast<const jbyte*>(entry.getBlob().data()));
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get content for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-
-  return data;
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPart(
-    JNIEnv* env, jobject obj, jstring url, jint offset, jint len, jobject sizeObj)
-{
-  jbyteArray data = env->NewByteArray(0);
-  setIntObjValue(0, sizeObj, env);
-
-  /* Default values */
-  /* Retrieve the content */
-  std::string cUrl = jni2c(url, env);
-  unsigned int cOffset = jni2c(offset, env);
-  unsigned int cLen = jni2c(len, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-
-    if (cLen == 0) {
-      setIntObjValue(entry.getSize(), sizeObj, env);
-    } else if (cOffset+cLen < entry.getSize()) {
-      auto blob = entry.getBlob(cOffset, cLen);
-      data = env->NewByteArray(cLen);
-      env->SetByteArrayRegion(
-          data, 0, cLen, reinterpret_cast<const jbyte*>(blob.data()));
-      setIntObjValue(cLen, sizeObj, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get partial content for url: %s (%u : %u)", cUrl.c_str(), cOffset, cLen);
-    LOG(e.what());
-  }
-  return data;
-}
-
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
-    JNIEnv* env, jobject obj, jstring url)
-{
-  std::string cUrl = jni2c(url, env);
-  try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    return c2jni(entry.getSize(), env);
-  } catch(std::exception& e) {
-    LOG("Unable to get size for url : %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return c2jni(0, env);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
-    JNIEnv* env, jobject obj, jstring url)
-{
-   jclass daiClass = env->FindClass("org/kiwix/kiwixlib/DirectAccessInfo");
-   jmethodID daiInitMethod = env->GetMethodID(daiClass, "<init>", "()V");
-   jobject dai = env->NewObject(daiClass, daiInitMethod);
-   setDaiObjValue("", 0, dai, env);
-
-   std::string cUrl = jni2c(url, env);
-   try {
-    auto entry = READER->getEntryFromEncodedPath(cUrl);
-    entry = entry.getFinalEntry();
-    auto part_info = entry.getDirectAccessInfo();
-    setDaiObjValue(part_info.first, part_info.second, dai, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get direct access info for url: %s", cUrl.c_str());
-    LOG(e.what());
-  }
-  return dai;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
-                                                         jobject obj,
-                                                         jstring prefix,
-                                                         jint count)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cPrefix = jni2c(prefix, env);
-  unsigned int cCount = jni2c(count, env);
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
-                                                         jobject obj,
-                                                         jobject titleObj,
-                                                         jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cTitle;
-  std::string cUrl;
-
-  try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    if (READER->getNextSuggestion(cTitle, cUrl)) {
-      setStringObjValue(cTitle, titleObj, env);
-      setStringObjValue(cUrl, urlObj, env);
-      retVal = JNI_TRUE;
-    }
-#pragma GCC diagnostic pop
-  } catch (std::exception& e) {
-    LOG("Unable to get next suggestion");
-    LOG(e.what());
-  }
-
-  return retVal;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getPageUrlFromTitle(JNIEnv* env,
-                                                           jobject obj,
-                                                           jstring title,
-                                                           jobject urlObj)
-{
-  std::string cTitle = jni2c(title, env);
-
-  try {
-    auto entry = READER->getEntryFromTitle(cTitle);
-    entry = entry.getFinalEntry();
-    setStringObjValue(entry.getPath(), urlObj, env);
-    return JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get url for title %s: ", cTitle.c_str());
-    LOG(e.what());
-  }
-
-  return JNI_FALSE;
-}
-
-JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getTitle(
-    JNIEnv* env, jobject obj)
-{
-  jstring title;
-
-  try {
-    std::string cTitle = READER->getTitle();
-    title = c2jni(cTitle, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim title");
-    LOG(e.what());
-    title = NULL;
-  }
-  return title;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getDescription(JNIEnv* env, jobject obj)
-{
-  jstring description;
-
-  try {
-    std::string cDescription = READER->getDescription();
-    description = c2jni(cDescription, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get zim description");
-    LOG(e.what());
-    description = NULL;
-  }
-  return description;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleCount(JNIEnv* env, jobject obj)
-{
-  jint articleCount = 0;
-  try {
-    auto cArticleCount = READER->getArticleCount();
-    articleCount = c2jni(cArticleCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get article count.");
-    LOG(e.what());
-  }
-  return articleCount;
-}
-
-JNIEXPORT jint JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixReader_getMediaCount(JNIEnv* env, jobject obj)
-{
-  jint mediaCount = 0;
-  try {
-    auto cMediaCount = READER->getMediaCount();
-    mediaCount = c2jni(cMediaCount, env);
-  } catch (std::exception& e) {
-    LOG("Unable to get media count.");
-    LOG(e.what());
-  }
-  return mediaCount;
-}
-
-
-JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getRandomPage(
-    JNIEnv* env, jobject obj, jobject urlObj)
-{
-  jboolean retVal = JNI_FALSE;
-  std::string cUrl;
-
-  try {
-    std::string cUrl = READER->getRandomPage().getPath();
-    setStringObjValue(cUrl, urlObj, env);
-    retVal = JNI_TRUE;
-  } catch (std::exception& e) {
-    LOG("Unable to get random page");
-    LOG(e.what());
-  }
-  return retVal;
-}

src/wrapper/java/kiwixsearcher.cpp

@@ -1,123 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher.h"
-#include "org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h"
-
-#include "reader.h"
-#include "searcher.h"
-#include "utils.h"
-
-#define SEARCHER (Handle<kiwix::Searcher>::getHandle(env, obj))
-#define RESULT (Handle<kiwix::Result>::getHandle(env, obj))
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Searcher>::dispose(env, obj);
-}
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNativeHandle(JNIEnv* env,
-                                                          jobject obj)
-{
-  kiwix::Searcher* searcher = new kiwix::Searcher();
-  return reinterpret_cast<jlong>(new Handle<kiwix::Searcher>(searcher));
-}
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
-    JNIEnv* env, jobject obj, jobject reader)
-{
-  auto searcher = SEARCHER;
-
-  searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)));
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(
-    JNIEnv* env, jobject obj, jstring query, jint count)
-{
-  std::string cquery = jni2c(query, env);
-  unsigned int ccount = jni2c(count, env);
-
-  SEARCHER->search(cquery, 0, ccount);
-}
-
-JNIEXPORT jobject JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_getNextResult(JNIEnv* env,
-                                                         jobject obj)
-{
-  jobject result = nullptr;
-
-  kiwix::Result* cresult = SEARCHER->getNextResult();
-  if (cresult != nullptr) {
-    jclass resultclass
-        = env->FindClass("org/kiwix/kiwixlib/JNIKiwixSearcher$Result");
-    jmethodID ctor = env->GetMethodID(
-        resultclass, "<init>", "(Lorg/kiwix/kiwixlib/JNIKiwixSearcher;JLorg/kiwix/kiwixlib/JNIKiwixSearcher;)V");
-    result = env->NewObject(resultclass, ctor, obj, reinterpret_cast<jlong>(new Handle<kiwix::Result>(cresult)), obj);
-  }
-  return result;
-}
-
-JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_dispose(
-    JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Result>::dispose(env, obj);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getUrl(JNIEnv* env,
-                                                        jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_url(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getTitle(JNIEnv* env,
-                                                          jobject obj)
-{
-  try {
-    return c2jni(RESULT->get_title(), env);
-  } catch (...) {
-    return nullptr;
-  }
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getSnippet(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_snippet(), env);
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixSearcher_00024Result_getContent(JNIEnv* env,
-                                                            jobject obj)
-{
-  return c2jni(RESULT->get_content(), env);
-}

src/wrapper/java/kiwixserver.cpp

@@ -1,103 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_JNIKiwixServer.h"
-
-#include "tools/base64.h"
-#include "server.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
-    JNIEnv* env, jobject obj, jobject jLibrary)
-{
-  LOG("Attempting to create server");
-  Lock l;
-  try {
-    auto library = getPtr<kiwix::Library>(env, jLibrary);
-    kiwix::Server* server = new kiwix::Server(library);
-    return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
-  } catch (std::exception& e) {
-    LOG("Error creating the server");
-    LOG(e.what());
-    return 0;
-  }
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_dispose(JNIEnv* env, jobject obj)
-{
-  Handle<kiwix::Server>::dispose(env, obj);
-}
-
-#define SERVER (Handle<kiwix::Server>::getHandle(env, obj))
-
-/* Kiwix library functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setRoot(JNIEnv* env, jobject obj, jstring jRoot)
-{
-  std::string root = jni2c(jRoot, env);
-  SERVER->setRoot(root);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setAddress(JNIEnv* env, jobject obj, jstring jAddress)
-{
-  std::string address = jni2c(jAddress, env);
-  SERVER->setAddress(address);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setPort(JNIEnv* env, jobject obj, int port)
-{
-  SERVER->setPort(port);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setNbThreads(JNIEnv* env, jobject obj, int threads)
-{
-  SERVER->setNbThreads(threads);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboolean withTaskbar, jboolean withLibraryButton)
-{
-  SERVER->setTaskbar(withTaskbar, withLibraryButton);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
-{
-  SERVER->setBlockExternalLinks(blockExternalLinks);
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
-{
-  return SERVER->start();
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_JNIKiwixServer_stop(JNIEnv* env, jobject obj)
-{
-  SERVER->stop();
-}

src/wrapper/java/library.cpp

@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Library.h"
-
-#include "library.h"
-#include "reader.h"
-#include "utils.h"
-
-/* Kiwix Reader JNI functions */
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_allocate(
-    JNIEnv* env, jobject thisObj)
-{
-  allocate<kiwix::Library>(env, thisObj);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Library_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Library>(env, thisObj);
-}
-
-#define LIBRARY (getPtr<kiwix::Library>(env, thisObj))
-
-/* Kiwix library functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Library_addBook(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    kiwix::Reader reader(cPath);
-    kiwix::Book book;
-    book.update(reader);
-    return LIBRARY->addBook(book);
-  } catch (std::exception& e) {
-    LOG("Unable to add the book");
-    LOG(e.what()); }
-  return false;
-}
-
-METHOD(jobject, Library, getBookById, jstring id) {
-  auto cId = jni2c(id, env);
-  auto cBook = new kiwix::Book(LIBRARY->getBookById(cId));
-  jclass cls = env->FindClass("org/kiwix/kiwixlib/Book");
-  jmethodID constructorId = env->GetMethodID(cls, "<init>", "()V");
-  jobject book = env->NewObject(cls, constructorId);
-  setPtr(env, book, cBook);
-  return book;
-}
-
-METHOD(jint, Library, getBookCount, jboolean localBooks, jboolean remoteBooks) {
-  return LIBRARY->getBookCount(localBooks, remoteBooks);
-}
-
-METHOD0(jobjectArray, Library, getBooksIds) {
-  return c2jni(LIBRARY->getBooksIds(), env);
-}
-
-METHOD(jobjectArray, Library, filter, jobject filterObj) {
-  auto filter = getPtr<kiwix::Filter>(env, filterObj);
-  return c2jni(LIBRARY->filter(*filter), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksLanguages) {
-  return c2jni(LIBRARY->getBooksLanguages(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksCreators) {
-  return c2jni(LIBRARY->getBooksCreators(), env);
-}
-
-METHOD0(jobjectArray, Library, getBooksPublisher) {
-  return c2jni(LIBRARY->getBooksPublishers(), env);
-}
-

src/wrapper/java/manager.cpp

@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#include <jni.h>
-#include "org_kiwix_kiwixlib_Manager.h"
-
-#include "manager.h"
-#include "utils.h"
-
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_allocate(
-  JNIEnv* env, jobject thisObj, jobject libraryObj)
-{
-  auto lib = getPtr<kiwix::Library>(env, libraryObj);
-  allocate<kiwix::Manager>(env, thisObj, lib);
-}
-
-JNIEXPORT void JNICALL
-Java_org_kiwix_kiwixlib_Manager_dispose(JNIEnv* env, jobject thisObj)
-{
-  dispose<kiwix::Manager>(env, thisObj);
-}
-
-#define MANAGER (getPtr<kiwix::Manager>(env, thisObj))
-
-/* Kiwix manager functions */
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get readFile");
-    LOG(e.what());
-  }
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readXml(
-  JNIEnv* env, jobject thisObj, jstring content, jstring libraryPath)
-{
-  auto cContent = jni2c(content, env);
-  auto cPath = jni2c(libraryPath, env);
-
-  try {
-    return MANAGER->readXml(cContent, false, cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readOpds(
-  JNIEnv* env, jobject thisObj, jstring content, jstring urlHost)
-{
-  auto cContent = jni2c(content, env);
-  auto cUrl = jni2c(urlHost, env);
-
-  try {
-    return MANAGER->readOpds(cContent, cUrl);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jboolean JNICALL
-Java_org_kiwix_kiwixlib_Manager_readBookmarkFile(
-  JNIEnv* env, jobject thisObj, jstring path)
-{
-  auto cPath = jni2c(path, env);
-
-  try {
-    return MANAGER->readBookmarkFile(cPath);
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM id");
-    LOG(e.what());
-  }
-
-  return false;
-}
-
-JNIEXPORT jstring JNICALL
-Java_org_kiwix_kiwixlib_Manager_addBookFromPath(
-  JNIEnv* env, jobject thisObj,
-  jstring pathToOpen, jstring pathToSave, jstring url, jboolean checkMetaData)
-{
-  auto cPathToOpen = jni2c(pathToOpen, env);
-  auto cPathToSave = jni2c(pathToSave, env);
-  auto cUrl = jni2c(url, env);
-  jstring id = NULL;
-
-  try {
-    auto cId = MANAGER->addBookFromPathAndGetId(cPathToOpen, cPathToSave, cUrl, checkMetaData);
-    if ( !cId.empty() ) {
-      id = c2jni(cId, env);
-    }
-  } catch (std::exception& e) {
-    LOG("Unable to get ZIM file size");
-    LOG(e.what());
-  }
-
-  return id;
-}

src/wrapper/java/meson.build

@@ -1,55 +0,0 @@
-
-java_sources = files([
-  'org/kiwix/kiwixlib/JNIICU.java',
-  'org/kiwix/kiwixlib/Book.java',
-  'org/kiwix/kiwixlib/JNIKiwixReader.java',
-  'org/kiwix/kiwixlib/Library.java',
-  'org/kiwix/kiwixlib/Manager.java',
-  'org/kiwix/kiwixlib/Filter.java',
-  'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
-  'org/kiwix/kiwixlib/JNIKiwixServer.java',
-  'org/kiwix/kiwixlib/JNIKiwixInt.java',
-  'org/kiwix/kiwixlib/JNIKiwixString.java',
-  'org/kiwix/kiwixlib/JNIKiwixBool.java',
-  'org/kiwix/kiwixlib/JNIKiwixException.java',
-  'org/kiwix/kiwixlib/DirectAccessInfo.java'
-])
-
-kiwix_jni = custom_target('jni',
-  input: java_sources,
-  output: ['org_kiwix_kiwixlib_JNIKiwix.h',
-           'org_kiwix_kiwixlib_Book.h',
-           'org_kiwix_kiwixlib_JNIKiwixReader.h',
-           'org_kiwix_kiwixlib_Library.h',
-           'org_kiwix_kiwixlib_Manager.h',
-           'org_kiwix_kiwixlib_Filter.h',
-           'org_kiwix_kiwixlib_JNIKiwixServer.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
-           'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
-  command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
-)
-
-jni_sources = files([
-     'kiwixicu.cpp',
-     'book.cpp',
-     'kiwixreader.cpp',
-     'library.cpp',
-     'manager.cpp',
-     'filter.cpp',
-     'kiwixsearcher.cpp',
-     'kiwixserver.cpp',
-])
-
-kiwix_sources += jni_sources + [kiwix_jni]
-
-if 'java' in wrapper
-  kiwix_jar = jar('kiwixlib', java_sources)
-  #junit_jar = files('org/kiwix/testing/junit-4.13.jar')
-  #test_jar = jar('testing', 'org/kiwix/testing/test.java',
-  #               link_with: [kiwix_jar, junit_jar])
-  #test('javatest', test_jar)
-endif
-
-install_subdir('org', install_dir: 'kiwix-lib/java', exclude_directories: ['kiwix/testing'])
-install_subdir('res', install_dir: 'kiwix-lib')
-install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

src/wrapper/java/org/kiwix/kiwixlib/Book.java

@@ -1,48 +0,0 @@
-
-package org.kiwix.kiwixlib;
-
-public class Book
-{
-  public Book() { allocate(); }
-
-
-  public native void update(Book book);
-  public native void update(JNIKiwixReader reader);
-
-  @Override
-  protected void finalize() { dispose();  }
-
-  public native String getId();
-  public native String getPath();
-  public native boolean isPathValid();
-  public native String getTitle();
-  public native String getDescription();
-  public native String getLanguage();
-  public native String getCreator();
-  public native String getPublisher();
-  public native String getDate();
-  public native String getUrl();
-  public native String getName();
-  public native String getFlavour();
-  public native String getCategory();
-  public native String getTags();
-  /**
-   * Return the value associated to the tag tagName
-   *
-   * @param tagName the tag name to search for.
-   * @return The value of the tag. If the tag is not found, return empty string.
-   */
-  public native String getTagStr(String tagName);
-
-  public native long getArticleCount();
-  public native long getMediaCount();
-  public native long getSize();
-
-  public native String getFavicon();
-  public native String getFaviconUrl();
-  public native String getFaviconMimeType();
-
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class DirectAccessInfo
-{
-  public String filename;
-  public long offset;
-}

src/wrapper/java/org/kiwix/kiwixlib/Filter.java

@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class Filter
-{
-
-  public native Filter local(boolean accept);
-  public native Filter remote(boolean accept);
-  public native Filter valid(boolean accept);
-  public native Filter acceptTags(String[] tags);
-  public native Filter rejectTags(String[] tags);
-  public native Filter lang(String lang);
-  public native Filter publisher(String publisher);
-  public native Filter creator(String creator);
-  public native Filter maxSize(long size);
-  public native Filter query(String query);
-
-
-  public Filter() { allocate(); }
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIICU.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIICU
-{
-  static public native void setDataDirectory(String icuDataDir);
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwix.java

@@ -1,36 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import android.content.Context;
-import com.getkeepsafe.relinker.ReLinker;
-import org.kiwix.kiwixlib.JNIICU;
-
-public class JNIKiwix
-{
-  public JNIKiwix(final Context context){
-    ReLinker.loadLibrary(context, "kiwix");
-  }
-
-  public void setDataDirectory(String icuDataDir) {
-    JNIICU.setDataDirectory(icuDataDir);
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixBool.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixBool
-{
-  public boolean value;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixException.java

@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixException extends Exception
-{
-  public JNIKiwixException(String message) {
-    super(message);
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixInt.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixInt
-{
-  public int value;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -1,182 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.JNIKiwixString;
-import org.kiwix.kiwixlib.JNIKiwixInt;
-import org.kiwix.kiwixlib.JNIKiwixSearcher;
-import org.kiwix.kiwixlib.DirectAccessInfo;
-import java.io.FileDescriptor;
-
-public class JNIKiwixReader
-{
-  public native String getMainPage();
-
-  public native String getTitle();
-
-  public native String getId();
-
-  public native String getLanguage();
-
-  public native String getMimeType(String url);
-
-  /**
-   * Check if a url exists and is a redirect or not.
-   *
-   * Return an empty string if the url doesn't exist in the reader.
-   * Return the url of the "final" entry.
-   * - equal to the input url if the entry is not a redirection.
-   * - different if the url is a redirection (and the webview should redirect to it).
-   */
-  public native String checkUrl(String url);
-
-  /**
-   * Get the content of a article.
-   *
-   * Return a byte array of the content of the article.
-   * Set the title, mimeType to the title and mimeType of the article.
-   * Set the size to the size of the returned array.
-   *
-   * If the entry doesn't exist :
-   *  - return a empty byte array
-   *  - set all arguments (except url) to empty/0.
-   * If the entry exist but is a redirection :
-   *  - return an empty byte array
-   *  - set all arguments (including url) to information of the targeted article.
-   */
-  public native byte[] getContent(JNIKiwixString url,
-                                  JNIKiwixString title,
-                                  JNIKiwixString mimeType,
-                                  JNIKiwixInt size);
-
-  /**
-   * getContentPart.
-   *
-   * Get only a part of the content of the article.
-   * Return a byte array of `len` size starting from offset `offset`.
-   * Set `size` to the number of bytes read
-   * (`len` if everything is ok, 0 in case of error).
-   * If `len` == 0, no bytes are read but `size` is set to the total size of the
-   * article.
-   */
-  public native byte[] getContentPart(String url,
-                                      int offest,
-                                      int len,
-                                      JNIKiwixInt size);
-
-  /**
-   *
-   * Get the size of an article.
-   *
-   * @param   url The url of the article.
-   * @return  The size of the final (redirections are resolved) article (in byte).
-   *          Return 0 if the article is not found.
-   */
-  public native long getArticleSize(String url);
-
-  /**
-   * getDirectAccessInformation.
-   *
-   * Return information giving where the content is located in the zim file.
-   *
-   * Some contents (binary content) are stored uncompressed in the zim file.
-   * Knowing this information, it could be interesting to directly open
-   * the zim file (or zim part) and directly read the content from it (and so
-   * bypassing the libzim).
-   *
-   * Return a `DirectAccessInfo` (filename, offset) where the content is located.
-   *
-   * If the content cannot be directly accessed (content is compressed or zim
-   * file is cut in the middle of the content), the filename is an empty string
-   * and offset is zero.
-   */
-  public native DirectAccessInfo getDirectAccessInformation(String url);
-
-  public native boolean searchSuggestions(String prefix, int count);
-
-  public native boolean getNextSuggestion(JNIKiwixString title, JNIKiwixString url);
-
-  public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
-
-  public native String getDescription();
-
-  public native String getDate();
-
-  public native String getFavicon();
-
-  public native String getCreator();
-
-  public native String getPublisher();
-
-  public native String getName();
-
-  public native int getFileSize();
-
-  public native int getArticleCount();
-
-  public native int getMediaCount();
-
-  public native boolean getRandomPage(JNIKiwixString url);
-
-  public JNIKiwixSearcher search(String query, int count)
-  {
-    JNIKiwixSearcher searcher = new JNIKiwixSearcher();
-    searcher.addKiwixReader(this);
-    searcher.search(query, count);
-    return searcher;
-  }
-
-  public JNIKiwixReader(String filename) throws JNIKiwixException
-  {
-    nativeHandle = getNativeReader(filename);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException("Cannot open zimfile "+filename);
-    }
-  }
-
-  public JNIKiwixReader(FileDescriptor fd) throws JNIKiwixException
-  {
-    nativeHandle = getNativeReaderByFD(fd);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException("Cannot open zimfile by fd "+fd.toString());
-    }
-  }
-
-  public JNIKiwixReader(FileDescriptor fd, long offset, long size)
-      throws JNIKiwixException
-  {
-    nativeHandle = getNativeReaderEmbedded(fd, offset, size);
-    if (nativeHandle == 0) {
-        throw new JNIKiwixException(String.format("Cannot open embedded zimfile (fd=%s, offset=%d, size=%d)", fd, offset, size));
-    }
-  }
-
-  public JNIKiwixReader() {
-
-  }
-  public native void dispose();
-
-  private native long getNativeReader(String filename);
-  private native long getNativeReaderByFD(FileDescriptor fd);
-  private native long getNativeReaderEmbedded(FileDescriptor fd, long offset, long size);
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixSearcher.java

@@ -1,67 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixReader;
-import java.util.Vector;
-
-public class JNIKiwixSearcher
-{
-  public class Result
-  {
-    private long nativeHandle;
-    private JNIKiwixSearcher searcher;
-    public Result(long handle, JNIKiwixSearcher _searcher)
-    {
-      nativeHandle = handle;
-      searcher = _searcher;
-    }
-    public native String getUrl();
-    public native String getTitle();
-    public native String getContent();
-    public native String getSnippet();
-    public native void dispose();
-  }
-
-  public JNIKiwixSearcher()
-  {
-    nativeHandle = getNativeHandle();
-    usedReaders = new Vector();
-  }
-  public native void dispose();
-
-  private native long getNativeHandle();
-  private long nativeHandle;
-  private Vector usedReaders;
-
-  public native void addReader(JNIKiwixReader reader);
-  public void addKiwixReader(JNIKiwixReader reader)
-  {
-    addReader(reader);
-    usedReaders.addElement(reader);
-  };
-
-  public native void search(String query, int count);
-
-  public native Result getNextResult();
-  public native boolean hasMoreResult();
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixServer.java

@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.JNIKiwixException;
-import org.kiwix.kiwixlib.Library;
-
-public class JNIKiwixServer
-{
-  public native void setRoot(String root);
-
-  public native void setAddress(String address);
-
-  public native void setPort(int port);
-
-  public native void setNbThreads(int nbTreads);
-
-  public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
-
-  public native void setBlockExternalLinks(boolean blockExternalLinks);
-
-  public native boolean start();
-
-  public native void stop();
-
-  public JNIKiwixServer(Library library)
-  {
-    nativeHandle = getNativeServer(library);
-  }
-
-  private native long getNativeServer(Library library);
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixString.java

@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-public class JNIKiwixString
-{
-  public String value;
-
-  public JNIKiwixString(String value) {
-    this.value = value;
-  }
-
-  public JNIKiwixString() {
-    this("");
-  }
-
-  public String getValue() {
-    return value;
-  }
-}

src/wrapper/java/org/kiwix/kiwixlib/Library.java

@@ -1,49 +0,0 @@
-/*
- * Copyright (C) 2019-2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.Book;
-import org.kiwix.kiwixlib.JNIKiwixException;
-
-public class Library
-{
-  public native boolean addBook(String path) throws JNIKiwixException;
-
-  public native Book getBookById(String id);
-  public native int  getBookCount(boolean localBooks, boolean remoteBooks);
-
-  public native String[] getBooksIds();
-  public native String[] filter(Filter filter);
-
-  public native String[] getBooksLanguages();
-  public native String[] getBooksCreators();
-  public native String[] getBooksPublishers();
-
-  public Library()
-  {
-    allocate();
-  }
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate();
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/kiwixlib/Manager.java

@@ -1,90 +0,0 @@
-/*
- * Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-package org.kiwix.kiwixlib;
-
-import org.kiwix.kiwixlib.Library;
-
-public class Manager
-{
-  /**
-   * Read a `library.xml` file and add books in the library.
-   *
-   * @param path The (utf8) path to the `library.xml` file.
-   * @return True if the file has been properly parsed.
-   */
-  public native boolean readFile(String path);
-
-  /**
-   * Load a library content stored in a string (at `library.xml` format).
-   *
-   * @param content The content corresponding of the library xml.
-   * @param libraryPath The library path (used to resolve relative paths)
-   * @return True if the content has been properly parsed.
-   */
-  public native boolean readXml(String content, String libraryPath);
-
-  /**
-   * Load a library content stored in a string (at OPDS stream format)
-   *
-   * @param content the content of the OPDS stream.
-   * @param urlHost the url of the stream (used to resolve relative url)
-   * @return True if the content has been properly parsed.
-   */
-  public native boolean readOpds(String content, String urlHost);
-
-  /**
-   * Load a bookmark file
-   *
-   * @param path The path of the file to read.
-   * @return True if the content has been properly parsed
-   */
-  public native boolean readBookmarkFile(String path);
-
-  /**
-   * Add a book to the library.
-   *
-   * @param pathToOpen    The path of the zim file to add.
-   * @param pathToSave    The path to store in the library in place of
-   *                      pathToOpen.
-   * @param url           The url of the book to store in the library
-   *                      (useful for kiiwix-serve catalog)
-   * @param checkMetaData Tell if we check metadata before adding a book to the
-   *                      library.
-   * @return The id of te book if the book has been added to the library.
-   *         Empty string if not.
-   */
-  public native String addBookFromPath(String pathToOpen,
-                                       String pathToSave,
-                                       String url,
-                                       boolean checkMetaData);
-
-  public Manager(Library library) {
-    allocate(library);
-    _library = library;
-  }
-
-  private Library _library;
-
-  @Override
-  protected void finalize() { dispose(); }
-  private native void allocate(Library library);
-  private native void dispose();
-  private long nativeHandle;
-}

src/wrapper/java/org/kiwix/testing/catalog.xml

@@ -1,19 +0,0 @@
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
-  <id>00000000-0000-0000-0000-000000000000</id>
-  <entry>
-    <title>Test ZIM file</title>
-    <id>urn:uuid:86c91e51-55bf-8882-464e-072aca37a3e8</id>
-    <icon>/meta?name=favicon&amp;content=small</icon>
-    <updated>2020-11-27:00::00:Z</updated>
-    <language>en</language>
-    <summary>This is a ZIM file used in libzim unit-tests</summary>
-    <tags>unit;test</tags>
-    <link type="text/html" href="/small" />
-    <author>
-      <name>Kiwix</name>
-    </author>
-    <link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="http://localhost/small.zim" length="78982" />
-    <link rel="http://opds-spec.org/image/thumbnail" type="image/png" href="/meta?name=favicon&amp;content=small" />
-  </entry>
-</feed>
-

src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/bash
-
-# This script compiles and runs the unit test to test the java wrapper.
-# This is not integrated in meson because ... this is not so easy.
-
-die()
-{
-  echo >&2 "!!! ERROR: $*"
-  exit 1
-}
-
-
-KIWIX_LIB_JAR=$1
-if [ -z $KIWIX_LIB_JAR ]
-then
-  die "You must give the path to the kiwixlib.jar as first argument"
-fi
-
-KIWIX_LIB_DIR=$2
-if [ -z $KIWIX_LIB_DIR ]
-then
-  die "You must give the path to directory containing libkiwix.so as second argument"
-fi
-
-KIWIX_LIB_JAR=$(readlink -f "$KIWIX_LIB_JAR")
-KIWIX_LIB_DIR=$(readlink -f "$KIWIX_LIB_DIR")
-TEST_SOURCE_DIR=$(dirname "$(readlink -f $0)")
-
-cd "$TEST_SOURCE_DIR"
-
-javac -g -d . -s . -cp "junit-4.13.jar:$KIWIX_LIB_JAR" test.java \
-  || die "Compilation failed"
-
-java -Djava.library.path="$KIWIX_LIB_DIR" \
-     -cp "junit-4.13.jar:hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:." \
-     org.junit.runner.JUnitCore test \
-  || die "Unit test failed"

src/wrapper/java/org/kiwix/testing/create_test_zimfiles

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-die()
-{
-  echo >&2 "!!! ERROR: $*"
-  exit 1
-}
-
-cd "$(dirname "$0")"
-rm -f small.zim
-zimwriterfs --withoutFTIndex \
-            -w main.html \
-            -f favicon.png \
-            -l en \
-            -t "Test ZIM file" \
-            -d "N/A" \
-            -c "N/A" \
-            -p "N/A" \
-            small_zimfile_data \
-            small.zim \
-&& echo 'small.zim was successfully created' \
-|| die 'Failed to create small.zim'
-
-printf "BEGINZIM" > small.zim.embedded \
-&& cat small.zim >> small.zim.embedded \
-&& printf "ENDZIM" >> small.zim.embedded \
-&& echo 'small.zim.embedded was successfully created' \
-|| die 'Failed to create small.zim.embedded'

src/wrapper/java/org/kiwix/testing/small_zimfile_data/main.html

@@ -1,11 +0,0 @@
-<html>
-  <head>
-    <meta charset="UTF-8">
-    <title>Test ZIM file</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  </head>
-
-  <body>
-    Test ZIM file
-  </body>
-</html>

src/wrapper/java/org/kiwix/testing/test.java

@@ -1,160 +0,0 @@
-
-import java.io.*;
-import java.util.*;
-import org.junit.Test;
-import static org.junit.Assert.*;
-import org.kiwix.kiwixlib.*;
-
-public class test {
-static {
-  System.loadLibrary("kiwix");
-}
-
-private static byte[] getFileContent(String path)
-throws IOException
-{
-  File file = new File(path);
-  DataInputStream in = new DataInputStream(
-                         new BufferedInputStream(
-                           new FileInputStream(file)));
-  byte[] data = new byte[(int)file.length()];
-  in.read(data);
-  return data;
-}
-
-private static byte[] getFileContentPartial(String path, int offset, int size)
-throws IOException
-{
-  File file = new File(path);
-  DataInputStream in = new DataInputStream(
-                         new BufferedInputStream(
-                           new FileInputStream(file)));
-  byte[] data = new byte[size];
-  in.skipBytes(offset);
-  in.read(data, 0, size);
-  return data;
-}
-
-private static String getTextFileContent(String path)
-throws IOException
-{
-  return new String(getFileContent(path));
-}
-
-@Test
-public void testReader()
-throws JNIKiwixException, IOException
-{
-  JNIKiwixReader reader = new JNIKiwixReader("small.zim");
-  assertEquals("Test ZIM file", reader.getTitle());
-  assertEquals(45, reader.getFileSize()); // The file size is in KiB
-  assertEquals("A/main.html", reader.getMainPage());
-  String s = getTextFileContent("small_zimfile_data/main.html");
-  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertEquals(s, new String(c));
-
-  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
-  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
-  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertTrue(Arrays.equals(faviconData, c));
-
-  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
-  assertNotEquals("", dai.filename);
-  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
-  assertTrue(Arrays.equals(faviconData, c));
-}
-
-@Test
-public void testReaderByFd()
-throws JNIKiwixException, IOException
-{
-  FileInputStream fis = new FileInputStream("small.zim");
-  JNIKiwixReader reader = new JNIKiwixReader(fis.getFD());
-  assertEquals("Test ZIM file", reader.getTitle());
-  assertEquals(45, reader.getFileSize()); // The file size is in KiB
-  assertEquals("A/main.html", reader.getMainPage());
-  String s = getTextFileContent("small_zimfile_data/main.html");
-  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertEquals(s, new String(c));
-
-  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
-  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
-  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertTrue(Arrays.equals(faviconData, c));
-
-  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
-  assertNotEquals("", dai.filename);
-  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
-  assertTrue(Arrays.equals(faviconData, c));
-}
-
-@Test
-public void testReaderWithAnEmbeddedArchive()
-throws JNIKiwixException, IOException
-{
-  File plainArchive = new File("small.zim");
-  FileInputStream fis = new FileInputStream("small.zim.embedded");
-  JNIKiwixReader reader = new JNIKiwixReader(fis.getFD(), 8, plainArchive.length());
-  assertEquals("Test ZIM file", reader.getTitle());
-  assertEquals(45, reader.getFileSize()); // The file size is in KiB
-  assertEquals("A/main.html", reader.getMainPage());
-  String s = getTextFileContent("small_zimfile_data/main.html");
-  byte[] c = reader.getContent(new JNIKiwixString("A/main.html"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertEquals(s, new String(c));
-
-  byte[] faviconData = getFileContent("small_zimfile_data/favicon.png");
-  assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png"));
-  c = reader.getContent(new JNIKiwixString("I/favicon.png"),
-                                new JNIKiwixString(),
-                                new JNIKiwixString(),
-                                new JNIKiwixInt());
-  assertTrue(Arrays.equals(faviconData, c));
-
-  DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png");
-  assertNotEquals("", dai.filename);
-  c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length);
-  assertTrue(Arrays.equals(faviconData, c));
-}
-
-@Test
-public void testLibrary()
-throws IOException
-{
-  Library lib = new Library();
-  Manager manager = new Manager(lib);
-  String content = getTextFileContent("catalog.xml");
-  manager.readOpds(content, "http://localhost");
-  assertEquals(lib.getBookCount(true, true), 1);
-  String[] bookIds = lib.getBooksIds();
-  assertEquals(bookIds.length, 1);
-  Book book = lib.getBookById(bookIds[0]);
-  assertEquals(book.getTitle(), "Test ZIM file");
-  assertEquals(book.getTags(), "unit;test");
-  assertEquals(book.getFaviconUrl(), "http://localhost/meta?name=favicon&content=small");
-  assertEquals(book.getUrl(), "http://localhost/small.zim");
-}
-
-static
-public void main(String[] args) {
-  Library lib = new Library();
-  lib.getBookCount(true, true);
-}
-
-
-
-}

src/wrapper/java/res/values/strings.xml

@@ -1,3 +0,0 @@
-<resources>
-  <string name="app_name">Kiwix Lib</string>
-</resources>

src/wrapper/java/utils.h

@@ -1,259 +0,0 @@
-/*
- * Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
- * Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-
-#ifndef _ANDROID_JNI_UTILS_H
-#define _ANDROID_JNI_UTILS_H
-
-#include <jni.h>
-
-#include <mutex>
-#include <string>
-#include <vector>
-#include <iostream>
-
-#if __ANDROID__
- #include <android/log.h>
- #define LOG(...) __android_log_print(ANDROID_LOG_ERROR, "kiwix", __VA_ARGS__)
-#else
- #define LOG(...)
-#endif
-
-extern std::mutex globalLock;
-
-template<typename T>
-void setPtr(JNIEnv* env, jobject thisObj, T* ptr)
-{
-  jclass thisClass = env->GetObjectClass(thisObj);
-  jfieldID fieldId = env->GetFieldID(thisClass, "nativeHandle", "J");
-  env->SetLongField(thisObj, fieldId, reinterpret_cast<jlong>(ptr));
-}
-
-template<typename T, typename ...Args>
-void allocate(JNIEnv* env, jobject thisObj, Args && ...args)
-{
-  T* ptr = new T(std::forward<Args>(args)...);
-  setPtr(env, thisObj, ptr);
-}
-
-template<typename T>
-T* getPtr(JNIEnv* env, jobject thisObj)
-{
-  jclass thisClass = env->GetObjectClass(thisObj);
-  jfieldID fidNumber = env->GetFieldID(thisClass, "nativeHandle", "J");
-  return reinterpret_cast<T*>(env->GetLongField(thisObj, fidNumber));
-}
-
-template<typename T>
-void dispose(JNIEnv* env, jobject thisObj)
-{
-  delete getPtr<T>(env, thisObj);
-}
-
-#define METHOD0(retType, class, name) \
-JNIEXPORT retType JNICALL Java_org_kiwix_kiwixlib_##class##_##name( \
- JNIEnv* env, jobject thisObj)
-
-#define METHOD(retType, class, name, ...) \
-JNIEXPORT retType JNICALL Java_org_kiwix_kiwixlib_##class##_##name( \
-  JNIEnv* env, jobject thisObj, __VA_ARGS__)
-
-inline jfieldID getHandleField(JNIEnv* env, jobject obj)
-{
-  jclass c = env->GetObjectClass(obj);
-  // J is the type signature for long:
-  return env->GetFieldID(c, "nativeHandle", "J");
-}
-
-inline jobjectArray createArray(JNIEnv* env, size_t length, const std::string& type_sig)
-{
-  jclass c = env->FindClass(type_sig.c_str());
-  return env->NewObjectArray(length, c, NULL);
-}
-
-class Lock : public std::unique_lock<std::mutex>
-{
- public:
-  Lock() : std::unique_lock<std::mutex>(globalLock) { }
-};
-
-template <class T>
-class LockedHandle;
-
-template <class T>
-class Handle
-{
- protected:
-  T* h;
-
- public:
-  Handle(T* h) : h(h){};
-
-  // No destructor. This must and will be handled by dispose method.
-
-  static LockedHandle<T> getHandle(JNIEnv* env, jobject obj)
-  {
-    jlong handle = env->GetLongField(obj, getHandleField(env, obj));
-    return LockedHandle<T>(reinterpret_cast<Handle<T>*>(handle));
-  }
-
-  static void dispose(JNIEnv* env, jobject obj)
-  {
-    auto lHandle = getHandle(env, obj);
-    auto handle = lHandle.h;
-    delete handle->h;
-    delete handle;
-  }
-  friend class LockedHandle<T>;
-};
-
-template <class T>
-struct LockedHandle : public Lock {
-  Handle<T>* h;
-  LockedHandle(Handle<T>* h) : h(h) {}
-  T* operator->() { return h->h; }
-  T* operator*() { return h->h; }
-  operator bool() const { return (h->h != nullptr); }
-  operator T*() const { return h->h; }
-};
-
-template<typename T>
-struct JType { };
-
-template<> struct JType<bool>{ typedef jboolean type_t; };
-template<> struct JType<int>{ typedef jint type_t; };
-template<> struct JType<long>{ typedef jlong type_t; };
-template<> struct JType<uint64_t> { typedef jlong type_t; };
-template<> struct JType<uint32_t> { typedef jlong type_t; };
-template<> struct JType<std::string>{ typedef jstring type_t; };
-template<> struct JType<std::vector<std::string>>{ typedef jobjectArray type_t; };
-
-template<typename T>
-inline typename JType<T>::type_t c2jni(const T& val, JNIEnv* env) {
-  return static_cast<typename JType<T>::type_t>(val);
-}
-
-template<>
-inline jboolean c2jni(const bool& val, JNIEnv* env) { return val ? JNI_TRUE : JNI_FALSE; }
-
-template<>
-inline jstring c2jni(const std::string& val, JNIEnv* env)
-{
-  return env->NewStringUTF(val.c_str());
-}
-
-template<>
-inline jobjectArray c2jni(const std::vector<std::string>& val, JNIEnv* env)
-{
-  auto array = createArray(env, val.size(), "java/lang/String");
-  size_t index = 0;
-  for (auto& elem: val) {
-    auto jElem = c2jni(elem, env);
-    env->SetObjectArrayElement(array, index++, jElem);
-  }
-  return array;
-}
-
-template<typename T, typename U=T>
-struct CType { };
-
-template<> struct CType<jboolean>{ typedef bool type_t; };
-template<> struct CType<jint>{ typedef int type_t; };
-template<> struct CType<jlong>{ typedef long type_t; };
-template<> struct CType<jstring>{ typedef std::string type_t; };
-
-template<typename U>
-struct CType<jobjectArray, U>{ typedef std::vector<typename CType<U>::type_t> type_t; };
-
-/* jni2c type conversion functions */
-template<typename T, typename U=T>
-inline typename CType<T>::type_t jni2c(const T& val, JNIEnv* env) {
-  return static_cast<typename CType<T>::type_t>(val);
-}
-
-template<>
-inline bool jni2c(const jboolean& val, JNIEnv* env) { return val == JNI_TRUE; }
-
-template<>
-inline std::string jni2c(const jstring& val, JNIEnv* env)
-{
-  const char* chars = env->GetStringUTFChars(val, 0);
-  std::string ret(chars);
-  env->ReleaseStringUTFChars(val, chars);
-  return ret;
-}
-
-template<typename U>
-inline typename CType<jobjectArray, U>::type_t jni2c(const jobjectArray& val, JNIEnv* env)
-{
-  jsize length = env->GetArrayLength(val);
-  typename CType<jobjectArray, U>::type_t v(length);
-
-  int i;
-  for (i = 0; i < length; i++) {
-    U obj = (U) env->GetObjectArrayElement(val, i);
-    auto cobj = jni2c<U>(obj, env);
-    v.push_back(cobj);
-  }
-  return v;
-}
-
-/* Method to deal with variable passed by reference */
-inline std::string getStringObjValue(const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
-  jstring jstr = (jstring)env->GetObjectField(obj, objFid);
-  return jni2c(jstr, env);
-}
-inline void setStringObjValue(const std::string& value,
-                              const jobject obj,
-                              JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
-  env->SetObjectField(obj, objFid, c2jni(value, env));
-}
-
-inline void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "I");
-  env->SetIntField(obj, objFid, value);
-}
-
-inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
-  env->SetIntField(obj, objFid, c2jni(value, env));
-}
-
-inline void setDaiObjValue(const std::string& filename, const long offset,
-                            const jobject obj, JNIEnv* env)
-{
-  jclass objClass = env->GetObjectClass(obj);
-  jfieldID filenameFid = env->GetFieldID(objClass, "filename", "Ljava/lang/String;");
-  env->SetObjectField(obj, filenameFid, c2jni(filename, env));
-  jfieldID offsetFid = env->GetFieldID(objClass, "offset", "J");
-  env->SetLongField(obj, offsetFid, offset);
-}
-
-#endif // _ANDROID_JNI_UTILS_H

static/ft_opensearchdescription.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+  <ShortName>Fulltext articles search</ShortName>
+  <Description>Search for articles in the Library.</Description>
+  <Url type="application/atom+xml;profile=opds-catalog"
+       xmlns:atom="http://www.w3.org/2005/Atom"
+       xmlns:k="http://kiwix.org/opensearchextension/1.0"
+       indexOffset="0"
+       template="{{root}}/search?format=xml&amp;pattern={searchTerms}&amp;books.filter.lang={language?}&amp;books.name={k:name?}&amp;pageLength={count?}&amp;start={startIndex?}"/>
+</OpenSearchDescription>

static/generate_i18n_resources_list.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 Matthieu Gautier <mgautier@kymeria.fr>
+#
+# This file is part of libkiwix.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from pathlib import Path
+
+script_path = Path(__file__)
+
+resource_file = script_path.parent / "i18n_resources_list.txt"
+translation_dir = script_path.parent / "i18n"
+
+json_files = translation_dir.glob("*.json")
+with open(resource_file, 'w', encoding="utf-8") as f:
+    for json in sorted(translation_dir.glob("*.json")):
+        if json.name == "qqq.json":
+            continue
+        f.write(str(json.relative_to(script_path.parent)) + '\n')

static/i18n/bn.json

@@ -0,0 +1,14 @@
+{
+	"@metadata": {
+		"authors": [
+			"আফতাবুজ্জামান"
+		]
+	},
+	"name": "বাংলা",
+	"404-page-heading": "পাওয়া যায়নি",
+	"500-page-title": "অভ্যন্তরীণ সার্ভার ত্রুটি",
+	"500-page-heading": "অভ্যন্তরীণ সার্ভার ত্রুটি",
+	"library-button-text": "স্বাগত পাতায় চলুন",
+	"home-button-text": "'{{BOOK_TITLE}}'-এর প্রধান পাতায় চলুন",
+	"searchbox-tooltip": "'{{BOOK_TITLE}}' অনুসন্ধান করুন"
+}

static/i18n/en.json

@@ -0,0 +1,30 @@
+{
+	"@metadata": {
+		"authors": [
+		]
+	},
+	"name":"English",
+	"suggest-full-text-search" : "containing '{{{SEARCH_TERMS}}}'..."
+	, "no-such-book" : "No such book: {{BOOK_NAME}}"
+	, "too-many-books" : "Too many books requested ({{NB_BOOKS}}) where limit is {{LIMIT}}"
+	, "no-book-found" : "No book matches selection criteria"
+	, "url-not-found" : "The requested URL \"{{url}}\" was not found on this server."
+	, "suggest-search" : "Make a full text search for <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>"
+	, "random-article-failure" :  "Oops! Failed to pick a random article :("
+	, "invalid-raw-data-type" : "{{DATATYPE}} is not a valid request for raw content."
+	, "no-value-for-arg": "No value provided for argument {{ARGUMENT}}"
+	, "no-query" : "No query provided."
+	, "raw-entry-not-found" : "Cannot find {{DATATYPE}} entry {{ENTRY}}"
+	, "400-page-title" : "Invalid request"
+	, "400-page-heading" : "Invalid request"
+	, "404-page-title" : "Content not found"
+	, "404-page-heading" : "Not Found"
+	, "500-page-title" : "Internal Server Error"
+	, "500-page-heading" : "Internal Server Error"
+	, "fulltext-search-unavailable" : "Fulltext search unavailable"
+	, "no-search-results": "The fulltext search engine is not available for this content."
+	, "library-button-text": "Go to welcome page"
+	, "home-button-text": "Go to the main page of '{{BOOK_TITLE}}'"
+	, "random-page-button-text": "Go to a randomly selected page"
+	, "searchbox-tooltip": "Search '{{BOOK_TITLE}}'"
+}

static/i18n/fr.json

@@ -0,0 +1,33 @@
+{
+	"@metadata": {
+		"authors": [
+			"Gomoko",
+			"Thibaut120094",
+			"Verdy p"
+		]
+	},
+	"name": "français",
+	"suggest-full-text-search": "contenant « {{{SEARCH_TERMS}}} »...",
+	"no-such-book": "Aucun livre avec ce nom : {{BOOK_NAME}}",
+	"too-many-books": "Trop de livres demandés ({{NB_BOOKS}}) alors que la limite est de {{LIMIT}}",
+	"no-book-found": "Aucun livre ne correspond à ces critères de sélection",
+	"url-not-found": "L’URL demandée « {{url}} » est introuvable sur ce serveur.",
+	"suggest-search": "Faire une recherche en texte intégral de « <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> »",
+	"random-article-failure": "Oups ! Échec de sélection d’un article aléatoire :(",
+	"invalid-raw-data-type": "{{DATATYPE}} n’est pas une requête valide pour du contenu brut.",
+	"no-value-for-arg": "Aucune valeur fournie pour l’argument {{ARGUMENT}}",
+	"no-query": "Aucune requête fournie.",
+	"raw-entry-not-found": "Impossible de trouver l’entrée « {{ENTRY}} » de type « {{DATATYPE}} »",
+	"400-page-title": "Requête non valide",
+	"400-page-heading": "Requête non valide",
+	"404-page-title": "Contenu non trouvé",
+	"404-page-heading": "Non trouvé",
+	"500-page-title": "Erreur interne du serveur",
+	"500-page-heading": "Erreur interne du serveur",
+	"fulltext-search-unavailable": "Recherche en texte intégral non disponible",
+	"no-search-results": "Le moteur de recherche en texte intégral n’est pas disponible pour ce contenu.",
+	"library-button-text": "Aller à la page de bienvenue",
+	"home-button-text": "Aller à la page principale de « {{BOOK_TITLE}} »",
+	"random-page-button-text": "Aller à une page sélectionnée aléatoirement",
+	"searchbox-tooltip": "Rechercher « {{BOOK_TITLE}} »"
+}

static/i18n/he.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Amire80"
+		]
+	},
+	"name": "עברית",
+	"suggest-full-text-search": "מכיל '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "אין ספר כזה: {{BOOK_NAME}}",
+	"too-many-books": "נתבקשו יותר ספרים ({{NB_BOOKS}}) והמגבלה היא {{LIMIT}}",
+	"no-book-found": "אין ספר שמתאים לתנאים שנבחרו",
+	"url-not-found": "הכתובת המבוקשת \"{{url}}\" לא נמצאה בשרת הזה.",
+	"suggest-search": "לעשות חיפוש טקסט מלא עבור <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "אוי! לא עבדה בחירת ערך אקראי :(",
+	"invalid-raw-data-type": "{{DATATYPE}} הוא לא בקשה תקינה של תוכן גולמי.",
+	"no-value-for-arg": "לא סופק ערך לארגומנט {{ARGUMENT}}",
+	"no-query": "לא סופקה שאילתה.",
+	"raw-entry-not-found": "לא ניתן למצוא את רשומת ה־{{DATATYPE}} בשם {{ENTRY}}",
+	"400-page-title": "בקשה בלתי־תקינה",
+	"400-page-heading": "בקשה בלתי־תקינה",
+	"404-page-title": "התוכן לא נמצא",
+	"404-page-heading": "לא נמצא",
+	"500-page-title": "שגיאת שרת פנימית",
+	"500-page-heading": "שגיאת שרת פנימית",
+	"fulltext-search-unavailable": "חיפוש בטקסט מלא אינו זמין",
+	"no-search-results": "מנוע החיפוש בטקסט מלא אינו זמין עבור התוכן הזה.",
+	"library-button-text": "מעבר לדף הבית \"ברוך בואך\"",
+	"home-button-text": "מעבר לדף הראשי של \"{{BOOK_TITLE}}\"",
+	"random-page-button-text": "מעבר לדף שנבחר אקראית",
+	"searchbox-tooltip": "חיפוש \"{{BOOK_TITLE}}\""
+}

static/i18n/hy.json

@@ -0,0 +1,16 @@
+{
+	"@metadata": {
+		"authors": []
+	},
+	"name": "Հայերեն",
+	"suggest-full-text-search": "որոնել '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Գիրքը բացակայում է՝ {{BOOK_NAME}}",
+	"url-not-found": "Սխալ հասցե՝ {{url}}",
+	"suggest-search": "Որոնել <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"404-page-title": "Սխալ հասցե",
+	"404-page-heading": "Սխալ հասցե",
+	"library-button-text": "Գրադարանի էջ",
+	"home-button-text": "Դեպի '{{BOOK_TITLE}}'֊ի գլխավոր էջը",
+	"random-page-button-text": "Բացել պատահական էջ",
+	"searchbox-tooltip": "Որոնել '{{BOOK_TITLE}}'֊ում"
+}

static/i18n/it.json

@@ -0,0 +1,15 @@
+{
+	"@metadata": {
+		"authors": [
+			"Beta16"
+		]
+	},
+	"name": "italiano",
+	"suggest-full-text-search": "contenente '{{{SEARCH_TERMS}}}'...",
+	"url-not-found": "L'URL richiesto \"{{url}}\" non è stato trovato in questo server.",
+	"400-page-title": "Richiesta non valida",
+	"400-page-heading": "Richiesta non valida",
+	"404-page-title": "Contenuto non trovato",
+	"404-page-heading": "Non trovato",
+	"home-button-text": "Vai alla pagina principale di '{{BOOK_TITLE}}'"
+}

static/i18n/ja.json

@@ -0,0 +1,18 @@
+{
+	"@metadata": {
+		"authors": [
+			"MathXplore"
+		]
+	},
+	"no-query": "クエリを指定していません。",
+	"400-page-title": "無効なリクエストです",
+	"400-page-heading": "無効なリクエストです",
+	"404-page-title": "コンテンツが見つかりませんでした",
+	"404-page-heading": "見つかりません",
+	"500-page-title": "内部サーバーエラー",
+	"500-page-heading": "内部サーバーエラー",
+	"fulltext-search-unavailable": "全文検索は利用できません",
+	"no-search-results": "このコンテンツでは全文検索エンジンが利用できません",
+	"library-button-text": "ウェルカムページに移動",
+	"random-page-button-text": "無作為に選ばれたページに移動する"
+}

static/i18n/ko.json

@@ -0,0 +1,18 @@
+{
+	"@metadata": {
+		"authors": [
+			"Ykhwong"
+		]
+	},
+	"name": "한국어",
+	"suggest-full-text-search": "'{{{SEARCH_TERMS}}}' 포함...",
+	"no-such-book": "해당 책이 없습니다: {{BOOK_NAME}}",
+	"400-page-title": "잘못된 요청",
+	"400-page-heading": "잘못된 요청",
+	"404-page-title": "내용이 없습니다",
+	"404-page-heading": "찾을 수 없음",
+	"500-page-title": "내부 서버 오류",
+	"500-page-heading": "내부 서버 오류",
+	"fulltext-search-unavailable": "전문 검색을 사용할 수 없습니다",
+	"random-page-button-text": "무작위로 선택된 문서로 이동"
+}

static/i18n/ku-latn.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Bikarhêner"
+		]
+	},
+	"name": "kurdî",
+	"suggest-full-text-search": "'{{{SEARCH_TERMS}}}' dihewîne...",
+	"no-such-book": "Kitêbeke wisa nîne: {{BOOK_NAME}}",
+	"too-many-books": "Pir zêde kitêb hatiye xwestin ({{NB_BOOKS}}) ku sînor {{LIMIT}} ye",
+	"no-book-found": "Ti kitêbekê ku li krîterên te bê peyda nebû",
+	"url-not-found": "URLya xwestî \"{{url}}\" li ser vê serverê nehate dîtin.",
+	"suggest-search": "Ji bo <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> lêgerîneke nivîsa temamî pêk bîne",
+	"random-article-failure": "Ha ho! Bi ser neket ku gotareke ketober bibijêre :(",
+	"invalid-raw-data-type": "{{DATATYPE}} ne daxwazeke derbasdar e ji bo naveroka xav.",
+	"no-value-for-arg": "Ti tişt nehatiye diyarkirin ji bo mijara {{ARGUMENT}}",
+	"no-query": "Ti tiştekî xwestî nîne.",
+	"raw-entry-not-found": "Hêmana {{ENTRY}} {{DATATYPE}} peyda nebû",
+	"400-page-title": "Xwesteka nederbasdar",
+	"400-page-heading": "Xwesteka nederbasdar",
+	"404-page-title": "Naverok nehat dîtin",
+	"404-page-heading": "Nehat dîtin",
+	"500-page-title": "Çewtiya Serverê yê Daxilî",
+	"500-page-heading": "Çewtiya Serverê yê Daxilî",
+	"fulltext-search-unavailable": "Lêgerîna Temamê Nivîsê Neberdest e",
+	"no-search-results": "Motora lêgerînê yê temamê metnê ne berdest e ji bo vê naverokê.",
+	"library-button-text": "Here rûpela xêrhatinê",
+	"home-button-text": "Here rûpela destpêkê yê {{BOOK_TITLE}}",
+	"random-page-button-text": "Here rûpeleke ketober bijartî",
+	"searchbox-tooltip": "Li {{BOOK_TITLE}} bigere"
+}

static/i18n/mk.json

@@ -0,0 +1,31 @@
+{
+	"@metadata": {
+		"authors": [
+			"Bjankuloski06"
+		]
+	},
+	"name": "македонски",
+	"suggest-full-text-search": "содржи „{{{SEARCH_TERMS}}}“...",
+	"no-such-book": "Нема книга нарчена {{BOOK_NAME}}",
+	"too-many-books": "Побаравте премногу книги ({{NB_BOOKS}}). Ограничени сте на {{LIMIT}}",
+	"no-book-found": "Ниедна книга не одговара на избраното",
+	"url-not-found": "Не ја пронајдов побараната адреса „{{url}}“ на опслужувачот.",
+	"suggest-search": "Побарајте го <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> по целиот текст",
+	"random-article-failure": "Упс! Не успеав да изберам случајна статија :(",
+	"invalid-raw-data-type": "{{DATATYPE}} не претставува важечко барање за сирова содржина.",
+	"no-value-for-arg": "Нема укажано вредност за аргументот {{ARGUMENT}}",
+	"no-query": "Не е укажано барање.",
+	"raw-entry-not-found": "Не можам да ја најдам {{DATATYPE}}-ставката {{ENTRY}}",
+	"400-page-title": "Неважечко барање",
+	"400-page-heading": "Неважечко барање",
+	"404-page-title": "Содржината не е најдена",
+	"404-page-heading": "Не е најдено",
+	"500-page-title": "Внатрешна грешка во опслужувачот",
+	"500-page-heading": "Внатрешна грешка во опслужувачот",
+	"fulltext-search-unavailable": "Целотекстното пребарување е недостапно",
+	"no-search-results": "Погонот за целотекстно пребарување не е достапен за оваа содржина.",
+	"library-button-text": "Оди на воведната страница",
+	"home-button-text": "Оди на главната страница на „{{BOOK_TITLE}}“",
+	"random-page-button-text": "Оди на случајно избрана страница",
+	"searchbox-tooltip": "Пребарај го „{{BOOK_TITLE}}“"
+}

static/i18n/pl.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"Strebski"
+		]
+	},
+	"name": "Polski",
+	"suggest-full-text-search": "zawierający '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Brak takiej książki: {{BOOK_NAME}}",
+	"url-not-found": "Żądany adres URL „{{url}}” nie został znaleziony na tym serwerze.",
+	"suggest-search": "Wykonaj wyszukiwanie pełnotekstowe dla <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ups! Nie udało się wybrać losowego artykułu :(",
+	"invalid-raw-data-type": "{{DATATYPE}} nie jest prawidłowym żądaniem zawartości nieprzetworzonej.",
+	"raw-entry-not-found": "Nie można znaleźć {{DATATYPE}} wpisu {{ENTRY}}",
+	"400-page-title": "Nieprawidłowe żądanie",
+	"400-page-heading": "Nieprawidłowe żądanie",
+	"404-page-title": "Zawartość nie została znaleziona",
+	"404-page-heading": "Nie znaleziono",
+	"500-page-title": "Wewnętrzny błąd serwera",
+	"500-page-heading": "Wewnętrzny błąd serwera",
+	"fulltext-search-unavailable": "Wyszukiwanie pełnotekstowe jest niedostępne",
+	"no-search-results": "Wyszukiwarka pełnotekstowa nie jest dostępna dla tej zawartości.",
+	"library-button-text": "Przejdź do strony powitalnej",
+	"home-button-text": "Przejdź do głównej strony '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Przejdź do losowo wybranej strony",
+	"searchbox-tooltip": "Szukaj '{{BOOK_TITLE}}'"
+}

static/i18n/qqq.json

@@ -0,0 +1,33 @@
+{
+	"@metadata": {
+		"authors": [
+			"Matthieu Gautier",
+			"Veloman Yunkan",
+			"Verdy p"
+		]
+	},
+	"name": "{{Doc-important|Don't write \"English\" in your language!}}\n\n'''Write the name of ''your'' language in its native script.'''\n\nCurrent language to which the string is being translated to.\n\nFor example, write \"français\" when translating to French, or \"Deutsch\" when translating to German.\n\n'''Important:''' Do not use your language’s word for “English”. Use the word that your language uses to refer to itself. If you translate this message to mean “English” in your language, your change will be reverted.",
+	"suggest-full-text-search": "Text appearing in the suggestion list that, when selected, runs a full text search instead of the title search",
+	"no-such-book": "Error text when the requested book is not found in the library",
+	"too-many-books": "Error text when user request more books than the limit set by the administrator",
+	"no-book-found": "Error text when no book matches the selection criteria",
+	"url-not-found": "Error text about wrong URL for an HTTP 404 error",
+	"suggest-search": "Suggest a search when the URL points to a non existing article",
+	"random-article-failure": "Failure of the random article selection procedure",
+	"invalid-raw-data-type": "Invalid DATATYPE was used with the /raw endpoint (/raw/<book>/DATATYPE/...); allowed values are 'meta' and 'content'",
+	"no-value-for-arg": "Error text when no value has been provided for ARGUMENT in the request's query string",
+	"no-query": "Error text when no query has been provided for fulltext search",
+	"raw-entry-not-found": "Entry requested via the /raw endpoint was not found",
+	"400-page-title": "Title of the 400 error page",
+	"400-page-heading": "Heading of the 400 error page",
+	"404-page-title": "Title of the 404 error page",
+	"404-page-heading": "Heading of the 404 error page",
+	"500-page-title": "Title of the 500 error page",
+	"500-page-heading": "Heading of the 500 error page",
+	"fulltext-search-unavailable": "Title of the error page returned when search is attempted in a book without fulltext search database",
+	"no-search-results": "Text of the error page returned when search is attempted in a book without fulltext search database",
+	"library-button-text": "Tooltip of the button leading to the welcome page",
+	"home-button-text": "Tooltip of the button leading to the main page of a book",
+	"random-page-button-text": "Tooltip of the button opening a randomly selected page",
+	"searchbox-tooltip": "Tooltip displayed for the search box"
+}

static/i18n/ru.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"Fenixs-ru",
+			"Kareyac",
+			"Pacha Tchernof"
+		]
+	},
+	"name": "русский",
+	"suggest-full-text-search": "содержащее '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Такой книги нет: {{BOOK_NAME}}",
+	"url-not-found": "Запрошенный URL \"{{url}}\" не найден на этом сервере.",
+	"suggest-search": "Выполните полнотекстовый поиск для <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Ой! Не удалось выбрать случайную статью :(",
+	"400-page-title": "Недействительный запрос",
+	"400-page-heading": "Недействительный запрос",
+	"404-page-title": "Содержание не найдено",
+	"404-page-heading": "Не найдено",
+	"500-page-title": "Внутренняя ошибка сервера",
+	"500-page-heading": "Внутренняя ошибка сервера",
+	"fulltext-search-unavailable": "Полнотекстовый поиск недоступен",
+	"no-search-results": "Полнотекстовая поисковая система недоступна для этого содержания.",
+	"library-button-text": "Перейти на страницу-приветствие",
+	"home-button-text": "Перейти на главную страницу '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Перейти на случайно выбранную страницу",
+	"searchbox-tooltip": "Искать '{{BOOK_TITLE}}'"
+}

static/i18n/sc.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"L2212"
+		]
+	},
+	"name": "Sardu",
+	"suggest-full-text-search": "chi cuntenet '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Perunu libru cun custu nùmene: {{BOOK_NAME}}",
+	"url-not-found": "S'URL pedidu \"{{url}}\" non s'est atzapadu in custu serbidore.",
+	"suggest-search": "Faghe una chirca de testu intreu pro <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Oops! Sa seletzione de un'artìculu a casu est fallida :(",
+	"invalid-raw-data-type": "{{DATATYPE}} no est una rechesta vàlida pro cuntenutu puru.",
+	"raw-entry-not-found": "Non faghet a atzapare s'elementu {{ENTRY}} de genia {{DATATYPE}}",
+	"400-page-title": "Rechesta non vàlida",
+	"400-page-heading": "Rechesta non vàlida",
+	"404-page-title": "Cuntentu no agatadu",
+	"404-page-heading": "No agatadu",
+	"500-page-title": "Errore internu de su serbidore",
+	"500-page-heading": "Errore internu de su serbidore",
+	"fulltext-search-unavailable": "Chirca de testu integrale no a disponimentu",
+	"no-search-results": "Su motore de chirca de testu integrale no est a disponimentu pro custu cuntenutu.",
+	"library-button-text": "Bae a sa pàgina de bene bènnidu",
+	"home-button-text": "Bae a sa pàgina printzipale de '{{BOOK_TITLE}}'",
+	"random-page-button-text": "Bae a una pàgina seletzionada a manera casuale",
+	"searchbox-tooltip": "Chirca '{{BOOK_TITLE}}'"
+}

static/i18n/sk.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"Yardom78"
+		]
+	},
+	"name": "slovenčina",
+	"suggest-full-text-search": "obsahuje '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Žiadna kniha ako: {{BOOK_NAME}}",
+	"url-not-found": "Požadovaná adresa URL \"{{url}}\" na tomto serveri nebola nájdená.",
+	"suggest-search": "Spustite hľadanie celého textu <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Nepodarilo sa vybrať náhodný článok :(",
+	"invalid-raw-data-type": "{{DATATYPE}} nie je platná požiadavka pre surový obsah.",
+	"raw-entry-not-found": "Nepodarilo sa nájsť {{DATATYPE}} položka {{ENTRY}}",
+	"400-page-title": "Neplatná požiadavka",
+	"400-page-heading": "Neplatná požiadavka",
+	"404-page-title": "Obsah nebol nájdený",
+	"404-page-heading": "Nenájdené",
+	"500-page-title": "Chyba interného servera",
+	"500-page-heading": "Chyba interného servera",
+	"fulltext-search-unavailable": "Vyhľadávanie celého textu nie je dostupné",
+	"no-search-results": "Funkcia na vyhľadávanie celého textu nie je pre tento obsah dostupná.",
+	"library-button-text": "Prejsť na uvítaciu stránku",
+	"home-button-text": "Prejsť na hlavnú stránku knihy {{BOOK_TITLE}}",
+	"random-page-button-text": "Prejsť na náhodne vybranú stránku",
+	"searchbox-tooltip": "Vyhľadať {{BOOK_TITLE}}"
+}

static/i18n/sv.json

@@ -0,0 +1,26 @@
+{
+	"@metadata": {
+		"authors": [
+			"Sabelöga"
+		]
+	},
+	"suggest-full-text-search": "innehåller '{{{SEARCH_TERMS}}}'...",
+	"no-such-book": "Ingen sådan bok: {{BOOK_NAME}}",
+	"url-not-found": "Den begärda webbadressen \"{{url}}\" hittades inte på denna server.",
+	"suggest-search": "Utför en fulltextsökning för <a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a>",
+	"random-article-failure": "Hoppsan! Kunde inte välja en slumpartikel :(",
+	"invalid-raw-data-type": "{{DATATYPE}} är ingen giltig begäran för oformaterat innehåll.",
+	"raw-entry-not-found": "Kunde inte hitta {{DATATYPE}}-inlägget {{ENTRY}}",
+	"400-page-title": "Ogiltig begäran",
+	"400-page-heading": "Ogiltig begäran",
+	"404-page-title": "Innehållet hittades inte",
+	"404-page-heading": "Hittades inte",
+	"500-page-title": "Internt serverfel",
+	"500-page-heading": "Internt serverfel",
+	"fulltext-search-unavailable": "Fulltextsökning är inte tillgänglig",
+	"no-search-results": "Sökmaskinen för fulltext är inte tillgänglig för detta innehåll.",
+	"library-button-text": "Gå till hemsidan",
+	"home-button-text": "Gå till huvudsidan för \"{{BOOK_TITLE}}\"",
+	"random-page-button-text": "Gå till en slumpmässigt utvald sida",
+	"searchbox-tooltip": "Sök efter \"{{BOOK_TITLE}}\""
+}

static/i18n/tr.json

@@ -0,0 +1,27 @@
+{
+	"@metadata": {
+		"authors": [
+			"Hedda"
+		]
+	},
+	"name": "Türkçe",
+	"suggest-full-text-search": "'{{{SEARCH_TERMS}}}' içeriyor...",
+	"no-such-book": "Böyle bir kitap yok: {{BOOK_NAME}}",
+	"url-not-found": "İstenen \"{{url}}\" URL'si bu sunucuda bulunamadı.",
+	"suggest-search": "<a href=\"{{{SEARCH_URL}}}\">{{PATTERN}}</a> için tam metin araması yapın",
+	"random-article-failure": "Hata! Rastgele bir madde seçilemedi :(",
+	"invalid-raw-data-type": "{{DATATYPE}}, ham içerik için geçerli bir istek değil.",
+	"raw-entry-not-found": "{{DATATYPE}} {{ENTRY}} girişi bulunamadı",
+	"400-page-title": "Geçersiz istek",
+	"400-page-heading": "Geçersiz istek",
+	"404-page-title": "içerik bulunamadı",
+	"404-page-heading": "Bulunamadı",
+	"500-page-title": "İç Sunucu Hatası",
+	"500-page-heading": "İç Sunucu Hatası",
+	"fulltext-search-unavailable": "Tam metin araması kullanılamıyor",
+	"no-search-results": "Tam metin arama motoru bu içerik için kullanılamaz.",
+	"library-button-text": "Karşılama sayfasına git",
+	"home-button-text": "'{{BOOK_TITLE}}' anasayfasına gidin",
+	"random-page-button-text": "Rastgele seçilen bir sayfaya git",
+	"searchbox-tooltip": "'{{BOOK_TITLE}}' ara"
+}

static/i18n/zh-hant.json

@@ -0,0 +1,15 @@
+{
+	"@metadata": {
+		"authors": [
+			"Winston Sung"
+		]
+	},
+	"name": "繁體中文",
+	"400-page-title": "無效請求",
+	"400-page-heading": "無效請求",
+	"404-page-title": "查無內容",
+	"404-page-heading": "查無頁面",
+	"500-page-title": "內部伺服器錯誤",
+	"500-page-heading": "內部伺服器錯誤",
+	"searchbox-tooltip": "在{{BOOK_TITLE}}搜尋"
+}

static/i18n_resources_list.txt

@@ -0,0 +1,17 @@
+i18n/bn.json
+i18n/en.json
+i18n/fr.json
+i18n/he.json
+i18n/hy.json
+i18n/it.json
+i18n/ja.json
+i18n/ko.json
+i18n/ku-latn.json
+i18n/mk.json
+i18n/pl.json
+i18n/ru.json
+i18n/sc.json
+i18n/sk.json
+i18n/sv.json
+i18n/tr.json
+i18n/zh-hant.json

static/meson.build

@@ -1,16 +1,45 @@
-resource_files = run_command(find_program('python3'),
-                    '-c',
-                    'import sys; f=open(sys.argv[1]); print(f.read())',
+resource_files = run_command(res_manager,
+                    '--list-all',
                     files('resources_list.txt')
                  ).stdout().strip().split('\n')
 
-lib_resources = custom_target('resources',
+preprocessed_resources = custom_target('preprocessed_resource_files',
     input: 'resources_list.txt',
+    output: ['resources_list.txt'],
+    command:[res_manager,
+             '--preprocess',
+             '--outdir', '@OUTDIR@',
+             '@INPUT@'],
+    depend_files: resource_files
+)
+
+lib_resources = custom_target('resources',
+    input: preprocessed_resources,
     output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
     command:[res_compiler,
              '--cxxfile', '@OUTPUT0@',
              '--hfile', '@OUTPUT1@',
              '--source_dir', '@OUTDIR@',
              '@INPUT@'],
-    depend_files: resource_files
+    depends: preprocessed_resources
+)
+
+# This could be replaced with
+# ```
+# fs = import('fs')
+# i18n_resource_files = fs.read('i18n_resources_list.txt').strip().split('\n')
+# ```
+# once we move to meson >= 0.57.0
+i18n_resource_files = run_command(find_program('python3'),
+                                    '-c',
+                                    'import sys; f=open(sys.argv[1]); print(f.read())',
+                                    files('i18n_resources_list.txt')
+                                 ).stdout().strip().split('\n')
+
+i18n_resources = custom_target('i18n_resources',
+    input: i18n_resource_files,
+    output: ['libkiwix-i18n-resources.cpp'],
+    command:[i18n_compiler,
+             '--cxxfile', '@OUTPUT0@',
+             '@INPUT@'],
 )