New version 9.4.1

libzim were dependent of zlib and we were (wrongly) using its
dependency declaration to link with zlib.

Now that libzim doesn't depends on zlib, we need to fix our build system
and explicitly depend on it.

.github/workflows/ci.yml

@@ -13,9 +13,8 @@ jobs:
         with:
           python-version: '3.5'
       - name: Install packages
-        uses: mstksg/get-package@v1
-        with:
-          brew: gcovr pkg-config ninja
+        run:
+          brew install gcovr pkg-config ninja
       - name: Install python modules
         run: pip3 install meson==0.49.2 pytest
       - name: Install deps
@@ -53,30 +52,41 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        name:
           - native_static
           - native_dyn
+          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
           - win32_dyn
         include:
-          - target: native_static
+          - name: native_static
+            target: native_static
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: native_dyn
+          - name: native_dyn
+            target: native_dyn
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm
+          - name: native_dyn_bionic
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm64
+          - name: android_arm64
+            target: android_arm64
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: win32_static
+          - name: win32_static
+            target: win32_static
             image_variant: f31
             lib_postfix: '64'
-          - target: win32_dyn
+          - name: win32_dyn
+            target: win32_dyn
             image_variant: f31
             lib_postfix: '64'
     env:
@@ -147,6 +157,6 @@ jobs:
         curl https://codecov.io/bash -o codecov.sh
         bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
         rm codecov.sh
-      if: startsWith(matrix.target, 'native_')
+      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -0,0 +1,79 @@
+name: Packages
+on: [push, pull_request]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distro: [ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
+    steps:
+      - uses: actions/checkout@v2
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-groovy
+        if: matrix.distro == 'ubuntu-groovy'
+        name: Build package for ubuntu-groovy
+        id: build-ubuntu-groovy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to master
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.gitignore

@@ -2,3 +2,5 @@
 *.swp
 subprojects/googletest-release*
 *.class
+build/
+.vscode/

ChangeLog

@@ -1,3 +1,64 @@
+kiwix-lib 9.4.1
+===============
+
+ * Fix `M/Counter` parsing.
+ * [SERVER] Adjust body padding-top for taskbar
+ * Fix potential crash when stoping a server not started.
+ * Various fix in build system and the CI.
+
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
 kiwix-lib 9.2
 =============
 

README.md

@@ -6,12 +6,13 @@ suite core. It contains the code shared by all Kiwix ports (Windows,
 GNU/Linux, macOS, Android, iOS, ...).
 
 [![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
-[![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
 [![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
 [![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
+[![Packaging status](https://repology.org/badge/vertical-allrepos/kiwix-lib.svg)](https://repology.org/project/kiwix-lib/versions)
+
 Disclaimer
 ----------
 
@@ -38,10 +39,15 @@ libraries need to be available:
 * [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
 * [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
 * [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
-* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
 set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
+* [libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
+* [microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
+* [zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
+
+The following dependency needs to be available at runtime:
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
@@ -56,7 +62,7 @@ Environment
 -------------
 
 The Kiwix library builds using [Meson](https://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '9.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '9.4.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,46 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+ meson,
+ pkg-config,
+ libzim-dev (>= 6.1.8),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/kiwix-lib
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3,
+ libzim-dev (>= 6.0.0),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev,
+ libmicrohttpd-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix9
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
+Conflicts: libkiwix0, libkiwix3
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix-dev.manpages

@@ -0,0 +1 @@
+usr/share/man/man1/kiwix-compile-resources.1*

debian/libkiwix9.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

include/reader.h

@@ -43,6 +43,8 @@ namespace kiwix
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
  */
+
+using SuggestionsList_t = std::vector<std::vector<std::string>>;
 class Reader
 {
  public:
@@ -419,6 +421,10 @@ class Reader
    *
    * Suggestions are stored in an internal vector and can be retrieved using
    * `getNextSuggestion` method.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestions(const string& prefix,
+   *                        unsigned int suggestionsCount,
+   *                        SuggestionsList_t& results);
    *
    * @param prefix The prefix to search.
    * @param suggestionsCount How many suggestions to search for.
@@ -426,12 +432,49 @@ class Reader
    *              If false, add suggestions to the internal vector
    *              (until internal vector size is suggestionCount (or no more
    *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
+   * @return True if some suggestions have been added to the internal vector.
    */
-  bool searchSuggestions(const string& prefix,
+  DEPRECATED bool searchSuggestions(const string& prefix,
                          unsigned int suggestionsCount,
                          const bool reset = true);
 
+  /**
+   * Search for entries with title starting with prefix (case sensitive).
+   *
+   * Suggestions are added to the `result` vector.
+   *
+   * @param prefix The prefix to search.
+   * @param suggestionsCount How many suggestions to search for.
+   * @param result The vector where to store the suggestions.
+   * @return True if some suggestions have been added to the vector.
+   */
+
+  bool searchSuggestions(const string& prefix,
+                         unsigned int suggestionsCount,
+                         SuggestionsList_t& resuls);
+
+  /**
+   * Search for entries for the given prefix.
+   *
+   * If the zim file has a internal fulltext index, the suggestions will be
+   * searched using it.
+   * Else the suggestions will be search using `searchSuggestions` while trying
+   * to be smart about case sensitivity (using `getTitleVariants`).
+   *
+   * In any case, suggestions are stored in an internal vector and can be
+   * retrieved using `getNextSuggestion` method.
+   * The internal vector will be reset.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestionsSmart(const string& prefix,
+   *                             unsigned int suggestionsCount,
+   *                             SuggestionsList_t& results);
+   *
+   * @param prefix The prefix to search for.
+   * @param suggestionsCount How many suggestions to search for.
+   */
+  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount);
+
   /**
    * Search for entries for the given prefix.
    *
@@ -446,9 +489,13 @@ class Reader
    *
    * @param prefix The prefix to search for.
    * @param suggestionsCount How many suggestions to search for.
+   * @param results The vector where to store the suggestions
+   * @return True if some suggestions have been added to the results.
    */
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
+   bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount,
+                              SuggestionsList_t& results);
+
 
   /**
    * Check if the url exists in the zim file.
@@ -490,7 +537,7 @@ class Reader
    * @param[out] title the title of the suggestion.
    * @return True if title has been set.
    */
-  bool getNextSuggestion(string& title);
+  DEPRECATED bool getNextSuggestion(string& title);
 
   /**
    * Get the next suggestion title and url.
@@ -499,7 +546,7 @@ class Reader
    * @param[out] url the url of the suggestion.
    * @return True if title and url have been set.
    */
-  bool getNextSuggestion(string& title, string& url);
+  DEPRECATED bool getNextSuggestion(string& title, string& url);
 
   /**
    * Get if we can check zim file integrity (has a checksum).
@@ -559,8 +606,8 @@ class Reader
   zim::size_type nsICount;
   std::string zimFilePath;
 
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
+  SuggestionsList_t suggestions;
+  SuggestionsList_t::iterator suggestionsOffset;
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;

include/search_renderer.h

@@ -60,6 +60,13 @@ class SearchRenderer
    */
   void setSearchProtocolPrefix(const std::string& prefix);
 
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
   /**
    * Generate the html page with the resutls of the search.
    */
@@ -73,10 +80,9 @@ class SearchRenderer
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
+  unsigned int pageLength;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
-  unsigned int resultEnd;
 };
 
 

include/searcher.h

@@ -31,7 +31,6 @@
 #include <vector>
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
-#include "kiwix_config.h"
 
 using namespace std;
 

include/tools/otherTools.h

@@ -22,6 +22,8 @@
 
 #include <string>
 #include <vector>
+#include <map>
+#include <zim/zim.h>
 
 namespace pugi {
   class xml_node;
@@ -41,6 +43,8 @@ namespace kiwix
                                      const std::string& tagName);
   bool convertStrToBool(const std::string& value);
 
+  using MimeCounterType = std::map<const std::string, zim::article_index_type>;
+  MimeCounterType parseMimetypeCounter(const std::string& counterData);
 }
 
 #endif

include/tools/stringTools.h

@@ -43,7 +43,7 @@ void loadICUExternalTables();
 std::string urlEncode(const std::string& value, bool encodeReserved = false);
 std::string urlDecode(const std::string& value, bool component = false);
 
-std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
+std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
 std::string join(const std::vector<std::string>& list, const std::string& sep);
 
 std::string ucAll(const std::string& word);

meson.build

@@ -1,31 +1,41 @@
 project('kiwix-lib', 'cpp',
-  version : '9.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
-  license : 'GPL',
+  version : '9.4.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
 wrapper = get_option('wrapper')
 
-static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
-if 'android' in wrapper
+static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
+if wrapper.contains('android')
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
-if 'java' in wrapper
+if wrapper.contains('java')
   add_languages('java')
 endif
 
+# See https://github.com/kiwix/kiwix-lib/issues/371
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
+  extra_libs += '-latomic'
+endif
+
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.3.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
+zlib_dep = dependency('zlib', static:static_deps)
 
-if not compiler.has_header('mustache.hpp')
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
   error('Cannot found header mustache.hpp')
 endif
 
@@ -35,9 +45,9 @@ if target_machine.system() == 'windows' and static_deps
   extra_cflags += '-DCURL_STATICLIB'
 endif
 
-all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
+all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep]
 
-inc = include_directories('include')
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))
@@ -54,7 +64,7 @@ subdir('static')
 subdir('src')
 subdir('test')
 
-pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
+pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd']
 
 pkg_conf = configuration_data()
 pkg_conf.set('prefix', get_option('prefix'))

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/meson.build

@@ -2,3 +2,5 @@
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')

src/aria2.cpp

@@ -19,6 +19,11 @@
 #endif
 
 
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+  }
 
 namespace kiwix {
 
@@ -26,8 +31,8 @@ Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
   m_secret("kiwixariarpc"),
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
+  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
+  mp_curl(nullptr)
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -84,27 +89,21 @@ Aria2::Aria2():
   }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
-  char errbuf[CURL_ERROR_SIZE];
+
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
-    errbuf[0] = 0;
+    m_curlErrorBuffer[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
     } else if (watchdog == 1) {
-      std::cerr <<" curl_easy_perform() failed." << std::endl;
-      fprintf(stderr, "\nlibcurl: (%d) ", res);
-      if (errbuf[0] != 0) {
-        std::cerr << errbuf << std::endl;
-      } else {
-        std::cerr << curl_easy_strerror(res) << std::endl;
-      }
+      LOG_ARIA_ERROR();
     }
   }
   if (!watchdog) {
@@ -115,6 +114,7 @@ Aria2::Aria2():
 
 Aria2::~Aria2()
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   curl_easy_cleanup(mp_curl);
 }
 
@@ -126,38 +126,44 @@ void Aria2::close()
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
+  auto outStream = static_cast<std::stringstream*>(userdata);
+  outStream->write(ptr, nmemb);
   return nmemb;
 }
 
 std::string Aria2::doRequest(const MethodCall& methodCall)
 {
-  pthread_mutex_lock(&m_lock);
   auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
+  std::stringstream outStream;
   CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
-  res = curl_easy_perform(mp_curl);
-  if (res == CURLE_OK) {
-    long response_code;
+  long response_code;
+  {
+    std::unique_lock<std::mutex> lock(m_lock);
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
+    m_curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(mp_curl);
+    if (res != CURLE_OK) {
+      LOG_ARIA_ERROR();
+      throw std::runtime_error("Cannot perform request");
+    }
     curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
-    pthread_mutex_unlock(&m_lock);
-    if (response_code != 200) {
-      throw std::runtime_error("Invalid return code from aria");
-    }
-    auto responseContent = stringstream.str();
-    MethodResponse response(responseContent);
-    if (response.isFault()) {
-      throw AriaError(response.getFault().getFaultString());
-    }
-    return responseContent;
   }
-  pthread_mutex_unlock(&m_lock);
-  throw std::runtime_error("Cannot perform request");
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
 }
 
 std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)

src/aria2.h

@@ -12,8 +12,8 @@
 #include "xmlrpc.h"
 
 #include <memory>
+#include <mutex>
 #include <curl/curl.h>
-#include <pthread.h>
 
 namespace kiwix {
 
@@ -24,8 +24,9 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
+    std::unique_ptr<char[]> m_curlErrorBuffer;
     CURL* mp_curl;
-    pthread_mutex_t m_lock;
+    std::mutex m_lock;
 
     std::string doRequest(const MethodCall& methodCall);
 

src/downloader.cpp

@@ -133,7 +133,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what();
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
   }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
@@ -141,7 +141,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellWaiting failed : " << e.what();
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
 }
 

src/meson.build

@@ -21,9 +21,11 @@ kiwix_sources = [
   'tools/otherTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
+  'server/byte_range.cpp',
   'server/etag.cpp',
   'server/request_context.cpp',
-  'server/response.cpp'
+  'server/response.cpp',
+  'server/internalServer.cpp'
 ]
 kiwix_sources += lib_resources
 
@@ -33,13 +35,13 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if 'android' in wrapper
+if wrapper.contains('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
-if 'android' in wrapper or 'java' in wrapper
+if wrapper.contains('android') or wrapper.contains('java')
   subdir('wrapper/java')
 endif
 
@@ -52,6 +54,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
                    install_dir: install_dir)

src/microhttpd_wrapper.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <microhttpd.h>
+
+#if MHD_VERSION < 0x00097002
+typedef int MHD_Result;
+#endif

src/reader.cpp

@@ -103,29 +103,16 @@ zim::File* Reader::getZimFileHandler() const
 {
   return this->zimFileHandler;
 }
-std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
-{
-  std::map<const std::string, unsigned int> counters;
-  string mimeType, item, counterString;
-  unsigned int counter;
 
+MimeCounterType Reader::parseCounterMetadata() const
+{
   zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
 
   if (article.good()) {
-    stringstream ssContent(article.getData());
-
-    while (getline(ssContent, item, ';')) {
-      stringstream ssItem(item);
-      getline(ssItem, mimeType, '=');
-      getline(ssItem, counterString, '=');
-      if (!counterString.empty() && !mimeType.empty()) {
-        sscanf(counterString.c_str(), "%u", &counter);
-        counters.insert(pair<string, int>(mimeType, counter));
-      }
-    }
+    return parseMimetypeCounter(article.getData());
   }
 
-  return counters;
+  return MimeCounterType();
 }
 
 /* Get the count of articles which can be indexed/displayed */
@@ -138,9 +125,10 @@ unsigned int Reader::getArticleCount() const
   if (counterMap.empty()) {
     counter = this->nsACount;
   } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
+    for(auto &pair:counterMap) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
     }
   }
 
@@ -709,12 +697,11 @@ bool Reader::hasFulltextIndex() const
 }
 
 /* Search titles by prefix */
+
 bool Reader::searchSuggestions(const string& prefix,
                                unsigned int suggestionsCount,
                                const bool reset)
 {
-  bool retVal = false;
-
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
   if (reset) {
@@ -726,6 +713,21 @@ bool Reader::searchSuggestions(const string& prefix,
     }
   }
 
+  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
+
+  /* Set the cursor to the begining */
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+
+bool Reader::searchSuggestions(const string& prefix,
+                               unsigned int suggestionsCount,
+                               SuggestionsList_t& results)
+{
+  bool retVal = false;
+
   /* Return if no prefix */
   if (prefix.size() == 0) {
     return false;
@@ -734,7 +736,7 @@ bool Reader::searchSuggestions(const string& prefix,
   for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
+       && results.size() < suggestionsCount;
        ++articleItr) {
     /* Extract the interesting part of article title & url */
     std::string normalizedArticleTitle
@@ -754,8 +756,8 @@ bool Reader::searchSuggestions(const string& prefix,
        title) */
     bool insert = true;
     std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
+    for (suggestionItr = results.begin();
+         suggestionItr != results.end();
          suggestionItr++) {
       int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
       if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
@@ -772,16 +774,13 @@ bool Reader::searchSuggestions(const string& prefix,
       suggestion.push_back(articleItr->getTitle());
       suggestion.push_back(articleFinalUrl);
       suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
+      results.insert(suggestionItr, suggestion);
     }
 
     /* Suggestions where found */
     retVal = true;
   }
 
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
   return retVal;
 }
 
@@ -796,15 +795,28 @@ std::vector<std::string> Reader::getTitleVariants(
   return variants;
 }
 
-/* Try also a few variations of the prefix to have better results */
+
 bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
+{
+  this->suggestions.clear();
+  this->suggestionsOffset = this->suggestions.begin();
+
+  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
+
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+/* Try also a few variations of the prefix to have better results */
+bool Reader::searchSuggestionsSmart(const string& prefix,
+                                    unsigned int suggestionsCount,
+                                    SuggestionsList_t& results)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
   bool retVal = false;
 
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
   const auto suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
@@ -820,15 +832,14 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
       suggestion.push_back(current->getTitle());
       suggestion.push_back("/A/" + current->getUrl());
       suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
+      results.push_back(suggestion);
     }
-    this->suggestionsOffset = this->suggestions.begin();
     retVal = true;
   } else {
     for (std::vector<std::string>::iterator variantsItr = variants.begin();
          variantsItr != variants.end();
          variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
+      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
                || retVal;
     }
   }

src/search_renderer.cpp

@@ -93,30 +93,30 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
   auto resultStart = mp_searcher->getResultStart();
-  auto resultEnd = mp_searcher->getResultEnd();
-  auto resultCountPerPage = resultEnd - resultStart;
+  auto resultEnd = 0U;
   auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
   auto currentPage = 0U;
   auto pageStart = 0U;
   auto pageEnd = 0U;
   auto lastPageStart = 0U;
-  if (resultCountPerPage) {
-    currentPage = resultStart/resultCountPerPage;
+  if (pageLength) {
+    currentPage = resultStart/pageLength;
     pageStart = currentPage > 4 ? currentPage-4 : 0;
     pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / resultCountPerPage) {
-      pageEnd = estimatedResultCount / resultCountPerPage;
+    if (pageEnd > estimatedResultCount / pageLength) {
+      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
     }
-    if (estimatedResultCount > resultCountPerPage) {
-      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
+    if (estimatedResultCount > pageLength) {
+      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
     }
   }
 
+  resultEnd = resultStart+pageLength; //setting result end
+
   for (unsigned int i = pageStart; i < pageEnd; i++) {
     kainjow::mustache::data page;
     page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * resultCountPerPage));
-    page.set("end", to_string((i + 1) * resultCountPerPage));
+    page.set("start", to_string(i * pageLength));
 
     if (i == currentPage) {
       page.set("selected", true);
@@ -137,9 +137,8 @@ std::string SearchRenderer::getHtml()
   allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
   allData.set("resultStart", to_string(resultStart + 1));
   allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("resultRange", to_string(resultCountPerPage));
+  allData.set("pageLength", to_string(pageLength));
   allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("lastResult", to_string(estimatedResultCount));
   allData.set("protocolPrefix", this->protocolPrefix);
   allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
   allData.set("contentId", this->searchContent);
@@ -149,4 +148,4 @@ std::string SearchRenderer::getHtml()
   return ss.str();
 }
 
-}
+}

src/server.cpp

@@ -19,140 +19,15 @@
 
 #include "server.h"
 
-#ifdef _WIN32
-# if !defined(__MINGW32__) && (_MSC_VER < 1600)
-#   include "stdint4win.h"
-# endif
-# include <winsock2.h>
-# include <ws2tcpip.h>
-# ifdef __GNUC__
-  // inet_pton is not declared in mingw, even if the function exists.
-  extern "C" {
-    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
-  }
-# endif
-  typedef UINT64 uint64_t;
-  typedef UINT16 uint16_t;
-#endif
-
-extern "C" {
-#include <microhttpd.h>
-}
-
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
 #include "library.h"
 #include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
-#include "search_renderer.h"
-#include "opds_dumper.h"
 
-#include <zim/uuid.h>
-
-#include <mustache.hpp>
-
-#include <pthread.h>
-#include <atomic>
 #include <string>
-#include <vector>
-#include <chrono>
-#include "kiwixlib-resources.h"
 
-#ifndef _WIN32
-# include <arpa/inet.h>
-#endif
-
-#include "server/request_context.h"
-#include "server/response.h"
-
-#define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#include "server/internalServer.h"
 
 namespace kiwix {
 
-static IdNameMapper defaultNameMapper;
-
-typedef kainjow::mustache::data MustacheData;
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls);
-
-
-class InternalServer {
-  public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
-                   int port,
-                   std::string root,
-                   int nbThreads,
-                   bool verbose,
-                   bool withTaskbar,
-                   bool withLibraryButton,
-                   bool blockExternalLinks);
-    virtual ~InternalServer() = default;
-
-    int handlerCallback(struct MHD_Connection* connection,
-                        const char* url,
-                        const char* method,
-                        const char* version,
-                        const char* upload_data,
-                        size_t* upload_data_size,
-                        void** cont_cls);
-    bool start();
-    void stop();
-
-  private: // functions
-    Response handle_request(const RequestContext& request);
-    Response build_500(const std::string& msg);
-    Response build_404(const RequestContext& request, const std::string& zimName);
-    Response build_304(const RequestContext& request, const ETag& etag) const;
-    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
-    Response build_homepage(const RequestContext& request);
-    Response handle_skin(const RequestContext& request);
-    Response handle_catalog(const RequestContext& request);
-    Response handle_meta(const RequestContext& request);
-    Response handle_search(const RequestContext& request);
-    Response handle_suggest(const RequestContext& request);
-    Response handle_random(const RequestContext& request);
-    Response handle_captured_external(const RequestContext& request);
-    Response handle_content(const RequestContext& request);
-
-    MustacheData get_default_data() const;
-    MustacheData homepage_data() const;
-    Response get_default_response() const;
-
-    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
-
-  private: // data
-    std::string m_addr;
-    int m_port;
-    std::string m_root;
-    int m_nbThreads;
-    std::atomic_bool m_verbose;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    struct MHD_Daemon* mp_daemon;
-
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
-
-    std::string m_server_id;
-};
-
-
 Server::Server(Library* library, NameMapper* nameMapper) :
   mp_library(library),
   mp_nameMapper(nameMapper),
@@ -178,8 +53,10 @@ bool Server::start() {
 }
 
 void Server::stop() {
-  mp_server->stop();
-  mp_server.reset(nullptr);
+  if (mp_server) {
+    mp_server->stop();
+    mp_server.reset(nullptr);
+  }
 }
 
 void Server::setRoot(const std::string& root)
@@ -193,773 +70,4 @@ void Server::setRoot(const std::string& root)
   }
 }
 
-
-InternalServer::InternalServer(Library* library,
-                               NameMapper* nameMapper,
-                               std::string addr,
-                               int port,
-                               std::string root,
-                               int nbThreads,
-                               bool verbose,
-                               bool withTaskbar,
-                               bool withLibraryButton,
-                               bool blockExternalLinks) :
-  m_addr(addr),
-  m_port(port),
-  m_root(root),
-  m_nbThreads(nbThreads),
-  m_verbose(verbose),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  mp_daemon(nullptr),
-  mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
-{}
-
-bool InternalServer::start() {
-#ifdef _WIN32
-  int flags = MHD_USE_SELECT_INTERNALLY;
-#else
-  int flags = MHD_USE_POLL_INTERNALLY;
-#endif
-  if (m_verbose.load())
-    flags |= MHD_USE_DEBUG;
-
-
-  struct sockaddr_in sockAddr;
-  memset(&sockAddr, 0, sizeof(sockAddr));
-  sockAddr.sin_family = AF_INET;
-  sockAddr.sin_port = htons(m_port);
-  if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
-      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-  } else {
-    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
-      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
-      return false;
-    }
-  }
-
-  mp_daemon = MHD_start_daemon(flags,
-                            m_port,
-                            NULL,
-                            NULL,
-                            &staticHandlerCallback,
-                            this,
-                            MHD_OPTION_SOCK_ADDR, &sockAddr,
-                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
-                            MHD_OPTION_END);
-  if (mp_daemon == nullptr) {
-    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
-              << " is maybe already occupied or need more permissions to be open. "
-                 "Please try as root or with a port number higher or equal to 1024."
-              << std::endl;
-    return false;
-  }
-  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
-  m_server_id = kiwix::to_string(server_start_time.count());
-  return true;
-}
-
-void InternalServer::stop()
-{
-  MHD_stop_daemon(mp_daemon);
-}
-
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls)
-{
-  InternalServer* _this = static_cast<InternalServer*>(cls);
-
-  return _this->handlerCallback(connection,
-                                url,
-                                method,
-                                version,
-                                upload_data,
-                                upload_data_size,
-                                cont_cls);
-}
-
-int InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                    const char* url,
-                                    const char* method,
-                                    const char* version,
-                                    const char* upload_data,
-                                    size_t* upload_data_size,
-                                    void** cont_cls)
-{
-  auto start_time = std::chrono::steady_clock::now();
-  if (m_verbose.load() ) {
-    printf("======================\n");
-    printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
-  }
-  RequestContext request(connection, m_root, url, method, version);
-
-  if (m_verbose.load() ) {
-    request.print_debug_info();
-  }
-  /* Unexpected method */
-  if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST
-   && request.get_method() != RequestMethod::HEAD) {
-    printf("Reject request because of unhandled request method.\n");
-    printf("----------------------\n");
-    return MHD_NO;
-  }
-
-  auto response = handle_request(request);
-
-  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
-    printf("========== INTERNAL ERROR !! ============\n");
-    if (!m_verbose.load()) {
-      printf("Requesting : \n");
-      printf("full_url : %s\n", url);
-      request.print_debug_info();
-    }
-  }
-
-  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response.set_server_id(m_server_id);
-
-  auto ret = response.send(request, connection);
-  auto end_time = std::chrono::steady_clock::now();
-  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
-  if (m_verbose.load()) {
-    printf("Request time : %fs\n", time_span.count());
-    printf("----------------------\n");
-  }
-  return ret;
-}
-
-Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
-{
-  auto response = get_default_response();
-  response.set_code(MHD_HTTP_NOT_MODIFIED);
-  response.set_etag(etag);
-  response.set_content("");
-  return response;
-}
-
-Response InternalServer::handle_request(const RequestContext& request)
-{
-  try {
-    if (! request.is_valid_url())
-      return build_404(request, "");
-
-    const ETag etag = get_matching_if_none_match_etag(request);
-    if ( etag )
-      return build_304(request, etag);
-
-    if (kiwix::startsWith(request.get_url(), "/skin/"))
-      return handle_skin(request);
-
-    if (startsWith(request.get_url(), "/catalog"))
-      return handle_catalog(request);
-
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
-
-    if (request.get_url() == "/search")
-      return handle_search(request);
-
-    if (request.get_url() == "/suggest")
-     return handle_suggest(request);
-
-    if (request.get_url() == "/random")
-      return handle_random(request);
-
-    if (request.get_url() == "/catch/external")
-      return handle_captured_external(request);
-
-    return handle_content(request);
-  } catch (std::exception& e) {
-    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return build_500(e.what());
-  } catch (...) {
-    fprintf(stderr, "===== Unhandled unknown error\n");
-    return build_500("Unknown error");
-  }
-}
-
-MustacheData InternalServer::get_default_data() const
-{
-  MustacheData data;
-  data.set("root", m_root);
-  return data;
-}
-
-Response InternalServer::get_default_response() const
-{
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
-}
-
-
-Response InternalServer::build_404(const RequestContext& request,
-                                   const std::string& bookName)
-{
-  MustacheData results;
-  results.set("url", request.get_full_url());
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::_404_html, results);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_NOT_FOUND);
-  response.set_compress(true);
-  response.set_taskbar(bookName, "");
-
-  return response;
-}
-
-Response InternalServer::build_500(const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  Response response(m_root, true, false, false, false);
-  response.set_template(RESOURCE::templates::_500_html, data);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
-MustacheData InternalServer::homepage_data() const
-{
-  auto data = get_default_data();
-
-  MustacheData books{MustacheData::type::list};
-  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = mp_library->getBookById(bookId);
-
-    MustacheData book;
-    book.set("name", mp_nameMapper->getNameForId(bookId));
-    book.set("title", currentBook.getTitle());
-    book.set("description", currentBook.getDescription());
-    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
-    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
-    books.push_back(book);
-  }
-
-  data.set("books", books);
-  return data;
-}
-
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
-Response InternalServer::build_homepage(const RequestContext& request)
-{
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, homepage_data());
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_meta(const RequestContext& request)
-{
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
-  } else {
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-  response.set_content(content);
-  response.set_mimeType(mimeType);
-  response.set_compress(false);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_suggest(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_suggest\n");
-  }
-
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-  std::string suggestion;
-
-  std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
-  }
-
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    reader->searchSuggestionsSmart(term, maxSuggestionCount);
-    while (reader->getNextSuggestion(suggestion)) {
-      MustacheData result;
-      result.set("label", suggestion);
-      result.set("value", suggestion);
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
-    }
-  }
-
-  /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
-    MustacheData result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term + " ");
-    result.set("first", first);
-    results.push_back(result);
-  }
-
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::suggestion_json, data);
-  response.set_mimeType("application/json; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_skin(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_skin\n");
-  }
-
-  auto response = get_default_response();
-  auto resourceName = request.get_url().substr(1);
-  try {
-    response.set_content(getResource(resourceName));
-  } catch (const ResourceNotFound& e) {
-    return build_404(request, "");
-  }
-  response.set_mimeType(getMimeTypeForFile(resourceName));
-  response.set_compress(true);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_search(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_search\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
-
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
-
-  std::shared_ptr<Reader> reader(nullptr);
-  try {
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Try first to load directly the article */
-  if (reader != nullptr && !patternString.empty()) {
-    std::string patternCorrespondingUrl;
-    auto variants = reader->getTitleVariants(patternString);
-    auto variantsItr = variants.begin();
-
-    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
-      try {
-        auto entry = reader->getEntryFromTitle(*variantsItr);
-        entry = entry.getFinalEntry();
-        patternCorrespondingUrl = entry.getPath();
-        break;
-      } catch(kiwix::NoEntry& e) {
-        variantsItr++;
-      }
-    }
-
-    /* If article found then redirect directly to it */
-    if (!patternCorrespondingUrl.empty()) {
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
-      return response;
-    }
-  }
-
-  /* Make the search */
-  auto response = get_default_response();
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
-  response.set_compress(true);
-
-  if ( (!reader && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    response.set_template(RESOURCE::templates::no_search_result_html, data);
-    response.set_code(MHD_HTTP_NOT_FOUND);
-    return response;
-  }
-
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-  auto end = 25;
-  try {
-    end = request.get_argument<unsigned int>("end");
-  } catch (const std::exception&) {}
-  if (start>end) {
-    auto tmp = start;
-    start = end;
-    end = tmp;
-  }
-  if (end > start + MAX_SEARCH_LEN) {
-    end = start + MAX_SEARCH_LEN;
-  }
-
-  /* Get the results */
-  try {
-    if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
-    } else {
-      searcher.search(patternString,
-                       start, end, m_verbose.load());
-    }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
-    response.set_content(renderer.getHtml());
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-  return response;
-}
-
-Response InternalServer::handle_random(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_random\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  try {
-    auto entry = reader->getRandomPage();
-    return build_redirect(bookName, entry.getFinalEntry());
-  } catch(kiwix::NoEntry& e) {
-    return build_404(request, bookName);
-  }
-}
-
-Response InternalServer::handle_captured_external(const RequestContext& request)
-{
-  std::string source = "";
-  try {
-    source = kiwix::urlDecode(request.get_argument("source"));
-  } catch (const std::out_of_range& e) {}
-
-  if (source.empty())
-    return build_404(request, "");
-
-  auto data = get_default_data();
-  data.set("source", source);
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::captured_external_html, data);
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  response.set_taskbar("", "");
-  return response;
-}
-
-Response InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return build_404(request, "");
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return build_404(request, "");
-  }
-
-  auto response = get_default_response();
-  response.set_compress(true);
-  if (url == "searchdescription.xml") {
-    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
-    response.set_mimeType("application/opensearchdescription+xml");
-    return response;
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper;
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    opdsDumper.setTitle("All zims");
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
-    string query("<Empty query>");
-    size_t count(10);
-    size_t startIndex(0);
-    try {
-      query = request.get_argument("q");
-      filter.query(query);
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      count = extractFromString<unsigned long>(request.get_argument("count"));
-    } catch (...) {}
-    try {
-      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
-    } catch (...) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    opdsDumper.setTitle("Search result for " + query);
-    uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(filter);
-    auto totalResults = bookIdsToDump.size();
-    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
-    if (count>0 && bookIdsToDump.size() > count) {
-      bookIdsToDump.resize(count);
-    }
-    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
-  }
-
-  opdsDumper.setId(kiwix::to_string(uuid));
-  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
-  return response;
-}
-
-namespace
-{
-
-std::string get_book_name(const RequestContext& request)
-{
-  try {
-    return request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return std::string();
-  }
-}
-
-} // unnamed namespace
-
-std::shared_ptr<Reader>
-InternalServer::get_reader(const std::string& bookName) const
-{
-  std::shared_ptr<Reader> reader;
-  try {
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-  }
-  return reader;
-}
-
-Response
-InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
-{
-  auto response = get_default_response();
-  response.set_redirection(m_root + "/" + bookName + "/" +
-                           kiwix::urlEncode(entry.getPath()));
-  return response;
-}
-
-Response InternalServer::handle_content(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_content\n");
-  }
-
-  const std::string bookName = get_book_name(request);
-  if (bookName.empty())
-    return build_homepage(request);
-
-  const std::shared_ptr<Reader> reader = get_reader(bookName);
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  auto urlStr = request.get_url().substr(bookName.size()+1);
-  if (urlStr[0] == '/') {
-    urlStr = urlStr.substr(1);
-  }
-
-  kiwix::Entry entry;
-
-  try {
-    entry = reader->getEntryFromPath(urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
-      return build_redirect(bookName, entry.getFinalEntry());
-    }
-  } catch(kiwix::NoEntry& e) {
-    if (m_verbose.load())
-      printf("Failed to find %s\n", urlStr.c_str());
-
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-
-  response.set_entry(entry, request);
-
-  if (m_verbose.load()) {
-    printf("Found %s\n", entry.getPath().c_str());
-    printf("mimeType: %s\n", response.get_mimeType().c_str());
-  }
-
-  if (response.get_mimeType().find("text/html") != string::npos)
-    response.set_taskbar(bookName, reader->getTitle());
-
-  return response;
-}
-
 }

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/internalServer.cpp

@@ -0,0 +1,798 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "internalServer.h"
+
+#ifdef _WIN32
+# if !defined(__MINGW32__) && (_MSC_VER < 1600)
+#   include "stdint4win.h"
+# endif
+# include <winsock2.h>
+# include <ws2tcpip.h>
+# ifdef __GNUC__
+  // inet_pton is not declared in mingw, even if the function exists.
+  extern "C" {
+    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
+  }
+# endif
+  typedef UINT64 uint64_t;
+  typedef UINT16 uint16_t;
+#endif
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+#include "library.h"
+#include "name_mapper.h"
+#include "entry.h"
+#include "searcher.h"
+#include "search_renderer.h"
+#include "opds_dumper.h"
+
+#include <zim/uuid.h>
+
+#include <mustache.hpp>
+
+#include <pthread.h>
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+#include "kiwixlib-resources.h"
+
+#ifndef _WIN32
+# include <arpa/inet.h>
+#endif
+
+#include "request_context.h"
+#include "response.h"
+
+#define MAX_SEARCH_LEN 140
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+static IdNameMapper defaultNameMapper;
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls);
+
+
+InternalServer::InternalServer(Library* library,
+                               NameMapper* nameMapper,
+                               std::string addr,
+                               int port,
+                               std::string root,
+                               int nbThreads,
+                               bool verbose,
+                               bool withTaskbar,
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
+  m_addr(addr),
+  m_port(port),
+  m_root(root),
+  m_nbThreads(nbThreads),
+  m_verbose(verbose),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  mp_daemon(nullptr),
+  mp_library(library),
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+{}
+
+bool InternalServer::start() {
+#ifdef _WIN32
+  int flags = MHD_USE_SELECT_INTERNALLY;
+#else
+  int flags = MHD_USE_POLL_INTERNALLY;
+#endif
+  if (m_verbose.load())
+    flags |= MHD_USE_DEBUG;
+
+
+  struct sockaddr_in sockAddr;
+  memset(&sockAddr, 0, sizeof(sockAddr));
+  sockAddr.sin_family = AF_INET;
+  sockAddr.sin_port = htons(m_port);
+  if (m_addr.empty()) {
+    if (0 != INADDR_ANY)
+      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+  } else {
+    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
+      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
+      return false;
+    }
+  }
+
+  mp_daemon = MHD_start_daemon(flags,
+                            m_port,
+                            NULL,
+                            NULL,
+                            &staticHandlerCallback,
+                            this,
+                            MHD_OPTION_SOCK_ADDR, &sockAddr,
+                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_END);
+  if (mp_daemon == nullptr) {
+    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
+              << " is maybe already occupied or need more permissions to be open. "
+                 "Please try as root or with a port number higher or equal to 1024."
+              << std::endl;
+    return false;
+  }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
+  return true;
+}
+
+void InternalServer::stop()
+{
+  MHD_stop_daemon(mp_daemon);
+}
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls)
+{
+  InternalServer* _this = static_cast<InternalServer*>(cls);
+
+  return _this->handlerCallback(connection,
+                                url,
+                                method,
+                                version,
+                                upload_data,
+                                upload_data_size,
+                                cont_cls);
+}
+
+MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                           const char* url,
+                                           const char* method,
+                                           const char* version,
+                                           const char* upload_data,
+                                           size_t* upload_data_size,
+                                           void** cont_cls)
+{
+  auto start_time = std::chrono::steady_clock::now();
+  if (m_verbose.load() ) {
+    printf("======================\n");
+    printf("Requesting : \n");
+    printf("full_url  : %s\n", url);
+  }
+  RequestContext request(connection, m_root, url, method, version);
+
+  if (m_verbose.load() ) {
+    request.print_debug_info();
+  }
+  /* Unexpected method */
+  if (request.get_method() != RequestMethod::GET
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
+    printf("Reject request because of unhandled request method.\n");
+    printf("----------------------\n");
+    return MHD_NO;
+  }
+
+  auto response = handle_request(request);
+
+  if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
+    printf("========== INTERNAL ERROR !! ============\n");
+    if (!m_verbose.load()) {
+      printf("Requesting : \n");
+      printf("full_url : %s\n", url);
+      request.print_debug_info();
+    }
+  }
+
+  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response->set_server_id(m_server_id);
+
+  auto ret = response->send(request, connection);
+  auto end_time = std::chrono::steady_clock::now();
+  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
+  if (m_verbose.load()) {
+    printf("Request time : %fs\n", time_span.count());
+    printf("----------------------\n");
+  }
+  return ret;
+}
+
+std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
+{
+  try {
+    if (! request.is_valid_url())
+      return Response::build_404(*this, request, "");
+
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return Response::build_304(*this, etag);
+
+    if (kiwix::startsWith(request.get_url(), "/skin/"))
+      return handle_skin(request);
+
+    if (startsWith(request.get_url(), "/catalog"))
+      return handle_catalog(request);
+
+    if (request.get_url() == "/meta")
+      return handle_meta(request);
+
+    if (request.get_url() == "/search")
+      return handle_search(request);
+
+    if (request.get_url() == "/suggest")
+     return handle_suggest(request);
+
+    if (request.get_url() == "/random")
+      return handle_random(request);
+
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
+    return handle_content(request);
+  } catch (std::exception& e) {
+    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
+    return Response::build_500(*this, e.what());
+  } catch (...) {
+    fprintf(stderr, "===== Unhandled unknown error\n");
+    return Response::build_500(*this, "Unknown error");
+  }
+}
+
+MustacheData InternalServer::get_default_data() const
+{
+  MustacheData data;
+  data.set("root", m_root);
+  return data;
+}
+
+MustacheData InternalServer::homepage_data() const
+{
+  auto data = get_default_data();
+
+  MustacheData books{MustacheData::type::list};
+  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = mp_library->getBookById(bookId);
+
+    MustacheData book;
+    book.set("name", mp_nameMapper->getNameForId(bookId));
+    book.set("title", currentBook.getTitle());
+    book.set("description", currentBook.getDescription());
+    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
+    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
+    books.push_back(book);
+  }
+
+  data.set("books", books);
+  return data;
+}
+
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
+{
+  return ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+{
+  std::string bookName;
+  std::string bookId;
+  std::string meta_name;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    meta_name = request.get_argument("name");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = ContentResponse::build(*this, content, mimeType);
+  response->set_cacheable();
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_suggest\n");
+  }
+
+  std::string content;
+  std::string mimeType;
+  unsigned int maxSuggestionCount = 10;
+  unsigned int suggestionCount = 0;
+
+  std::string bookName;
+  std::string bookId;
+  std::string term;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    term = request.get_argument("term");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (m_verbose.load()) {
+    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+  }
+
+  MustacheData results{MustacheData::type::list};
+
+  bool first = true;
+  if (reader != nullptr) {
+    /* Get the suggestions */
+    SuggestionsList_t suggestions;
+    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
+    for(auto& suggestion:suggestions) {
+      MustacheData result;
+      result.set("label", suggestion[0]);
+      result.set("value", suggestion[0]);
+      result.set("first", first);
+      first = false;
+      results.push_back(result);
+      suggestionCount++;
+    }
+  }
+
+  /* Propose the fulltext search if possible */
+  if (reader->hasFulltextIndex()) {
+    MustacheData result;
+    result.set("label", "containing '" + term + "'...");
+    result.set("value", term + " ");
+    result.set("first", first);
+    results.push_back(result);
+  }
+
+  auto data = get_default_data();
+  data.set("suggestions", results);
+
+  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_skin\n");
+  }
+
+  auto resourceName = request.get_url().substr(1);
+  try {
+    auto response = ContentResponse::build(
+        *this,
+        getResource(resourceName),
+        getMimeTypeForFile(resourceName));
+    response->set_cacheable();
+    return std::move(response);
+  } catch (const ResourceNotFound& e) {
+    return Response::build_404(*this, request, "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_search\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+  } catch (const std::out_of_range&) {}
+
+  std::string patternString;
+  try {
+    patternString = request.get_argument("pattern");
+  } catch (const std::out_of_range&) {}
+
+  /* Retrive geo search */
+  bool has_geo_query = false;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
+  try {
+    latitude = request.get_argument<float>("latitude");
+    longitude = request.get_argument<float>("longitude");
+    distance = request.get_argument<float>("distance");
+    has_geo_query = true;
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  std::shared_ptr<Reader> reader(nullptr);
+  try {
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {}
+
+  /* Try first to load directly the article */
+  if (reader != nullptr && !patternString.empty()) {
+    std::string patternCorrespondingUrl;
+    auto variants = reader->getTitleVariants(patternString);
+    auto variantsItr = variants.begin();
+
+    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
+      try {
+        auto entry = reader->getEntryFromTitle(*variantsItr);
+        entry = entry.getFinalEntry();
+        patternCorrespondingUrl = entry.getPath();
+        break;
+      } catch(kiwix::NoEntry& e) {
+        variantsItr++;
+      }
+    }
+
+    /* If article found then redirect directly to it */
+    if (!patternCorrespondingUrl.empty()) {
+      auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl;
+      return Response::build_redirect(*this, redirectUrl);
+    }
+  }
+
+  /* Make the search */
+  if ( (!reader && !bookName.empty())
+    || (patternString.empty() && ! has_geo_query) ) {
+    auto data = get_default_data();
+    data.set("pattern", encodeDiples(patternString));
+    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_code(MHD_HTTP_NOT_FOUND);
+    return std::move(response);
+  }
+
+  Searcher searcher;
+  if (reader) {
+    searcher.add_reader(reader.get());
+  } else {
+    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+      auto currentReader = mp_library->getReaderById(bookId);
+      if (currentReader) {
+        searcher.add_reader(currentReader.get());
+      }
+    }
+  }
+
+  auto start = 0;
+  try {
+    start = request.get_argument<unsigned int>("start");
+  } catch (const std::exception&) {}
+
+  auto pageLength = 25;
+  try {
+    pageLength = request.get_argument<unsigned int>("pageLength");
+  } catch (const std::exception&) {}
+  if (pageLength > MAX_SEARCH_LEN) {
+    pageLength = MAX_SEARCH_LEN;
+  }
+  if (pageLength == 0) {
+    pageLength = 25;
+  }
+
+  auto end = start + pageLength;
+
+  /* Get the results */
+  try {
+    if (patternString.empty()) {
+      searcher.geo_search(latitude, longitude, distance,
+                           start, end, m_verbose.load());
+    } else {
+      searcher.search(patternString,
+                       start, end, m_verbose.load());
+    }
+    SearchRenderer renderer(&searcher, mp_nameMapper);
+    renderer.setSearchPattern(patternString);
+    renderer.setSearchContent(bookName);
+    renderer.setProtocolPrefix(m_root + "/");
+    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setPageLength(pageLength);
+    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    //changing status code if no result obtained
+    if(searcher.getEstimatedResultCount() == 0)
+    {
+      response->set_code(MHD_HTTP_NO_CONTENT);
+    }
+
+    return std::move(response);
+  } catch (const std::exception& e) {
+    std::cerr << e.what() << std::endl;
+    return Response::build_500(*this, e.what());
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_random\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  try {
+    auto entry = reader->getRandomPage();
+    return build_redirect(bookName, entry.getFinalEntry());
+  } catch(kiwix::NoEntry& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return Response::build_404(*this, request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper;
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+  opdsDumper.setLibrary(mp_library);
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    opdsDumper.setTitle("All zims");
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
+    string query("<Empty query>");
+    size_t count(10);
+    size_t startIndex(0);
+    try {
+      query = request.get_argument("q");
+      filter.query(query);
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      count = extractFromString<unsigned long>(request.get_argument("count"));
+    } catch (...) {}
+    try {
+      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
+    } catch (...) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
+    } catch (...) {}
+    opdsDumper.setTitle("Search result for " + query);
+    uuid = zim::Uuid::generate();
+    bookIdsToDump = mp_library->filter(filter);
+    auto totalResults = bookIdsToDump.size();
+    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
+    if (count>0 && bookIdsToDump.size() > count) {
+      bookIdsToDump.resize(count);
+    }
+    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
+  }
+
+  opdsDumper.setId(kiwix::to_string(uuid));
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump),
+      "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
+  return std::move(response);
+}
+
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+std::unique_ptr<Response>
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
+  return Response::build_redirect(*this, redirectUrl);
+}
+
+std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_content\n");
+  }
+
+  const std::string bookName = get_book_name(request);
+  if (bookName.empty())
+    return build_homepage(request);
+
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto urlStr = request.get_url().substr(bookName.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
+
+  kiwix::Entry entry;
+
+  try {
+    entry = reader->getEntryFromPath(urlStr);
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      return build_redirect(bookName, entry.getFinalEntry());
+    }
+  } catch(kiwix::NoEntry& e) {
+    if (m_verbose.load())
+      printf("Failed to find %s\n", urlStr.c_str());
+
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = EntryResponse::build(*this, request, entry);
+  try {
+    dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
+  } catch (std::bad_cast& e) {}
+
+  if (m_verbose.load()) {
+    printf("Found %s\n", entry.getPath().c_str());
+    printf("mimeType: %s\n", entry.getMimetype().c_str());
+  }
+
+  return response;
+}
+
+}

src/server/internalServer.h

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H
+#define KIWIXLIB_SERVER_INTERNALSERVER_H
+
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "library.h"
+#include "name_mapper.h"
+
+#include <mustache.hpp>
+
+#include <atomic>
+#include <string>
+
+#include "server/request_context.h"
+#include "server/response.h"
+
+namespace kiwix {
+
+typedef kainjow::mustache::data MustacheData;
+
+class Entry;
+
+class InternalServer {
+  public:
+    InternalServer(Library* library,
+                   NameMapper* nameMapper,
+                   std::string addr,
+                   int port,
+                   std::string root,
+                   int nbThreads,
+                   bool verbose,
+                   bool withTaskbar,
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
+    virtual ~InternalServer() = default;
+
+    MHD_Result handlerCallback(struct MHD_Connection* connection,
+                               const char* url,
+                               const char* method,
+                               const char* version,
+                               const char* upload_data,
+                               size_t* upload_data_size,
+                               void** cont_cls);
+    bool start();
+    void stop();
+
+  private: // functions
+    std::unique_ptr<Response> handle_request(const RequestContext& request);
+    std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_skin(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog(const RequestContext& request);
+    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_suggest(const RequestContext& request);
+    std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
+    std::unique_ptr<Response> handle_content(const RequestContext& request);
+
+    MustacheData get_default_data() const;
+    MustacheData homepage_data() const;
+
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
+    std::string m_addr;
+    int m_port;
+    std::string m_root;
+    int m_nbThreads;
+    std::atomic_bool m_verbose;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    struct MHD_Daemon* mp_daemon;
+
+    Library* mp_library;
+    NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
+
+    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
+
+};
+
+}
+
+#endif //KIWIXLIB_SERVER_INTERNALSERVER_H

src/server/request_context.cpp

@@ -26,6 +26,8 @@
 #include <cstdio>
 #include <atomic>
 
+#include "tools/stringTools.h"
+
 namespace kiwix {
 
 static std::atomic_ullong s_requestIndex(0);
@@ -74,8 +76,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingDeflate(false),
-  accept_range(false),
-  range_pair(0, -1)
+  byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
@@ -85,43 +86,24 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
         (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
-  /*Check if range is requested. */
   try {
-    auto range = get_header(MHD_HTTP_HEADER_RANGE);
-    int start = 0;
-    int end = -1;
-    std::istringstream iss(range);
-    char c;
-
-    iss >> start >> c;
-    if (iss.good() && c=='-') {
-      iss >> end;
-      if (iss.fail()) {
-        // Something went wrong will extracting.
-        end = -1;
-      }
-      if (iss.eof()) {
-        accept_range = true;
-        range_pair = std::pair<int, int>(start, end);
-      }
-    }
+    byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
 }
 
 RequestContext::~RequestContext()
 {}
 
-
-int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                 const char *key, const char *value)
+MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
+                                       const char *key, const char *value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[key] = value;
+  _this->headers[lcAll(key)] = value;
   return MHD_YES;
 }
 
-int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                   const char *key, const char* value)
+MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key] = value == nullptr ? "" : value;
@@ -146,7 +128,7 @@ void RequestContext::print_debug_info() const {
   printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
-  printf("has_range : %d\n", accept_range);
+  printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
 }
@@ -188,12 +170,8 @@ bool RequestContext::is_valid_url() const {
   return !url.empty();
 }
 
-bool RequestContext::has_range() const {
-  return accept_range;
-}
-
-std::pair<int, int> RequestContext::get_range() const {
-  return range_pair;
+ByteRange RequestContext::get_range() const {
+  return byteRange_;
 }
 
 template<>
@@ -202,7 +180,7 @@ std::string RequestContext::get_argument(const std::string& name) const {
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
-  return headers.at(name);
+  return headers.at(lcAll(name));
 }
 
 }

src/server/request_context.h

@@ -27,8 +27,10 @@
 #include <map>
 #include <stdexcept>
 
+#include "byte_range.h"
+
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
@@ -51,9 +53,6 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
-  public: // types
-  typedef std::pair<int, int> ByteRange;
-
   public: // functions
     RequestContext(struct MHD_Connection* connection,
                    std::string rootLocation,
@@ -81,7 +80,6 @@ class RequestContext {
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
 
-    bool has_range() const;
     ByteRange get_range() const;
 
     bool can_compress() const { return acceptEncodingDeflate; }
@@ -95,14 +93,13 @@ class RequestContext {
 
     bool acceptEncodingDeflate;
 
-    bool accept_range;
-    ByteRange range_pair;
+    ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::string> arguments;
 
   private: // functions
-    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 
 template<> std::string RequestContext::get_argument(const std::string& name) const;

src/server/response.cpp

@@ -3,6 +3,7 @@
 
 #include "response.h"
 #include "request_context.h"
+#include "internalServer.h"
 #include "kiwixlib-resources.h"
 
 #include "tools/regexTools.h"
@@ -21,6 +22,17 @@ namespace
 {
 // some utilities
 
+std::string render_template(const std::string& template_str, kainjow::mustache::data data)
+{
+  kainjow::mustache::mustache tmpl(template_str);
+  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
+                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
+  data.set("urlencoded", urlencode);
+  std::stringstream ss;
+  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}
+
 std::string get_mime_type(const kiwix::Entry& entry)
 {
   try {
@@ -39,34 +51,79 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/json") != string::npos;
 }
 
-int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
-{
-  return range.second == -1
-       ? entry.getSize() - range.first
-       : range.second - range.first;
-}
 
 } // unnamed namespace
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
+Response::Response(bool verbose)
   : m_verbose(verbose),
-    m_root(root),
-    m_content(""),
-    m_mimeType(""),
-    m_returnCode(MHD_HTTP_OK),
-    m_withTaskbar(withTaskbar),
-    m_withLibraryButton(withLibraryButton),
-    m_blockExternalLinks(blockExternalLinks),
-    m_addTaskbar(false),
-    m_bookName(""),
-    m_startRange(0),
-    m_lenRange(0)
+    m_returnCode(MHD_HTTP_OK)
 {
+  add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+}
+
+std::unique_ptr<Response> Response::build(const InternalServer& server)
+{
+  return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
+}
+
+std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
+{
+  auto response = Response::build(server);
+  response->set_code(MHD_HTTP_NOT_MODIFIED);
+  response->m_etag = etag;
+  if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName)
+{
+  MustacheData results;
+  results.set("url", request.get_full_url());
+
+  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
+  response->set_code(MHD_HTTP_NOT_FOUND);
+  response->set_taskbar(bookName, "");
+
+  return std::move(response);
+}
+
+std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
+{
+  auto response = Response::build(server);
+// [FIXME] (compile with recent enough version of libmicrohttpd)
+//  response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
+  response->set_code(416);
+  std::ostringstream oss;
+  oss << "bytes */" << resourceLength;
+  response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str());
+
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
+{
+  MustacheData data;
+  data.set("error", msg);
+  auto content = render_template(RESOURCE::templates::_500_html, data);
+  std::unique_ptr<Response> response (
+      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
+  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
+  return response;
 }
 
 
-static int print_key_value (void *cls, enum MHD_ValueKind kind,
-                            const char *key, const char *value)
+std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
+{
+  auto response = Response::build(server);
+  response->m_returnCode = MHD_HTTP_FOUND;
+  response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
+  return response;
+}
+
+static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
+                                   const char *key, const char *value)
 {
   printf (" - %s: '%s'\n", key, value);
   return MHD_YES;
@@ -121,23 +178,10 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-std::string render_template(const std::string& template_str, kainjow::mustache::data data)
-{
-  kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
-  std::stringstream ss;
-  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
 
 
-void Response::introduce_taskbar()
+void ContentResponse::introduce_taskbar()
 {
-  if (! m_withTaskbar)
-    // Taskbar is globally disabled.
-    return;
   kainjow::mustache::data data;
   data.set("root", m_root);
   data.set("content", m_bookName);
@@ -158,7 +202,7 @@ void Response::introduce_taskbar()
 }
 
 
-void Response::inject_externallinks_blocker()
+void ContentResponse::inject_externallinks_blocker()
 {
   kainjow::mustache::data data;
   data.set("root", m_root);
@@ -170,24 +214,40 @@ void Response::inject_externallinks_blocker()
 }
 
 bool
-Response::can_compress(const RequestContext& request) const
+ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
 }
 
-MHD_Response*
-Response::create_raw_content_mhd_response(const RequestContext& request)
+bool
+ContentResponse::contentDecorationAllowed() const
 {
-  if (m_addTaskbar) {
-    introduce_taskbar();
-  }
-  if ( m_blockExternalLinks ) {
-    inject_externallinks_blocker();
+    return (startsWith(m_mimeType, "text/html")
+         && m_mimeType.find(";raw=true") == std::string::npos);
+}
+
+MHD_Response*
+Response::create_mhd_response(const RequestContext& request)
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT);
+  return response;
+}
+
+MHD_Response*
+ContentResponse::create_mhd_response(const RequestContext& request)
+{
+  if (contentDecorationAllowed()) {
+    if (m_withTaskbar) {
+      introduce_taskbar();
+    }
+    if (m_blockExternalLinks) {
+      inject_externallinks_blocker();
+    }
   }
 
-  bool shouldCompress = m_compress && can_compress(request);
+  bool shouldCompress = can_compress(request);
   if (shouldCompress) {
     std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
     uLongf comprLen = compr_buffer.capacity();
@@ -211,83 +271,29 @@ Response::create_raw_content_mhd_response(const RequestContext& request)
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
-  // shouldCompress can have different values. This can happen for a 304 (Not
-  // Modified) response generated while handling a conditional If-None-Match
-  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
-  // option) is obtained from the ETag list of the If-None-Match header and the
-  // response has no body (which shouldn't be compressed).
-  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-  }
-  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
   }
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
   return response;
 }
 
-MHD_Response*
-Response::create_redirection_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_entry_mhd_response() const
-{
-  MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
-                                               16384,
-                                               callback_reader_from_entry,
-                                               new RunningResponse(m_entry, m_startRange),
-                                               callback_free_response);
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
-  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-  std::ostringstream oss;
-  oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-      << "/" << m_entry.getSize();
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
-  return response;
-}
-
-MHD_Response*
-Response::create_mhd_response(const RequestContext& request)
-{
-  switch (m_mode) {
-    case ResponseMode::RAW_CONTENT :
-      return create_raw_content_mhd_response(request);
-
-    case ResponseMode::REDIRECTION :
-      return create_redirection_mhd_response();
-
-    case ResponseMode::ENTRY :
-      return create_entry_mhd_response();
-  }
-  return nullptr;
-}
-
-int Response::send(const RequestContext& request, MHD_Connection* connection)
+MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
 {
   MHD_Response* response = create_mhd_response(request);
 
-  MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
     m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+    MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  for(auto& p: m_customHeaders) {
+    MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
+  }
 
-  if (m_returnCode == MHD_HTTP_OK && request.has_range())
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
   if (m_verbose)
@@ -298,48 +304,105 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
   return ret;
 }
 
-void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
-  set_content(render_template(template_str, data));
-}
-
-void Response::set_content(const std::string& content) {
-  m_content = content;
-  m_mode = ResponseMode::RAW_CONTENT;
-}
-
-void Response::set_redirection(const std::string& url) {
-  m_content = url;
-  m_mode = ResponseMode::REDIRECTION;
-  m_returnCode = MHD_HTTP_FOUND;
-}
-
-void Response::set_entry(const Entry& entry, const RequestContext& request) {
-  m_entry = entry;
-  m_mode = ResponseMode::ENTRY;
-
-  const std::string mimeType = get_mime_type(entry);
-  set_mimeType(mimeType);
-  set_cacheable();
-
-  if ( is_compressible_mime_type(mimeType) ) {
-    zim::Blob raw_content = entry.getBlob();
-    const std::string content = string(raw_content.data(), raw_content.size());
-
-    set_content(content);
-    set_compress(true);
-  } else {
-    const int range_len = get_range_len(entry, request.get_range());
-    set_range_first(request.get_range().first);
-    set_range_len(range_len);
-  }
-}
-
-void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
 {
-  m_addTaskbar = true;
   m_bookName = bookName;
   m_bookTitle = bookTitle;
 }
 
 
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+  Response(verbose),
+  m_root(root),
+  m_content(content),
+  m_mimeType(mimetype),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  m_bookName(""),
+  m_bookTitle("")
+{
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype)
+{
+   return std::unique_ptr<ContentResponse>(new ContentResponse(
+        server.m_root,
+        server.m_verbose.load(),
+        server.m_withTaskbar,
+        server.m_withLibraryButton,
+        server.m_blockExternalLinks,
+        content,
+        mimetype));
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) {
+  auto content = render_template(template_str, data);
+  return ContentResponse::build(server, content, mimetype);
+}
+
+EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) :
+  Response(verbose),
+  m_entry(entry),
+  m_mimeType(mimetype)
+{
+  m_byteRange = byterange;
+  set_cacheable();
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry)
+{
+  const std::string mimetype = get_mime_type(entry);
+  auto byteRange = request.get_range().resolve(entry.getSize());
+  const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if (noRange && is_compressible_mime_type(mimetype)) {
+    // Return a contentResponse
+    zim::Blob raw_content = entry.getBlob();
+    const std::string content = string(raw_content.data(), raw_content.size());
+    auto response = ContentResponse::build(server, content, mimetype);
+    response->set_cacheable();
+    response->m_byteRange = byteRange;
+    return std::move(response);
+  }
+
+  if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
+    auto response = Response::build_416(server, entry.getSize());
+    response->set_cacheable();
+    return response;
+  }
+
+  return std::unique_ptr<Response>(new EntryResponse(
+        server.m_verbose.load(),
+        entry,
+        mimetype,
+        byteRange));
+}
+
+MHD_Response*
+EntryResponse::create_mhd_response(const RequestContext& request)
+{
+  const auto content_length = m_byteRange.length();
+  MHD_Response* response = MHD_create_response_from_callback(content_length,
+                                               16384,
+                                               callback_reader_from_entry,
+                                               new RunningResponse(m_entry, m_byteRange.first()),
+                                               callback_free_response);
+  MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
+  if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
+    std::ostringstream oss;
+    oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
+        << "/" << m_entry.getSize();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+
+  MHD_add_response_header(response,
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
+  return response;
+}
+
+
 }

src/server/response.h

@@ -22,80 +22,98 @@
 #define KIWIXLIB_SERVER_RESPONSE_H
 
 #include <string>
+#include <map>
 
 #include <mustache.hpp>
+#include "byte_range.h"
 #include "entry.h"
 #include "etag.h"
 
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
 
-enum class ResponseMode {
-  RAW_CONTENT,
-  REDIRECTION,
-  ENTRY
-};
-
+class InternalServer;
 class RequestContext;
 
+class EntryResponse;
+
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
-    ~Response() = default;
+    Response(bool verbose);
+    virtual ~Response() = default;
 
-    int send(const RequestContext& request, MHD_Connection* connection);
+    static std::unique_ptr<Response> build(const InternalServer& server);
+    static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
+    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName);
+    static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
+    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
+    static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
-    void set_template(const std::string& template_str, kainjow::mustache::data data);
-    void set_content(const std::string& content);
-    void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry, const RequestContext& request);
+    MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
-
-    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
     void set_code(int code) { m_returnCode = code; }
     void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
     void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
-    void set_etag(const ETag& etag) { m_etag = etag; }
-    void set_compress(bool compress) { m_compress = compress; }
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
-    void set_range_first(uint64_t start) { m_startRange = start; }
-    void set_range_len(uint64_t len) { m_lenRange = len; }
+    void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
-    std::string get_mimeType() const { return m_mimeType; }
+
+  private: // functions
+    virtual MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
+
+  protected: // data
+    bool m_verbose;
+    int m_returnCode;
+    ByteRange m_byteRange;
+    ETag m_etag;
+    std::map<std::string, std::string> m_customHeaders;
+
+    friend class EntryResponse; // temporary to allow the builder to change m_mode
+};
+
+
+class ContentResponse : public Response {
+  public:
+    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype);
+
+    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
 
     void introduce_taskbar();
     void inject_externallinks_blocker();
-
     bool can_compress(const RequestContext& request) const;
+    bool contentDecorationAllowed() const;
 
-  private: // functions
-    MHD_Response* create_mhd_response(const RequestContext& request);
-    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
-    MHD_Response* create_redirection_mhd_response() const;
-    MHD_Response* create_entry_mhd_response() const;
 
-  private: // data
-    bool m_verbose;
-    ResponseMode m_mode;
+  private:
     std::string m_root;
     std::string m_content;
-    Entry m_entry;
     std::string m_mimeType;
-    int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
-    bool m_compress;
-    bool m_addTaskbar;
     std::string m_bookName;
     std::string m_bookTitle;
-    uint64_t m_startRange;
-    uint64_t m_lenRange;
-    ETag m_etag;
+ };
+
+class EntryResponse : public Response {
+  public:
+    EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
+
+    Entry m_entry;
+    std::string m_mimeType;
 };
 
 }

src/subprocess_unix.cpp

@@ -26,6 +26,7 @@ UnixImpl::~UnixImpl()
 #else
   pthread_cancel(m_waitingThread);
 #endif
+  pthread_join(m_waitingThread, nullptr);
 }
 
 #ifdef __ANDROID__

src/subprocess_windows.cpp

@@ -11,7 +11,8 @@
 WinImpl::WinImpl():
   m_pid(0),
   m_running(false),
-  m_handle(INVALID_HANDLE_VALUE)
+  m_subprocessHandle(INVALID_HANDLE_VALUE),
+  m_waitingThreadHandle(INVALID_HANDLE_VALUE)
 {
   InitializeCriticalSection(&m_criticalSection);
 }
@@ -19,14 +20,15 @@ WinImpl::WinImpl():
 WinImpl::~WinImpl()
 {
   kill();
-  CloseHandle(m_handle);
+  WaitForSingleObject(m_waitingThreadHandle, INFINITE);
+  CloseHandle(m_subprocessHandle);
   DeleteCriticalSection(&m_criticalSection);
 }
 
 DWORD WINAPI WinImpl::waitForPID(void* _self)
 {
   WinImpl* self = static_cast<WinImpl*>(_self);
-  WaitForSingleObject(self->m_handle, INFINITE);
+  WaitForSingleObject(self->m_subprocessHandle, INFINITE);
 
   EnterCriticalSection(&self->m_criticalSection);
   self->m_running = false;
@@ -79,16 +81,16 @@ void WinImpl::run(commandLine_t& commandLine)
     &procInfo))
   {
     m_pid = procInfo.dwProcessId;
-    m_handle = procInfo.hProcess;
+    m_subprocessHandle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);
     m_running = true;
-    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
+    m_waitingThreadHandle = CreateThread(NULL, 0, &waitForPID, this, 0, NULL);
   }
 }
 
 bool WinImpl::kill()
 {
-  return TerminateProcess(m_handle, 0);
+  return TerminateProcess(m_subprocessHandle, 0);
 }
 
 bool WinImpl::isRunning()

src/subprocess_windows.h

@@ -11,7 +11,8 @@ class WinImpl : public SubprocessImpl
   private:
     int m_pid;
     bool m_running;
-    HANDLE m_handle;
+    HANDLE m_subprocessHandle;
+    HANDLE  m_waitingThreadHandle;
     CRITICAL_SECTION m_criticalSection;
 
   public:

src/tools/otherTools.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "tools/otherTools.h"
+#include <algorithm>
 
 #ifdef _WIN32
 #include <windows.h>
@@ -280,3 +281,63 @@ bool kiwix::convertStrToBool(const std::string& value)
   throw std::domain_error(ss.str());
 }
 
+namespace
+{
+// The counter metadata format is a list of item separated by a `;` :
+// item0;item1;item2
+// Each item is a "tuple" mimetype=number.
+// However, the mimetype may contains parameters:
+// text/html;raw=true;foo=bar
+// So the final format may be complex to parse:
+// key0=value0;key1;foo=bar=value1;key2=value2
+
+typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
+
+std::string readFullMimetypeAndCounterString(std::istream& in)
+{
+  std::string mtcStr, params;
+  getline(in, mtcStr, ';');
+  if ( mtcStr.find('=') == std::string::npos )
+  {
+    do
+    {
+      if ( !getline(in, params, ';' ) )
+        return std::string();
+      mtcStr += ";" + params;
+    }
+    while ( std::count(params.begin(), params.end(), '=') != 2 );
+  }
+  return mtcStr;
+}
+
+MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
+{
+  const std::string::size_type k = s.find_last_of("=");
+  if ( k != std::string::npos )
+  {
+    const std::string mimeType = s.substr(0, k);
+    std::istringstream counterSS(s.substr(k+1));
+    unsigned int counter;
+    if (counterSS >> counter && counterSS.eof())
+      return MimetypeAndCounter{mimeType, counter};
+  }
+  return MimetypeAndCounter{"", 0};
+}
+
+} // unnamed namespace
+
+kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
+{
+  kiwix::MimeCounterType counters;
+  std::istringstream ss(counterData);
+
+  while (ss)
+  {
+    const std::string mtcStr = readFullMimetypeAndCounterString(ss);
+    const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
+    if ( !mtc.first.empty() )
+      counters.insert(mtc);
+  }
+
+  return counters;
+}

src/tools/pathTools.cpp

@@ -81,13 +81,19 @@ std::wstring Utf8ToWide(const std::string& str)
 bool isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+  if (path.size() < 3 ) {
+    return true;
+  }
+  if (path.substr(1, 2) == ":\\" || path.substr(0, 2) == "\\\\") {
+    return false;
+  }
+  return true;
 #else
   return path.empty() || path.substr(0, 1) == "/" ? false : true;
 #endif
 }
 
-std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute)
 {
   std::vector<std::string> ret;
 #ifdef _WIN32
@@ -95,10 +101,35 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
   //Starts from there.
   auto it = find_if(parts.rbegin(), parts.rend(),
                     [](const std::string& p) ->bool
-                      { return p.length() == 2 && p[1] == ':'; });
+                      { return ((p.length() == 2 && p[1] == ':')
+                             || (p.length() > 2 && p[0] == '\\' && p[1] == '\\')); });
   if (it != parts.rend()) {
     parts.erase(parts.begin(), it.base()-1);
   }
+  // Special case for samba mount point starting with two "\\" ("\\\\samba\\foo")
+  if (parts.size() > 2  && parts[0].empty() && parts[1].empty()) {
+    parts.erase(parts.begin(), parts.begin()+2);
+    parts[0] = "\\\\" + parts[0];
+  }
+  // Special case if we have a samba drive not at first.
+  // Path is "..\\\\sambdadrive\\..\\.." So we will have an empty part.
+  auto previous_empty = false;
+  for (it = parts.rbegin(); it!=parts.rend(); it++) {
+    if(it->empty()) {
+      if (previous_empty) {
+        it++;
+        break;
+      } else {
+        previous_empty = true;
+      }
+    } else {
+      previous_empty = false;
+    }
+  }
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+    parts[0] = "\\\\" + parts[0];
+  }
 #endif
 
   size_t index = 0;
@@ -144,8 +175,10 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
 
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
-  auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
-  auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto parts = kiwix::split(path, SEPARATOR, false);
+  auto pathParts = normalizeParts(parts, false);
+  parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absolutePathParts = normalizeParts(parts, true);
 
   unsigned int commonCount = 0;
   while (commonCount < pathParts.size()
@@ -172,8 +205,10 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
     absolutePath = getCurrentDirectory();
   }
 
-  auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
-  auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
+  auto parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absoluteParts = normalizeParts(parts, true);
+  parts = kiwix::split(relativePath, SEPARATOR, false);
+  auto relativeParts = normalizeParts(parts, false);
 
   absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
   auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
@@ -182,7 +217,8 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
 
 std::string removeLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  auto parts_ = kiwix::split(path, SEPARATOR, false);
+  auto parts = normalizeParts(parts_, false);
   if (!parts.empty()) {
     parts.pop_back();
   }
@@ -202,7 +238,8 @@ std::string appendToDirectory(const std::string& directoryPath, const std::strin
 
 std::string getLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
+  auto parts_ = kiwix::split(path, SEPARATOR);
+  auto parts = normalizeParts(parts_, false);
   if (parts.empty()) {
     return "";
   }

src/tools/stringTools.cpp

@@ -268,7 +268,8 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
 /* Split string in a token array */
 std::vector<std::string> kiwix::split(const std::string& str,
                                       const std::string& delims,
-                                      bool trimEmpty)
+                                      bool trimEmpty,
+                                      bool keepDelim)
 {
   std::string::size_type lastPos = 0;
   std::string::size_type pos = 0;
@@ -279,6 +280,9 @@ std::vector<std::string> kiwix::split(const std::string& str,
     if (!trimEmpty || !token.empty()) {
       tokens.push_back(token);
     }
+    if (keepDelim) {
+      tokens.push_back(str.substr(pos, 1));
+    }
     lastPos = pos + 1;
   }
 

src/wrapper/java/kiwixreader.cpp

@@ -355,9 +355,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
   unsigned int cCount = jni2c(count, env);
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
     LOG(e.what());
@@ -377,11 +380,14 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
   std::string cUrl;
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
       setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get next suggestion");
     LOG(e.what());

src/xmlrpc.h

@@ -3,6 +3,7 @@
 #ifndef KIWIX_XMLRPC_H_
 #define KIWIX_XMLRPC_H_
 
+#include <stdexcept>
 #include <tools/otherTools.h>
 #include <pugixml.hpp>
 

static/skin/block_external.js

@@ -1,3 +1,4 @@
+// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
 var block_path = "/catch/external";
 // called only on external links
 function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }

static/skin/taskbar.css

@@ -124,7 +124,7 @@ label[for=kiwixsearchbox] {
 }
 
 body {
-    padding-top: 3em !important;
+    padding-top: calc(3em - 5px) !important;
 }
 
 /* Try to fix buggy stuff in jquery-ui autocomplete */

static/templates/index.html

@@ -29,7 +29,7 @@
                 box-shadow: 2px 2px 5px 0px #ccc;
         }
         .book:hover { background-color: #F9F9F9; box-shadow: none;}
-        .book__background { background-repeat: no-repeat; background-size: auto; background-position: top right; }
+        .book__background { background-repeat: no-repeat; background-size: 48px 48px; background-position: top right; }
         .book__title {
                 padding: 0 55px 0 0;overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
                 font-size: 18px; color: #0645ad; line-height: 1em;

static/templates/search_result.html

@@ -133,7 +133,7 @@
         <ul>
           {{#resultLastPageStart}}
             <li>
-              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&pageLength={{pageLength}}">
                 ◀
               </a>
             </li>
@@ -141,14 +141,14 @@
           {{#pages}}
             <li>
               <a {{#selected}}class="selected"{{/selected}}
-                 href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
+                 href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&pageLength={{pageLength}}">
                 {{label}}
               </a>
             </li>
           {{/pages}}
           {{#resultLastPageStart}}
             <li>
-              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&pageLength={{pageLength}}">
                 ▶
               </a>
             </li>

test/counterParsing.cpp

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2019 Matthieu Gautier
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
+ * NON-INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#include "gtest/gtest.h"
+#include <string>
+#include <vector>
+#include <map>
+#include <zim/zim.h>
+
+namespace kiwix {
+using CounterType = std::map<const std::string, zim::article_index_type>;
+CounterType parseMimetypeCounter(const std::string& counterData);
+};
+
+using namespace kiwix;
+#define parse parseMimetypeCounter
+
+namespace
+{
+TEST(ParseCounterTest, simpleMimeType)
+{
+  {
+    std::string counterStr = "";
+    CounterType counterMap = {};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "foo=1";
+    CounterType counterMap = {{"foo", 1}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "foo=1;text/html=50;";
+    CounterType counterMap = {{"foo", 1}, {"text/html", 50}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+}
+
+TEST(ParseCounterTest, paramMimeType)
+{
+  {
+    std::string counterStr = "text/html;raw=true=1";
+    CounterType counterMap = {{"text/html;raw=true", 1}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "foo=1;text/html;raw=true=50;bar=2";
+    CounterType counterMap = {{"foo", 1}, {"text/html;raw=true", 50}, {"bar", 2}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "foo=1;text/html;raw=true;param=value=50;bar=2";
+    CounterType counterMap = {{"foo", 1}, {"text/html;raw=true;param=value", 50}, {"bar", 2}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "foo=1;text/html;raw=true=50;bar=2";
+    CounterType counterMap = {{"foo", 1}, {"text/html;raw=true", 50}, {"bar", 2}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "application/javascript=8;text/html=3;application/warc-headers=28364;text/html;raw=true=6336;text/css=47;text/javascript=98;image/png=968;image/webp=24;application/json=3694;image/gif=10274;image/jpeg=1582;font/woff2=25;text/plain=284;application/atom+xml=247;application/x-www-form-urlencoded=9;video/mp4=9;application/x-javascript=7;application/xml=1;image/svg+xml=5";
+    CounterType counterMap = {
+      {"application/javascript", 8},
+      {"text/html", 3},
+      {"application/warc-headers", 28364},
+      {"text/html;raw=true", 6336},
+      {"text/css", 47},
+      {"text/javascript", 98},
+      {"image/png", 968},
+      {"image/webp", 24},
+      {"application/json", 3694},
+      {"image/gif", 10274},
+      {"image/jpeg", 1582},
+      {"font/woff2", 25},
+      {"text/plain", 284},
+      {"application/atom+xml", 247},
+      {"application/x-www-form-urlencoded", 9},
+      {"video/mp4", 9},
+      {"application/x-javascript", 7},
+      {"application/xml", 1},
+      {"image/svg+xml", 5}
+    };
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+}
+
+TEST(ParseCounterTest, wrongType)
+{
+  CounterType empty = {};
+  {
+    std::string counterStr = "text/html";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html=";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html=foo";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html=123foo";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html=50;foo";
+    CounterType counterMap = {{"text/html", 50}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html;foo=20";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html;foo=20;";
+    ASSERT_EQ(parse(counterStr), empty) << counterStr;
+  }
+  {
+    std::string counterStr = "text/html=50;;foo";
+    CounterType counterMap = {{"text/html", 50}};
+    ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
+  }
+}
+
+};

test/data/create_corner_cases_zim_file

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")"
+rm -f corner_cases.zim
+zimwriterfs -w empty.html \
+            -f empty.png \
+            -l=en \
+            -t="ZIM corner cases" \
+            -d="" \
+            -c="" \
+            -p="" \
+            corner_cases \
+            corner_cases.zim \
+&& echo 'corner_cases.zim was successfully created' \
+|| echo '!!! Failed to create corner_cases.zim !!!' >&2

test/library.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Tommi Maekitalo
+ * Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -262,8 +262,13 @@ TEST_F(LibraryTest, filterCheck)
 TEST_F(LibraryTest, getBookByPath)
 {
   auto& book = lib.getBookById(lib.getBooksIds()[0]);
-  book.setPath("/some/abs/path.zim");
-  EXPECT_EQ(lib.getBookByPath("/some/abs/path.zim").getId(), book.getId());
+#ifdef _WIN32
+  auto path = "C:\\some\\abs\\path.zim";
+#else
+  auto path = "/some/abs/path.zim";
+#endif
+  book.setPath(path);
+  EXPECT_EQ(lib.getBookByPath(path).getId(), book.getId());
   EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
 }
 };

test/meson.build

@@ -1,12 +1,9 @@
-configure_file(input : 'data/example.zim',
-               output : 'example.zim',
-               copy: true )
-
 tests = [
     'parseUrl',
     'library',
     'regex',
     'tagParsing',
+    'counterParsing',
     'stringTools',
     'pathTools',
     'kiwixserve',
@@ -26,9 +23,32 @@ gtest_dep = dependency('gtest',
                        required:false)
 
 if gtest_dep.found() and not meson.is_cross_build()
-    configure_file(input : 'data/wikipedia_en_ray_charles_mini_2020-03.zim',
-                   output : 'zimfile.zim',
-                   copy: true )
+    data_files = [
+      'example.zim',
+      'zimfile.zim',
+      'corner_cases.zim'
+    ]
+    foreach file : data_files
+        # configure_file(input : 'data/' + file,
+        #                output : file,
+        #                copy: true )
+        #
+        # Above (commented) command doesn't work with Meson versions below 0.47
+        # (in which the 'copy' keyword was first introduced). We want to keep
+        # compatibility with Ubuntu 18.04 Bionic (which has Meson version 0.45)
+        # until its EOL.
+        #
+        # Below is a python based workaround.
+        configure_file(input : 'data/' + file,
+                       output : file,
+                       command: [
+                         find_program('python3'),
+                          '-c',
+                          'import sys; import shutil; shutil.copy(sys.argv[1], sys.argv[2])',
+                          '@INPUT@',
+                          '@OUTPUT@'
+                       ])
+    endforeach
 
     foreach test_name : tests
         # XXX: implicit_include_directories must be set to false, otherwise
@@ -39,6 +59,6 @@ if gtest_dep.found() and not meson.is_cross_build()
                               link_args: extra_link_args,
                               dependencies : all_deps + [gtest_dep],
                              build_rpath : '$ORIGIN')
-        test(test_name, test_exe)
+        test(test_name, test_exe, timeout : 160)
     endforeach
 endif

test/pathTools.cpp

@@ -25,6 +25,7 @@
 #ifdef _WIN32
 # define S "\\"
 # define AS "c:"
+# define A_SAMBA "\\\\sambadir"
 #else
 # define S "/"
 # define AS ""
@@ -42,7 +43,10 @@
 #define A4(a, b, c, d) A1(P4(a, b, c, d))
 #define A5(a, b, c, d, e) A1(P5(a, b, c, d, e))
 
-std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute);
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute);
+std::vector<std::string> nParts(std::vector<std::string> parts, bool absolute) {
+  return normalizeParts(parts, absolute);
+}
 #ifdef _WIN32
 std::wstring Utf8ToWide(const std::string& str);
 std::string WideToUtf8(const std::wstring& wstr);
@@ -54,7 +58,7 @@ namespace
 #define V std::vector<std::string>
 TEST(pathTools, normalizePartsAbsolute)
 {
-#define N(...) normalizeParts(__VA_ARGS__, true)
+#define N(...) nParts(__VA_ARGS__, true)
   ASSERT_EQ(N({}), V({}));
 #ifdef _WIN32
   ASSERT_EQ(N({"c:"}), V({"c:"}));
@@ -75,13 +79,14 @@ TEST(pathTools, normalizePartsAbsolute)
 
 #ifdef _WIN32
   ASSERT_EQ(N({"c:", "a", "b", ".", "c", "d:", "..", "foo"}), V({"d:", "foo"}));
+  ASSERT_EQ(N({"","","samba","a","b"}), V({"\\\\samba", "a", "b"}));
 #endif
 #undef N
 }
 
 TEST(pathTools, normalizePartsRelative)
 {
-#define N(...) normalizeParts(__VA_ARGS__, false)
+#define N(...) nParts(__VA_ARGS__, false)
   ASSERT_EQ(N({}), V({}));
   ASSERT_EQ(N({""}), V({}));
   ASSERT_EQ(N({"a"}), V({"a"}));
@@ -103,6 +108,10 @@ TEST(pathTools, isRelativePath)
   ASSERT_TRUE(isRelativePath(P4("foo","","bar","")));
   ASSERT_FALSE(isRelativePath(A1("foo")));
   ASSERT_FALSE(isRelativePath(A2("foo", "bar")));
+#ifdef _WIN32
+  ASSERT_FALSE(isRelativePath(P2(A_SAMBA, "foo")));
+  ASSERT_FALSE(isRelativePath(P3(A_SAMBA, "foo", "bar")));
+#endif
 }
 
 TEST(pathTools, computeAbsolutePath)
@@ -121,6 +130,12 @@ TEST(pathTools, computeAbsolutePath)
             A5("a","b","c","d","foo"));
   ASSERT_EQ(computeAbsolutePath(A5("a","b","c","d","e"), P5("..","..","..","g","foo")),
             A4("a","b","g","foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b",""), P2("..","foo")),
+            P3(A_SAMBA,"a","foo"));
+  ASSERT_EQ(computeAbsolutePath(P6(A_SAMBA,"a","b","c","d","e"), P5("..","..","..","g","foo")),
+            P5(A_SAMBA,"a","b","g","foo"));
+#endif
 }
 
 TEST(pathTools, computeRelativePath)
@@ -137,6 +152,13 @@ TEST(pathTools, computeRelativePath)
             P2("..","foo"));
   ASSERT_EQ(computeRelativePath(A5("a","b","c","d","e"), A4("a","b","g","foo")),
             P5("..","..","..","g","foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeRelativePath(P3(A_SAMBA,"a","b"), P3(A_SAMBA,"a","foo")),
+            P2("..","foo"));
+  ASSERT_EQ(computeRelativePath(P6(A_SAMBA,"a","b","c","d","e"), P5(A_SAMBA,"a","b","g","foo")),
+            P5("..","..","..","g","foo"));
+
+#endif
 }
 
 TEST(pathTools, removeLastPathElement)
@@ -179,6 +201,15 @@ TEST(pathTools, goUp)
             "c:");
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
             "c:");
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), ".."),
+            P3(A_SAMBA,"a", "b"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P2("..","..")),
+            P2(A_SAMBA,"a"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P3("..","..","..")),
+            A_SAMBA);
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","..")),
+            A_SAMBA);
+
 #else
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
             "/");
@@ -195,6 +226,12 @@ TEST(pathTools, goUp)
             A1("foo"));
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P5("..","..","..","..","foo")),
             A1("foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","foo")),
+            P2(A_SAMBA,"foo"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P5("..","..","..","..","foo")),
+            P2(A_SAMBA,"foo"));
+#endif
 }
 
 
@@ -203,11 +240,22 @@ TEST(pathTools, dirChange)
 {
   std::string p1("c:\\a\\b\\c");
   std::string p2("d:\\d\\e\\foo.xml");
-  std::string relative_path = computeRelativePath(p1, p2);
-  ASSERT_EQ(relative_path, "d:\\d\\e\\foo.xml");
-  std::string abs_path = computeAbsolutePath(p1, relative_path);
-  ASSERT_EQ(abs_path, p2);
-  ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
+  {
+    std::string relative_path = computeRelativePath(p1, p2);
+    ASSERT_EQ(relative_path, p2);
+    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    ASSERT_EQ(abs_path, p2);
+    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
+  }
+  std::string ps("\\\\samba\\d\\e\\foo.xml");
+  {
+    std::string relative_path = computeRelativePath(p1, ps);
+    ASSERT_EQ(relative_path, ps);
+    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    ASSERT_EQ(abs_path, ps);
+    // I'm not sure this test is valid on windows :/
+//    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\\\samba\\d\\e\\foo.xml"), ps);
+  }
 }
 
 TEST(pathTools, Utf8ToWide)

test/server.cpp

@@ -50,9 +50,10 @@ class ZimFileServer
 {
 public: // types
   typedef std::shared_ptr<httplib::Response>  Response;
+  typedef std::vector<std::string> FilePathCollection;
 
 public: // functions
-  ZimFileServer(int serverPort, std::string zimpath);
+  ZimFileServer(int serverPort, const FilePathCollection& zimpaths);
   ~ZimFileServer();
 
   Response GET(const char* path, const Headers& headers = Headers())
@@ -73,11 +74,13 @@ private: // data
   std::unique_ptr<httplib::Client> client;
 };
 
-ZimFileServer::ZimFileServer(int serverPort, std::string zimpath)
+ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths)
 : manager(&this->library)
 {
-  if (!manager.addBookFromPath(zimpath, zimpath, "", false))
-    throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
+  for ( const auto& zimpath : zimpaths ) {
+    if (!manager.addBookFromPath(zimpath, zimpath, "", false))
+      throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
+  }
 
   const std::string address = "127.0.0.1";
   nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false));
@@ -104,11 +107,14 @@ protected:
   std::unique_ptr<ZimFileServer>   zfs1_;
 
   const int PORT = 8001;
-  const std::string ZIMFILE = "./test/zimfile.zim";
+  const ZimFileServer::FilePathCollection ZIMFILES {
+    "./test/zimfile.zim",
+    "./test/corner_cases.zim"
+  };
 
 protected:
   void SetUp() override {
-    zfs1_.reset(new ZimFileServer(PORT, ZIMFILE));
+    zfs1_.reset(new ZimFileServer(PORT, ZIMFILES));
   }
 
   void TearDown() override {
@@ -149,7 +155,7 @@ const ResourceCollection resources200Compressible{
   { NO_ETAG,   "/catalog/searchdescription.xml" },
   { NO_ETAG,   "/catalog/search" },
 
-  { NO_ETAG,   "/search?content=zimfile&pattern=abcd" },
+  { NO_ETAG,   "/search?content=zimfile&pattern=a" },
 
   { NO_ETAG,   "/suggest?content=zimfile&term=ray" },
 
@@ -174,6 +180,10 @@ const ResourceCollection resources200Uncompressible{
   { WITH_ETAG, "/meta?content=zimfile&name=favicon" },
 
   { WITH_ETAG, "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
+
+  { WITH_ETAG, "/corner_cases/A/empty.html" },
+  { WITH_ETAG, "/corner_cases/-/empty.css" },
+  { WITH_ETAG, "/corner_cases/-/empty.js" },
 };
 
 ResourceCollection all200Resources()
@@ -187,6 +197,15 @@ TEST_F(ServerTest, 200)
     EXPECT_EQ(200, zfs1_->GET(res.url)->status) << "res.url: " << res.url;
 }
 
+// seperate test for 204 code
+
+TEST_F(ServerTest, EmptySearchReturnsA204StatusCode)
+{
+  const char* url="/search?content=zimfile&pattern=abcd";
+  auto res=zfs1_->GET(url);
+  EXPECT_EQ(204, res->status) << "res.url: " << url;
+}
+
 TEST_F(ServerTest, CompressibleContentIsCompressedIfAcceptable)
 {
   for ( const Resource& res : resources200Compressible ) {
@@ -307,7 +326,7 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet)
 
 TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags)
 {
-  ZimFileServer zfs2(PORT + 1, ZIMFILE);
+  ZimFileServer zfs2(PORT + 1, ZIMFILES);
   for ( const Resource& res : all200Resources() ) {
     if ( !res.etag_expected ) continue;
     const auto h1 = zfs1_->HEAD(res.url);
@@ -408,3 +427,122 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
     EXPECT_EQ(200, g2->status);
   }
 }
+
+TEST_F(ServerTest, ValidSingleRangeByteRangeRequestsAreHandledProperly)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const auto full = zfs1_->GET(url);
+  EXPECT_FALSE(full->has_header("Content-Range"));
+  EXPECT_EQ("bytes", full->get_header_value("Accept-Ranges"));
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=0-100000"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body, p->body);
+    EXPECT_EQ("bytes 0-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=0-10"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ("bytes 0-10/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ(11U, p->body.size());
+    EXPECT_EQ(full->body.substr(0, 11), p->body);
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=123-456"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ("bytes 123-456/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ(334U, p->body.size());
+    EXPECT_EQ(full->body.substr(123, 334), p->body);
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=20000-"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body.substr(20000), p->body);
+    EXPECT_EQ("bytes 20000-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=-100"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body.substr(19977), p->body);
+    EXPECT_EQ("bytes 19977-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+}
+
+TEST_F(ServerTest, InvalidAndMultiRangeByteRangeRequestsResultIn416Responses)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+
+  const char* invalidRanges[] = {
+    "0-10", "bytes=", "bytes=123", "bytes=-10-20", "bytes=10-20xxx",
+    "bytes=10-0", // reversed range
+    "bytes=10-20, 30-40", // multi-range
+    "bytes=1000000-", "bytes=30000-30100" // unsatisfiable ranges
+  };
+
+  for( const char* range : invalidRanges )
+  {
+    const TestContext ctx{ {"Range", range} };
+    const auto p = zfs1_->GET(url, { {"Range", range } } );
+    EXPECT_EQ(416, p->status) << ctx;
+    EXPECT_TRUE(p->body.empty()) << ctx;
+    EXPECT_EQ("bytes */20077", p->get_header_value("Content-Range")) << ctx;
+  }
+}
+
+TEST_F(ServerTest, ValidByteRangeRequestsOfZeroSizedEntriesResultIn416Responses)
+{
+  const char url[] = "/corner_cases/-/empty.js";
+
+  const char* ranges[] = {
+    "bytes=0-",
+    "bytes=-100"
+  };
+
+  for( const char* range : ranges )
+  {
+    const TestContext ctx{ {"Range", range} };
+    const auto p = zfs1_->GET(url, { {"Range", range } } );
+    EXPECT_EQ(416, p->status) << ctx;
+    EXPECT_TRUE(p->body.empty()) << ctx;
+    EXPECT_EQ("bytes */0", p->get_header_value("Content-Range")) << ctx;
+  }
+}
+
+TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+
+  const Headers onlyRange{ {"Range", "bytes=123-456"} };
+  Headers rangeAndCompression(onlyRange);
+  rangeAndCompression.insert({"Accept-Encoding", "deflate"});
+
+  const auto p1 = zfs1_->GET(url, onlyRange);
+  const auto p2 = zfs1_->GET(url, rangeAndCompression);
+  EXPECT_EQ(p1->status, p2->status);
+  EXPECT_EQ(invariantHeaders(p1->headers), invariantHeaders(p2->headers));
+  EXPECT_EQ(p1->body, p2->body);
+}
+
+TEST_F(ServerTest, RangeHeaderIsCaseInsensitive)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const auto r0 = zfs1_->GET(url, { {"Range", "bytes=100-200"} } );
+
+  const char* header_variations[] = { "RANGE", "range", "rAnGe", "RaNgE" };
+  for ( const char* header : header_variations ) {
+    const auto r = zfs1_->GET(url, { {header, "bytes=100-200"} } );
+    EXPECT_EQ(206, r->status);
+    EXPECT_EQ("bytes 100-200/20077", r->get_header_value("Content-Range"));
+    EXPECT_EQ(r0->body, r->body);
+  }
+}

test/stringTools.cpp

@@ -23,7 +23,7 @@
 
 namespace kiwix {
 std::string join(const std::vector<std::string>& list, const std::string& sep);
-std::vector<std::string> split(const std::string&  base, const std::string& sep, bool trimEmpty);
+std::vector<std::string> split(const std::string&  base, const std::string& sep, bool trimEmpty, bool keepDelim);
 };
 
 using namespace kiwix;
@@ -40,17 +40,22 @@ TEST(stringTools, join)
 TEST(stringTools, split)
 {
   std::vector<std::string> list1 = { "a", "b", "c" };
-  ASSERT_EQ(split("a;b;c", ";", false), list1);
-  ASSERT_EQ(split("a;b;c", ";", true), list1);
+  ASSERT_EQ(split("a;b;c", ";", false, false), list1);
+  ASSERT_EQ(split("a;b;c", ";", true, false), list1);
   std::vector<std::string> list2 = { "", "a", "b", "c" };
-  ASSERT_EQ(split(";a;b;c", ";", false), list2);
-  ASSERT_EQ(split(";a;b;c", ";", true), list1);
+  ASSERT_EQ(split(";a;b;c", ";", false, false), list2);
+  ASSERT_EQ(split(";a;b;c", ";", true, false), list1);
   std::vector<std::string> list3 = { "", "a", "b", "c", ""};
-  ASSERT_EQ(split(";a;b;c;", ";", false), list3);
-  ASSERT_EQ(split(";a;b;c;", ";", true), list1);
+  ASSERT_EQ(split(";a;b;c;", ";", false, false), list3);
+  ASSERT_EQ(split(";a;b;c;", ";", true, false), list1);
   std::vector<std::string> list4 = { "", "a", "b", "", "c", ""};
-  ASSERT_EQ(split(";a;b;;c;", ";", false), list4);
-  ASSERT_EQ(split(";a;b;;c;", ";", true), list1);
+  ASSERT_EQ(split(";a;b;;c;", ";", false, false), list4);
+  ASSERT_EQ(split(";a;b;;c;", ";", true, false), list1);
+
+  std::vector<std::string> list5 = { ";", "a", ";", "b", "=", ";", "c", "=", "d", ";"};
+  ASSERT_EQ(split(";a;b=;c=d;", ";=", true, true), list5);
+  std::vector<std::string> list6 = { "", ";", "a", ";", "b", "=", "", ";", "c", "=", "d", ";", ""};
+  ASSERT_EQ(split(";a;b=;c=d;", ";=", false, true), list6);
 }
 
 };