New version 9.3.1

Have meson manage/install the kiwix-compile-resources.1 man page

.github/workflows/ci.yml

@@ -53,30 +53,41 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        name:
           - native_static
           - native_dyn
+          - native_dyn_bionic
           - android_arm
           - android_arm64
           - win32_static
           - win32_dyn
         include:
-          - target: native_static
+          - name: native_static
+            target: native_static
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: native_dyn
+          - name: native_dyn
+            target: native_dyn
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm
+          - name: native_dyn_bionic
+            target: native_dyn
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - name: android_arm
+            target: android_arm
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: android_arm64
+          - name: android_arm64
+            target: android_arm64
             image_variant: xenial
             lib_postfix: '/x86_64-linux-gnu'
-          - target: win32_static
+          - name: win32_static
+            target: win32_static
             image_variant: f31
             lib_postfix: '64'
-          - target: win32_dyn
+          - name: win32_dyn
+            target: win32_dyn
             image_variant: f31
             lib_postfix: '64'
     env:
@@ -147,6 +158,6 @@ jobs:
         curl https://codecov.io/bash -o codecov.sh
         bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
         rm codecov.sh
-      if: startsWith(matrix.target, 'native_')
+      if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/package.yml

@@ -0,0 +1,78 @@
+name: Packages
+on: [push, pull_request]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro: [ubuntu-groovy, ubuntu-focal, ubuntu-eoan]
+    steps:
+      - uses: actions/checkout@v2
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "::set-output name=ppa::kiwixteam/release"
+          else
+            echo "::set-output name=ppa::kiwixteam/dev"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@master
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-groovy
+        if: matrix.distro == 'ubuntu-groovy'
+        name: Build package for ubuntu-groovy
+        id: build-ubuntu-groovy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-focal
+        if: matrix.distro == 'ubuntu-focal'
+        name: Build package for ubuntu-focal
+        id: build-ubuntu-focal
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-eoan
+        if: matrix.distro == 'ubuntu-eoan'
+        name: Build package for ubuntu-eoan
+        id: build-ubuntu-eoan
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload dev package
+        # Only upload on pushes to master
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@master
+        name: Upload release package
+        # Only upload on pushes to master or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

ChangeLog

@@ -1,3 +1,39 @@
+kiwix-lib 9.3.1
+===============
+
+ * Fix handling of samba path on windows.
+ * Do not include `kiwix_config.h` in public header.
+ * Fix compilation with libmicrohttpd v0.97.1
+ * Increase default test timeout to 160seconds/test.
+ * Add automatic debian packaging.
+ * Use non-minified version of jquery-ui.js
+ * Pass `-latomic` compile option for sh4 architecture.
+ * Make mesion install `kiwix-compile-resources` man page.
+
+kiwix-lib 9.3.0
+===============
+
+ * Add a thread safe method to search suggestions.
+   Old methods are now deprecated.
+
+kiwix-lib 9.2.3
+===============
+
+ * Add test on byte-range
+ * Fix compilation on bionic and windows.
+ * Allow building using debian packaged kainjow-mustache
+ * Pass `-latomic` compile option for architectures that need it
+
+kiwix-lib 9.2.2
+===============
+
+ * Fix handling on empty content in byte range management (wrong assert)
+
+kiwix-lib 9.2.1
+===============
+
+ * Fix support of byte range request.
+
 kiwix-lib 9.2
 =============
 

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '9.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '9.3.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

debian/changelog

@@ -0,0 +1,5 @@
+libkiwix (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Wed, 08 Jul 2020 18:12:32 -0700

debian/control

@@ -0,0 +1,45 @@
+Source: libkiwix
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 12),
+ meson,
+ pkg-config,
+ libzim-dev (>= 6.0.0),
+ libcurl4-gnutls-dev,
+ libicu-dev,
+ libgtest-dev,
+ libkainjow-mustache-dev,
+ liblzma-dev,
+ libmicrohttpd-dev,
+ libpugixml-dev,
+ zlib1g-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://github.com/kiwix/kiwix-lib
+Rules-Requires-Root: no
+
+Package: libkiwix-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3, aria2,
+ libzim-dev (>= 6.0.0),
+ libicu-dev,
+ libpugixml-dev,
+ libcurl4-gnutls-dev
+Description: library of common code for Kiwix (development)
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).
+ .
+ This package contains development files.
+
+Package: libkiwix9
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Conflicts: libkiwix0, libkiwix3
+Description: library of common code for Kiwix
+ Kiwix is an offline Wikipedia reader. libkiwix provides the
+ software core for Kiwix, and contains the code shared by all
+ Kiwix ports (Windows, Linux, OSX, Android, etc.).

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/libkiwix-dev.install

@@ -0,0 +1,4 @@
+usr/include
+usr/lib/*/libkiwix.so
+usr/lib/*/pkgconfig
+usr/bin

debian/libkiwix9.install

@@ -0,0 +1 @@
+usr/lib/*/libkiwix.so.*

debian/rules

@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@ --buildsystem=meson

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

include/reader.h

@@ -43,6 +43,8 @@ namespace kiwix
  * The Reader class is the class who allow to get an entry content from a zim
  * file.
  */
+
+using SuggestionsList_t = std::vector<std::vector<std::string>>;
 class Reader
 {
  public:
@@ -419,6 +421,10 @@ class Reader
    *
    * Suggestions are stored in an internal vector and can be retrieved using
    * `getNextSuggestion` method.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestions(const string& prefix,
+   *                        unsigned int suggestionsCount,
+   *                        SuggestionsList_t& results);
    *
    * @param prefix The prefix to search.
    * @param suggestionsCount How many suggestions to search for.
@@ -426,12 +432,49 @@ class Reader
    *              If false, add suggestions to the internal vector
    *              (until internal vector size is suggestionCount (or no more
    *               suggestion))
-   * @return True if some suggestions where added to the internal vector.
+   * @return True if some suggestions have been added to the internal vector.
    */
-  bool searchSuggestions(const string& prefix,
+  DEPRECATED bool searchSuggestions(const string& prefix,
                          unsigned int suggestionsCount,
                          const bool reset = true);
 
+  /**
+   * Search for entries with title starting with prefix (case sensitive).
+   *
+   * Suggestions are added to the `result` vector.
+   *
+   * @param prefix The prefix to search.
+   * @param suggestionsCount How many suggestions to search for.
+   * @param result The vector where to store the suggestions.
+   * @return True if some suggestions have been added to the vector.
+   */
+
+  bool searchSuggestions(const string& prefix,
+                         unsigned int suggestionsCount,
+                         SuggestionsList_t& resuls);
+
+  /**
+   * Search for entries for the given prefix.
+   *
+   * If the zim file has a internal fulltext index, the suggestions will be
+   * searched using it.
+   * Else the suggestions will be search using `searchSuggestions` while trying
+   * to be smart about case sensitivity (using `getTitleVariants`).
+   *
+   * In any case, suggestions are stored in an internal vector and can be
+   * retrieved using `getNextSuggestion` method.
+   * The internal vector will be reset.
+   * This method is not thread safe and is deprecated. Use :
+   * bool searchSuggestionsSmart(const string& prefix,
+   *                             unsigned int suggestionsCount,
+   *                             SuggestionsList_t& results);
+   *
+   * @param prefix The prefix to search for.
+   * @param suggestionsCount How many suggestions to search for.
+   */
+  DEPRECATED bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount);
+
   /**
    * Search for entries for the given prefix.
    *
@@ -446,9 +489,13 @@ class Reader
    *
    * @param prefix The prefix to search for.
    * @param suggestionsCount How many suggestions to search for.
+   * @param results The vector where to store the suggestions
+   * @return True if some suggestions have been added to the results.
    */
-  bool searchSuggestionsSmart(const string& prefix,
-                              unsigned int suggestionsCount);
+   bool searchSuggestionsSmart(const string& prefix,
+                              unsigned int suggestionsCount,
+                              SuggestionsList_t& results);
+
 
   /**
    * Check if the url exists in the zim file.
@@ -490,7 +537,7 @@ class Reader
    * @param[out] title the title of the suggestion.
    * @return True if title has been set.
    */
-  bool getNextSuggestion(string& title);
+  DEPRECATED bool getNextSuggestion(string& title);
 
   /**
    * Get the next suggestion title and url.
@@ -499,7 +546,7 @@ class Reader
    * @param[out] url the url of the suggestion.
    * @return True if title and url have been set.
    */
-  bool getNextSuggestion(string& title, string& url);
+  DEPRECATED bool getNextSuggestion(string& title, string& url);
 
   /**
    * Get if we can check zim file integrity (has a checksum).
@@ -559,8 +606,8 @@ class Reader
   zim::size_type nsICount;
   std::string zimFilePath;
 
-  std::vector<std::vector<std::string>> suggestions;
-  std::vector<std::vector<std::string>>::iterator suggestionsOffset;
+  SuggestionsList_t suggestions;
+  SuggestionsList_t::iterator suggestionsOffset;
 
  private:
   std::map<const std::string, unsigned int> parseCounterMetadata() const;

include/searcher.h

@@ -31,7 +31,6 @@
 #include <vector>
 #include "tools/pathTools.h"
 #include "tools/stringTools.h"
-#include "kiwix_config.h"
 
 using namespace std;
 

meson.build

@@ -1,6 +1,6 @@
 project('kiwix-lib', 'cpp',
-  version : '9.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
-  license : 'GPL',
+  version : '9.3.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
@@ -18,14 +18,23 @@ if 'java' in wrapper
   add_languages('java')
 endif
 
+# See https://github.com/kiwix/kiwix-lib/issues/371
+if target_machine.cpu_family() in ['arm', 'mips', 'm68k', 'ppc', 'sh4']
+  extra_libs += '-latomic'
+endif
+
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=6.1.1', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.1.8', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
 
-if not compiler.has_header('mustache.hpp')
+if compiler.has_header('mustache.hpp')
+  extra_include = []
+elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
+  extra_include = ['/usr/include/kainjow']
+else
   error('Cannot found header mustache.hpp')
 endif
 
@@ -37,7 +46,7 @@ endif
 
 all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
 
-inc = include_directories('include')
+inc = include_directories('include', extra_include)
 
 conf = configuration_data()
 conf.set('VERSION', '"@0@"'.format(meson.project_version()))

scripts/kiwix-compile-resources.1

@@ -0,0 +1,20 @@
+.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
+.SH NAME
+kiwix-compile-resources \- helper to compile and generate some Kiwix resources
+.SH SYNOPSIS
+\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
+.SH DESCRIPTION
+.TP
+resource_file
+The list of resources to compile.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show a help message and exit
+.TP
+\fB\-\-cxxfile\fR CXXFILE
+The Cpp file name to generate
+.TP
+\fB\-\-hfile\fR HFILE
+The h file name to generate
+.SH AUTHOR
+Matthieu Gautier <mgautier@kymeria.fr>

scripts/meson.build

@@ -2,3 +2,5 @@
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))
+
+install_man('kiwix-compile-resources.1')

src/meson.build

@@ -21,6 +21,7 @@ kiwix_sources = [
   'tools/otherTools.cpp',
   'kiwixserve.cpp',
   'name_mapper.cpp',
+  'server/byte_range.cpp',
   'server/etag.cpp',
   'server/request_context.cpp',
   'server/response.cpp'
@@ -52,6 +53,7 @@ kiwixlib = library('kiwix',
                    kiwix_sources,
                    include_directories : inc,
                    dependencies : all_deps,
+                   link_args: extra_libs,
                    version: meson.project_version(),
                    install: true,
                    install_dir: install_dir)

src/microhttpd_wrapper.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include <microhttpd.h>
+
+#if MHD_VERSION < 0x00097002
+typedef int MHD_Result;
+#endif

src/reader.cpp

@@ -709,12 +709,11 @@ bool Reader::hasFulltextIndex() const
 }
 
 /* Search titles by prefix */
+
 bool Reader::searchSuggestions(const string& prefix,
                                unsigned int suggestionsCount,
                                const bool reset)
 {
-  bool retVal = false;
-
   /* Reset the suggestions otherwise check if the suggestions number is less
    * than the suggestionsCount */
   if (reset) {
@@ -726,6 +725,21 @@ bool Reader::searchSuggestions(const string& prefix,
     }
   }
 
+  auto ret =  searchSuggestions(prefix, suggestionsCount, this->suggestions);
+
+  /* Set the cursor to the begining */
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+
+bool Reader::searchSuggestions(const string& prefix,
+                               unsigned int suggestionsCount,
+                               SuggestionsList_t& results)
+{
+  bool retVal = false;
+
   /* Return if no prefix */
   if (prefix.size() == 0) {
     return false;
@@ -734,7 +748,7 @@ bool Reader::searchSuggestions(const string& prefix,
   for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
        articleItr != zimFileHandler->end()
        && articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
-       && this->suggestions.size() < suggestionsCount;
+       && results.size() < suggestionsCount;
        ++articleItr) {
     /* Extract the interesting part of article title & url */
     std::string normalizedArticleTitle
@@ -754,8 +768,8 @@ bool Reader::searchSuggestions(const string& prefix,
        title) */
     bool insert = true;
     std::vector<std::vector<std::string>>::iterator suggestionItr;
-    for (suggestionItr = this->suggestions.begin();
-         suggestionItr != this->suggestions.end();
+    for (suggestionItr = results.begin();
+         suggestionItr != results.end();
          suggestionItr++) {
       int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
       if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
@@ -772,16 +786,13 @@ bool Reader::searchSuggestions(const string& prefix,
       suggestion.push_back(articleItr->getTitle());
       suggestion.push_back(articleFinalUrl);
       suggestion.push_back(normalizedArticleTitle);
-      this->suggestions.insert(suggestionItr, suggestion);
+      results.insert(suggestionItr, suggestion);
     }
 
     /* Suggestions where found */
     retVal = true;
   }
 
-  /* Set the cursor to the begining */
-  this->suggestionsOffset = this->suggestions.begin();
-
   return retVal;
 }
 
@@ -796,15 +807,28 @@ std::vector<std::string> Reader::getTitleVariants(
   return variants;
 }
 
-/* Try also a few variations of the prefix to have better results */
+
 bool Reader::searchSuggestionsSmart(const string& prefix,
                                     unsigned int suggestionsCount)
+{
+  this->suggestions.clear();
+  this->suggestionsOffset = this->suggestions.begin();
+
+  auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
+
+  this->suggestionsOffset = this->suggestions.begin();
+
+  return ret;
+}
+
+/* Try also a few variations of the prefix to have better results */
+bool Reader::searchSuggestionsSmart(const string& prefix,
+                                    unsigned int suggestionsCount,
+                                    SuggestionsList_t& results)
 {
   std::vector<std::string> variants = this->getTitleVariants(prefix);
   bool retVal = false;
 
-  this->suggestions.clear();
-  this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
   const auto suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
@@ -820,15 +844,14 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
       suggestion.push_back(current->getTitle());
       suggestion.push_back("/A/" + current->getUrl());
       suggestion.push_back(kiwix::normalize(current->getTitle()));
-      this->suggestions.push_back(suggestion);
+      results.push_back(suggestion);
     }
-    this->suggestionsOffset = this->suggestions.begin();
     retVal = true;
   } else {
     for (std::vector<std::string>::iterator variantsItr = variants.begin();
          variantsItr != variants.end();
          variantsItr++) {
-      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
+      retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
                || retVal;
     }
   }

src/server.cpp

@@ -36,7 +36,7 @@
 #endif
 
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 #include "tools/otherTools.h"
@@ -77,14 +77,14 @@ static IdNameMapper defaultNameMapper;
 
 typedef kainjow::mustache::data MustacheData;
 
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls);
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls);
 
 
 class InternalServer {
@@ -101,13 +101,13 @@ class InternalServer {
                    bool blockExternalLinks);
     virtual ~InternalServer() = default;
 
-    int handlerCallback(struct MHD_Connection* connection,
-                        const char* url,
-                        const char* method,
-                        const char* version,
-                        const char* upload_data,
-                        size_t* upload_data_size,
-                        void** cont_cls);
+    MHD_Result handlerCallback(struct MHD_Connection* connection,
+                               const char* url,
+                               const char* method,
+                               const char* version,
+                               const char* upload_data,
+                               size_t* upload_data_size,
+                               void** cont_cls);
     bool start();
     void stop();
 
@@ -267,14 +267,14 @@ void InternalServer::stop()
   MHD_stop_daemon(mp_daemon);
 }
 
-static int staticHandlerCallback(void* cls,
-                                 struct MHD_Connection* connection,
-                                 const char* url,
-                                 const char* method,
-                                 const char* version,
-                                 const char* upload_data,
-                                 size_t* upload_data_size,
-                                 void** cont_cls)
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls)
 {
   InternalServer* _this = static_cast<InternalServer*>(cls);
 
@@ -287,13 +287,13 @@ static int staticHandlerCallback(void* cls,
                                 cont_cls);
 }
 
-int InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                    const char* url,
-                                    const char* method,
-                                    const char* version,
-                                    const char* upload_data,
-                                    size_t* upload_data_size,
-                                    void** cont_cls)
+MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                           const char* url,
+                                           const char* method,
+                                           const char* version,
+                                           const char* upload_data,
+                                           size_t* upload_data_size,
+                                           void** cont_cls)
 {
   auto start_time = std::chrono::steady_clock::now();
   if (m_verbose.load() ) {
@@ -543,7 +543,6 @@ Response InternalServer::handle_suggest(const RequestContext& request)
   std::string mimeType;
   unsigned int maxSuggestionCount = 10;
   unsigned int suggestionCount = 0;
-  std::string suggestion;
 
   std::string bookName;
   std::string bookId;
@@ -567,11 +566,12 @@ Response InternalServer::handle_suggest(const RequestContext& request)
   bool first = true;
   if (reader != nullptr) {
     /* Get the suggestions */
-    reader->searchSuggestionsSmart(term, maxSuggestionCount);
-    while (reader->getNextSuggestion(suggestion)) {
+    SuggestionsList_t suggestions;
+    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
+    for(auto& suggestion:suggestions) {
       MustacheData result;
-      result.set("label", suggestion);
-      result.set("value", suggestion);
+      result.set("label", suggestion[0]);
+      result.set("value", suggestion[0]);
       result.set("first", first);
       first = false;
       results.push_back(result);

src/server/byte_range.cpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#include "byte_range.h"
+
+#include "tools/stringTools.h"
+
+#include <cassert>
+#include <algorithm>
+
+namespace kiwix {
+
+namespace {
+
+ByteRange parseByteRange(const std::string& rangeStr)
+{
+  std::istringstream iss(rangeStr);
+
+  int64_t start, end = INT64_MAX;
+  if (iss >> start) {
+    if ( start < 0 ) {
+      if ( iss.eof() )
+        return ByteRange(-start);
+    } else {
+      char c;
+      if (iss >> c && c=='-') {
+        iss >> end; // if this fails, end is not modified, which is OK
+        if (iss.eof() && start <= end)
+          return ByteRange(ByteRange::PARSED, start, end);
+      }
+    }
+  }
+
+  return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
+}
+
+} // unnamed namespace
+
+ByteRange::ByteRange()
+  : kind_(NONE)
+  , first_(0)
+  , last_(INT64_MAX)
+{}
+
+ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
+  : kind_(kind)
+  , first_(first)
+  , last_(last)
+{
+  assert(kind != NONE);
+  assert(first >= 0);
+  assert(last >= first || (first == 0 && last == -1));
+}
+
+ByteRange::ByteRange(int64_t suffix_length)
+  : kind_(PARSED)
+  , first_(-suffix_length)
+  , last_(INT64_MAX)
+{
+  assert(suffix_length > 0);
+}
+
+int64_t ByteRange::first() const
+{
+  assert(kind_ > PARSED);
+  return first_;
+}
+
+int64_t ByteRange::last() const
+{
+  assert(kind_ > PARSED);
+  return last_;
+}
+
+int64_t ByteRange::length() const
+{
+  assert(kind_ > PARSED);
+  return last_ + 1 - first_;
+}
+
+ByteRange ByteRange::parse(const std::string& rangeStr)
+{
+  const std::string byteUnitSpec("bytes=");
+  if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
+    return ByteRange(INVALID, 0, INT64_MAX);
+
+  return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
+}
+
+ByteRange ByteRange::resolve(int64_t contentSize) const
+{
+  if ( kind() == NONE )
+    return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
+
+  if ( kind() == INVALID )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  const int64_t resolved_first = first_ < 0
+                               ? std::max(int64_t(0), contentSize + first_)
+                               : first_;
+
+  const int64_t resolved_last = std::min(contentSize-1, last_);
+
+  if ( resolved_first > resolved_last )
+    return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
+
+  return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
+}
+
+} // namespace kiwix

src/server/byte_range.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+
+#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
+#define KIWIXLIB_SERVER_BYTE_RANGE_H
+
+#include <cstdint>
+#include <string>
+
+namespace kiwix {
+
+class ByteRange
+{
+  public: // types
+    // ByteRange is parsed in a request, then it must be resolved (taking
+    // into account the actual size of the requested resource) before
+    // being applied in the response.
+    // The Kind enum represents possible states in such a lifecycle.
+    enum Kind {
+      // The request is not a range request (no Range header)
+      NONE,
+
+      // The value of the Range header is not a valid continuous
+      // range. Note that a valid (according to RFC7233) sequence of multiple
+      // byte ranges is considered invalid in the current implementation
+      // (i.e. only single-range partial requests are supported).
+      INVALID,
+
+      // This byte-range has been successfully parsed from the request
+      PARSED,
+
+      // This is a response to a regular (non-range) request
+      RESOLVED_FULL_CONTENT,
+
+      // The range request is invalid or unsatisfiable
+      RESOLVED_UNSATISFIABLE,
+
+      // This is a response to a (satisfiable) range request
+      RESOLVED_PARTIAL_CONTENT,
+    };
+
+  public: // functions
+    // Constructs a ByteRange object of NONE kind
+    ByteRange();
+
+    // Constructs a ByteRange object of the given kind (except NONE)
+    ByteRange(Kind kind, int64_t first, int64_t last);
+
+    // Constructs a ByteRange object of PARSED kind corresponding to a
+    // range request of the form "Range: bytes=-suffix_length"
+    explicit ByteRange(int64_t suffix_length);
+
+    Kind kind() const { return kind_; }
+    int64_t first() const;
+    int64_t last() const;
+    int64_t length() const;
+
+    static ByteRange parse(const std::string& rangeStr);
+    ByteRange resolve(int64_t contentSize) const;
+
+  private: // data
+    Kind kind_;
+    int64_t first_;
+    int64_t last_;
+};
+
+} // namespace kiwix
+
+#endif //KIWIXLIB_SERVER_BYTE_RANGE_H

src/server/request_context.cpp

@@ -74,8 +74,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
   version(version),
   requestIndex(s_requestIndex++),
   acceptEncodingDeflate(false),
-  accept_range(false),
-  range_pair(0, -1)
+  byteRange_()
 {
   MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
@@ -85,43 +84,24 @@ RequestContext::RequestContext(struct MHD_Connection* connection,
         (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
   } catch (const std::out_of_range&) {}
 
-  /*Check if range is requested. */
   try {
-    auto range = get_header(MHD_HTTP_HEADER_RANGE);
-    int start = 0;
-    int end = -1;
-    std::istringstream iss(range);
-    char c;
-
-    iss >> start >> c;
-    if (iss.good() && c=='-') {
-      iss >> end;
-      if (iss.fail()) {
-        // Something went wrong will extracting.
-        end = -1;
-      }
-      if (iss.eof()) {
-        accept_range = true;
-        range_pair = std::pair<int, int>(start, end);
-      }
-    }
+    byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
   } catch (const std::out_of_range&) {}
 }
 
 RequestContext::~RequestContext()
 {}
 
-
-int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
-                                 const char *key, const char *value)
+MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
+                                       const char *key, const char *value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->headers[key] = value;
   return MHD_YES;
 }
 
-int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
-                                   const char *key, const char* value)
+MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
+                                         const char *key, const char* value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
   _this->arguments[key] = value == nullptr ? "" : value;
@@ -146,7 +126,7 @@ void RequestContext::print_debug_info() const {
   printf("full_url: %s\n", full_url.c_str());
   printf("url   : %s\n", url.c_str());
   printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
-  printf("has_range : %d\n", accept_range);
+  printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
   printf("is_valid_url : %d\n", is_valid_url());
   printf(".............\n");
 }
@@ -188,12 +168,8 @@ bool RequestContext::is_valid_url() const {
   return !url.empty();
 }
 
-bool RequestContext::has_range() const {
-  return accept_range;
-}
-
-std::pair<int, int> RequestContext::get_range() const {
-  return range_pair;
+ByteRange RequestContext::get_range() const {
+  return byteRange_;
 }
 
 template<>

src/server/request_context.h

@@ -27,8 +27,10 @@
 #include <map>
 #include <stdexcept>
 
+#include "byte_range.h"
+
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
@@ -51,9 +53,6 @@ class IndexError: public std::runtime_error {};
 
 
 class RequestContext {
-  public: // types
-  typedef std::pair<int, int> ByteRange;
-
   public: // functions
     RequestContext(struct MHD_Connection* connection,
                    std::string rootLocation,
@@ -81,7 +80,6 @@ class RequestContext {
     std::string get_url_part(int part) const;
     std::string get_full_url() const;
 
-    bool has_range() const;
     ByteRange get_range() const;
 
     bool can_compress() const { return acceptEncodingDeflate; }
@@ -95,14 +93,13 @@ class RequestContext {
 
     bool acceptEncodingDeflate;
 
-    bool accept_range;
-    ByteRange range_pair;
+    ByteRange byteRange_;
     std::map<std::string, std::string> headers;
     std::map<std::string, std::string> arguments;
 
   private: // functions
-    static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
-    static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
+    static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
 };
 
 template<> std::string RequestContext::get_argument(const std::string& name) const;

src/server/response.cpp

@@ -39,12 +39,6 @@ bool is_compressible_mime_type(const std::string& mimeType)
       || mimeType.find("application/json") != string::npos;
 }
 
-int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
-{
-  return range.second == -1
-       ? entry.getSize() - range.first
-       : range.second - range.first;
-}
 
 } // unnamed namespace
 
@@ -58,15 +52,13 @@ Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool
     m_withLibraryButton(withLibraryButton),
     m_blockExternalLinks(blockExternalLinks),
     m_addTaskbar(false),
-    m_bookName(""),
-    m_startRange(0),
-    m_lenRange(0)
+    m_bookName("")
 {
 }
 
 
-static int print_key_value (void *cls, enum MHD_ValueKind kind,
-                            const char *key, const char *value)
+static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
+                                   const char *key, const char *value)
 {
   printf (" - %s: '%s'\n", key, value);
   return MHD_YES;
@@ -177,6 +169,20 @@ Response::can_compress(const RequestContext& request) const
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
 }
 
+MHD_Response*
+Response::create_error_response(const RequestContext& request) const
+{
+  MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT);
+  if ( m_returnCode == 416 ) {
+    std::ostringstream oss;
+    oss << "bytes */" << m_byteRange.length();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
+  return response;
+}
+
 MHD_Response*
 Response::create_raw_content_mhd_response(const RequestContext& request)
 {
@@ -240,23 +246,26 @@ Response::create_redirection_mhd_response() const
 MHD_Response*
 Response::create_entry_mhd_response() const
 {
-  MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
+  const auto content_length = m_byteRange.length();
+  MHD_Response* response = MHD_create_response_from_callback(content_length,
                                                16384,
                                                callback_reader_from_entry,
-                                               new RunningResponse(m_entry, m_startRange),
+                                               new RunningResponse(m_entry, m_byteRange.first()),
                                                callback_free_response);
   MHD_add_response_header(response,
     MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
   MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
-  std::ostringstream oss;
-  oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
-      << "/" << m_entry.getSize();
+  if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
+    std::ostringstream oss;
+    oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
+        << "/" << m_entry.getSize();
+
+    MHD_add_response_header(response,
+      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
+  }
 
   MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
+    MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
   return response;
 }
 
@@ -264,6 +273,9 @@ MHD_Response*
 Response::create_mhd_response(const RequestContext& request)
 {
   switch (m_mode) {
+    case ResponseMode::ERROR_RESPONSE:
+      return create_error_response(request);
+
     case ResponseMode::RAW_CONTENT :
       return create_raw_content_mhd_response(request);
 
@@ -276,18 +288,20 @@ Response::create_mhd_response(const RequestContext& request)
   return nullptr;
 }
 
-int Response::send(const RequestContext& request, MHD_Connection* connection)
+MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
 {
   MHD_Response* response = create_mhd_response(request);
 
-  MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
-  const std::string etag = m_etag.get_etag();
-  if ( ! etag.empty() )
-      MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  if ( m_mode != ResponseMode::ERROR_RESPONSE ) {
+    MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
+    MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
+      m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+    const std::string etag = m_etag.get_etag();
+    if ( ! etag.empty() )
+        MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  }
 
-  if (m_returnCode == MHD_HTTP_OK && request.has_range())
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
     m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
 
   if (m_verbose)
@@ -321,16 +335,18 @@ void Response::set_entry(const Entry& entry, const RequestContext& request) {
   set_mimeType(mimeType);
   set_cacheable();
 
-  if ( is_compressible_mime_type(mimeType) ) {
+  m_byteRange = request.get_range().resolve(entry.getSize());
+  const bool noRange = m_byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if ( noRange && is_compressible_mime_type(mimeType) ) {
     zim::Blob raw_content = entry.getBlob();
     const std::string content = string(raw_content.data(), raw_content.size());
 
     set_content(content);
     set_compress(true);
-  } else {
-    const int range_len = get_range_len(entry, request.get_range());
-    set_range_first(request.get_range().first);
-    set_range_len(range_len);
+  } else if ( m_byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE ) {
+    set_code(416);
+    set_content("");
+    m_mode = ResponseMode::ERROR_RESPONSE;
   }
 }
 

src/server/response.h

@@ -24,16 +24,18 @@
 #include <string>
 
 #include <mustache.hpp>
+#include "byte_range.h"
 #include "entry.h"
 #include "etag.h"
 
 extern "C" {
-#include <microhttpd.h>
+#include "microhttpd_wrapper.h"
 }
 
 namespace kiwix {
 
 enum class ResponseMode {
+  ERROR_RESPONSE,
   RAW_CONTENT,
   REDIRECTION,
   ENTRY
@@ -46,7 +48,7 @@ class Response {
     Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
     ~Response() = default;
 
-    int send(const RequestContext& request, MHD_Connection* connection);
+    MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
     void set_template(const std::string& template_str, kainjow::mustache::data data);
     void set_content(const std::string& content);
@@ -61,8 +63,6 @@ class Response {
     void set_etag(const ETag& etag) { m_etag = etag; }
     void set_compress(bool compress) { m_compress = compress; }
     void set_taskbar(const std::string& bookName, const std::string& bookTitle);
-    void set_range_first(uint64_t start) { m_startRange = start; }
-    void set_range_len(uint64_t len) { m_lenRange = len; }
 
     int getReturnCode() const { return m_returnCode; }
     std::string get_mimeType() const { return m_mimeType; }
@@ -74,6 +74,7 @@ class Response {
 
   private: // functions
     MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
     MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
     MHD_Response* create_redirection_mhd_response() const;
     MHD_Response* create_entry_mhd_response() const;
@@ -93,8 +94,7 @@ class Response {
     bool m_addTaskbar;
     std::string m_bookName;
     std::string m_bookTitle;
-    uint64_t m_startRange;
-    uint64_t m_lenRange;
+    ByteRange m_byteRange;
     ETag m_etag;
 };
 

src/tools/pathTools.cpp

@@ -81,13 +81,19 @@ std::wstring Utf8ToWide(const std::string& str)
 bool isRelativePath(const std::string& path)
 {
 #ifdef _WIN32
-  return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
+  if (path.size() < 3 ) {
+    return true;
+  }
+  if (path.substr(1, 2) == ":\\" || path.substr(0, 2) == "\\\\") {
+    return false;
+  }
+  return true;
 #else
   return path.empty() || path.substr(0, 1) == "/" ? false : true;
 #endif
 }
 
-std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute)
 {
   std::vector<std::string> ret;
 #ifdef _WIN32
@@ -95,10 +101,35 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
   //Starts from there.
   auto it = find_if(parts.rbegin(), parts.rend(),
                     [](const std::string& p) ->bool
-                      { return p.length() == 2 && p[1] == ':'; });
+                      { return ((p.length() == 2 && p[1] == ':')
+                             || (p.length() > 2 && p[0] == '\\' && p[1] == '\\')); });
   if (it != parts.rend()) {
     parts.erase(parts.begin(), it.base()-1);
   }
+  // Special case for samba mount point starting with two "\\" ("\\\\samba\\foo")
+  if (parts.size() > 2  && parts[0].empty() && parts[1].empty()) {
+    parts.erase(parts.begin(), parts.begin()+2);
+    parts[0] = "\\\\" + parts[0];
+  }
+  // Special case if we have a samba drive not at first.
+  // Path is "..\\\\sambdadrive\\..\\.." So we will have an empty part.
+  auto previous_empty = false;
+  for (it = parts.rbegin(); it!=parts.rend(); it++) {
+    if(it->empty()) {
+      if (previous_empty) {
+        it++;
+        break;
+      } else {
+        previous_empty = true;
+      }
+    } else {
+      previous_empty = false;
+    }
+  }
+  if (it != parts.rend()) {
+    parts.erase(parts.begin(), it.base()-1);
+    parts[0] = "\\\\" + parts[0];
+  }
 #endif
 
   size_t index = 0;
@@ -144,8 +175,10 @@ std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool abs
 
 std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
 {
-  auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
-  auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto parts = kiwix::split(path, SEPARATOR, false);
+  auto pathParts = normalizeParts(parts, false);
+  parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absolutePathParts = normalizeParts(parts, true);
 
   unsigned int commonCount = 0;
   while (commonCount < pathParts.size()
@@ -172,8 +205,10 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
     absolutePath = getCurrentDirectory();
   }
 
-  auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
-  auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
+  auto parts = kiwix::split(absolutePath, SEPARATOR, false);
+  auto absoluteParts = normalizeParts(parts, true);
+  parts = kiwix::split(relativePath, SEPARATOR, false);
+  auto relativeParts = normalizeParts(parts, false);
 
   absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
   auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
@@ -182,7 +217,8 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
 
 std::string removeLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
+  auto parts_ = kiwix::split(path, SEPARATOR, false);
+  auto parts = normalizeParts(parts_, false);
   if (!parts.empty()) {
     parts.pop_back();
   }
@@ -202,7 +238,8 @@ std::string appendToDirectory(const std::string& directoryPath, const std::strin
 
 std::string getLastPathElement(const std::string& path)
 {
-  auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
+  auto parts_ = kiwix::split(path, SEPARATOR);
+  auto parts = normalizeParts(parts_, false);
   if (parts.empty()) {
     return "";
   }

src/wrapper/java/kiwixreader.cpp

@@ -355,9 +355,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
   unsigned int cCount = jni2c(count, env);
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
     LOG(e.what());
@@ -377,11 +380,14 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
   std::string cUrl;
 
   try {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
       setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
+#pragma GCC diagnostic pop
   } catch (std::exception& e) {
     LOG("Unable to get next suggestion");
     LOG(e.what());

test/data/create_corner_cases_zim_file

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")"
+rm -f corner_cases.zim
+zimwriterfs -w empty.html \
+            -f empty.png \
+            -l=en \
+            -t="ZIM corner cases" \
+            -d="" \
+            -c="" \
+            -p="" \
+            corner_cases \
+            corner_cases.zim \
+&& echo 'corner_cases.zim was successfully created' \
+|| echo '!!! Failed to create corner_cases.zim !!!' >&2

test/library.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Tommi Maekitalo
+ * Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -262,8 +262,13 @@ TEST_F(LibraryTest, filterCheck)
 TEST_F(LibraryTest, getBookByPath)
 {
   auto& book = lib.getBookById(lib.getBooksIds()[0]);
-  book.setPath("/some/abs/path.zim");
-  EXPECT_EQ(lib.getBookByPath("/some/abs/path.zim").getId(), book.getId());
+#ifdef _WIN32
+  auto path = "C:\\some\\abs\\path.zim";
+#else
+  auto path = "/some/abs/path.zim";
+#endif
+  book.setPath(path);
+  EXPECT_EQ(lib.getBookByPath(path).getId(), book.getId());
   EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
 }
 };

test/meson.build

@@ -29,6 +29,9 @@ if gtest_dep.found() and not meson.is_cross_build()
     configure_file(input : 'data/wikipedia_en_ray_charles_mini_2020-03.zim',
                    output : 'zimfile.zim',
                    copy: true )
+    configure_file(input : 'data/corner_cases.zim',
+                   output : 'corner_cases.zim',
+                   copy: true )
 
     foreach test_name : tests
         # XXX: implicit_include_directories must be set to false, otherwise
@@ -39,6 +42,6 @@ if gtest_dep.found() and not meson.is_cross_build()
                               link_args: extra_link_args,
                               dependencies : all_deps + [gtest_dep],
                              build_rpath : '$ORIGIN')
-        test(test_name, test_exe)
+        test(test_name, test_exe, timeout : 160)
     endforeach
 endif

test/pathTools.cpp

@@ -25,6 +25,7 @@
 #ifdef _WIN32
 # define S "\\"
 # define AS "c:"
+# define A_SAMBA "\\\\sambadir"
 #else
 # define S "/"
 # define AS ""
@@ -42,7 +43,10 @@
 #define A4(a, b, c, d) A1(P4(a, b, c, d))
 #define A5(a, b, c, d, e) A1(P5(a, b, c, d, e))
 
-std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute);
+std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute);
+std::vector<std::string> nParts(std::vector<std::string> parts, bool absolute) {
+  return normalizeParts(parts, absolute);
+}
 #ifdef _WIN32
 std::wstring Utf8ToWide(const std::string& str);
 std::string WideToUtf8(const std::wstring& wstr);
@@ -54,7 +58,7 @@ namespace
 #define V std::vector<std::string>
 TEST(pathTools, normalizePartsAbsolute)
 {
-#define N(...) normalizeParts(__VA_ARGS__, true)
+#define N(...) nParts(__VA_ARGS__, true)
   ASSERT_EQ(N({}), V({}));
 #ifdef _WIN32
   ASSERT_EQ(N({"c:"}), V({"c:"}));
@@ -75,13 +79,14 @@ TEST(pathTools, normalizePartsAbsolute)
 
 #ifdef _WIN32
   ASSERT_EQ(N({"c:", "a", "b", ".", "c", "d:", "..", "foo"}), V({"d:", "foo"}));
+  ASSERT_EQ(N({"","","samba","a","b"}), V({"\\\\samba", "a", "b"}));
 #endif
 #undef N
 }
 
 TEST(pathTools, normalizePartsRelative)
 {
-#define N(...) normalizeParts(__VA_ARGS__, false)
+#define N(...) nParts(__VA_ARGS__, false)
   ASSERT_EQ(N({}), V({}));
   ASSERT_EQ(N({""}), V({}));
   ASSERT_EQ(N({"a"}), V({"a"}));
@@ -103,6 +108,10 @@ TEST(pathTools, isRelativePath)
   ASSERT_TRUE(isRelativePath(P4("foo","","bar","")));
   ASSERT_FALSE(isRelativePath(A1("foo")));
   ASSERT_FALSE(isRelativePath(A2("foo", "bar")));
+#ifdef _WIN32
+  ASSERT_FALSE(isRelativePath(P2(A_SAMBA, "foo")));
+  ASSERT_FALSE(isRelativePath(P3(A_SAMBA, "foo", "bar")));
+#endif
 }
 
 TEST(pathTools, computeAbsolutePath)
@@ -121,6 +130,12 @@ TEST(pathTools, computeAbsolutePath)
             A5("a","b","c","d","foo"));
   ASSERT_EQ(computeAbsolutePath(A5("a","b","c","d","e"), P5("..","..","..","g","foo")),
             A4("a","b","g","foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b",""), P2("..","foo")),
+            P3(A_SAMBA,"a","foo"));
+  ASSERT_EQ(computeAbsolutePath(P6(A_SAMBA,"a","b","c","d","e"), P5("..","..","..","g","foo")),
+            P5(A_SAMBA,"a","b","g","foo"));
+#endif
 }
 
 TEST(pathTools, computeRelativePath)
@@ -137,6 +152,13 @@ TEST(pathTools, computeRelativePath)
             P2("..","foo"));
   ASSERT_EQ(computeRelativePath(A5("a","b","c","d","e"), A4("a","b","g","foo")),
             P5("..","..","..","g","foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeRelativePath(P3(A_SAMBA,"a","b"), P3(A_SAMBA,"a","foo")),
+            P2("..","foo"));
+  ASSERT_EQ(computeRelativePath(P6(A_SAMBA,"a","b","c","d","e"), P5(A_SAMBA,"a","b","g","foo")),
+            P5("..","..","..","g","foo"));
+
+#endif
 }
 
 TEST(pathTools, removeLastPathElement)
@@ -179,6 +201,15 @@ TEST(pathTools, goUp)
             "c:");
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
             "c:");
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), ".."),
+            P3(A_SAMBA,"a", "b"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P2("..","..")),
+            P2(A_SAMBA,"a"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P3("..","..","..")),
+            A_SAMBA);
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","..")),
+            A_SAMBA);
+
 #else
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
             "/");
@@ -195,6 +226,12 @@ TEST(pathTools, goUp)
             A1("foo"));
   ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P5("..","..","..","..","foo")),
             A1("foo"));
+#ifdef _WIN32
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","foo")),
+            P2(A_SAMBA,"foo"));
+  ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P5("..","..","..","..","foo")),
+            P2(A_SAMBA,"foo"));
+#endif
 }
 
 
@@ -203,11 +240,22 @@ TEST(pathTools, dirChange)
 {
   std::string p1("c:\\a\\b\\c");
   std::string p2("d:\\d\\e\\foo.xml");
-  std::string relative_path = computeRelativePath(p1, p2);
-  ASSERT_EQ(relative_path, "d:\\d\\e\\foo.xml");
-  std::string abs_path = computeAbsolutePath(p1, relative_path);
-  ASSERT_EQ(abs_path, p2);
-  ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
+  {
+    std::string relative_path = computeRelativePath(p1, p2);
+    ASSERT_EQ(relative_path, p2);
+    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    ASSERT_EQ(abs_path, p2);
+    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
+  }
+  std::string ps("\\\\samba\\d\\e\\foo.xml");
+  {
+    std::string relative_path = computeRelativePath(p1, ps);
+    ASSERT_EQ(relative_path, ps);
+    std::string abs_path = computeAbsolutePath(p1, relative_path);
+    ASSERT_EQ(abs_path, ps);
+    // I'm not sure this test is valid on windows :/
+//    ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\\\samba\\d\\e\\foo.xml"), ps);
+  }
 }
 
 TEST(pathTools, Utf8ToWide)

test/server.cpp

@@ -50,9 +50,10 @@ class ZimFileServer
 {
 public: // types
   typedef std::shared_ptr<httplib::Response>  Response;
+  typedef std::vector<std::string> FilePathCollection;
 
 public: // functions
-  ZimFileServer(int serverPort, std::string zimpath);
+  ZimFileServer(int serverPort, const FilePathCollection& zimpaths);
   ~ZimFileServer();
 
   Response GET(const char* path, const Headers& headers = Headers())
@@ -73,11 +74,13 @@ private: // data
   std::unique_ptr<httplib::Client> client;
 };
 
-ZimFileServer::ZimFileServer(int serverPort, std::string zimpath)
+ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths)
 : manager(&this->library)
 {
-  if (!manager.addBookFromPath(zimpath, zimpath, "", false))
-    throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
+  for ( const auto zimpath : zimpaths ) {
+    if (!manager.addBookFromPath(zimpath, zimpath, "", false))
+      throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
+  }
 
   const std::string address = "127.0.0.1";
   nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false));
@@ -104,11 +107,14 @@ protected:
   std::unique_ptr<ZimFileServer>   zfs1_;
 
   const int PORT = 8001;
-  const std::string ZIMFILE = "./test/zimfile.zim";
+  const ZimFileServer::FilePathCollection ZIMFILES {
+    "./test/zimfile.zim",
+    "./test/corner_cases.zim"
+  };
 
 protected:
   void SetUp() override {
-    zfs1_.reset(new ZimFileServer(PORT, ZIMFILE));
+    zfs1_.reset(new ZimFileServer(PORT, ZIMFILES));
   }
 
   void TearDown() override {
@@ -174,6 +180,10 @@ const ResourceCollection resources200Uncompressible{
   { WITH_ETAG, "/meta?content=zimfile&name=favicon" },
 
   { WITH_ETAG, "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
+
+  { WITH_ETAG, "/corner_cases/A/empty.html" },
+  { WITH_ETAG, "/corner_cases/-/empty.css" },
+  { WITH_ETAG, "/corner_cases/-/empty.js" },
 };
 
 ResourceCollection all200Resources()
@@ -307,7 +317,7 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet)
 
 TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags)
 {
-  ZimFileServer zfs2(PORT + 1, ZIMFILE);
+  ZimFileServer zfs2(PORT + 1, ZIMFILES);
   for ( const Resource& res : all200Resources() ) {
     if ( !res.etag_expected ) continue;
     const auto h1 = zfs1_->HEAD(res.url);
@@ -408,3 +418,108 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
     EXPECT_EQ(200, g2->status);
   }
 }
+
+TEST_F(ServerTest, ValidSingleRangeByteRangeRequestsAreHandledProperly)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const auto full = zfs1_->GET(url);
+  EXPECT_FALSE(full->has_header("Content-Range"));
+  EXPECT_EQ("bytes", full->get_header_value("Accept-Ranges"));
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=0-100000"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body, p->body);
+    EXPECT_EQ("bytes 0-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=0-10"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ("bytes 0-10/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ(11U, p->body.size());
+    EXPECT_EQ(full->body.substr(0, 11), p->body);
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=123-456"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ("bytes 123-456/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ(334U, p->body.size());
+    EXPECT_EQ(full->body.substr(123, 334), p->body);
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=20000-"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body.substr(20000), p->body);
+    EXPECT_EQ("bytes 20000-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+
+  {
+    const auto p = zfs1_->GET(url, { {"Range", "bytes=-100"} } );
+    EXPECT_EQ(206, p->status);
+    EXPECT_EQ(full->body.substr(19977), p->body);
+    EXPECT_EQ("bytes 19977-20076/20077", p->get_header_value("Content-Range"));
+    EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
+  }
+}
+
+TEST_F(ServerTest, InvalidAndMultiRangeByteRangeRequestsResultIn416Responses)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+
+  const char* invalidRanges[] = {
+    "0-10", "bytes=", "bytes=123", "bytes=-10-20", "bytes=10-20xxx",
+    "bytes=10-0", // reversed range
+    "bytes=10-20, 30-40", // multi-range
+    "bytes=1000000-", "bytes=30000-30100" // unsatisfiable ranges
+  };
+
+  for( const char* range : invalidRanges )
+  {
+    const TestContext ctx{ {"Range", range} };
+    const auto p = zfs1_->GET(url, { {"Range", range } } );
+    EXPECT_EQ(416, p->status) << ctx;
+    EXPECT_TRUE(p->body.empty()) << ctx;
+    EXPECT_EQ("bytes */20077", p->get_header_value("Content-Range")) << ctx;
+  }
+}
+
+TEST_F(ServerTest, ValidByteRangeRequestsOfZeroSizedEntriesResultIn416Responses)
+{
+  const char url[] = "/corner_cases/-/empty.js";
+
+  const char* ranges[] = {
+    "bytes=0-",
+    "bytes=-100"
+  };
+
+  for( const char* range : ranges )
+  {
+    const TestContext ctx{ {"Range", range} };
+    const auto p = zfs1_->GET(url, { {"Range", range } } );
+    EXPECT_EQ(416, p->status) << ctx;
+    EXPECT_TRUE(p->body.empty()) << ctx;
+    EXPECT_EQ("bytes */0", p->get_header_value("Content-Range")) << ctx;
+  }
+}
+
+TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+
+  const Headers onlyRange{ {"Range", "bytes=123-456"} };
+  Headers rangeAndCompression(onlyRange);
+  rangeAndCompression.insert({"Accept-Encoding", "deflate"});
+
+  const auto p1 = zfs1_->GET(url, onlyRange);
+  const auto p2 = zfs1_->GET(url, rangeAndCompression);
+  EXPECT_EQ(p1->status, p2->status);
+  EXPECT_EQ(invariantHeaders(p1->headers), invariantHeaders(p2->headers));
+  EXPECT_EQ(p1->body, p2->body);
+}