Run the CI on pull request.

This is needed to have the CI run on pull request created from of fork
respository.

We don't need to launch the CI on push :
- It will also be build on pull_request.
- The CI does check code only (no publication,...)

.github/workflows/check_pr.yml

@@ -1,6 +1,6 @@
-name: CI
+name: Check PR
 
-on: [push]
+on: [pull_request]
 
 jobs:
   Macos:

.github/workflows/package.yml

@@ -5,8 +5,9 @@ jobs:
   build-deb:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        distro: [ubuntu-groovy, ubuntu-focal, ubuntu-eoan]
+        distro: [ubuntu-groovy, ubuntu-focal]
     steps:
       - uses: actions/checkout@v2
 
@@ -45,14 +46,6 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-eoan
-        if: matrix.distro == 'ubuntu-eoan'
-        name: Build package for ubuntu-eoan
-        id: build-ubuntu-eoan
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
       - uses: actions/upload-artifact@v2
         with:
           name: Packages for ${{ matrix.distro }}

.gitignore

@@ -2,3 +2,5 @@
 *.swp
 subprojects/googletest-release*
 *.class
+build/
+.vscode/

README.md

@@ -38,11 +38,13 @@ libraries need to be available:
 * [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
 * [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
 * [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
-* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
 set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
 
+The following dependency needs to be available at runtime:
+* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
+
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The
 compilation script will tell you if one of them is missing or too old.
@@ -56,7 +58,7 @@ Environment
 -------------
 
 The Kiwix library builds using [Meson](https://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
 compilation tools.
 
 Install first the few common compilation tools:

debian/control

@@ -1,10 +1,10 @@
 Source: libkiwix
 Priority: optional
 Maintainer: Kiwix team <kiwix@kiwix.org>
-Build-Depends: debhelper-compat (= 12),
+Build-Depends: debhelper-compat (= 13),
  meson,
  pkg-config,
- libzim-dev (>= 6.0.0),
+ libzim-dev (>= 6.1.8),
  libcurl4-gnutls-dev,
  libicu-dev,
  libgtest-dev,
@@ -22,7 +22,7 @@ Package: libkiwix-dev
 Section: libdevel
 Architecture: any
 Multi-Arch: same
-Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3, aria2,
+Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3,
  libzim-dev (>= 6.0.0),
  libicu-dev,
  libpugixml-dev,
@@ -37,7 +37,7 @@ Description: library of common code for Kiwix (development)
 Package: libkiwix9
 Architecture: any
 Multi-Arch: same
-Depends: ${shlibs:Depends}, ${misc:Depends}
+Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
 Conflicts: libkiwix0, libkiwix3
 Description: library of common code for Kiwix
  Kiwix is an offline Wikipedia reader. libkiwix provides the

debian/libkiwix-dev.manpages

@@ -0,0 +1 @@
+usr/share/man/man1/kiwix-compile-resources.1

debian/rules

@@ -3,3 +3,6 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
 %:
 	dh $@ --buildsystem=meson
+
+override_dh_auto_test:
+	dh_auto_test -- -t 3

include/search_renderer.h

@@ -60,6 +60,13 @@ class SearchRenderer
    */
   void setSearchProtocolPrefix(const std::string& prefix);
 
+  /**
+   * set result count per page
+   */
+  void setPageLength(unsigned int pageLength){
+    this->pageLength  = pageLength;
+  }
+
   /**
    * Generate the html page with the resutls of the search.
    */
@@ -73,10 +80,9 @@ class SearchRenderer
   std::string searchPattern;
   std::string protocolPrefix;
   std::string searchProtocolPrefix;
-  unsigned int resultCountPerPage;
+  unsigned int pageLength;
   unsigned int estimatedResultCount;
   unsigned int resultStart;
-  unsigned int resultEnd;
 };
 
 

meson.build

@@ -7,19 +7,19 @@ compiler = meson.get_compiler('cpp')
 
 wrapper = get_option('wrapper')
 
-static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
-if 'android' in wrapper
+static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
+if wrapper.contains('android')
   extra_libs = ['-llog']
 else
   extra_libs = []
 endif
 
-if 'java' in wrapper
+if wrapper.contains('java')
   add_languages('java')
 endif
 
 # See https://github.com/kiwix/kiwix-lib/issues/371
-if target_machine.cpu_family() in ['arm', 'mips', 'm68k', 'ppc', 'sh4']
+if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
   extra_libs += '-latomic'
 endif
 

src/meson.build

@@ -34,13 +34,13 @@ else
   kiwix_sources += 'subprocess_unix.cpp'
 endif
 
-if 'android' in wrapper
+if wrapper.contains('android')
   install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
 else
   install_dir = get_option('libdir')
 endif
 
-if 'android' in wrapper or 'java' in wrapper
+if wrapper.contains('android') or wrapper.contains('java')
   subdir('wrapper/java')
 endif
 

src/reader.cpp

@@ -138,9 +138,10 @@ unsigned int Reader::getArticleCount() const
   if (counterMap.empty()) {
     counter = this->nsACount;
   } else {
-    auto it = counterMap.find("text/html");
-    if (it != counterMap.end()) {
-      counter = it->second;
+    for(auto &pair:counterMap) {
+      if (startsWith(pair.first, "text/html")) {
+        counter += pair.second;
+      }
     }
   }
 

src/search_renderer.cpp

@@ -93,30 +93,30 @@ std::string SearchRenderer::getHtml()
   kainjow::mustache::data pages{kainjow::mustache::data::type::list};
 
   auto resultStart = mp_searcher->getResultStart();
-  auto resultEnd = mp_searcher->getResultEnd();
-  auto resultCountPerPage = resultEnd - resultStart;
+  auto resultEnd = 0U;
   auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
   auto currentPage = 0U;
   auto pageStart = 0U;
   auto pageEnd = 0U;
   auto lastPageStart = 0U;
-  if (resultCountPerPage) {
-    currentPage = resultStart/resultCountPerPage;
+  if (pageLength) {
+    currentPage = resultStart/pageLength;
     pageStart = currentPage > 4 ? currentPage-4 : 0;
     pageEnd = currentPage + 5;
-    if (pageEnd > estimatedResultCount / resultCountPerPage) {
-      pageEnd = estimatedResultCount / resultCountPerPage;
+    if (pageEnd > estimatedResultCount / pageLength) {
+      pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
     }
-    if (estimatedResultCount > resultCountPerPage) {
-      lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
+    if (estimatedResultCount > pageLength) {
+      lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
     }
   }
 
+  resultEnd = resultStart+pageLength; //setting result end
+
   for (unsigned int i = pageStart; i < pageEnd; i++) {
     kainjow::mustache::data page;
     page.set("label", to_string(i + 1));
-    page.set("start", to_string(i * resultCountPerPage));
-    page.set("end", to_string((i + 1) * resultCountPerPage));
+    page.set("start", to_string(i * pageLength));
 
     if (i == currentPage) {
       page.set("selected", true);
@@ -137,9 +137,8 @@ std::string SearchRenderer::getHtml()
   allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
   allData.set("resultStart", to_string(resultStart + 1));
   allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
-  allData.set("resultRange", to_string(resultCountPerPage));
+  allData.set("pageLength", to_string(pageLength));
   allData.set("resultLastPageStart", to_string(lastPageStart));
-  allData.set("lastResult", to_string(estimatedResultCount));
   allData.set("protocolPrefix", this->protocolPrefix);
   allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
   allData.set("contentId", this->searchContent);
@@ -149,4 +148,4 @@ std::string SearchRenderer::getHtml()
   return ss.str();
 }
 
-}
+}

src/server.cpp

@@ -477,7 +477,6 @@ Response InternalServer::build_homepage(const RequestContext& request)
   response.set_template(RESOURCE::templates::index_html, homepage_data());
   response.set_mimeType("text/html; charset=utf-8");
   response.set_compress(true);
-  response.set_taskbar("", "");
   return response;
 }
 
@@ -709,37 +708,47 @@ Response InternalServer::handle_search(const RequestContext& request)
   try {
     start = request.get_argument<unsigned int>("start");
   } catch (const std::exception&) {}
-  auto end = 25;
-  try {
-    end = request.get_argument<unsigned int>("end");
-  } catch (const std::exception&) {}
-  if (start>end) {
-    auto tmp = start;
-    start = end;
-    end = tmp;
+  auto pageLength=25;
+  try{
+    pageLength=request.get_argument<unsigned int>("pageLength");
+  }catch(const std::exception&){}
+  if (pageLength > MAX_SEARCH_LEN) {
+    pageLength = MAX_SEARCH_LEN;
   }
-  if (end > start + MAX_SEARCH_LEN) {
-    end = start + MAX_SEARCH_LEN;
+
+  if(pageLength==0)
+  {
+    pageLength=25;
   }
 
+  auto end=start+pageLength;
+
   /* Get the results */
   try {
     if (patternString.empty()) {
       searcher.geo_search(latitude, longitude, distance,
-                           start, end, m_verbose.load());
+                           start, end , m_verbose.load());
     } else {
       searcher.search(patternString,
-                       start, end, m_verbose.load());
+                       start, end , m_verbose.load());
     }
     SearchRenderer renderer(&searcher, mp_nameMapper);
     renderer.setSearchPattern(patternString);
     renderer.setSearchContent(bookName);
     renderer.setProtocolPrefix(m_root + "/");
     renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setPageLength(pageLength);
     response.set_content(renderer.getHtml());
   } catch (const std::exception& e) {
     std::cerr << e.what() << std::endl;
   }
+
+  //changing status code if no result obtained
+  if(searcher.getEstimatedResultCount() == 0)
+  {
+    response.set_code(MHD_HTTP_NO_CONTENT);
+  }
+
   return response;
 }
 
@@ -788,7 +797,6 @@ Response InternalServer::handle_captured_external(const RequestContext& request)
   response.set_template(RESOURCE::templates::captured_external_html, data);
   response.set_mimeType("text/html; charset=utf-8");
   response.set_compress(true);
-  response.set_taskbar("", "");
   return response;
 }
 
@@ -950,14 +958,13 @@ Response InternalServer::handle_content(const RequestContext& request)
   auto response = get_default_response();
 
   response.set_entry(entry, request);
+  response.set_taskbar(bookName, reader->getTitle());
 
   if (m_verbose.load()) {
     printf("Found %s\n", entry.getPath().c_str());
     printf("mimeType: %s\n", response.get_mimeType().c_str());
   }
 
-  if (response.get_mimeType().find("text/html") != string::npos)
-    response.set_taskbar(bookName, reader->getTitle());
 
   return response;
 }

src/server/request_context.cpp

@@ -26,6 +26,8 @@
 #include <cstdio>
 #include <atomic>
 
+#include "tools/stringTools.h"
+
 namespace kiwix {
 
 static std::atomic_ullong s_requestIndex(0);
@@ -96,7 +98,7 @@ MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
                                        const char *key, const char *value)
 {
   RequestContext *_this = static_cast<RequestContext*>(__this);
-  _this->headers[key] = value;
+  _this->headers[lcAll(key)] = value;
   return MHD_YES;
 }
 
@@ -178,7 +180,7 @@ std::string RequestContext::get_argument(const std::string& name) const {
 }
 
 std::string RequestContext::get_header(const std::string& name) const {
-  return headers.at(name);
+  return headers.at(lcAll(name));
 }
 
 }

src/server/response.cpp

@@ -51,8 +51,8 @@ Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool
     m_withTaskbar(withTaskbar),
     m_withLibraryButton(withLibraryButton),
     m_blockExternalLinks(blockExternalLinks),
-    m_addTaskbar(false),
-    m_bookName("")
+    m_bookName(""),
+    m_bookTitle("")
 {
 }
 
@@ -127,9 +127,6 @@ std::string render_template(const std::string& template_str, kainjow::mustache::
 
 void Response::introduce_taskbar()
 {
-  if (! m_withTaskbar)
-    // Taskbar is globally disabled.
-    return;
   kainjow::mustache::data data;
   data.set("root", m_root);
   data.set("content", m_bookName);
@@ -169,6 +166,13 @@ Response::can_compress(const RequestContext& request) const
       && (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
 }
 
+bool
+Response::contentDecorationAllowed() const
+{
+    return (startsWith(m_mimeType, "text/html")
+         && m_mimeType.find(";raw=true") == std::string::npos);
+}
+
 MHD_Response*
 Response::create_error_response(const RequestContext& request) const
 {
@@ -186,11 +190,13 @@ Response::create_error_response(const RequestContext& request) const
 MHD_Response*
 Response::create_raw_content_mhd_response(const RequestContext& request)
 {
-  if (m_addTaskbar) {
-    introduce_taskbar();
-  }
-  if ( m_blockExternalLinks ) {
-    inject_externallinks_blocker();
+  if (contentDecorationAllowed()) {
+    if (m_withTaskbar) {
+      introduce_taskbar();
+    }
+    if (m_blockExternalLinks) {
+      inject_externallinks_blocker();
+    }
   }
 
   bool shouldCompress = m_compress && can_compress(request);
@@ -352,7 +358,6 @@ void Response::set_entry(const Entry& entry, const RequestContext& request) {
 
 void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
 {
-  m_addTaskbar = true;
   m_bookName = bookName;
   m_bookTitle = bookTitle;
 }

src/server/response.h

@@ -71,6 +71,7 @@ class Response {
     void inject_externallinks_blocker();
 
     bool can_compress(const RequestContext& request) const;
+    bool contentDecorationAllowed() const;
 
   private: // functions
     MHD_Response* create_mhd_response(const RequestContext& request);
@@ -91,7 +92,6 @@ class Response {
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
     bool m_compress;
-    bool m_addTaskbar;
     std::string m_bookName;
     std::string m_bookTitle;
     ByteRange m_byteRange;

static/templates/search_result.html

@@ -133,7 +133,7 @@
         <ul>
           {{#resultLastPageStart}}
             <li>
-              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&pageLength={{pageLength}}">
                 ◀
               </a>
             </li>
@@ -141,14 +141,14 @@
           {{#pages}}
             <li>
               <a {{#selected}}class="selected"{{/selected}}
-                 href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
+                 href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&pageLength={{pageLength}}">
                 {{label}}
               </a>
             </li>
           {{/pages}}
           {{#resultLastPageStart}}
             <li>
-              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
+              <a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&pageLength={{pageLength}}">
                 ▶
               </a>
             </li>

test/meson.build

@@ -1,7 +1,3 @@
-configure_file(input : 'data/example.zim',
-               output : 'example.zim',
-               copy: true )
-
 tests = [
     'parseUrl',
     'library',
@@ -26,12 +22,32 @@ gtest_dep = dependency('gtest',
                        required:false)
 
 if gtest_dep.found() and not meson.is_cross_build()
-    configure_file(input : 'data/wikipedia_en_ray_charles_mini_2020-03.zim',
-                   output : 'zimfile.zim',
-                   copy: true )
-    configure_file(input : 'data/corner_cases.zim',
-                   output : 'corner_cases.zim',
-                   copy: true )
+    data_files = [
+      'example.zim',
+      'zimfile.zim',
+      'corner_cases.zim'
+    ]
+    foreach file : data_files
+        # configure_file(input : 'data/' + file,
+        #                output : file,
+        #                copy: true )
+        #
+        # Above (commented) command doesn't work with Meson versions below 0.47
+        # (in which the 'copy' keyword was first introduced). We want to keep
+        # compatibility with Ubuntu 18.04 Bionic (which has Meson version 0.45)
+        # until its EOL.
+        #
+        # Below is a python based workaround.
+        configure_file(input : 'data/' + file,
+                       output : file,
+                       command: [
+                         find_program('python3'),
+                          '-c',
+                          'import sys; import shutil; shutil.copy(sys.argv[1], sys.argv[2])',
+                          '@INPUT@',
+                          '@OUTPUT@'
+                       ])
+    endforeach
 
     foreach test_name : tests
         # XXX: implicit_include_directories must be set to false, otherwise

test/server.cpp

@@ -155,7 +155,7 @@ const ResourceCollection resources200Compressible{
   { NO_ETAG,   "/catalog/searchdescription.xml" },
   { NO_ETAG,   "/catalog/search" },
 
-  { NO_ETAG,   "/search?content=zimfile&pattern=abcd" },
+  { NO_ETAG,   "/search?content=zimfile&pattern=a" },
 
   { NO_ETAG,   "/suggest?content=zimfile&term=ray" },
 
@@ -197,6 +197,15 @@ TEST_F(ServerTest, 200)
     EXPECT_EQ(200, zfs1_->GET(res.url)->status) << "res.url: " << res.url;
 }
 
+// seperate test for 204 code
+
+TEST_F(ServerTest, EmptySearchReturnsA204StatusCode)
+{
+  const char* url="/search?content=zimfile&pattern=abcd";
+  auto res=zfs1_->GET(url);
+  EXPECT_EQ(204, res->status) << "res.url: " << url;
+}
+
 TEST_F(ServerTest, CompressibleContentIsCompressedIfAcceptable)
 {
   for ( const Resource& res : resources200Compressible ) {
@@ -523,3 +532,17 @@ TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
   EXPECT_EQ(invariantHeaders(p1->headers), invariantHeaders(p2->headers));
   EXPECT_EQ(p1->body, p2->body);
 }
+
+TEST_F(ServerTest, RangeHeaderIsCaseInsensitive)
+{
+  const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
+  const auto r0 = zfs1_->GET(url, { {"Range", "bytes=100-200"} } );
+
+  const char* header_variations[] = { "RANGE", "range", "rAnGe", "RaNgE" };
+  for ( const char* header : header_variations ) {
+    const auto r = zfs1_->GET(url, { {header, "bytes=100-200"} } );
+    EXPECT_EQ(206, r->status);
+    EXPECT_EQ("bytes 100-200/20077", r->get_header_value("Content-Range"));
+    EXPECT_EQ(r0->body, r->body);
+  }
+}