New version 9.4.0

Use a heap allocated buffer (with lifetime of Aria2 class) instead of
a stack allocated one.

Original fix made by @ZaWertun. Kudos to him.

Fix #kiwix/kiwix-desktop#123, kiwix/kiwix-desktop#513
and kiwix/kiwix-desktop#423

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
-name: Check PR
+name: CI
 
-on: [pull_request]
+on: [push]
 
 jobs:
   Macos:

.github/workflows/package.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        distro: [ubuntu-groovy, ubuntu-focal]
+        distro: [ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
     steps:
       - uses: actions/checkout@v2
 
@@ -46,6 +46,14 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
+      - uses: legoktm/gh-action-build-deb@ubuntu-bionic
+        if: matrix.distro == 'ubuntu-bionic'
+        name: Build package for ubuntu-bionic
+        id: build-ubuntu-bionic
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
       - uses: actions/upload-artifact@v2
         with:
           name: Packages for ${{ matrix.distro }}

ChangeLog

@@ -1,3 +1,20 @@
+kiwix-lib 9.4.0
+===============
+
+ * [SERVER] Make the headers handling case insensitive.
+ * [SERVER] Make server answer 204 http status code for empty search
+ * [PACKAGING] Made CI build deb packages.
+ * [SERVER] Add a way to prevent taskbar and external link bloquer at article
+   level.
+ * Fix meson file to be compatible with meson 0.45
+ * [SERVER] Update search requests to use pageStart/pageLength instead of
+   pageStart/pageEnd arguments.
+ * [SERVER] Set a fixed favicon size in the main page.
+ * [SERVER] Refactor the response system code to better handling future new
+   libzim api.
+ * Fix segmentation fault around exchange with aria2 process making
+   kiwix-desktop crash at exit.
+
 kiwix-lib 9.3.1
 ===============
 

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '9.3.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '9.4.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

debian/libkiwix-dev.manpages

@@ -1 +1 @@
-usr/share/man/man1/kiwix-compile-resources.1
+usr/share/man/man1/kiwix-compile-resources.1*

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-lib', 'cpp',
-  version : '9.3.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '9.4.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPLv3+',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 

src/aria2.cpp

@@ -19,6 +19,11 @@
 #endif
 
 
+#define LOG_ARIA_ERROR() \
+  { \
+    std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
+    std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
+  }
 
 namespace kiwix {
 
@@ -26,8 +31,8 @@ Aria2::Aria2():
   mp_aria(nullptr),
   m_port(42042),
   m_secret("kiwixariarpc"),
-  mp_curl(nullptr),
-  m_lock(PTHREAD_MUTEX_INITIALIZER)
+  m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
+  mp_curl(nullptr)
 {
   m_downloadDir = getDataDirectory();
   makeDirectory(m_downloadDir);
@@ -84,27 +89,21 @@ Aria2::Aria2():
   }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
-  char errbuf[CURL_ERROR_SIZE];
+
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
-  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
-    errbuf[0] = 0;
+    m_curlErrorBuffer[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
     } else if (watchdog == 1) {
-      std::cerr <<" curl_easy_perform() failed." << std::endl;
-      fprintf(stderr, "\nlibcurl: (%d) ", res);
-      if (errbuf[0] != 0) {
-        std::cerr << errbuf << std::endl;
-      } else {
-        std::cerr << curl_easy_strerror(res) << std::endl;
-      }
+      LOG_ARIA_ERROR();
     }
   }
   if (!watchdog) {
@@ -115,6 +114,7 @@ Aria2::Aria2():
 
 Aria2::~Aria2()
 {
+  std::unique_lock<std::mutex> lock(m_lock);
   curl_easy_cleanup(mp_curl);
 }
 
@@ -126,38 +126,44 @@ void Aria2::close()
 
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
-  auto str = static_cast<std::stringstream*>(userdata);
-  str->write(ptr, nmemb);
+  auto outStream = static_cast<std::stringstream*>(userdata);
+  outStream->write(ptr, nmemb);
   return nmemb;
 }
 
 std::string Aria2::doRequest(const MethodCall& methodCall)
 {
-  pthread_mutex_lock(&m_lock);
   auto requestContent = methodCall.toString();
-  std::stringstream stringstream;
+  std::stringstream outStream;
   CURLcode res;
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
-  curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
-  curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
-  res = curl_easy_perform(mp_curl);
-  if (res == CURLE_OK) {
-    long response_code;
+  long response_code;
+  {
+    std::unique_lock<std::mutex> lock(m_lock);
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
+    curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
+    curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
+    m_curlErrorBuffer[0] = 0;
+    res = curl_easy_perform(mp_curl);
+    if (res != CURLE_OK) {
+      LOG_ARIA_ERROR();
+      throw std::runtime_error("Cannot perform request");
+    }
     curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
-    pthread_mutex_unlock(&m_lock);
-    if (response_code != 200) {
-      throw std::runtime_error("Invalid return code from aria");
-    }
-    auto responseContent = stringstream.str();
-    MethodResponse response(responseContent);
-    if (response.isFault()) {
-      throw AriaError(response.getFault().getFaultString());
-    }
-    return responseContent;
   }
-  pthread_mutex_unlock(&m_lock);
-  throw std::runtime_error("Cannot perform request");
+
+  auto responseContent = outStream.str();
+  if (response_code != 200) {
+    std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
+    std::cerr << responseContent << std::endl;
+    throw std::runtime_error("Invalid return code from aria");
+  }
+
+  MethodResponse response(responseContent);
+  if (response.isFault()) {
+    throw AriaError(response.getFault().getFaultString());
+  }
+  return responseContent;
 }
 
 std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)

src/aria2.h

@@ -12,8 +12,8 @@
 #include "xmlrpc.h"
 
 #include <memory>
+#include <mutex>
 #include <curl/curl.h>
-#include <pthread.h>
 
 namespace kiwix {
 
@@ -24,8 +24,9 @@ class Aria2
     int m_port;
     std::string m_secret;
     std::string m_downloadDir;
+    std::unique_ptr<char[]> m_curlErrorBuffer;
     CURL* mp_curl;
-    pthread_mutex_t m_lock;
+    std::mutex m_lock;
 
     std::string doRequest(const MethodCall& methodCall);
 

src/downloader.cpp

@@ -133,7 +133,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellActive failed : " << e.what();
+    std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
   }
   try {
     for (auto gid : mp_aria->tellWaiting()) {
@@ -141,7 +141,7 @@ Downloader::Downloader() :
       m_knownDownloads[gid]->updateStatus();
     }
   } catch (std::exception& e) {
-    std::cerr << "aria2 tellWaiting failed : " << e.what();
+    std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
   }
 }
 

src/meson.build

@@ -24,7 +24,8 @@ kiwix_sources = [
   'server/byte_range.cpp',
   'server/etag.cpp',
   'server/request_context.cpp',
-  'server/response.cpp'
+  'server/response.cpp',
+  'server/internalServer.cpp'
 ]
 kiwix_sources += lib_resources
 

src/server.cpp

@@ -19,140 +19,15 @@
 
 #include "server.h"
 
-#ifdef _WIN32
-# if !defined(__MINGW32__) && (_MSC_VER < 1600)
-#   include "stdint4win.h"
-# endif
-# include <winsock2.h>
-# include <ws2tcpip.h>
-# ifdef __GNUC__
-  // inet_pton is not declared in mingw, even if the function exists.
-  extern "C" {
-    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
-  }
-# endif
-  typedef UINT64 uint64_t;
-  typedef UINT16 uint16_t;
-#endif
-
-extern "C" {
-#include "microhttpd_wrapper.h"
-}
-
-#include "tools/otherTools.h"
-#include "tools/pathTools.h"
-#include "tools/regexTools.h"
-#include "tools/stringTools.h"
 #include "library.h"
 #include "name_mapper.h"
-#include "entry.h"
-#include "searcher.h"
-#include "search_renderer.h"
-#include "opds_dumper.h"
 
-#include <zim/uuid.h>
-
-#include <mustache.hpp>
-
-#include <pthread.h>
-#include <atomic>
 #include <string>
-#include <vector>
-#include <chrono>
-#include "kiwixlib-resources.h"
 
-#ifndef _WIN32
-# include <arpa/inet.h>
-#endif
-
-#include "server/request_context.h"
-#include "server/response.h"
-
-#define MAX_SEARCH_LEN 140
-#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+#include "server/internalServer.h"
 
 namespace kiwix {
 
-static IdNameMapper defaultNameMapper;
-
-typedef kainjow::mustache::data MustacheData;
-
-static MHD_Result staticHandlerCallback(void* cls,
-                                        struct MHD_Connection* connection,
-                                        const char* url,
-                                        const char* method,
-                                        const char* version,
-                                        const char* upload_data,
-                                        size_t* upload_data_size,
-                                        void** cont_cls);
-
-
-class InternalServer {
-  public:
-    InternalServer(Library* library,
-                   NameMapper* nameMapper,
-                   std::string addr,
-                   int port,
-                   std::string root,
-                   int nbThreads,
-                   bool verbose,
-                   bool withTaskbar,
-                   bool withLibraryButton,
-                   bool blockExternalLinks);
-    virtual ~InternalServer() = default;
-
-    MHD_Result handlerCallback(struct MHD_Connection* connection,
-                               const char* url,
-                               const char* method,
-                               const char* version,
-                               const char* upload_data,
-                               size_t* upload_data_size,
-                               void** cont_cls);
-    bool start();
-    void stop();
-
-  private: // functions
-    Response handle_request(const RequestContext& request);
-    Response build_500(const std::string& msg);
-    Response build_404(const RequestContext& request, const std::string& zimName);
-    Response build_304(const RequestContext& request, const ETag& etag) const;
-    Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
-    Response build_homepage(const RequestContext& request);
-    Response handle_skin(const RequestContext& request);
-    Response handle_catalog(const RequestContext& request);
-    Response handle_meta(const RequestContext& request);
-    Response handle_search(const RequestContext& request);
-    Response handle_suggest(const RequestContext& request);
-    Response handle_random(const RequestContext& request);
-    Response handle_captured_external(const RequestContext& request);
-    Response handle_content(const RequestContext& request);
-
-    MustacheData get_default_data() const;
-    MustacheData homepage_data() const;
-    Response get_default_response() const;
-
-    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
-
-  private: // data
-    std::string m_addr;
-    int m_port;
-    std::string m_root;
-    int m_nbThreads;
-    std::atomic_bool m_verbose;
-    bool m_withTaskbar;
-    bool m_withLibraryButton;
-    bool m_blockExternalLinks;
-    struct MHD_Daemon* mp_daemon;
-
-    Library* mp_library;
-    NameMapper* mp_nameMapper;
-
-    std::string m_server_id;
-};
-
-
 Server::Server(Library* library, NameMapper* nameMapper) :
   mp_library(library),
   mp_nameMapper(nameMapper),
@@ -193,780 +68,4 @@ void Server::setRoot(const std::string& root)
   }
 }
 
-
-InternalServer::InternalServer(Library* library,
-                               NameMapper* nameMapper,
-                               std::string addr,
-                               int port,
-                               std::string root,
-                               int nbThreads,
-                               bool verbose,
-                               bool withTaskbar,
-                               bool withLibraryButton,
-                               bool blockExternalLinks) :
-  m_addr(addr),
-  m_port(port),
-  m_root(root),
-  m_nbThreads(nbThreads),
-  m_verbose(verbose),
-  m_withTaskbar(withTaskbar),
-  m_withLibraryButton(withLibraryButton),
-  m_blockExternalLinks(blockExternalLinks),
-  mp_daemon(nullptr),
-  mp_library(library),
-  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
-{}
-
-bool InternalServer::start() {
-#ifdef _WIN32
-  int flags = MHD_USE_SELECT_INTERNALLY;
-#else
-  int flags = MHD_USE_POLL_INTERNALLY;
-#endif
-  if (m_verbose.load())
-    flags |= MHD_USE_DEBUG;
-
-
-  struct sockaddr_in sockAddr;
-  memset(&sockAddr, 0, sizeof(sockAddr));
-  sockAddr.sin_family = AF_INET;
-  sockAddr.sin_port = htons(m_port);
-  if (m_addr.empty()) {
-    if (0 != INADDR_ANY)
-      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-  } else {
-    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
-      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
-      return false;
-    }
-  }
-
-  mp_daemon = MHD_start_daemon(flags,
-                            m_port,
-                            NULL,
-                            NULL,
-                            &staticHandlerCallback,
-                            this,
-                            MHD_OPTION_SOCK_ADDR, &sockAddr,
-                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
-                            MHD_OPTION_END);
-  if (mp_daemon == nullptr) {
-    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
-              << " is maybe already occupied or need more permissions to be open. "
-                 "Please try as root or with a port number higher or equal to 1024."
-              << std::endl;
-    return false;
-  }
-  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
-  m_server_id = kiwix::to_string(server_start_time.count());
-  return true;
-}
-
-void InternalServer::stop()
-{
-  MHD_stop_daemon(mp_daemon);
-}
-
-static MHD_Result staticHandlerCallback(void* cls,
-                                        struct MHD_Connection* connection,
-                                        const char* url,
-                                        const char* method,
-                                        const char* version,
-                                        const char* upload_data,
-                                        size_t* upload_data_size,
-                                        void** cont_cls)
-{
-  InternalServer* _this = static_cast<InternalServer*>(cls);
-
-  return _this->handlerCallback(connection,
-                                url,
-                                method,
-                                version,
-                                upload_data,
-                                upload_data_size,
-                                cont_cls);
-}
-
-MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
-                                           const char* url,
-                                           const char* method,
-                                           const char* version,
-                                           const char* upload_data,
-                                           size_t* upload_data_size,
-                                           void** cont_cls)
-{
-  auto start_time = std::chrono::steady_clock::now();
-  if (m_verbose.load() ) {
-    printf("======================\n");
-    printf("Requesting : \n");
-    printf("full_url  : %s\n", url);
-  }
-  RequestContext request(connection, m_root, url, method, version);
-
-  if (m_verbose.load() ) {
-    request.print_debug_info();
-  }
-  /* Unexpected method */
-  if (request.get_method() != RequestMethod::GET
-   && request.get_method() != RequestMethod::POST
-   && request.get_method() != RequestMethod::HEAD) {
-    printf("Reject request because of unhandled request method.\n");
-    printf("----------------------\n");
-    return MHD_NO;
-  }
-
-  auto response = handle_request(request);
-
-  if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
-    printf("========== INTERNAL ERROR !! ============\n");
-    if (!m_verbose.load()) {
-      printf("Requesting : \n");
-      printf("full_url : %s\n", url);
-      request.print_debug_info();
-    }
-  }
-
-  if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
-    response.set_server_id(m_server_id);
-
-  auto ret = response.send(request, connection);
-  auto end_time = std::chrono::steady_clock::now();
-  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
-  if (m_verbose.load()) {
-    printf("Request time : %fs\n", time_span.count());
-    printf("----------------------\n");
-  }
-  return ret;
-}
-
-Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const
-{
-  auto response = get_default_response();
-  response.set_code(MHD_HTTP_NOT_MODIFIED);
-  response.set_etag(etag);
-  response.set_content("");
-  return response;
-}
-
-Response InternalServer::handle_request(const RequestContext& request)
-{
-  try {
-    if (! request.is_valid_url())
-      return build_404(request, "");
-
-    const ETag etag = get_matching_if_none_match_etag(request);
-    if ( etag )
-      return build_304(request, etag);
-
-    if (kiwix::startsWith(request.get_url(), "/skin/"))
-      return handle_skin(request);
-
-    if (startsWith(request.get_url(), "/catalog"))
-      return handle_catalog(request);
-
-    if (request.get_url() == "/meta")
-      return handle_meta(request);
-
-    if (request.get_url() == "/search")
-      return handle_search(request);
-
-    if (request.get_url() == "/suggest")
-     return handle_suggest(request);
-
-    if (request.get_url() == "/random")
-      return handle_random(request);
-
-    if (request.get_url() == "/catch/external")
-      return handle_captured_external(request);
-
-    return handle_content(request);
-  } catch (std::exception& e) {
-    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
-    return build_500(e.what());
-  } catch (...) {
-    fprintf(stderr, "===== Unhandled unknown error\n");
-    return build_500("Unknown error");
-  }
-}
-
-MustacheData InternalServer::get_default_data() const
-{
-  MustacheData data;
-  data.set("root", m_root);
-  return data;
-}
-
-Response InternalServer::get_default_response() const
-{
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
-}
-
-
-Response InternalServer::build_404(const RequestContext& request,
-                                   const std::string& bookName)
-{
-  MustacheData results;
-  results.set("url", request.get_full_url());
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::_404_html, results);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_NOT_FOUND);
-  response.set_compress(true);
-  response.set_taskbar(bookName, "");
-
-  return response;
-}
-
-Response InternalServer::build_500(const std::string& msg)
-{
-  MustacheData data;
-  data.set("error", msg);
-  Response response(m_root, true, false, false, false);
-  response.set_template(RESOURCE::templates::_500_html, data);
-  response.set_mimeType("text/html");
-  response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
-  return response;
-}
-
-MustacheData InternalServer::homepage_data() const
-{
-  auto data = get_default_data();
-
-  MustacheData books{MustacheData::type::list};
-  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-    auto& currentBook = mp_library->getBookById(bookId);
-
-    MustacheData book;
-    book.set("name", mp_nameMapper->getNameForId(bookId));
-    book.set("title", currentBook.getTitle());
-    book.set("description", currentBook.getDescription());
-    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
-    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
-    books.push_back(book);
-  }
-
-  data.set("books", books);
-  return data;
-}
-
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
-Response InternalServer::build_homepage(const RequestContext& request)
-{
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::index_html, homepage_data());
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_meta(const RequestContext& request)
-{
-  std::string bookName;
-  std::string bookId;
-  std::string meta_name;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    meta_name = request.get_argument("name");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  std::string content;
-  std::string mimeType = "text";
-
-  if (meta_name == "title") {
-    content = reader->getTitle();
-  } else if (meta_name == "description") {
-    content = reader->getDescription();
-  } else if (meta_name == "language") {
-    content = reader->getLanguage();
-  } else if (meta_name == "name") {
-    content = reader->getName();
-  } else if (meta_name == "tags") {
-    content = reader->getTags();
-  } else if (meta_name == "date") {
-    content = reader->getDate();
-  } else if (meta_name == "creator") {
-    content = reader->getCreator();
-  } else if (meta_name == "publisher") {
-    content = reader->getPublisher();
-  } else if (meta_name == "favicon") {
-    reader->getFavicon(content, mimeType);
-  } else {
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-  response.set_content(content);
-  response.set_mimeType(mimeType);
-  response.set_compress(false);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_suggest(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_suggest\n");
-  }
-
-  std::string content;
-  std::string mimeType;
-  unsigned int maxSuggestionCount = 10;
-  unsigned int suggestionCount = 0;
-
-  std::string bookName;
-  std::string bookId;
-  std::string term;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    term = request.get_argument("term");
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (m_verbose.load()) {
-    printf("Searching suggestions for: \"%s\"\n", term.c_str());
-  }
-
-  MustacheData results{MustacheData::type::list};
-
-  bool first = true;
-  if (reader != nullptr) {
-    /* Get the suggestions */
-    SuggestionsList_t suggestions;
-    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
-    for(auto& suggestion:suggestions) {
-      MustacheData result;
-      result.set("label", suggestion[0]);
-      result.set("value", suggestion[0]);
-      result.set("first", first);
-      first = false;
-      results.push_back(result);
-      suggestionCount++;
-    }
-  }
-
-  /* Propose the fulltext search if possible */
-  if (reader->hasFulltextIndex()) {
-    MustacheData result;
-    result.set("label", "containing '" + term + "'...");
-    result.set("value", term + " ");
-    result.set("first", first);
-    results.push_back(result);
-  }
-
-  auto data = get_default_data();
-  data.set("suggestions", results);
-
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::suggestion_json, data);
-  response.set_mimeType("application/json; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_skin(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_skin\n");
-  }
-
-  auto response = get_default_response();
-  auto resourceName = request.get_url().substr(1);
-  try {
-    response.set_content(getResource(resourceName));
-  } catch (const ResourceNotFound& e) {
-    return build_404(request, "");
-  }
-  response.set_mimeType(getMimeTypeForFile(resourceName));
-  response.set_compress(true);
-  response.set_cacheable();
-  return response;
-}
-
-Response InternalServer::handle_search(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_search\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-  } catch (const std::out_of_range&) {}
-
-  std::string patternString;
-  try {
-    patternString = request.get_argument("pattern");
-  } catch (const std::out_of_range&) {}
-
-  /* Retrive geo search */
-  bool has_geo_query = false;
-  float latitude = 0;
-  float longitude = 0;
-  float distance = 0;
-  try {
-    latitude = request.get_argument<float>("latitude");
-    longitude = request.get_argument<float>("longitude");
-    distance = request.get_argument<float>("distance");
-    has_geo_query = true;
-  } catch(const std::out_of_range&) {}
-    catch(const std::invalid_argument&) {}
-
-  std::shared_ptr<Reader> reader(nullptr);
-  try {
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {}
-
-  /* Try first to load directly the article */
-  if (reader != nullptr && !patternString.empty()) {
-    std::string patternCorrespondingUrl;
-    auto variants = reader->getTitleVariants(patternString);
-    auto variantsItr = variants.begin();
-
-    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
-      try {
-        auto entry = reader->getEntryFromTitle(*variantsItr);
-        entry = entry.getFinalEntry();
-        patternCorrespondingUrl = entry.getPath();
-        break;
-      } catch(kiwix::NoEntry& e) {
-        variantsItr++;
-      }
-    }
-
-    /* If article found then redirect directly to it */
-    if (!patternCorrespondingUrl.empty()) {
-      auto response = get_default_response();
-      response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
-      return response;
-    }
-  }
-
-  /* Make the search */
-  auto response = get_default_response();
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_taskbar(bookName, reader ? reader->getTitle() : "");
-  response.set_compress(true);
-
-  if ( (!reader && !bookName.empty())
-    || (patternString.empty() && ! has_geo_query) ) {
-    auto data = get_default_data();
-    data.set("pattern", encodeDiples(patternString));
-    response.set_template(RESOURCE::templates::no_search_result_html, data);
-    response.set_code(MHD_HTTP_NOT_FOUND);
-    return response;
-  }
-
-  Searcher searcher;
-  if (reader) {
-    searcher.add_reader(reader.get());
-  } else {
-    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
-      auto currentReader = mp_library->getReaderById(bookId);
-      if (currentReader) {
-        searcher.add_reader(currentReader.get());
-      }
-    }
-  }
-
-  auto start = 0;
-  try {
-    start = request.get_argument<unsigned int>("start");
-  } catch (const std::exception&) {}
-  auto pageLength=25;
-  try{
-    pageLength=request.get_argument<unsigned int>("pageLength");
-  }catch(const std::exception&){}
-  if (pageLength > MAX_SEARCH_LEN) {
-    pageLength = MAX_SEARCH_LEN;
-  }
-
-  if(pageLength==0)
-  {
-    pageLength=25;
-  }
-
-  auto end=start+pageLength;
-
-  /* Get the results */
-  try {
-    if (patternString.empty()) {
-      searcher.geo_search(latitude, longitude, distance,
-                           start, end , m_verbose.load());
-    } else {
-      searcher.search(patternString,
-                       start, end , m_verbose.load());
-    }
-    SearchRenderer renderer(&searcher, mp_nameMapper);
-    renderer.setSearchPattern(patternString);
-    renderer.setSearchContent(bookName);
-    renderer.setProtocolPrefix(m_root + "/");
-    renderer.setSearchProtocolPrefix(m_root + "/search?");
-    renderer.setPageLength(pageLength);
-    response.set_content(renderer.getHtml());
-  } catch (const std::exception& e) {
-    std::cerr << e.what() << std::endl;
-  }
-
-  //changing status code if no result obtained
-  if(searcher.getEstimatedResultCount() == 0)
-  {
-    response.set_code(MHD_HTTP_NO_CONTENT);
-  }
-
-  return response;
-}
-
-Response InternalServer::handle_random(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_random\n");
-  }
-
-  std::string bookName;
-  std::string bookId;
-  std::shared_ptr<Reader> reader;
-  try {
-    bookName = request.get_argument("content");
-    bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range&) {
-    return build_404(request, bookName);
-  }
-
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  try {
-    auto entry = reader->getRandomPage();
-    return build_redirect(bookName, entry.getFinalEntry());
-  } catch(kiwix::NoEntry& e) {
-    return build_404(request, bookName);
-  }
-}
-
-Response InternalServer::handle_captured_external(const RequestContext& request)
-{
-  std::string source = "";
-  try {
-    source = kiwix::urlDecode(request.get_argument("source"));
-  } catch (const std::out_of_range& e) {}
-
-  if (source.empty())
-    return build_404(request, "");
-
-  auto data = get_default_data();
-  data.set("source", source);
-  auto response = get_default_response();
-  response.set_template(RESOURCE::templates::captured_external_html, data);
-  response.set_mimeType("text/html; charset=utf-8");
-  response.set_compress(true);
-  return response;
-}
-
-Response InternalServer::handle_catalog(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_catalog");
-  }
-
-  std::string host;
-  std::string url;
-  try {
-    host = request.get_header("Host");
-    url  = request.get_url_part(1);
-  } catch (const std::out_of_range&) {
-    return build_404(request, "");
-  }
-
-  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
-    return build_404(request, "");
-  }
-
-  auto response = get_default_response();
-  response.set_compress(true);
-  if (url == "searchdescription.xml") {
-    response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
-    response.set_mimeType("application/opensearchdescription+xml");
-    return response;
-  }
-
-  zim::Uuid uuid;
-  kiwix::OPDSDumper opdsDumper;
-  opdsDumper.setRootLocation(m_root);
-  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
-  opdsDumper.setLibrary(mp_library);
-  response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
-  std::vector<std::string> bookIdsToDump;
-  if (url == "root.xml") {
-    opdsDumper.setTitle("All zims");
-    uuid = zim::Uuid::generate(host);
-    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
-  } else if (url == "search") {
-    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
-    string query("<Empty query>");
-    size_t count(10);
-    size_t startIndex(0);
-    try {
-      query = request.get_argument("q");
-      filter.query(query);
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
-    } catch (...) {}
-    try {
-      filter.name(request.get_argument("name"));
-    } catch (const std::out_of_range&) {}
-    try {
-      filter.lang(request.get_argument("lang"));
-    } catch (const std::out_of_range&) {}
-    try {
-      count = extractFromString<unsigned long>(request.get_argument("count"));
-    } catch (...) {}
-    try {
-      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
-    } catch (...) {}
-    try {
-      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
-    } catch (...) {}
-    try {
-      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
-    } catch (...) {}
-    opdsDumper.setTitle("Search result for " + query);
-    uuid = zim::Uuid::generate();
-    bookIdsToDump = mp_library->filter(filter);
-    auto totalResults = bookIdsToDump.size();
-    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
-    if (count>0 && bookIdsToDump.size() > count) {
-      bookIdsToDump.resize(count);
-    }
-    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
-  }
-
-  opdsDumper.setId(kiwix::to_string(uuid));
-  response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
-  return response;
-}
-
-namespace
-{
-
-std::string get_book_name(const RequestContext& request)
-{
-  try {
-    return request.get_url_part(0);
-  } catch (const std::out_of_range& e) {
-    return std::string();
-  }
-}
-
-} // unnamed namespace
-
-std::shared_ptr<Reader>
-InternalServer::get_reader(const std::string& bookName) const
-{
-  std::shared_ptr<Reader> reader;
-  try {
-    const std::string bookId = mp_nameMapper->getIdForName(bookName);
-    reader = mp_library->getReaderById(bookId);
-  } catch (const std::out_of_range& e) {
-  }
-  return reader;
-}
-
-Response
-InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
-{
-  auto response = get_default_response();
-  response.set_redirection(m_root + "/" + bookName + "/" +
-                           kiwix::urlEncode(entry.getPath()));
-  return response;
-}
-
-Response InternalServer::handle_content(const RequestContext& request)
-{
-  if (m_verbose.load()) {
-    printf("** running handle_content\n");
-  }
-
-  const std::string bookName = get_book_name(request);
-  if (bookName.empty())
-    return build_homepage(request);
-
-  const std::shared_ptr<Reader> reader = get_reader(bookName);
-  if (reader == nullptr) {
-    return build_404(request, bookName);
-  }
-
-  auto urlStr = request.get_url().substr(bookName.size()+1);
-  if (urlStr[0] == '/') {
-    urlStr = urlStr.substr(1);
-  }
-
-  kiwix::Entry entry;
-
-  try {
-    entry = reader->getEntryFromPath(urlStr);
-    if (entry.isRedirect() || urlStr.empty()) {
-      // If urlStr is empty, we want to mainPage.
-      // We must do a redirection to the real page.
-      return build_redirect(bookName, entry.getFinalEntry());
-    }
-  } catch(kiwix::NoEntry& e) {
-    if (m_verbose.load())
-      printf("Failed to find %s\n", urlStr.c_str());
-
-    return build_404(request, bookName);
-  }
-
-  auto response = get_default_response();
-
-  response.set_entry(entry, request);
-  response.set_taskbar(bookName, reader->getTitle());
-
-  if (m_verbose.load()) {
-    printf("Found %s\n", entry.getPath().c_str());
-    printf("mimeType: %s\n", response.get_mimeType().c_str());
-  }
-
-
-  return response;
-}
-
 }

src/server/internalServer.cpp

@@ -0,0 +1,798 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "internalServer.h"
+
+#ifdef _WIN32
+# if !defined(__MINGW32__) && (_MSC_VER < 1600)
+#   include "stdint4win.h"
+# endif
+# include <winsock2.h>
+# include <ws2tcpip.h>
+# ifdef __GNUC__
+  // inet_pton is not declared in mingw, even if the function exists.
+  extern "C" {
+    WINSOCK_API_LINKAGE  INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
+  }
+# endif
+  typedef UINT64 uint64_t;
+  typedef UINT16 uint16_t;
+#endif
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "tools/otherTools.h"
+#include "tools/pathTools.h"
+#include "tools/regexTools.h"
+#include "tools/stringTools.h"
+#include "library.h"
+#include "name_mapper.h"
+#include "entry.h"
+#include "searcher.h"
+#include "search_renderer.h"
+#include "opds_dumper.h"
+
+#include <zim/uuid.h>
+
+#include <mustache.hpp>
+
+#include <pthread.h>
+#include <atomic>
+#include <string>
+#include <vector>
+#include <chrono>
+#include "kiwixlib-resources.h"
+
+#ifndef _WIN32
+# include <arpa/inet.h>
+#endif
+
+#include "request_context.h"
+#include "response.h"
+
+#define MAX_SEARCH_LEN 140
+#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
+
+namespace kiwix {
+
+static IdNameMapper defaultNameMapper;
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls);
+
+
+InternalServer::InternalServer(Library* library,
+                               NameMapper* nameMapper,
+                               std::string addr,
+                               int port,
+                               std::string root,
+                               int nbThreads,
+                               bool verbose,
+                               bool withTaskbar,
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
+  m_addr(addr),
+  m_port(port),
+  m_root(root),
+  m_nbThreads(nbThreads),
+  m_verbose(verbose),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  mp_daemon(nullptr),
+  mp_library(library),
+  mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
+{}
+
+bool InternalServer::start() {
+#ifdef _WIN32
+  int flags = MHD_USE_SELECT_INTERNALLY;
+#else
+  int flags = MHD_USE_POLL_INTERNALLY;
+#endif
+  if (m_verbose.load())
+    flags |= MHD_USE_DEBUG;
+
+
+  struct sockaddr_in sockAddr;
+  memset(&sockAddr, 0, sizeof(sockAddr));
+  sockAddr.sin_family = AF_INET;
+  sockAddr.sin_port = htons(m_port);
+  if (m_addr.empty()) {
+    if (0 != INADDR_ANY)
+      sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+  } else {
+    if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
+      std::cerr << "Ip address " << m_addr << "  is not a valid ip address" << std::endl;
+      return false;
+    }
+  }
+
+  mp_daemon = MHD_start_daemon(flags,
+                            m_port,
+                            NULL,
+                            NULL,
+                            &staticHandlerCallback,
+                            this,
+                            MHD_OPTION_SOCK_ADDR, &sockAddr,
+                            MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
+                            MHD_OPTION_END);
+  if (mp_daemon == nullptr) {
+    std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
+              << " is maybe already occupied or need more permissions to be open. "
+                 "Please try as root or with a port number higher or equal to 1024."
+              << std::endl;
+    return false;
+  }
+  auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
+  m_server_id = kiwix::to_string(server_start_time.count());
+  return true;
+}
+
+void InternalServer::stop()
+{
+  MHD_stop_daemon(mp_daemon);
+}
+
+static MHD_Result staticHandlerCallback(void* cls,
+                                        struct MHD_Connection* connection,
+                                        const char* url,
+                                        const char* method,
+                                        const char* version,
+                                        const char* upload_data,
+                                        size_t* upload_data_size,
+                                        void** cont_cls)
+{
+  InternalServer* _this = static_cast<InternalServer*>(cls);
+
+  return _this->handlerCallback(connection,
+                                url,
+                                method,
+                                version,
+                                upload_data,
+                                upload_data_size,
+                                cont_cls);
+}
+
+MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
+                                           const char* url,
+                                           const char* method,
+                                           const char* version,
+                                           const char* upload_data,
+                                           size_t* upload_data_size,
+                                           void** cont_cls)
+{
+  auto start_time = std::chrono::steady_clock::now();
+  if (m_verbose.load() ) {
+    printf("======================\n");
+    printf("Requesting : \n");
+    printf("full_url  : %s\n", url);
+  }
+  RequestContext request(connection, m_root, url, method, version);
+
+  if (m_verbose.load() ) {
+    request.print_debug_info();
+  }
+  /* Unexpected method */
+  if (request.get_method() != RequestMethod::GET
+   && request.get_method() != RequestMethod::POST
+   && request.get_method() != RequestMethod::HEAD) {
+    printf("Reject request because of unhandled request method.\n");
+    printf("----------------------\n");
+    return MHD_NO;
+  }
+
+  auto response = handle_request(request);
+
+  if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
+    printf("========== INTERNAL ERROR !! ============\n");
+    if (!m_verbose.load()) {
+      printf("Requesting : \n");
+      printf("full_url : %s\n", url);
+      request.print_debug_info();
+    }
+  }
+
+  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+    response->set_server_id(m_server_id);
+
+  auto ret = response->send(request, connection);
+  auto end_time = std::chrono::steady_clock::now();
+  auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
+  if (m_verbose.load()) {
+    printf("Request time : %fs\n", time_span.count());
+    printf("----------------------\n");
+  }
+  return ret;
+}
+
+std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
+{
+  try {
+    if (! request.is_valid_url())
+      return Response::build_404(*this, request, "");
+
+    const ETag etag = get_matching_if_none_match_etag(request);
+    if ( etag )
+      return Response::build_304(*this, etag);
+
+    if (kiwix::startsWith(request.get_url(), "/skin/"))
+      return handle_skin(request);
+
+    if (startsWith(request.get_url(), "/catalog"))
+      return handle_catalog(request);
+
+    if (request.get_url() == "/meta")
+      return handle_meta(request);
+
+    if (request.get_url() == "/search")
+      return handle_search(request);
+
+    if (request.get_url() == "/suggest")
+     return handle_suggest(request);
+
+    if (request.get_url() == "/random")
+      return handle_random(request);
+
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
+    return handle_content(request);
+  } catch (std::exception& e) {
+    fprintf(stderr, "===== Unhandled error : %s\n", e.what());
+    return Response::build_500(*this, e.what());
+  } catch (...) {
+    fprintf(stderr, "===== Unhandled unknown error\n");
+    return Response::build_500(*this, "Unknown error");
+  }
+}
+
+MustacheData InternalServer::get_default_data() const
+{
+  MustacheData data;
+  data.set("root", m_root);
+  return data;
+}
+
+MustacheData InternalServer::homepage_data() const
+{
+  auto data = get_default_data();
+
+  MustacheData books{MustacheData::type::list};
+  for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+    auto& currentBook = mp_library->getBookById(bookId);
+
+    MustacheData book;
+    book.set("name", mp_nameMapper->getNameForId(bookId));
+    book.set("title", currentBook.getTitle());
+    book.set("description", currentBook.getDescription());
+    book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
+    book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
+    books.push_back(book);
+  }
+
+  data.set("books", books);
+  return data;
+}
+
+bool InternalServer::etag_not_needed(const RequestContext& request) const
+{
+  const std::string url = request.get_url();
+  return kiwix::startsWith(url, "/catalog")
+      || url == "/search"
+      || url == "/suggest"
+      || url == "/random"
+      || url == "/catch/external";
+}
+
+ETag
+InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, m_server_id);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
+std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
+{
+  return ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
+{
+  std::string bookName;
+  std::string bookId;
+  std::string meta_name;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    meta_name = request.get_argument("name");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = ContentResponse::build(*this, content, mimeType);
+  response->set_cacheable();
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_suggest\n");
+  }
+
+  std::string content;
+  std::string mimeType;
+  unsigned int maxSuggestionCount = 10;
+  unsigned int suggestionCount = 0;
+
+  std::string bookName;
+  std::string bookId;
+  std::string term;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    term = request.get_argument("term");
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (m_verbose.load()) {
+    printf("Searching suggestions for: \"%s\"\n", term.c_str());
+  }
+
+  MustacheData results{MustacheData::type::list};
+
+  bool first = true;
+  if (reader != nullptr) {
+    /* Get the suggestions */
+    SuggestionsList_t suggestions;
+    reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
+    for(auto& suggestion:suggestions) {
+      MustacheData result;
+      result.set("label", suggestion[0]);
+      result.set("value", suggestion[0]);
+      result.set("first", first);
+      first = false;
+      results.push_back(result);
+      suggestionCount++;
+    }
+  }
+
+  /* Propose the fulltext search if possible */
+  if (reader->hasFulltextIndex()) {
+    MustacheData result;
+    result.set("label", "containing '" + term + "'...");
+    result.set("value", term + " ");
+    result.set("first", first);
+    results.push_back(result);
+  }
+
+  auto data = get_default_data();
+  data.set("suggestions", results);
+
+  auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
+  return std::move(response);
+}
+
+std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_skin\n");
+  }
+
+  auto resourceName = request.get_url().substr(1);
+  try {
+    auto response = ContentResponse::build(
+        *this,
+        getResource(resourceName),
+        getMimeTypeForFile(resourceName));
+    response->set_cacheable();
+    return std::move(response);
+  } catch (const ResourceNotFound& e) {
+    return Response::build_404(*this, request, "");
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_search\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+  } catch (const std::out_of_range&) {}
+
+  std::string patternString;
+  try {
+    patternString = request.get_argument("pattern");
+  } catch (const std::out_of_range&) {}
+
+  /* Retrive geo search */
+  bool has_geo_query = false;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
+  try {
+    latitude = request.get_argument<float>("latitude");
+    longitude = request.get_argument<float>("longitude");
+    distance = request.get_argument<float>("distance");
+    has_geo_query = true;
+  } catch(const std::out_of_range&) {}
+    catch(const std::invalid_argument&) {}
+
+  std::shared_ptr<Reader> reader(nullptr);
+  try {
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {}
+
+  /* Try first to load directly the article */
+  if (reader != nullptr && !patternString.empty()) {
+    std::string patternCorrespondingUrl;
+    auto variants = reader->getTitleVariants(patternString);
+    auto variantsItr = variants.begin();
+
+    while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
+      try {
+        auto entry = reader->getEntryFromTitle(*variantsItr);
+        entry = entry.getFinalEntry();
+        patternCorrespondingUrl = entry.getPath();
+        break;
+      } catch(kiwix::NoEntry& e) {
+        variantsItr++;
+      }
+    }
+
+    /* If article found then redirect directly to it */
+    if (!patternCorrespondingUrl.empty()) {
+      auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl;
+      return Response::build_redirect(*this, redirectUrl);
+    }
+  }
+
+  /* Make the search */
+  if ( (!reader && !bookName.empty())
+    || (patternString.empty() && ! has_geo_query) ) {
+    auto data = get_default_data();
+    data.set("pattern", encodeDiples(patternString));
+    auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    response->set_code(MHD_HTTP_NOT_FOUND);
+    return std::move(response);
+  }
+
+  Searcher searcher;
+  if (reader) {
+    searcher.add_reader(reader.get());
+  } else {
+    for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
+      auto currentReader = mp_library->getReaderById(bookId);
+      if (currentReader) {
+        searcher.add_reader(currentReader.get());
+      }
+    }
+  }
+
+  auto start = 0;
+  try {
+    start = request.get_argument<unsigned int>("start");
+  } catch (const std::exception&) {}
+
+  auto pageLength = 25;
+  try {
+    pageLength = request.get_argument<unsigned int>("pageLength");
+  } catch (const std::exception&) {}
+  if (pageLength > MAX_SEARCH_LEN) {
+    pageLength = MAX_SEARCH_LEN;
+  }
+  if (pageLength == 0) {
+    pageLength = 25;
+  }
+
+  auto end = start + pageLength;
+
+  /* Get the results */
+  try {
+    if (patternString.empty()) {
+      searcher.geo_search(latitude, longitude, distance,
+                           start, end, m_verbose.load());
+    } else {
+      searcher.search(patternString,
+                       start, end, m_verbose.load());
+    }
+    SearchRenderer renderer(&searcher, mp_nameMapper);
+    renderer.setSearchPattern(patternString);
+    renderer.setSearchContent(bookName);
+    renderer.setProtocolPrefix(m_root + "/");
+    renderer.setSearchProtocolPrefix(m_root + "/search?");
+    renderer.setPageLength(pageLength);
+    auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
+    response->set_taskbar(bookName, reader ? reader->getTitle() : "");
+    //changing status code if no result obtained
+    if(searcher.getEstimatedResultCount() == 0)
+    {
+      response->set_code(MHD_HTTP_NO_CONTENT);
+    }
+
+    return std::move(response);
+  } catch (const std::exception& e) {
+    std::cerr << e.what() << std::endl;
+    return Response::build_500(*this, e.what());
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_random\n");
+  }
+
+  std::string bookName;
+  std::string bookId;
+  std::shared_ptr<Reader> reader;
+  try {
+    bookName = request.get_argument("content");
+    bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  try {
+    auto entry = reader->getRandomPage();
+    return build_redirect(bookName, entry.getFinalEntry());
+  } catch(kiwix::NoEntry& e) {
+    return Response::build_404(*this, request, bookName);
+  }
+}
+
+std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return Response::build_404(*this, request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
+}
+
+std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request.get_header("Host");
+    url  = request.get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
+    return Response::build_404(*this, request, "");
+  }
+
+  if (url == "searchdescription.xml") {
+    auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
+    return std::move(response);
+  }
+
+  zim::Uuid uuid;
+  kiwix::OPDSDumper opdsDumper;
+  opdsDumper.setRootLocation(m_root);
+  opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+  opdsDumper.setLibrary(mp_library);
+  std::vector<std::string> bookIdsToDump;
+  if (url == "root.xml") {
+    opdsDumper.setTitle("All zims");
+    uuid = zim::Uuid::generate(host);
+    bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
+  } else if (url == "search") {
+    auto filter = kiwix::Filter().valid(true).local(true).remote(true);
+    string query("<Empty query>");
+    size_t count(10);
+    size_t startIndex(0);
+    try {
+      query = request.get_argument("q");
+      filter.query(query);
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
+    } catch (...) {}
+    try {
+      filter.name(request.get_argument("name"));
+    } catch (const std::out_of_range&) {}
+    try {
+      filter.lang(request.get_argument("lang"));
+    } catch (const std::out_of_range&) {}
+    try {
+      count = extractFromString<unsigned long>(request.get_argument("count"));
+    } catch (...) {}
+    try {
+      startIndex = extractFromString<unsigned long>(request.get_argument("start"));
+    } catch (...) {}
+    try {
+      filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
+    } catch (...) {}
+    try {
+      filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
+    } catch (...) {}
+    opdsDumper.setTitle("Search result for " + query);
+    uuid = zim::Uuid::generate();
+    bookIdsToDump = mp_library->filter(filter);
+    auto totalResults = bookIdsToDump.size();
+    bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
+    if (count>0 && bookIdsToDump.size() > count) {
+      bookIdsToDump.resize(count);
+    }
+    opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
+  }
+
+  opdsDumper.setId(kiwix::to_string(uuid));
+  auto response = ContentResponse::build(
+      *this,
+      opdsDumper.dumpOPDSFeed(bookIdsToDump),
+      "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
+  return std::move(response);
+}
+
+namespace
+{
+
+std::string get_book_name(const RequestContext& request)
+{
+  try {
+    return request.get_url_part(0);
+  } catch (const std::out_of_range& e) {
+    return std::string();
+  }
+}
+
+} // unnamed namespace
+
+std::shared_ptr<Reader>
+InternalServer::get_reader(const std::string& bookName) const
+{
+  std::shared_ptr<Reader> reader;
+  try {
+    const std::string bookId = mp_nameMapper->getIdForName(bookName);
+    reader = mp_library->getReaderById(bookId);
+  } catch (const std::out_of_range& e) {
+  }
+  return reader;
+}
+
+std::unique_ptr<Response>
+InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
+{
+  auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
+  return Response::build_redirect(*this, redirectUrl);
+}
+
+std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
+{
+  if (m_verbose.load()) {
+    printf("** running handle_content\n");
+  }
+
+  const std::string bookName = get_book_name(request);
+  if (bookName.empty())
+    return build_homepage(request);
+
+  const std::shared_ptr<Reader> reader = get_reader(bookName);
+  if (reader == nullptr) {
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto urlStr = request.get_url().substr(bookName.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
+
+  kiwix::Entry entry;
+
+  try {
+    entry = reader->getEntryFromPath(urlStr);
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      return build_redirect(bookName, entry.getFinalEntry());
+    }
+  } catch(kiwix::NoEntry& e) {
+    if (m_verbose.load())
+      printf("Failed to find %s\n", urlStr.c_str());
+
+    return Response::build_404(*this, request, bookName);
+  }
+
+  auto response = EntryResponse::build(*this, request, entry);
+  try {
+    dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
+  } catch (std::bad_cast& e) {}
+
+  if (m_verbose.load()) {
+    printf("Found %s\n", entry.getPath().c_str());
+    printf("mimeType: %s\n", entry.getMimetype().c_str());
+  }
+
+  return response;
+}
+
+}

src/server/internalServer.h

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU  General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H
+#define KIWIXLIB_SERVER_INTERNALSERVER_H
+
+
+extern "C" {
+#include "microhttpd_wrapper.h"
+}
+
+#include "library.h"
+#include "name_mapper.h"
+
+#include <mustache.hpp>
+
+#include <atomic>
+#include <string>
+
+#include "server/request_context.h"
+#include "server/response.h"
+
+namespace kiwix {
+
+typedef kainjow::mustache::data MustacheData;
+
+class Entry;
+
+class InternalServer {
+  public:
+    InternalServer(Library* library,
+                   NameMapper* nameMapper,
+                   std::string addr,
+                   int port,
+                   std::string root,
+                   int nbThreads,
+                   bool verbose,
+                   bool withTaskbar,
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
+    virtual ~InternalServer() = default;
+
+    MHD_Result handlerCallback(struct MHD_Connection* connection,
+                               const char* url,
+                               const char* method,
+                               const char* version,
+                               const char* upload_data,
+                               size_t* upload_data_size,
+                               void** cont_cls);
+    bool start();
+    void stop();
+
+  private: // functions
+    std::unique_ptr<Response> handle_request(const RequestContext& request);
+    std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
+    std::unique_ptr<Response> build_homepage(const RequestContext& request);
+    std::unique_ptr<Response> handle_skin(const RequestContext& request);
+    std::unique_ptr<Response> handle_catalog(const RequestContext& request);
+    std::unique_ptr<Response> handle_meta(const RequestContext& request);
+    std::unique_ptr<Response> handle_search(const RequestContext& request);
+    std::unique_ptr<Response> handle_suggest(const RequestContext& request);
+    std::unique_ptr<Response> handle_random(const RequestContext& request);
+    std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
+    std::unique_ptr<Response> handle_content(const RequestContext& request);
+
+    MustacheData get_default_data() const;
+    MustacheData homepage_data() const;
+
+    std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
+    bool etag_not_needed(const RequestContext& r) const;
+    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+
+  private: // data
+    std::string m_addr;
+    int m_port;
+    std::string m_root;
+    int m_nbThreads;
+    std::atomic_bool m_verbose;
+    bool m_withTaskbar;
+    bool m_withLibraryButton;
+    bool m_blockExternalLinks;
+    struct MHD_Daemon* mp_daemon;
+
+    Library* mp_library;
+    NameMapper* mp_nameMapper;
+
+    std::string m_server_id;
+
+    friend std::unique_ptr<Response> Response::build(const InternalServer& server);
+    friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    friend std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+    friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
+
+};
+
+}
+
+#endif //KIWIXLIB_SERVER_INTERNALSERVER_H

src/server/response.cpp

@@ -3,6 +3,7 @@
 
 #include "response.h"
 #include "request_context.h"
+#include "internalServer.h"
 #include "kiwixlib-resources.h"
 
 #include "tools/regexTools.h"
@@ -21,6 +22,17 @@ namespace
 {
 // some utilities
 
+std::string render_template(const std::string& template_str, kainjow::mustache::data data)
+{
+  kainjow::mustache::mustache tmpl(template_str);
+  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
+                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
+  data.set("urlencoded", urlencode);
+  std::stringstream ss;
+  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
+  return ss.str();
+}
+
 std::string get_mime_type(const kiwix::Entry& entry)
 {
   try {
@@ -42,20 +54,73 @@ bool is_compressible_mime_type(const std::string& mimeType)
 
 } // unnamed namespace
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
+Response::Response(bool verbose)
   : m_verbose(verbose),
-    m_root(root),
-    m_content(""),
-    m_mimeType(""),
-    m_returnCode(MHD_HTTP_OK),
-    m_withTaskbar(withTaskbar),
-    m_withLibraryButton(withLibraryButton),
-    m_blockExternalLinks(blockExternalLinks),
-    m_bookName(""),
-    m_bookTitle("")
+    m_returnCode(MHD_HTTP_OK)
 {
+  add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
+std::unique_ptr<Response> Response::build(const InternalServer& server)
+{
+  return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
+}
+
+std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
+{
+  auto response = Response::build(server);
+  response->set_code(MHD_HTTP_NOT_MODIFIED);
+  response->m_etag = etag;
+  if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+    response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
+  }
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName)
+{
+  MustacheData results;
+  results.set("url", request.get_full_url());
+
+  auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
+  response->set_code(MHD_HTTP_NOT_FOUND);
+  response->set_taskbar(bookName, "");
+
+  return std::move(response);
+}
+
+std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
+{
+  auto response = Response::build(server);
+// [FIXME] (compile with recent enough version of libmicrohttpd)
+//  response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
+  response->set_code(416);
+  std::ostringstream oss;
+  oss << "bytes */" << resourceLength;
+  response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str());
+
+  return response;
+}
+
+std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
+{
+  MustacheData data;
+  data.set("error", msg);
+  auto content = render_template(RESOURCE::templates::_500_html, data);
+  std::unique_ptr<Response> response (
+      new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
+  response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
+  return response;
+}
+
+
+std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
+{
+  auto response = Response::build(server);
+  response->m_returnCode = MHD_HTTP_FOUND;
+  response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
+  return response;
+}
 
 static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
                                    const char *key, const char *value)
@@ -113,19 +178,9 @@ void print_response_info(int retCode, MHD_Response* response)
 }
 
 
-std::string render_template(const std::string& template_str, kainjow::mustache::data data)
-{
-  kainjow::mustache::mustache tmpl(template_str);
-  kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
-                               [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
-  data.set("urlencoded", urlencode);
-  std::stringstream ss;
-  tmpl.render(data, [&ss](const std::string& str) { ss << str; });
-  return ss.str();
-}
 
 
-void Response::introduce_taskbar()
+void ContentResponse::introduce_taskbar()
 {
   kainjow::mustache::data data;
   data.set("root", m_root);
@@ -147,7 +202,7 @@ void Response::introduce_taskbar()
 }
 
 
-void Response::inject_externallinks_blocker()
+void ContentResponse::inject_externallinks_blocker()
 {
   kainjow::mustache::data data;
   data.set("root", m_root);
@@ -159,7 +214,7 @@ void Response::inject_externallinks_blocker()
 }
 
 bool
-Response::can_compress(const RequestContext& request) const
+ContentResponse::can_compress(const RequestContext& request) const
 {
   return request.can_compress()
       && is_compressible_mime_type(m_mimeType)
@@ -167,28 +222,21 @@ Response::can_compress(const RequestContext& request) const
 }
 
 bool
-Response::contentDecorationAllowed() const
+ContentResponse::contentDecorationAllowed() const
 {
     return (startsWith(m_mimeType, "text/html")
          && m_mimeType.find(";raw=true") == std::string::npos);
 }
 
 MHD_Response*
-Response::create_error_response(const RequestContext& request) const
+Response::create_mhd_response(const RequestContext& request)
 {
-  MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT);
-  if ( m_returnCode == 416 ) {
-    std::ostringstream oss;
-    oss << "bytes */" << m_byteRange.length();
-
-    MHD_add_response_header(response,
-      MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
-  }
+  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT);
   return response;
 }
 
 MHD_Response*
-Response::create_raw_content_mhd_response(const RequestContext& request)
+ContentResponse::create_mhd_response(const RequestContext& request)
 {
   if (contentDecorationAllowed()) {
     if (m_withTaskbar) {
@@ -199,7 +247,7 @@ Response::create_raw_content_mhd_response(const RequestContext& request)
     }
   }
 
-  bool shouldCompress = m_compress && can_compress(request);
+  bool shouldCompress = can_compress(request);
   if (shouldCompress) {
     std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
     uLongf comprLen = compr_buffer.capacity();
@@ -223,34 +271,117 @@ Response::create_raw_content_mhd_response(const RequestContext& request)
   MHD_Response* response = MHD_create_response_from_buffer(
     m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
 
-  // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and
-  // shouldCompress can have different values. This can happen for a 304 (Not
-  // Modified) response generated while handling a conditional If-None-Match
-  // request. In that case the m_etag (together with its COMPRESSED_CONTENT
-  // option) is obtained from the ETag list of the If-None-Match header and the
-  // response has no body (which shouldn't be compressed).
-  if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) {
+  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
-  }
-  if (shouldCompress) {
     MHD_add_response_header(
         response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
   }
-  MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
   return response;
 }
 
-MHD_Response*
-Response::create_redirection_mhd_response() const
+MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
 {
-  MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
-  MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
-  return response;
+  MHD_Response* response = create_mhd_response(request);
+
+  MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
+    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+  const std::string etag = m_etag.get_etag();
+  if ( ! etag.empty() )
+    MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
+  for(auto& p: m_customHeaders) {
+    MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
+  }
+
+  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
+    m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
+
+  if (m_verbose)
+    print_response_info(m_returnCode, response);
+
+  auto ret = MHD_queue_response(connection, m_returnCode, response);
+  MHD_destroy_response(response);
+  return ret;
+}
+
+void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
+{
+  m_bookName = bookName;
+  m_bookTitle = bookTitle;
+}
+
+
+ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
+  Response(verbose),
+  m_root(root),
+  m_content(content),
+  m_mimeType(mimetype),
+  m_withTaskbar(withTaskbar),
+  m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
+  m_bookName(""),
+  m_bookTitle("")
+{
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype)
+{
+   return std::unique_ptr<ContentResponse>(new ContentResponse(
+        server.m_root,
+        server.m_verbose.load(),
+        server.m_withTaskbar,
+        server.m_withLibraryButton,
+        server.m_blockExternalLinks,
+        content,
+        mimetype));
+}
+
+std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) {
+  auto content = render_template(template_str, data);
+  return ContentResponse::build(server, content, mimetype);
+}
+
+EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) :
+  Response(verbose),
+  m_entry(entry),
+  m_mimeType(mimetype)
+{
+  m_byteRange = byterange;
+  set_cacheable();
+  add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
+}
+
+std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry)
+{
+  const std::string mimetype = get_mime_type(entry);
+  auto byteRange = request.get_range().resolve(entry.getSize());
+  const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
+  if (noRange && is_compressible_mime_type(mimetype)) {
+    // Return a contentResponse
+    zim::Blob raw_content = entry.getBlob();
+    const std::string content = string(raw_content.data(), raw_content.size());
+    auto response = ContentResponse::build(server, content, mimetype);
+    response->set_cacheable();
+    response->m_byteRange = byteRange;
+    return std::move(response);
+  }
+
+  if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
+    auto response = Response::build_416(server, entry.getSize());
+    response->set_cacheable();
+    return response;
+  }
+
+  return std::unique_ptr<Response>(new EntryResponse(
+        server.m_verbose.load(),
+        entry,
+        mimetype,
+        byteRange));
 }
 
 MHD_Response*
-Response::create_entry_mhd_response() const
+EntryResponse::create_mhd_response(const RequestContext& request)
 {
   const auto content_length = m_byteRange.length();
   MHD_Response* response = MHD_create_response_from_callback(content_length,
@@ -258,8 +389,6 @@ Response::create_entry_mhd_response() const
                                                callback_reader_from_entry,
                                                new RunningResponse(m_entry, m_byteRange.first()),
                                                callback_free_response);
-  MHD_add_response_header(response,
-    MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
   MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
   if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
     std::ostringstream oss;
@@ -275,92 +404,5 @@ Response::create_entry_mhd_response() const
   return response;
 }
 
-MHD_Response*
-Response::create_mhd_response(const RequestContext& request)
-{
-  switch (m_mode) {
-    case ResponseMode::ERROR_RESPONSE:
-      return create_error_response(request);
-
-    case ResponseMode::RAW_CONTENT :
-      return create_raw_content_mhd_response(request);
-
-    case ResponseMode::REDIRECTION :
-      return create_redirection_mhd_response();
-
-    case ResponseMode::ENTRY :
-      return create_entry_mhd_response();
-  }
-  return nullptr;
-}
-
-MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
-{
-  MHD_Response* response = create_mhd_response(request);
-
-  if ( m_mode != ResponseMode::ERROR_RESPONSE ) {
-    MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
-    MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-      m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
-    const std::string etag = m_etag.get_etag();
-    if ( ! etag.empty() )
-        MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
-  }
-
-  if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
-    m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
-
-  if (m_verbose)
-    print_response_info(m_returnCode, response);
-
-  auto ret = MHD_queue_response(connection, m_returnCode, response);
-  MHD_destroy_response(response);
-  return ret;
-}
-
-void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
-  set_content(render_template(template_str, data));
-}
-
-void Response::set_content(const std::string& content) {
-  m_content = content;
-  m_mode = ResponseMode::RAW_CONTENT;
-}
-
-void Response::set_redirection(const std::string& url) {
-  m_content = url;
-  m_mode = ResponseMode::REDIRECTION;
-  m_returnCode = MHD_HTTP_FOUND;
-}
-
-void Response::set_entry(const Entry& entry, const RequestContext& request) {
-  m_entry = entry;
-  m_mode = ResponseMode::ENTRY;
-
-  const std::string mimeType = get_mime_type(entry);
-  set_mimeType(mimeType);
-  set_cacheable();
-
-  m_byteRange = request.get_range().resolve(entry.getSize());
-  const bool noRange = m_byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
-  if ( noRange && is_compressible_mime_type(mimeType) ) {
-    zim::Blob raw_content = entry.getBlob();
-    const std::string content = string(raw_content.data(), raw_content.size());
-
-    set_content(content);
-    set_compress(true);
-  } else if ( m_byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE ) {
-    set_code(416);
-    set_content("");
-    m_mode = ResponseMode::ERROR_RESPONSE;
-  }
-}
-
-void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
-{
-  m_bookName = bookName;
-  m_bookTitle = bookTitle;
-}
-
 
 }

src/server/response.h

@@ -22,6 +22,7 @@
 #define KIWIXLIB_SERVER_RESPONSE_H
 
 #include <string>
+#include <map>
 
 #include <mustache.hpp>
 #include "byte_range.h"
@@ -34,68 +35,85 @@ extern "C" {
 
 namespace kiwix {
 
-enum class ResponseMode {
-  ERROR_RESPONSE,
-  RAW_CONTENT,
-  REDIRECTION,
-  ENTRY
-};
-
+class InternalServer;
 class RequestContext;
 
+class EntryResponse;
+
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
-    ~Response() = default;
+    Response(bool verbose);
+    virtual ~Response() = default;
+
+    static std::unique_ptr<Response> build(const InternalServer& server);
+    static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
+    static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName);
+    static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
+    static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
+    static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
 
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
-    void set_template(const std::string& template_str, kainjow::mustache::data data);
-    void set_content(const std::string& content);
-    void set_redirection(const std::string& url);
-    void set_entry(const Entry& entry, const RequestContext& request);
-
-
-    void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
     void set_code(int code) { m_returnCode = code; }
     void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
     void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
-    void set_etag(const ETag& etag) { m_etag = etag; }
-    void set_compress(bool compress) { m_compress = compress; }
-    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+    void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
-    std::string get_mimeType() const { return m_mimeType; }
+
+  private: // functions
+    virtual MHD_Response* create_mhd_response(const RequestContext& request);
+    MHD_Response* create_error_response(const RequestContext& request) const;
+
+  protected: // data
+    bool m_verbose;
+    int m_returnCode;
+    ByteRange m_byteRange;
+    ETag m_etag;
+    std::map<std::string, std::string> m_customHeaders;
+
+    friend class EntryResponse; // temporary to allow the builder to change m_mode
+};
+
+
+class ContentResponse : public Response {
+  public:
+    ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype);
+    static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype);
+
+    void set_taskbar(const std::string& bookName, const std::string& bookTitle);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
 
     void introduce_taskbar();
     void inject_externallinks_blocker();
-
     bool can_compress(const RequestContext& request) const;
     bool contentDecorationAllowed() const;
 
-  private: // functions
-    MHD_Response* create_mhd_response(const RequestContext& request);
-    MHD_Response* create_error_response(const RequestContext& request) const;
-    MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
-    MHD_Response* create_redirection_mhd_response() const;
-    MHD_Response* create_entry_mhd_response() const;
 
-  private: // data
-    bool m_verbose;
-    ResponseMode m_mode;
+  private:
     std::string m_root;
     std::string m_content;
-    Entry m_entry;
     std::string m_mimeType;
-    int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
     bool m_blockExternalLinks;
-    bool m_compress;
     std::string m_bookName;
     std::string m_bookTitle;
-    ByteRange m_byteRange;
-    ETag m_etag;
+ };
+
+class EntryResponse : public Response {
+  public:
+    EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange);
+    static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const Entry& entry);
+
+  private:
+    MHD_Response* create_mhd_response(const RequestContext& request);
+
+    Entry m_entry;
+    std::string m_mimeType;
 };
 
 }

src/subprocess_unix.cpp

@@ -26,6 +26,7 @@ UnixImpl::~UnixImpl()
 #else
   pthread_cancel(m_waitingThread);
 #endif
+  pthread_join(m_waitingThread, nullptr);
 }
 
 #ifdef __ANDROID__

src/subprocess_windows.cpp

@@ -11,7 +11,8 @@
 WinImpl::WinImpl():
   m_pid(0),
   m_running(false),
-  m_handle(INVALID_HANDLE_VALUE)
+  m_subprocessHandle(INVALID_HANDLE_VALUE),
+  m_waitingThreadHandle(INVALID_HANDLE_VALUE)
 {
   InitializeCriticalSection(&m_criticalSection);
 }
@@ -19,14 +20,15 @@ WinImpl::WinImpl():
 WinImpl::~WinImpl()
 {
   kill();
-  CloseHandle(m_handle);
+  WaitForSingleObject(m_waitingThreadHandle, INFINITE);
+  CloseHandle(m_subprocessHandle);
   DeleteCriticalSection(&m_criticalSection);
 }
 
 DWORD WINAPI WinImpl::waitForPID(void* _self)
 {
   WinImpl* self = static_cast<WinImpl*>(_self);
-  WaitForSingleObject(self->m_handle, INFINITE);
+  WaitForSingleObject(self->m_subprocessHandle, INFINITE);
 
   EnterCriticalSection(&self->m_criticalSection);
   self->m_running = false;
@@ -79,16 +81,16 @@ void WinImpl::run(commandLine_t& commandLine)
     &procInfo))
   {
     m_pid = procInfo.dwProcessId;
-    m_handle = procInfo.hProcess;
+    m_subprocessHandle = procInfo.hProcess;
     CloseHandle(procInfo.hThread);
     m_running = true;
-    CreateThread(NULL, 0, &waitForPID, this, 0, NULL	);
+    m_waitingThreadHandle = CreateThread(NULL, 0, &waitForPID, this, 0, NULL);
   }
 }
 
 bool WinImpl::kill()
 {
-  return TerminateProcess(m_handle, 0);
+  return TerminateProcess(m_subprocessHandle, 0);
 }
 
 bool WinImpl::isRunning()

src/subprocess_windows.h

@@ -11,7 +11,8 @@ class WinImpl : public SubprocessImpl
   private:
     int m_pid;
     bool m_running;
-    HANDLE m_handle;
+    HANDLE m_subprocessHandle;
+    HANDLE  m_waitingThreadHandle;
     CRITICAL_SECTION m_criticalSection;
 
   public:

static/skin/block_external.js

@@ -1,3 +1,4 @@
+// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
 var block_path = "/catch/external";
 // called only on external links
 function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }

static/templates/index.html

@@ -29,7 +29,7 @@
                 box-shadow: 2px 2px 5px 0px #ccc;
         }
         .book:hover { background-color: #F9F9F9; box-shadow: none;}
-        .book__background { background-repeat: no-repeat; background-size: auto; background-position: top right; }
+        .book__background { background-repeat: no-repeat; background-size: 48px 48px; background-position: top right; }
         .book__title {
                 padding: 0 55px 0 0;overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
                 font-size: 18px; color: #0645ad; line-height: 1em;