Merge pull request #585 from kiwix/version_3.4.0

* Remove last reference to kiwix-read tool (@legoktm #569)

kiwix-serve
-----------

 * Fix broken indentation in usage (@kelson42 #560)
 * Exit if wrong arguments are passed (@kelson42 #567)
 * Do not allow multiple values for same option (@juuz0 #564)
 * Fix default location of "rootLocation" (@rgaudin #571)
 * [DOCKER] Change default port to 8080 (@neyder #581)
 * [DOCKER] Simplify dockerfile (@rgaudin #582)

kiwix-manage
------------

 * Fix man page (@kelson42 #576)

.github/workflows/ci.yml

@@ -14,16 +14,16 @@ jobs:
           - win32_dyn
         include:
           - target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: native_dyn
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
     env:
       HOME: /home/runner

.github/workflows/docker.yml

@@ -13,9 +13,9 @@ jobs:
     name: Deploy kiwix-tools Docker Image
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3.1.0
       - name: build and publish kiwix-tools
-        uses: openzim/docker-publish-action@v5
+        uses: openzim/docker-publish-action@v9
         with:
           image-name: kiwix/kiwix-tools
           credentials: |
@@ -33,15 +33,17 @@ jobs:
             linux/arm64
           restrict-to: kiwix/kiwix-tools
           manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix command line tools
 
   build-and-push-kiwix-serve:
     name: Deploy kiwix-serve Docker Image
     runs-on: ubuntu-20.04
     needs: build-and-push-kiwix-tools
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3.1.0
       - name: build and publish kiwix-serve
-        uses: openzim/docker-publish-action@v5
+        uses: openzim/docker-publish-action@v9
         with:
           image-name: kiwix/kiwix-serve
           credentials: |
@@ -59,3 +61,5 @@ jobs:
             linux/arm64
           restrict-to: kiwix/kiwix-tools
           manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix web-server

.github/workflows/package.yml

@@ -10,7 +10,6 @@ jobs:
         distro:
           - ubuntu-jammy
           - ubuntu-impish
-          - ubuntu-hirsute
           - ubuntu-focal
     steps:
       - uses: actions/checkout@v2
@@ -50,14 +49,6 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
       - uses: legoktm/gh-action-build-deb@ubuntu-focal
         if: matrix.distro == 'ubuntu-focal'
         name: Build package for ubuntu-focal

Changelog

@@ -1,3 +1,45 @@
+kiwix-tools 3.4.0
+=================
+
+ * Remove last reference to kiwix-read tool (@legoktm #569)
+
+kiwix-serve
+-----------
+
+ * Fix broken indentation in usage (@kelson42 #560)
+ * Exit if wrong arguments are passed (@kelson42 #567)
+ * Do not allow multiple values for same option (@juuz0 #564)
+ * Fix default location of "rootLocation" (@rgaudin #571)
+ * [DOCKER] Change default port to 8080 (@neyder #581)
+ * [DOCKER] Simplify dockerfile (@rgaudin #582)
+
+kiwix-manage
+------------
+
+ * Fix man page (@kelson42 #576)
+
+kiwix-tools 3.3.0
+=================
+
+ * Remove kiwix-read tool (@veloman-yunkan #535)
+
+kiwix-serve
+-----------
+
+ * Add an option to limit the number of connections for a same IP (@juuz0 #534)
+ * Add an option to limit the number of zim in a multizim fulltext search (@mgautierfr #558)
+
+kiwix-search
+------------
+
+ * Remove usage of libkiwix's deprecated api (@veloman-yunkan #535)
+
+kiwix-manage
+------------
+
+ * Correctly return a value !0 if something went wrong (@mgautierfr #553)
+
+
 kiwix-tools 3.2.0
 =================
 
@@ -12,7 +54,7 @@ kiwix-tools 3.2.0
 kiwix-serve
 -----------
 
- * Print the url on which a user can connect to on startup (@nikhil #499 #522)
+ * Print the url on which a user can connect to on startup (@juuz0 #499 #522)
  * Reload library on SIGHUP signal (@veloman-yunkan #497)
  * Add a option `--monitorLibrary` to monitor and automically reload the library
    (@veloman-yunkan #503)

README.md

@@ -4,7 +4,6 @@ Kiwix tools
 The Kiwix tools is a collection of [Kiwix](https://kiwix.org) related
 command line tools:
 * kiwix-manage: Manage XML based library of ZIM files
-* kiwix-read: Read ZIM file content
 * kiwix-search: Fulltext search in ZIM files
 * kiwix-serve: HTTP daemon serving ZIM files
 

debian/control

@@ -3,8 +3,8 @@ Section: utils
 Priority: optional
 Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
-               libkiwix-dev (>= 10.0.0),
-               libzim-dev (>= 7.2.0),
+               libkiwix-dev (>= 10.0.0~),
+               libzim-dev (>= 7.2.0~),
                meson,
                pkg-config,
 Standards-Version: 4.5.0
@@ -21,6 +21,5 @@ Description: collection of Kiwix tools
     over the network.
   * kiwix-manage allows one to manage the content of the Kiwix library (an
     XML file listing available ZIM files).
-  * kiwix-read allows for reading ZIM files from the command-line.
   * kiwix-search allows one to find articles in a ZIM file using fulltext
     search patterns.

docker/Dockerfile

@@ -1,46 +1,18 @@
-# dumb-init to use as entrypoint so it can forward signals to our CMD
-FROM debian:bullseye-slim as builder
-RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends \
-    build-essential ca-certificates musl-tools wget unzip make \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-# compile dumb-init
-RUN wget -nv https://github.com/Yelp/dumb-init/archive/refs/tags/v1.2.5.zip && \
-    unzip v1.2.5.zip && cd dumb-init-1.2.5 && CC=musl-gcc make
-
-# declare build option ARCH if not using buildx
-ARG ARCH=
-# declare version to build image for
-ARG VERSION=
-
-# alpine is a multi-arch image
-FROM alpine:3
+FROM alpine:3.16
 LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
-
-# TARGETARCH is injected by buildx
-ARG TARGETARCH
 ARG VERSION
-ARG ARCH
 
-# copy built dumb-init
-COPY --from=builder /dumb-init-1.2.5/dumb-init /usr/bin/dumb-init
-
-# find and store arch in image, using (in this order):
-# $TARGETARCH || --build-arg ARCH (not using buildx) || amd64
-RUN echo "amd64" > /etc/docker_arch && \
-    if [ ! -z "$ARCH" ] ; then echo "$ARCH" > /etc/docker_arch ; fi && \
-    if [ ! -z "$TARGETARCH" ] ; then echo "$TARGETARCH" > /etc/docker_arch ; fi
-
-# decide which kiwix arch to download later (`armhf` for all arm* and x86_64 otherwise)
-RUN if [ $(cut -c 1-3 /etc/docker_arch) = "arm" ] ; then echo "armhf" > /etc/kiwix_arch ; else echo "x86_64" > /etc/kiwix_arch ; fi
-
-# Install kiwix-tools
-RUN url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$(cat /etc/kiwix_arch)-$VERSION.tar.gz" && \
+RUN apk --no-cache add dumb-init curl && \
+    ARCH=$(cat /etc/apk/arch) && \
+    if [ "$ARCH" = "x86" ]; then ARCH="i586"; \
+    elif [ "$ARCH" = "aarch64" ]; then ARCH="armhf"; \
+    elif [ "$ARCH" = "armv7" ]; then ARCH="armhf"; fi && \
+    # download requested kiwix-tools version
+    url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$ARCH-$VERSION.tar.gz" && \
     echo "URL: $url" && \
-    wget -q -O - $url | tar -xz && \
-    mv kiwix-tools*/kiwix-* /usr/local/bin && \
-    rm -r kiwix-tools*
+    curl -k -L $url | tar -xz -C /usr/local/bin/ --strip-components 1 && \
+    # only needed in dockerfile
+    apk del curl
 
 # expose kiwix-serve default port
 EXPOSE 80

docker/README.md

@@ -11,13 +11,13 @@ Kiwix-tools Docker image
 $ docker run -it kiwix/kiwix-tools:3.1.2
 
 Welcome to kiwix-tools! The following binaries are available:
-kiwix-manage  kiwix-read    kiwix-search  kiwix-serve
+kiwix-manage  kiwix-search  kiwix-serve
 ```
 
 `kiwix-tools` operates on zim files. You shall mount a volume to access the files.
 
 ```sh
-docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-read --suggest="Mali" /data/wikipedia_fr_test.zim
+docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
 ```
 
 ## Building and reusing
@@ -29,4 +29,4 @@ docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-read --suggest="Mali" /da
 **Notes:**
 
 - `wget` in `alpine:3` on `arm/v7` (__inside github action only__) crashes when downloading from HTTPs locations. Keep http-only in Dockerfile.
-- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.
+- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.

docker/server/Dockerfile

@@ -5,7 +5,7 @@ FROM kiwix/kiwix-tools:$VERSION
 LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
 
 # expose kiwix-serve default port and workdir
-EXPOSE 80
+EXPOSE 8080
 VOLUME /data
 WORKDIR /data
 

docker/server/README.md

@@ -9,14 +9,23 @@ With local ZIM file(s)
 * Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following:
 
 ```bash
-docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
+docker run -v /tmp/zim:/data -p 8080:8080 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
 ```
 
 With remote ZIM file
 --------------------
 
 ```bash
-docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:80 kiwix/kiwix-serve
+docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 kiwix/kiwix-serve
+```
+
+Change default port
+-------------------
+
+You can change port to expose with environment variable PORT, useful if running on Podman, K8s or OpenShift
+
+```bash
+podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 kiwix/kiwix-serve
 ```
 
 ARM

docker/server/docker-compose.yml.example

@@ -2,7 +2,7 @@ version: '3.3'
 services:
   kiwix-serve:
       ports:
-        - 8080:80
+        - 8080:8080
       image: kiwix/kiwix-serve
       # uncomment next 4 lines to use it with local zim file in /tmp/zim
       # volumes:

docker/server/start.sh

@@ -13,7 +13,11 @@ then
     fi
 fi
 
-CMD="/usr/local/bin/kiwix-serve --port=80 $@"
+if [ -z "$PORT" ]
+then
+    PORT=8080
+fi
+CMD="/usr/local/bin/kiwix-serve --port=$PORT $@"
 echo $CMD
 $CMD
 

format_code.sh

@@ -3,7 +3,6 @@
 files=(
 "src/installer/kiwix-install.cpp"
 "src/searcher/kiwix-search.cpp"
-"src/reader/kiwix-read.cpp"
 "src/manager/kiwix-manage.cpp"
 "src/server/kiwix-serve.cpp"
 )

meson.build

@@ -1,7 +1,7 @@
 project('kiwix-tools', 'cpp',
-  version : '3.2.0',
+  version : '3.4.0',
   license : 'GPL',
-  default_options: ['c_std=c11', 'cpp_std=c++11'])
+  default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
@@ -13,8 +13,8 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', version:'>=10.0.0', static:static_linkage)
-libzim_dep = dependency('libzim', version:'>=7.2.0', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'>=12.0.0', static:static_linkage)
+libzim_dep = dependency('libzim', version:'>=8.1.0', static:static_linkage)
 
 all_deps = [thread_dep, kiwixlib_dep, libzim_dep]
 

src/man/kiwix-manage.1

@@ -1,62 +1,69 @@
-.TH KIWIX 1 "21 May 2012"
+.TH KIWIX-MANAGE 1 "21 May 2012"
+
 .SH NAME
 kiwix\-manage \- Kiwix Library Manager
+
 .SH SYNOPSIS
 .IX Header SYNOPSIS
-.B kiwix\-manage
-.br
-kiwix\-manage LIBRARY_PATH add ZIM_PATH ...
-.br
-	kiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ...
-.br
-	kiwix\-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]
-.SH DESCRIPTION
-.SS kiwix\-manage
-.PP
-\fBkiwix\-manage\fP tool for managing a kiwix library.
-.PP
-Allows one to manage the content of the kiwix library. The library file is a
-flat XML file listing ZIM files with all necessary information like favicon,
-date, creator, description, indexpath, filepath, title, source/metalink, etc.
-.
-.PP
- Example library file can be found at https://www.kiwix.org/library\.xml
-.br
 .TP
-\fBadd\fR
-Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP.
+\fBkiwix\-manage\fR LIBRARY_PATH \fBadd\fR ZIM_PATH ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBshow\fR [ZIM_ID_1] [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBremove\fR ZIM_ID_1 [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR --version
+.TP
+\fBkiwix\-manage\fR --help
+
+.SH DESCRIPTION
+.PP
+\fBkiwix\-manage\fP is a command line tool for manipulating a Kiwix XML library.
+.PP
+\fBkiwix\-manage\fP allows to manage the entries of the Kiwix
+library. The library file is a flat XML file listing ZIM files with
+all necessary information like id, favicon, date, creator,
+description, filepath, title, url, etc.
+
+.SH ACTIONS
 
 .TP
-\fBshow\fR
-Show given \fBCONTENT_ID\fP from \fBLIBRARY_FILE\fR. If no \fBCONTENT_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+\fBadd\fR
+Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP. Create the library file if necessary.
 
 .TP
 \fBremove\fR
-Remove the given \fBCONTENT_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBCONTENT_ID\fR should be specified.
+Remove the given \fBZIM_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBZIM_ID\fR should be specified.
 
 .TP
-\fB\-\-zimPathToSave=\fR
-Change the location reference of the ZIM file in the library.
+\fBshow\fR
+Show given \fBZIM_ID\fP from \fBLIBRARY_FILE\fR. If no \fBZIM_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+
+.SH OPTIONS
+.TP
+Options to be used with the action \fBadd\fR:
 
 .TP
-\fB\-\-current\fR
-Mark this content as the current (default) content in the library.
+\fB\-\-url=HTTP_URL\fR
+Set the ZIM online HTTP(S) URL
 
 .TP
-\fB\-\-backend=xapian|clucene\fR
-Select an indexing backend.
+\fB\-\-zimPathToSave=OTHER_FS_PATH\fR
+Set an arbitrary ZIM filesystem path (instead of the ZIM_PATH)
 
 .TP
-\fB\-\-indexPath=FULLTEXT_IDX_PATH\fR
-Path to full-text index for that ZIM file.
+Other options (to be used alone):
 
 .TP
-\fBurl\fR
-Set the content location of the ZIM file over the network for in\-kiwix download.
+\fB\-\-help | \-h\fR
+Display the kiwix-manage help
+
+.TP
+\fB\-\-version | \-v\fR
+Display the version of kiwix-manage and all dependences
 
 .SH SEE ALSO
-kiwix(1) kiwix\-serve(1)
-.SH AUTHOR
-Emmanuel Engelhart <kelson@kiwix.org>
-.br
-Vasudev Kamath <kamathvasudev@gmail.com> (Manual)
+kiwix\-serve(1)
+
+.SH AUTHORS
+Kiwix team <contact@kiwix.org>

src/man/kiwix-read.1

@@ -1,18 +0,0 @@
-.TH KIWIX-READ "1" "July 2020" "kiwix-tools" "User Commands"
-.SH NAME
-kiwix-read \- Read ZIM file content
-.SH SYNOPSIS
-\fBkiwix-read\fR [\-\-verbose] [\-\-version] \-\-suggest=PATTERN ZIM\fR
-.SH DESCRIPTION
-.TP
-ZIM
-ZIM file to search
-.TP
-\fB\-\-verbose\fR
-Enable extra output
-.TP
-\fB\-\-version\fR
-print software version
-.TP
-\fB\-\-suggest=PATTERN\fR
-Get search suggestions for the provided pattern

src/man/meson.build

@@ -1,5 +1,4 @@
 install_man('kiwix-manage.1',
-            'kiwix-read.1',
             'kiwix-search.1',
             'kiwix-serve.1')
 subdir('fr')

src/manager/kiwix-manage.cpp

@@ -169,7 +169,10 @@ int handle_add(kiwix::Library* library, const std::string& libraryPath,
     std::string zimPath = argv[i];
     if (!zimPath.empty()) {
       auto _zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false);
+      if (manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false).empty()) {
+        std::cerr << "Cannot add zim " << zimPath << " to the library." << std::endl;
+        resultCode = 1;
+      }
     } else {
       std::cerr << "Invalid zim file path" << std::endl;
       resultCode = 1;
@@ -251,7 +254,7 @@ int main(int argc, char** argv)
   /* Print usage)) if necessary */
   if (libraryPath == "" || action == NONE) {
     usage();
-    exit(1);
+    return -1;
   }
 
   /* Try to read the file */
@@ -259,7 +262,12 @@ int main(int argc, char** argv)
                     ? kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryPath)
                     : libraryPath;
   kiwix::Manager manager(&library);
-  manager.readFile(libraryPath, false);
+  if (!manager.readFile(libraryPath, false)) {
+    if (kiwix::fileExists(libraryPath) || action!=ADD) {
+      std::cerr << "Cannot read the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
 
   /* SHOW */
   int exitCode = 0;
@@ -277,10 +285,18 @@ int main(int argc, char** argv)
       break;
   }
 
-  /* Rewrite the library file */
-  if (action == REMOVE || action == ADD) {
-    library.writeToFile(libraryPath);
+  if (exitCode) {
+    return exitCode;
   }
 
-  exit(exitCode);
+  /* Rewrite the library file */
+  if (action == REMOVE || action == ADD) {
+    // writeToFile return true (1) if everything is ok => exitCode is 0
+    if (!library.writeToFile(libraryPath)) {
+      std::cerr << "Cannot write the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
 }

src/meson.build

@@ -1,6 +1,4 @@
-
 subdir('manager')
-subdir('reader')
 subdir('searcher')
 subdir('server')
 subdir('man')

src/reader/kiwix-read.cpp

@@ -1,107 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <getopt.h>
-#include <kiwix/reader.h>
-#include <map>
-#include <string>
-
-#include "../version.h"
-
-void usage()
-{
-  cout << "Usage: kiwix-read [--verbose] [--version] --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
-  exit(1);
-}
-
-int main(int argc, char** argv)
-{
-  /* Init the variables */
-  const char* filePath = NULL;
-  const char* pattern = NULL;
-  int option_index = 0;
-  int c = 0;
-
-  kiwix::Reader* reader = NULL;
-
-  /* Argument parsing */
-  while (42) {
-    static struct option long_options[]
-        = {{"verbose", no_argument, 0, 'v'},
-           {"version", no_argument, 0, 'V'},
-           {"suggest", required_argument, 0, 's'},
-           {0, 0, 0, 0}};
-
-    if (c != -1) {
-      c = getopt_long(argc, argv, "Vvs:", long_options, &option_index);
-
-      switch (c) {
-        case 'v':
-          break;
-        case 'V':
-          version();
-          return 0;
-        case 's':
-          pattern = optarg;
-          break;
-      }
-    } else {
-      if (optind < argc) {
-        if (filePath == NULL) {
-          filePath = argv[optind++];
-        } else {
-          usage();
-        }
-      } else {
-        break;
-      }
-    }
-  }
-
-  /* Check if we have enough arguments */
-  if (filePath == NULL) {
-    usage();
-  }
-
-  /* Instanciate the reader */
-  reader = new kiwix::Reader(filePath);
-
-  /* Start to read an article */
-  if (reader != NULL) {
-    string content;
-    string contentType;
-    string suggestion;
-
-    if (pattern != NULL) {
-      std::cout << "Searching suggestions for: " << pattern << std::endl;
-      kiwix::SuggestionsList_t suggestions;
-      reader->searchSuggestionsSmart(pattern, 10, suggestions);
-      for (auto& suggestion: suggestions) {
-        std::cout << suggestion.getTitle() << std::endl;
-      }
-    }
-
-    delete reader;
-  } else {
-    cerr << "Unable instanciate the Kiwix reader." << endl;
-    exit(1);
-  }
-
-  exit(0);
-}

src/reader/meson.build

@@ -1,3 +0,0 @@
-executable('kiwix-read', ['kiwix-read.cpp'],
-  dependencies:all_deps,
-  install:true)

src/searcher/kiwix-search.cpp

@@ -18,11 +18,16 @@
  */
 
 #include <getopt.h>
-#include <kiwix/reader.h>
-#include <kiwix/searcher.h>
+
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
+#include <iostream>
 
 #include "../version.h"
 
+using namespace std;
+
 void usage()
 {
   cout << "Usage: kiwix-search [OPTIONS] ZIM PATTERN" << endl << endl
@@ -48,9 +53,6 @@ int main(int argc, char** argv)
   int option_index = 0;
   int c = 0;
 
-  kiwix::Searcher* searcher = NULL;
-  kiwix::Reader* reader = NULL;
-
   /* Argument parsing */
   while (42) {
     static struct option long_options[]
@@ -96,47 +98,24 @@ int main(int argc, char** argv)
 
   /* Try to prepare the indexing */
   try {
-    reader = new kiwix::Reader(zimPath);
-  } catch (...) {
-    /* Cannot open the zimPath, maybe it is a plain old xapian database
-     * directory */
-  }
+    zim::Archive archive(zimPath);
 
-  if (reader) {
-    searcher = new kiwix::Searcher();
-    bool contians_FTIndex=searcher->add_reader(reader);
-    if(!contians_FTIndex){
-        std::cerr << "The Zim file does not contain a full-text index." << std::endl;
-        if(suggestionFlag){
-            exit(0);
-        }
-        exit(1);
-      }
-  } else {
-    cerr << "Unable to search through zim '" << zimPath << "'." << endl;
-    exit(1);
-  }
-
-  /* Start the indexing */
-  if (searcher != NULL) {
-    string searchString(search);
     if (suggestionFlag) {
-        searcher->suggestions(searchString, verboseFlag);
-    } else  {
-        searcher->search(searchString, 0, 10, verboseFlag);
+      zim::SuggestionSearcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      for (const auto& r : searcher.suggest(search).getResults(0, 10) ) {
+        cout << r.getTitle() << endl;
+      }
+    } else {
+      zim::Searcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      const zim::Query query(search);
+      for (const auto& r : searcher.search(query).getResults(0, 10) ) {
+        cout << r.getTitle() << endl;
+      }
     }
-    kiwix::Result* p_result;
-    while ((p_result = searcher->getNextResult())) {
-      cout << p_result->get_title() << endl;
-      delete p_result;
-    }
-
-    delete searcher;
-    delete reader;
-
-    //      kiwix::XapianSearcher::terminate();
-  } else {
-    cerr << "Unable instanciate the Kiwix searcher." << endl;
+  } catch ( const std::runtime_error& err)  {
+    cerr << err.what() << endl;
     exit(1);
   }
 

src/server/kiwix-serve.cpp

@@ -1,3 +1,4 @@
+
 /*
  * Copyright 2009-2019 Emmanuel Engelhart <kelson@kiwix.org>
  *
@@ -49,31 +50,33 @@ void usage()
             << std::endl
 
             << "Purpose:" << std::endl
-            << "\tDeliver ZIM file articles via HTTP"
+            << "\tDeliver ZIM file(s) articles via HTTP"
             << std::endl << std::endl
 
             << "Mandatory arguments:" << std::endl
-            << "\tLIBRARY_PATH\t\tis the XML library file path listing ZIM file to serve. To be used only with the --library argument."
+            << "\tLIBRARY_PATH\t\tXML library file path listing ZIM file to serve. To be used only with the --library argument."
             << std::endl
-            << "\tZIM_PATH\t\tis the path of a ZIM file."
+            << "\tZIM_PATH\t\tZIM file path(s)"
             << std::endl << std::endl
 
             << "Optional arguments:" << std::endl << std::endl
-            << "\t-h, --help\t\tprint this help" << std::endl << std::endl
-            << "\t-a, --attachToProcess\texit if given process id is not running anymore" << std::endl
-            << "\t-d, --daemon\t\tdetach the HTTP server daemon from the main process" << std::endl
-            << "\t-i, --address\t\tlisten only on this ip address, all available ones otherwise" << std::endl
-            << "\t-M, --monitorLibrary\t\tmonitor the XML library file and reload it automatically" << std::endl
-            << "\t-m, --nolibrarybutton\tdo not print the builtin home button in the builtin top bar overlay" << std::endl
-            << "\t-n, --nosearchbar\tdo not print the builtin bar overlay on the top of each served page" << std::endl
-            << "\t-b, --blockexternal\tprevent users from directly accessing external links" << std::endl
+            << "\t-h, --help\t\tPrint this help" << std::endl << std::endl
+            << "\t-a, --attachToProcess\tWxit if given process id is not running anymore" << std::endl
+            << "\t-d, --daemon\t\tDetach the HTTP server daemon from the main process" << std::endl
+            << "\t-i, --address\t\tListen only on this ip address, all available ones otherwise" << std::endl
+            << "\t-M, --monitorLibrary\tMonitor the XML library file and reload it automatically" << std::endl
+            << "\t-m, --nolibrarybutton\tDon't print the builtin home button in the builtin top bar overlay" << std::endl
+            << "\t-n, --nosearchbar\tDon't print the builtin bar overlay on the top of each served page" << std::endl
+            << "\t-b, --blockexternal\tPrevent users from directly accessing external links" << std::endl
             << "\t-p, --port\t\tTCP port on which to listen to HTTP requests (default: 80)" << std::endl
             << "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
-            << "\t-t, --threads\t\tnumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
-            << "\t-v, --verbose\t\tprint debug log to STDOUT" << std::endl
-            << "\t-V, --version\t\tprint software version" << std::endl
-            << "\t-z, --nodatealiases\tcreate URL aliases for each content by removing the date" << std::endl
-            << "\t-c, --customIndex\tadd path to custom index.html for welcome page" << std::endl
+            << "\t-s, --searchLimit\tMaximun number of zim in a fulltext multizim search (default: No limit)" << std::endl
+            << "\t-t, --threads\t\tNumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
+            << "\t-v, --verbose\t\tPrint debug log to STDOUT" << std::endl
+            << "\t-V, --version\t\tPrint software version" << std::endl
+            << "\t-z, --nodatealiases\tCreate URL aliases for each content by removing the date" << std::endl
+            << "\t-c, --customIndex\tAdd path to custom index.html for welcome page" << std::endl
+            << "\t-L, --ipConnectionLimit\tMax number of (concurrent) connections per IP (default: infinite, recommended: >= 6)" << std::endl
             << std::endl
 
             << "Documentation:" << std::endl
@@ -82,24 +85,34 @@ void usage()
             << std::endl;
 }
 
-string loadCustomTemplate (string customIndexPath) {
+std::string loadCustomTemplate (std::string customIndexPath) {
   customIndexPath = kiwix::isRelativePath(customIndexPath) ?
                       kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), customIndexPath) :
                       customIndexPath;
   if (!kiwix::fileReadable(customIndexPath)) {
-    throw runtime_error("No such file exist (or file is not readable) " + customIndexPath);
+    throw std::runtime_error("No such file exist (or file is not readable) " + customIndexPath);
   }
   if (kiwix::getMimeTypeForFile(customIndexPath) != "text/html") {
-    throw runtime_error("Invalid File Mime Type " + kiwix::getMimeTypeForFile(customIndexPath));
+    throw std::runtime_error("Invalid File Mime Type " + kiwix::getMimeTypeForFile(customIndexPath));
   }
-  string indexTemplateString = kiwix::getFileContent(customIndexPath);
+  std::string indexTemplateString = kiwix::getFileContent(customIndexPath);
 
   if (indexTemplateString.empty()) {
-    throw runtime_error("Unreadable or empty file " + customIndexPath);
+    throw std::runtime_error("Unreadable or empty file " + customIndexPath);
   }
   return indexTemplateString;
 }
 
+inline std::string normalizeRootUrl(std::string rootUrl)
+{
+  while ( !rootUrl.empty() && rootUrl.back() == '/' )
+    rootUrl.pop_back();
+
+  while ( !rootUrl.empty() && rootUrl.front() == '/' )
+    rootUrl = rootUrl.substr(1);
+  return rootUrl.empty() ? rootUrl : "/" + rootUrl;
+}
+
 volatile sig_atomic_t waiting = false;
 volatile sig_atomic_t libraryMustBeReloaded = false;
 
@@ -137,7 +150,7 @@ void setup_sighandlers()
 
 uint64_t fileModificationTime(const std::string& path)
 {
-#if defined(_WIN32)
+#if defined(_WIN32) && !defined(stat)
 #define stat _stat
 #endif
   struct stat fileStatData;
@@ -183,15 +196,15 @@ int main(int argc, char** argv)
   setup_sighandlers();
 #endif
 
-  std::string rootLocation = "";
+  std::string rootLocation = "/";
   kiwix::Library library;
   unsigned int nb_threads = DEFAULT_THREADS;
-  vector<string> zimPathes;
-  string libraryPath;
-  string rootPath;
-  string address;
-  string customIndexPath="";
-  string indexTemplateString="";
+  std::vector<std::string> zimPathes;
+  std::string libraryPath;
+  std::string rootPath;
+  std::string address;
+  std::string customIndexPath="";
+  std::string indexTemplateString="";
   int serverPort = 80;
   int daemonFlag [[gnu::unused]] = false;
   int libraryFlag = false;
@@ -202,6 +215,8 @@ int main(int argc, char** argv)
   bool isVerboseFlag = false;
   bool monitorLibrary = false;
   unsigned int PPID = 0;
+  int ipConnectionLimit = 0;
+  int searchLimit = 0;
 
   static struct option long_options[]
       = {{"daemon", no_argument, 0, 'd'},
@@ -220,15 +235,23 @@ int main(int argc, char** argv)
          {"urlRootLocation", required_argument, 0, 'r'},
          {"customIndex", required_argument, 0, 'c'},
          {"monitorLibrary", no_argument, 0, 'M'},
+         {"ipConnectionLimit", required_argument, 0, 'L'},
+         {"searchLimit", required_argument, 0, 's'},
          {0, 0, 0, 0}};
 
+  std::set<int> usedOptions;
   /* Argument parsing */
   while (true) {
     int option_index = 0;
     int c
-        = getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:M", long_options, &option_index);
+        = getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:ML:s:", long_options, &option_index);
 
     if (c != -1) {
+      auto insertRes = usedOptions.insert(c);
+      if (!insertRes.second) {
+        std::cerr << "Multiple values of same option are not allowed." << std::endl;
+        exit(1);
+      }
       switch (c) {
         case 'h':
           usage();
@@ -264,20 +287,29 @@ int main(int argc, char** argv)
           PPID = atoi(optarg);
           break;
         case 'i':
-          address = string(optarg);
+          address = std::string(optarg);
           break;
         case 't':
           nb_threads = atoi(optarg);
           break;
         case 'r':
-          rootLocation = string(optarg);
+          rootLocation = std::string(optarg);
           break;
         case 'c':
-          customIndexPath = string(optarg);
+          customIndexPath = std::string(optarg);
           break;
         case 'M':
           monitorLibrary = true;
           break;
+        case 'L':
+          ipConnectionLimit = atoi(optarg);
+          break;
+        case 's':
+          searchLimit = atoi(optarg);
+          break;
+        case '?':
+          usage();
+          return 2;
       }
     } else {
       if (optind < argc) {
@@ -300,7 +332,7 @@ int main(int argc, char** argv)
 
   /* Setup the library manager and get the list of books */
   kiwix::Manager manager(&library);
-  vector<string> libraryPaths;
+  std::vector<std::string> libraryPaths;
   if (libraryFlag) {
     libraryPaths = kiwix::split(libraryPath, ";");
     if ( !reloadLibrary(manager, libraryPaths) ) {
@@ -309,15 +341,15 @@ int main(int argc, char** argv)
 
     /* Check if the library is not empty (or only remote books)*/
     if (library.getBookCount(true, false) == 0) {
-      cerr << "The XML library file '" << libraryPath
-           << "' is empty (or has only remote books)." << endl;
+      std::cerr << "The XML library file '" << libraryPath
+           << "' is empty (or has only remote books)." << std::endl;
     }
   } else {
     std::vector<std::string>::iterator it;
     for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
       if (!manager.addBookFromPath(*it, *it, "", false)) {
-        cerr << "Unable to add the ZIM file '" << *it
-             << "' to the internal library." << endl;
+        std::cerr << "Unable to add the ZIM file '" << *it
+             << "' to the internal library." << std::endl;
         exit(1);
       }
     }
@@ -364,15 +396,17 @@ int main(int argc, char** argv)
   server.setTaskbar(!noSearchBarFlag, !noLibraryButtonFlag);
   server.setBlockExternalLinks(blockExternalLinks);
   server.setIndexTemplateString(indexTemplateString);
+  server.setIpConnectionLimit(ipConnectionLimit);
+  server.setMultiZimSearchLimit(searchLimit);
 
   if (! server.start()) {
     exit(1);
   }
-  
-  std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + "/" + rootLocation;
+
+  std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + normalizeRootUrl(rootLocation);
   std::cout << "The Kiwix server is running and can be accessed in the local network at: "
             << url << std::endl;
-            
+
   /* Run endless (until PPID dies) */
   waiting = true;
   do {
@@ -395,7 +429,7 @@ int main(int argc, char** argv)
       int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0);
       if (ret != -1 && len > 0) {
 #else /* Linux & co */
-      string procPath = "/proc/" + std::to_string(PPID);
+      std::string procPath = "/proc/" + std::to_string(PPID);
       if (access(procPath.c_str(), F_OK) != -1) {
 #endif
       } else {

src/version.h

@@ -26,6 +26,7 @@
 
 #include <kiwix/version.h>
 #include <zim/version.h>
+#include <iostream>
 
 void version()
 {