Merge pull request #787 from kiwix/better-deal-with-data-dir-permissions

Better deal with container /data dir permissions

.github/workflows/ci.yml

@@ -0,0 +1,106 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  Windows:
+    runs-on: windows-2022
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install packages
+        run:
+          choco install pkgconfiglite ninja
+
+      - name: Install python modules
+        run: pip3 install meson
+
+      - name: Setup MSVC compiler
+        uses: bus1/cabuild/action/msdevshell@v1
+        with:
+          architecture: x64
+
+      - name: Install dependencies
+        uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+        with:
+          target_platform: win-x86_64-static
+
+      - name: Compile
+        shell: cmd
+        run: |
+          set PKG_CONFIG_PATH=%cd%\BUILD_win-amd64\INSTALL\lib\pkgconfig
+          set CPPFLAGS=-I%cd%\BUILD_win-amd64\INSTALL\include
+          meson.exe setup . build -Dstatic-linkage=true --buildtype=release
+          cd build
+          ninja.exe
+
+      - name: Test
+        shell: cmd
+        run: |
+          cd build
+          meson.exe test --verbose
+        env:
+          WAIT_TIME_FACTOR_TEST: 10
+
+  Linux:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - linux-x86_64-static
+          - linux-x86_64-dyn
+        include:
+          - target: linux-x86_64-static
+            image_variant: jammy
+            lib_postfix: '/x86_64-linux-gnu'
+            arch_name: linux-x86_64
+          - target: linux-x86_64-dyn
+            image_variant: jammy
+            lib_postfix: '/x86_64-linux-gnu'
+            arch_name: linux-x86_64
+
+    env:
+      HOME: /home/runner
+
+    container:
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2025-06-07"
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install dependencies
+      uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+      with:
+        target_platform: ${{ matrix.target }}
+
+    - name: Compile
+      shell: bash
+      run: |
+        if [[ "${{matrix.target}}" =~ .*-static ]]; then
+          MESON_OPTION="-Dstatic-linkage=true"
+        else
+          MESON_OPTION=""
+        fi
+        if [ -e "$HOME/BUILD_${{matrix.arch_name}}/meson_cross_file.txt" ]; then
+          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.arch_name}}/meson_cross_file.txt"
+        fi
+        meson . build ${MESON_OPTION}
+        cd build
+        ninja
+      env:
+        PKG_CONFIG_PATH: "${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib/pkgconfig:${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
+        CPPFLAGS: "-I${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/include"

.github/workflows/docker.yml

@@ -0,0 +1,67 @@
+name: Docker
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Specific version to build (overrides on-master and tag-pattern)
+        required: false
+        default: ''
+
+jobs:
+  build-and-push-kiwix-tools:
+    name: Deploy kiwix-tools Docker Image
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: build and publish kiwix-tools
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: kiwix/kiwix-tools
+          registries: ghcr.io
+          credentials: |
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          context: docker
+          latest-on-tag: true
+          build-args:
+            VERSION={tag}
+          platforms: |
+            linux/amd64
+            linux/arm64
+            linux/arm/v7
+            linux/arm/v6
+            linux/386
+          restrict-to: kiwix/kiwix-tools
+          manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix command line tools
+
+  build-and-push-kiwix-serve:
+    name: Deploy kiwix-serve Docker Image
+    runs-on: ubuntu-22.04
+    needs: build-and-push-kiwix-tools
+    steps:
+      - uses: actions/checkout@v4
+      - name: build and publish kiwix-serve
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: kiwix/kiwix-serve
+          registries: ghcr.io
+          credentials: |
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          context: docker/server
+          latest-on-tag: true
+          build-args:
+            VERSION={tag}
+          platforms: |
+            linux/amd64
+            linux/arm64
+            linux/arm/v7
+            linux/arm/v6
+            linux/386
+          restrict-to: kiwix/kiwix-tools
+          manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix web-server

.github/workflows/package.yml

@@ -0,0 +1,112 @@
+name: Packages
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+#          - debian-unstable
+#          - debian-trixie
+#          - debian-bookworm
+#          - debian-bullseye
+          - ubuntu-noble
+          - ubuntu-jammy
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # Determine which PPA we should upload to
+      - name: PPA
+        id: ppa
+        run: |
+          if [[ $REF == refs/tags* ]]
+          then
+            echo "ppa=kiwixteam/release" >> $GITHUB_OUTPUT
+          else
+            echo "ppa=kiwixteam/dev" >> $GITHUB_OUTPUT
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - uses: legoktm/gh-action-auto-dch@main
+        with:
+          fullname: Kiwix builder
+          email: release+launchpad@kiwix.org
+          distro: ${{ matrix.distro }}
+
+#      - uses: legoktm/gh-action-build-deb@debian-unstable
+#        if: matrix.distro == 'debian-unstable'
+#        name: Build package for debian-unstable
+#        id: build-debian-unstable
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@b47978ba8498dc8b8153cc3b5f99a5fc1afa5de1 # pin@debian-trixie
+#        if: matrix.distro == 'debian-trixie'
+#        name: Build package for debian-trixie
+#        id: build-debian-trixie
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@1f4e86a6bb34aaad388167eaf5eb85d553935336 # pin@debian-bookworm
+#        if: matrix.distro == 'debian-bookworm'
+#        name: Build package for debian-bookworm
+#        id: build-debian-bookworm
+#        with:
+#          args: --no-sign
+#
+#      - uses: legoktm/gh-action-build-deb@084b4263209252ec80a75d2c78a586192c17f18d # pin@debian-bullseye
+#        if: matrix.distro == 'debian-bullseye'
+#        name: Build package for debian-bullseye
+#        id: build-debian-bullseye
+#        with:
+#          args: --no-sign
+
+      - uses: legoktm/gh-action-build-deb@9114a536498b65c40b932209b9833aa942bf108d # pin@ubuntu-noble
+        if: matrix.distro == 'ubuntu-noble'
+        name: Build package for ubuntu-noble
+        id: build-ubuntu-noble
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: legoktm/gh-action-build-deb@ubuntu-jammy
+        if: matrix.distro == 'ubuntu-jammy'
+        name: Build package for ubuntu-jammy
+        id: build-ubuntu-jammy
+        with:
+          args: --no-sign
+          ppa: ${{ steps.ppa.outputs.ppa }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Packages for ${{ matrix.distro }}
+          path: output
+
+      - uses: legoktm/gh-action-dput@main
+        name: Upload dev package
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/dev
+          packages: output/*_source.changes
+
+      - uses: legoktm/gh-action-dput@main
+        name: Upload release package
+        # Only upload on pushes to main or tag
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
+        with:
+          gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
+          repository: ppa:kiwixteam/release
+          packages: output/*_source.changes
+

.github/workflows/pr.yml

@@ -1,77 +0,0 @@
-name: CI
-
-on: [pull_request]
-
-jobs:
-  Linux:
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-          - native_static
-          - native_dyn
-          - win32_static
-          - win32_dyn
-        include:
-          - target: native_static
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: native_dyn
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: win32_static
-            image_variant: f31
-            lib_postfix: '64'
-          - target: win32_dyn
-            image_variant: f31
-            lib_postfix: '64'
-    env:
-      HOME: /home/runner
-    runs-on: ubuntu-latest
-    container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
-    steps:
-    - name: Checkout code
-      shell: python
-      run: |
-        import json
-        from subprocess import check_call
-        from os import environ
-        with open(environ['GITHUB_EVENT_PATH'], 'r') as f:
-            content = f.read()
-        event_data = json.loads(content)
-        try:
-            branch_ref = event_data['ref'].split('/')[-1]
-        except KeyError:
-            branch_ref = event_data['pull_request']['head']['ref']
-        print("Cloning branch", branch_ref)
-        command = [
-          'git', 'clone',
-          'https://github.com/${{github.repository}}',
-          '--depth=1',
-          '--branch', branch_ref
-        ]
-        check_call(command, cwd=environ['HOME'])
-    - name: Install deps
-      shell: bash
-      run: |
-        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-tools.tar.xz
-        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
-    - name: Compile
-      shell: bash
-      run: |
-        if [[ "${{matrix.target}}" =~ .*_static ]]; then
-          MESON_OPTION="-Dstatic-linkage=true"
-        else
-          MESON_OPTION=""
-        fi
-        if [[ ! "${{matrix.target}}" =~ native_.* ]]; then
-          MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
-        fi
-        cd $HOME/kiwix-tools
-        meson . build ${MESON_OPTION}
-        cd build
-        ninja
-      env:
-        PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
-        CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"

.readthedocs.yaml

@@ -0,0 +1,21 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: docs/requirements.txt

.well-known/funding-manifest-urls

@@ -0,0 +1 @@
+https://kiwix.org/funding.json

Changelog

@@ -1,3 +1,146 @@
+kiwix-tools 3.8.1
+=================
+
+ * Kiwix server
+   - Hide port number in URL when server is running on port 80 (@vighnesh-sawant #763)
+   - Better deal with container /data dir permissions (@kelson42 #787)
+
+ * Other
+   - Fix kiwix-manage docopt integration (@kelson42 #783)
+
+kiwix-tools 3.8.0
+=================
+
+ * Kiwix server
+   - Improve message when server is running on standard port (@vighnesh-sawant #763)
+   - Update container base image to latest Alpline version (@yashgoyal0110 #771)
+   - Container image to use unprivileged user (@Sedetisu #755)
+   - Empty urlRootLocation doesn't disable book preview links anymore (@veloman-yunkan #1224)
+   - Add support of IPv6 (@sgourdas #673 #704)
+   - Popups are allowed to escape the browser sandbox (@veloman-yunkan #1208)
+
+ * Other
+   - Stop publishing on Ubuntu 20.04 PPA (@kelson42 #748)
+   - Use Docopt for command line argument parsing (@mgautierfr #695)
+
+ * Compilation & Packaging
+   - Based on libkiwix 14.1.0 (@kelson42 #773)
+   - Improve CI/CD (@kelson42 #681 #698 #702 #705 #720 #722, @mgautierfr #695)
+
+kiwix-tools 3.7.0
+=================
+
+ * Fixed ZIM name vs Book name confusion in documentation (@veloman-yunkan #663)
+ * Fixes compilation dependencies to rely on appropriate version (@kelson42 #667)
+ * New --skipInvalid Kiwix Server command line option (@schuellerf @kelson42 #666)
+
+kiwix-tools 3.6.0
+=================
+
+ * Improved kiwix-serve man page (@iArchitSharma #626)
+ * C++17 compliant code base (@mgautierfr #636)
+ * Support of libkiwix13 (@mgautierfr #633)
+ * Additional docker images archs for armv6 and i386 (@rgaudin #622)
+
+kiwix-tools 3.5.0
+=================
+
+ * Do not use `--static` option when compiling on MacOs (@mgautierfr #615)
+ * Move main branch from `master` to `main`.
+ * Fix docker image (@jacroe #597)
+ * Various CI improvements (@kelson42)
+
+kiwix-serve
+-----------
+
+ * Add documentation about the kiwix-serve API (@veloman-yunkan #586)
+   https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html#http-api
+
+kiwix-tools 3.4.0
+=================
+
+ * Remove last reference to kiwix-read tool (@legoktm #569)
+
+kiwix-serve
+-----------
+
+ * Fix broken indentation in usage (@kelson42 #560)
+ * Exit if wrong arguments are passed (@kelson42 #567)
+ * Do not allow multiple values for same option (@juuz0 #564)
+ * Fix default location of "rootLocation" (@rgaudin #571)
+ * [DOCKER] Change default port to 8080 (@neyder #581)
+ * [DOCKER] Simplify dockerfile (@rgaudin #582)
+
+kiwix-manage
+------------
+
+ * Fix man page (@kelson42 #576)
+
+kiwix-tools 3.3.0
+=================
+
+ * Remove kiwix-read tool (@veloman-yunkan #535)
+
+kiwix-serve
+-----------
+
+ * Add an option to limit the number of connections for a same IP (@juuz0 #534)
+ * Add an option to limit the number of zim in a multizim fulltext search (@mgautierfr #558)
+
+kiwix-search
+------------
+
+ * Remove usage of libkiwix's deprecated api (@veloman-yunkan #535)
+
+kiwix-manage
+------------
+
+ * Correctly return a value !0 if something went wrong (@mgautierfr #553)
+
+
+kiwix-tools 3.2.0
+=================
+
+ * Print the version of all dependencies (@kelson42 #516)
+ * Better Docker images (@kelson42 @rgaudin)
+ * Update Readme (@kelson42)
+ * Build debian packages on CI (@legoktm #394)
+ * Add man pages for kiwix-read and kiwix-search (@legoktm #392)
+ * Various fixes (@legoktm @hashworks @mgautierfr)
+
+
+kiwix-serve
+-----------
+
+ * Print the url on which a user can connect to on startup (@juuz0 #499 #522)
+ * Reload library on SIGHUP signal (@veloman-yunkan #497)
+ * Add a option `--monitorLibrary` to monitor and automically reload the library
+   (@veloman-yunkan #503)
+ * Correct handling of SIGTERM and SIGINT (@veloman-yunkan #488)
+ * Add `--customIndexTemplate` option (@manan #477)
+ * Add `--help` option (@kelson42 #511)
+
+
+kiwix-tools 3.1.2
+=================
+
+ * Use new threadsafe API of kiwix-lib to do suggestions search.
+
+kiwix-tools 3.1.1
+=================
+
+ * Fix compilation on Windows' CI
+
+kiwix-tools 3.1.0
+=================
+
+ * [SERVER] Add option to block external links
+
+kiwix-tools 3.0.3
+=================
+
+ * [MANAGER] Fix broken --version argument parsing
+
 kiwix-tools 3.0.2
 =================
 

README.md

@@ -4,27 +4,18 @@ Kiwix tools
 The Kiwix tools is a collection of [Kiwix](https://kiwix.org) related
 command line tools:
 * kiwix-manage: Manage XML based library of ZIM files
-* kiwix-read: Read ZIM file content
-* kiwix-search: Fulltext search in ZIM files
+* kiwix-search: Full text search in ZIM files
 * kiwix-serve: HTTP daemon serving ZIM files
 
-[![Build Status](https://travis-ci.com/kiwix/kiwix-tools.svg?branch=master)](https://travis-ci.com/kiwix/kiwix-tools)
+[![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/)
+[![Repositories](https://img.shields.io/repology/repositories/kiwix-tools?label=repositories)](https://github.com/kiwix/kiwix-tools/wiki/Repology)
+[![Docker](https://ghcr-badge.egpl.dev/kiwix/kiwix-tools/latest_tag?label=docker)](https://ghcr.io/kiwix/kiwix-tools)
+[![Docker](https://ghcr-badge.egpl.dev/kiwix/kiwix-tools/latest_tag?label=docker%20(kiwix-serve))](https://ghcr.io/kiwix/kiwix-tools)
+[![Build Status](https://github.com/kiwix/kiwix-tools/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-tools/actions/workflows/ci.yml?query=branch%3Amain)
+[![Doc](https://readthedocs.org/projects/kiwix-tools/badge/?style=flat)](https://kiwix-tools.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-tools/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-tools)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
-Distribution
-------------
-
-#### Official
-
-[![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/)
-[![Docker](https://img.shields.io/docker/cloud/build/kiwix/kiwix-serve)](https://hub.docker.com/r/kiwix/kiwix-serve)
-
-#### Contribution
-
-[![AUR](https://img.shields.io/aur/version/kiwix-tools)](https://aur.archlinux.org/packages/kiwix-tools/)
-[![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h)
-
 Disclaimer
 ----------
 
@@ -37,8 +28,8 @@ Preamble
 --------
 
 Although the Kiwix tools can be compiled/cross-compiled on/for many
-sytems, the following documentation explains how to do it on POSIX
-ones. It is primarly thought for GNU/Linux systems and has been tested
+systems, the following documentation explains how to do it on POSIX
+ones. It is primarily thought for GNU/Linux systems and has been tested
 on recent releases of
 [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) and
 [Fedora](https://getfedora.org).
@@ -49,10 +40,8 @@ Dependencies
 The Kiwix tools rely on a few third party software libraries. They are
 prerequisites to the Kiwix tools compilation. Therefore, following
 libraries need to be available:
-* [Kiwix lib](https://github.com/kiwix/kiwix-lib) (no package so far)
-* [Libmicrohttpd](https://www.gnu.org/software/libmicrohttpd/)
-  (package `libmicrohttpd-dev` on Debian/Ubuntu)
-* [Zlib](https://www.zlib.net/) (package `zlib1g-dev` on Debian/Ubuntu)
+* [libkiwix](https://github.com/kiwix/libkiwix) (package `libkiwix` on Debian/Ubuntu)
+* [libzim](https://github.com/openzim/libzim) (package `libzim` on Debian/Ubuntu)
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. They
@@ -68,15 +57,11 @@ If you want to compile Kiwix tools statically, the dependencies should
 be compiled statically (provide a `lib...a` library), for example by
 using `--enable-static` with `./configure`.
 
-If you compile manually Libmicrohttpd, you might need to compile it
-without GNU TLS, a bug here will empeach further compilation of Kiwix
-tools otherwise.
-
 Environment
 -------------
 
 The Kiwix tools build using [Meson](http://mesonbuild.com/) version
-0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
+0.43 or higher. Meson relies itself on Ninja, pkg-config and a few other
 compilation tools. Install them first:
 * [Meson](http://mesonbuild.com/)
 * [Ninja](https://ninja-build.org/)
@@ -110,7 +95,7 @@ ninja -C build install
 ```
 
 You might need to run the command as `root` (or using `sudo`),
-depending where you want to install the Kiwix tools. After the
+depending on where you want to install the Kiwix tools. After the
 installation succeeded, you may need to run ldconfig (as `root`).
 
 Uninstallation
@@ -127,7 +112,10 @@ Like for the installation, you might need to run the command as `root`
 Docker
 ------
 
-An official Docker image of `kiwix-serve` can be found in the [Docker Hub](https://hub.docker.com/r/kiwix/kiwix-serve).
+An official Docker image of the Kiwix tools can be found on 
+[GHCR](https://ghcr.io/kiwix/kiwix-tools). A
+`kiwix-serve` dedicated Docker image [exists
+too](https://ghcr.io/kiwix/kiwix-serve).
 
 Troubleshooting
 ---------------

debian/changelog

@@ -0,0 +1,5 @@
+kiwix-tools (0.0.0) unstable; urgency=medium
+
+  * Initial release
+
+ -- Kunal Mehta <legoktm@debian.org>  Mon, 13 Jul 2020 17:21:11 -0700

debian/control

@@ -0,0 +1,27 @@
+Source: kiwix-tools
+Section: utils
+Priority: optional
+Maintainer: Kiwix team <kiwix@kiwix.org>
+Build-Depends: debhelper-compat (= 13),
+               libzim-dev (>= 9.0), libzim-dev (<< 10.0),
+               libkiwix-dev (>= 14.0), libkiwix-dev (<< 15.0),
+               cmake,
+               libdocopt-dev,
+               meson,
+               pkgconf,
+Standards-Version: 4.6.2
+Homepage: https://github.com/kiwix/kiwix-tools
+Rules-Requires-Root: no
+
+Package: kiwix-tools
+Architecture: any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: collection of Kiwix tools
+ kiwix-tools is a collection of various command-line tools used to help
+ users interact with and manage ZIM files. It includes:
+  * kiwix-serve is a standalone HTTP server for serving ZIM files
+    over the network.
+  * kiwix-manage allows one to manage the content of the Kiwix library (an
+    XML file listing available ZIM files).
+  * kiwix-search allows one to find articles in a ZIM file using fulltext
+    search patterns.

debian/copyright

@@ -0,0 +1 @@
+See COPYING in the repository root.

debian/rules

@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+%:
+	dh $@

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

docker/Dockerfile

@@ -0,0 +1,34 @@
+FROM alpine:3.22
+LABEL org.opencontainers.image.source=https://github.com/openzim/kiwix-tools
+
+# TARGETPLATFORM is injected by docker build
+ARG TARGETPLATFORM
+ARG VERSION
+
+RUN set -e && \
+    apk --no-cache add dumb-init curl && \
+    echo "TARGETPLATFORM: $TARGETPLATFORM" && \
+    if [ "$TARGETPLATFORM" = "linux/386" ]; then ARCH="i586"; \
+    # linux/arm64/v8 points to linux/arm64
+    elif [ "$TARGETPLATFORM" = "linux/arm64/v8" \
+        -o "$TARGETPLATFORM" = "linux/arm64" ]; then ARCH="aarch64"; \
+    # linux/arm translates to linux/arm/v7
+    elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then ARCH="armv8"; \
+    elif [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then ARCH="armv6"; \
+    elif [ "$TARGETPLATFORM" = "linux/amd64/v3" \
+        -o "$TARGETPLATFORM" = "linux/amd64/v2" \
+        -o "$TARGETPLATFORM" = "linux/amd64" ]; then ARCH="x86_64"; \
+    # we dont suppot any other arch so let it fail
+    else ARCH="unknown"; fi && \
+    # download requested kiwix-tools version
+    url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$ARCH-$VERSION.tar.gz" && \
+    echo "URL: $url" && \
+    curl -k -L $url | tar -xz -C /usr/local/bin/ --strip-components 1 && \
+    # only needed in dockerfile
+    apk del curl
+
+# expose kiwix-serve default port
+EXPOSE 80
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/bin/sh", "-c", "echo 'Welcome to kiwix-tools! The following binaries are available:' && ls /usr/local/bin/"]

docker/README.md

@@ -0,0 +1,36 @@
+Kiwix-tools Docker image
+===
+
+- Available on [ghcr.io](https://ghcr.io/kiwix/kiwix-tools).
+- multi-arch (`linux/amd64`, `linux/arm64`, `linux/arm/v7`)
+- based on official `kiwix-tools` binaries.
+
+## Usage
+
+``` sh
+$ docker run -it ghcr.io/kiwix/kiwix-tools:3.1.2
+
+Welcome to kiwix-tools! The following binaries are available:
+kiwix-manage  kiwix-search  kiwix-serve
+```
+
+`kiwix-tools` operates on zim files. You shall mount a volume to access the files.
+
+```sh
+docker run -v $(pwd):/data -it ghcr.io/kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
+```
+
+## Building and reusing
+
+- `kiwix/kiwix-tools` is multi-arch and is ideally built using `buildx`.
+- requires a `--build-arg VERSION=` with the kiwix-tools release.
+- can be built using `docker build` in which case it expects an additionnal `--build-arg ARCH=arm` for arm. Otherwise defaults to `amd64`.
+
+**Notes:**
+
+- `wget` in `alpine:3` on `arm/v7` (__inside github action only__) crashes when downloading from HTTPs locations. Keep http-only in Dockerfile.
+- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.
+
+## See also
+
+If you are interested by a Kiwix server only container image, [here it is](server/README.md).

docker/server/Dockerfile

@@ -1,18 +1,19 @@
-FROM alpine:latest
-LABEL maintainer Emmanuel Engelhart <kelson@kiwix.org>
+ARG VERSION=latest
 
-# Install kiwix-serve
-WORKDIR /
-RUN apk add --no-cache curl bzip2
-RUN curl -kL https://download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-x86_64.tar.gz | tar -xz && \
-    mv kiwix-tools*/kiwix-serve /usr/local/bin && \
-    rm -r kiwix-tools*
+# kiwix-tools is multi-arch
+FROM ghcr.io/kiwix/kiwix-tools:$VERSION
+LABEL org.opencontainers.image.source=https://github.com/openzim/kiwix-tools
 
-# Run kiwix-serve
-EXPOSE 80
+# expose kiwix-serve default port and workdir
+EXPOSE 8080
 VOLUME /data
 WORKDIR /data
 
+# running as a named unprivileged user
+RUN addgroup -S -g 1001 user && adduser -S -u 1001 user -G user
+RUN chown user:user /data
+USER user
+
 COPY ./start.sh /usr/local/bin/
 
-ENTRYPOINT ["/usr/local/bin/start.sh"]
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/start.sh"]

docker/server/README.md

@@ -1,24 +1,54 @@
 Kiwix serve Docker image
 ========================
 
-With local ZIM file
--------------------
+With local ZIM file(s)
+----------------------
 
 * Download a ZIM file from <https://wiki.kiwix.org/wiki/Content>
 
-* Given `wikipedia.zim` resides in `/tmp/zim/`, execute the following:
+* Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following:
 
 ```bash
-docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim
+docker run -v /tmp/zim:/data -p 8080:8080 ghcr.io/kiwix/kiwix-serve wikipedia.zim wiktionary.zim
+```
+
+or, if you want to load all ZIM files within a directory, then use globbing:
+
+```bash
+docker run -v /tmp/zim:/data -p 8080:8080 ghcr.io/kiwix/kiwix-serve '*.zim'
 ```
 
 With remote ZIM file
 --------------------
 
 ```bash
-docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:80 kiwix/kiwix-serve
+docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 ghcr.io/kiwix/kiwix-serve
 ```
 
+Change default port
+-------------------
+
+You can change port to expose with environment variable PORT, useful if running on Podman, K8s or OpenShift
+
+```bash
+podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 ghcr.io/kiwix/kiwix-serve
+```
+
+ARM
+---
+
+Build an image for an ARM based GNU/Linux:
+```bash
+docker build . -t ghcr.io/kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/"
+```
+
+Docker Compose
+--------------
+
+You can also deploy kiwix with
+[`docker-compose`](https://docs.docker.com/compose/). Check out a
+sample at [docker-compose.yml.example](docker-compose.yml.example).
+
 Screenshots
 -----------
 

docker/server/docker-compose.yml.example

@@ -0,0 +1,14 @@
+version: '3.3'
+services:
+  kiwix-serve:
+      ports:
+        - 8080:8080
+      image: ghcr.io/kiwix/kiwix-serve:latest
+      # uncomment next 4 lines to use it with local zim file in /tmp/zim
+      # volumes:
+      #   - /tmp/zim:/data
+      # command:
+      #   - '*.zim'
+      # uncomment next 2 lines to use it with remote zim file
+      # environment:
+      #   - 'DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim'

docker/server/start.sh

@@ -3,6 +3,14 @@
 # Download if necessary a file
 if [ ! -z "$DOWNLOAD" ]
 then
+    # Check if /data is writable
+    if [ ! -w /data ]
+    then
+        echo "'/data' directory is not writable by '$(id -n -u):$(id -n -g)' ($(id -u):$(id -g)). ZIM file(s) can not be written."
+        exit 1
+    fi
+
+    # Dwonload ZIM file
     ZIM=`basename $DOWNLOAD`
     wget $DOWNLOAD -O "$ZIM"
 
@@ -13,6 +21,17 @@ then
     fi
 fi
 
-CMD="/usr/local/bin/kiwix-serve --port=80 $@"
+if [ -z "$PORT" ]
+then
+    PORT=8080
+fi
+CMD="/usr/local/bin/kiwix-serve --port=$PORT $@"
 echo $CMD
 $CMD
+
+# If error, print the content of /data
+if [ $? -ne 0 ]
+then
+    echo "Here is the content of /data:"
+    find /data -type f
+fi

docs/conf.py

@@ -0,0 +1,50 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'kiwix-tools'
+copyright = '2024, kiwix-team'
+author = 'kiwix-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+        "sphinx_rtd_theme"
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []

docs/index.rst

@@ -0,0 +1,8 @@
+Welcome to kiwix-tools' documentation!
+======================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   kiwix-serve

docs/kiwix-serve.rst

@@ -0,0 +1,889 @@
+***********
+kiwix-serve
+***********
+
+Introduction
+============
+
+``kiwix-serve`` is a tool for serving ZIM file content over HTTP. It supports
+serving a library containing multiple ZIM files. In a large library served by a
+``kiwix-serve`` instance clients can look up/filter ZIM files of interest by
+words in their :term:`titles <ZIM title>` and/or descriptions, language, tags, etc.
+
+``kiwix-serve`` provides a ZIM file viewer for displaying inidividual pages
+from a ZIM file inside the user's web browser (without downloading the full ZIM
+file).
+
+Clients can also remotely search inside those ZIM files that contain a full-text
+search database.
+
+``kiwix-serve`` supports Web browsers `Firefox >= 70, Chrome >= 80, Edge >= 80, ChromeAndroid >= 80, Safari >= 14, iOS >= 14 <https://browsersl.ist/#q=Firefox+%3E%3D+70%2C+Chrome+%3E%3D+80%2C+Edge+%3E%3D+80%2C+ChromeAndroid+%3E%3D+80%2C+Safari+%3E%3D+14%2C+iOS+%3E%3D+14>`_.
+
+Usage
+=====
+
+.. code-block:: sh
+
+  kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
+  kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
+
+
+Arguments
+---------
+
+.. _cli-arg-library-file-path:
+
+``LIBRARY_FILE_PATH``: path of an XML library file listing ZIM files to serve.
+To be used only with the :option:`--library` option. Multiple library files can
+be provided as a semicolon (``;``) separated list.
+
+``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed).
+
+Options
+-------
+
+.. option:: --library
+
+  By default, ``kiwix-serve`` expects a list of ZIM files as command line
+  arguments. Providing the :option:`--library` option tells ``kiwix-serve``
+  that the command line argument is rather a :ref:`library XML file
+  <cli-arg-library-file-path>`.
+
+.. option:: --catalogOnly
+
+  In this mode ``kiwix-serve`` only serves the welcome (library) page and the
+  OPDS catalog. ZIM files referred by the :ref:`library XML file
+  <cli-arg-library-file-path>` need not be accessible.
+
+  This option may be combined with the :option:`--contentServerURL` option.
+
+.. option:: --contentServerURL=URL
+
+  In :option:`--catalogOnly` mode book content is not served by this instance
+  of `kiwix-serve`. If a separate instance of `kiwix-serve` is running for the
+  same library without that option and thus serves book content, then the root
+  URL of that server can be passed to this instance so that books can still be
+  previewed.
+
+  This option must be combined with the :option:`--catalogOnly` option.
+
+.. option:: -i ADDR, --address=ADDR
+
+  Listen only on this IP address. By default the server listens on all
+  available IP addresses. Alternatively, you can use special values to define which types of connections to accept:
+
+  - all : Listen for connections on all IP addresses (IPv4 and IPv6).
+  - ipv4 : Listen for connections on all IPv4 addresses.
+  - ipv6 : Listen for connections on all IPv6 addresses.
+
+
+.. option:: -p PORT, --port=PORT
+
+  TCP port on which to listen for HTTP requests (default: 80).
+
+
+.. option:: -r ROOT, --urlRootLocation=ROOT
+
+  URL prefix on which the content should be made available (default: empty).
+
+
+.. option:: -d, --daemon
+
+  Detach the HTTP server daemon from the main process.
+
+
+.. option:: -a PID, --attachToProcess=PID
+
+  Exit when the process with id PID stops running.
+
+
+.. option:: -M, --monitorLibrary
+
+  Monitor the XML library file and reload it automatically when it changes.
+
+  Library reloading can be forced anytime by sending a SIGHUP signal to the
+  ``kiwix-serve`` process (this works regardless of the presence of the
+  :option:`--monitorLibrary`/:option:`-M` option).
+
+
+.. option:: -m, --nolibrarybutton
+
+  Disable the library home button in the ZIM viewer toolbar.
+
+
+.. option:: -n, --nosearchbar
+
+  Disable the searchbox in the ZIM viewer toolbar.
+
+
+.. option:: -b, --blockexternal
+
+  Prevent the users from directly navigating to external resources via such
+  links in ZIM content.
+
+
+.. option:: -t N, --threads=N
+
+  Number of threads to run in parallel (default: 4).
+
+
+.. option:: -s N, --searchLimit=N
+
+  Maximum number of ZIM files in a fulltext multizim search (default: No limit).
+
+
+.. option:: -z, --nodatealiases
+
+  Create URL aliases for each content by removing the date embedded in the file
+  name. The expected format of the date in the filename is ``_YYYY-MM``. For
+  example, ZIM file ``wikipedia_en_all_2020-08.zim`` will be accessible both as
+  ``wikipedia_en_all_2020-08`` and ``wikipedia_en_all``.
+
+
+.. option:: -c PATH, --customIndex=PATH
+
+  Override the welcome page with a custom HTML file.
+
+
+.. option:: -L N, --ipConnectionLimit=N
+
+  Max number of (concurrent) connections per IP (default: infinite,
+  recommended: >= 6).
+
+
+.. option:: -v, --verbose
+
+  Print debug log to STDOUT.
+
+
+.. option:: -V, --version
+
+  Print the software version.
+
+
+.. option:: -h, --help
+
+  Print the help text.
+
+
+HTTP API
+========
+
+``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where
+``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the
+:option:`--address`/:option:`-i`, :option:`--port`/:option:`-p` and
+:option:`--urlRootLocation`/:option:`-r` options, respectively.
+
+HTTP API endpoints presented below are relative to that location, i.e.
+``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``.
+
+.. note::
+
+  The HTTP API is documented in its entirety in order to facilitate the work of
+  the Kiwix development team. Note, however, that only a subset of the HTTP API
+  constitutes ``kiwix-serves``'s public interface.
+
+  .. _public-api-endpoint:
+
+  **Public API endpoint**
+
+    A public HTTP API endpoint is intended to serve the outside world (in
+    addition to ``kiwix-serve``'s front-end and other Kiwix products). The
+    Kiwix development team will do its best to ensure gratifying experience for
+    clients of public API endpoints at all stages of the endpoint lifecycle.
+
+  .. _private-api-endpoint:
+
+  **Private API endpoint**
+
+    A private API endpoint is intended to be used only by ``kiwix-serve``'s
+    frontend or by other products maintained solely by the Kiwix team. Private
+    API comes without any guaranees. It may change as frequently and as
+    drasticaly as the Kiwix development team sees fit.
+
+  .. _deprecation:
+
+  **Deprecation**
+
+    Public API doesn't stay frozen once and forever. As the API evolves, Kiwix
+    team reserves the right to drop support for certain old functionality. In
+    such events, an advance notice will be issued and the users will be given
+    enough time to prepare for the change.
+
+  Currently, public endpoints are limited to the following list:
+
+    -  :ref:`OPDS API <new-opds-api>`
+    -  ``/raw``
+    -  ``/search`` (with ``/search/searchdescription.xml``)
+
+.. _welcome-page:
+
+``/``
+-----
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Welcome page is served under ``/``. By default this is the library page, where
+books are listed and can be looked up/filtered interactively. However, the
+welcome page can be overriden through the :option:`--customIndex`/:option:`-c`
+command line option of ``kiwix-serve``.
+
+
+.. _new-opds-api:
+
+``/catalog/v2`` (OPDS API)
+------------------------------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+The new OPDS API of ``kiwix-serve`` is based on the `OPDS Catalog specification
+v1.2 <https://specs.opds.io/opds-1.2>`_. All of its endpoints are grouped under
+``/catalog/v2``.
+
+:ref:`Legacy OPDS API <legacy-opds-api>` is preserved for backward
+compatibility.
+
+
+``/catalog/v2/root.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+The OPDS Catalog Root links to the OPDS acquisition and navigation feeds
+accessible through the other endpoints of the OPDS API.
+
+
+``/catalog/v2/searchdescription.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Describes the `/catalog/v2/entries`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+
+``/catalog/v2/categories``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full list of ZIM file categories as an `OPDS Navigation Feed
+<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
+
+
+``/catalog/v2/entries``
+^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns a full or filtered list of ZIM files as a paginated `OPDS acquisition
+feed <https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `complete
+entries
+<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
+
+**Pagination:**
+
+By default, no more than 10 first entries are returned from the library. To
+obtain the remaining entries the URL query parameters ``start`` and/or
+``count`` must be used. The output of ``/catalog/v2/entries?start=s&count=n``
+will contain at most ``n`` (default value: 10) results starting from entry #
+``s`` (default value: 0).  ``count`` with a negative value (e.g.  ``count=-1``)
+removes the limit on the number of results in the output.
+
+
+.. note::
+
+  Previously ``count=0`` also designated an unbounded query (i.e. worked
+  similarly to ``count=-1``). The response to a ``count=0`` query was changed
+  to consist of 0 results, as such a query/response combination is a good way
+  to find out the total number of results (when only that information is
+  needed) with minimal consumption of resources.
+
+Examples:
+
+.. code:: sh
+
+  # Returns the first 10 entries (internally numbered 0 through 9)
+  $ curl 'http://localhost:8080/catalog/v2/entries'
+
+  # Returns the next 10 entries (internally numbered 10 through 19)
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=10'
+
+  # Returns the first 50 entries
+  $ curl 'http://localhost:8080/catalog/v2/entries?count=50'
+
+  # Returns 50 entries starting from entry # 100 (i.e. entries ## 100-149)
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=50'
+
+  # Returns all entries
+  $ curl 'http://localhost:8080/catalog/v2/entries?count=-1'
+
+  # Returns all entries starting from entry # 100
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=-1'
+
+
+.. _library-filtering:
+
+**Filtering:**
+
+A filtered subset of the library can be requested by providing one or more
+filtering criteria, whereupon only entries matching *all* of the criteria are
+included in the response. Pagination is applied to the filtered list. The
+filtering criteria must be specified via the following URL parameters:
+
+* ``lang`` - filter by language (specified as a 3-letter language code).
+
+* ``category`` - filter by categories associated with the library entries.
+
+* ``tag`` - filter by tags associated with the library entries. Multiple tags
+  can be provided as a semicolon separated list (e.g
+  ``tag=wikipedia;_videos:no``). The result will contain only those entries
+  that contain *all* of the requested tags.
+
+* ``notag`` - filter out (exclude) entries with *any* of the specified tags
+  (example - ``notag=ted;youtube``).
+
+* ``maxsize`` - include in the results only entries whose size (in bytes)
+  doesn't exceed the provided value.
+
+* ``q`` - include in the results only entries that contain the specified text
+  in the title or description.
+
+* ``name`` - include in the results only entries with a matching
+  :term:`book name <Book name>`.
+
+
+Examples:
+
+.. code:: sh
+
+  # List only books in Italian (lang=ita) but
+  # return only results ## 100-149 (start=100&count=50)
+  $ curl 'http://localhost:8080/catalog/v2/entries?lang=ita&start=100&count=50'
+
+  # List only books with category of 'wikipedia' AND containing the word
+  # 'science' in the title or description. Return only the first 10 results.
+  $ curl 'http://localhost:8080/catalog/v2/entries?q=science&category=wikipedia'
+
+
+``/catalog/v2/entry/ZIMID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns full info about the library entry with :term:`UUID <ZIM UUID>`
+``ZIMID``.
+
+
+``/catalog/v2/illustration/ZIMID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+**Usage:**
+
+  ``/catalog/v2/illustration/ZIMID?size=N``
+
+Returns the illustration of size ``NxN`` pixels for the library entry with
+:term:`UUID <ZIM UUID>` ``ZIMID``.
+
+If no illustration of requested size is found a HTTP 404 error is returned.
+
+
+``/catalog/v2/languages``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full list of ZIM file languages as an `OPDS Navigation Feed
+<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
+
+
+``/catalog/v2/partial_entries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full or filtered list of ZIM files as an `OPDS acquisition feed
+<https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `partial entries
+<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
+
+Supported filters are the same as for the `/catalog/v2/entries`_ endpoint.
+
+
+.. _legacy-opds-api:
+
+``/catalog`` (Legacy OPDS API)
+------------------------------
+
+===== ===========
+Type: :ref:`deprecated <deprecation>`
+===== ===========
+
+The legacy OPDS API is preserved for backward compatibility and is deprecated.
+:ref:`New OPDS API <new-opds-api>` should be used instead.
+
+
+``/catalog/root.xml``
+^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Full library OPDS catalog (list of all ZIM files).
+
+
+``/catalog/searchdescription.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Describes the `/catalog/search`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+``/catalog/search``
+^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Returns the list of ZIM files (in OPDS catalog format) matching the
+search/filtering criteria. Supported filters are the same as for the
+`/catalog/v2/entries`_ endpoint.
+
+
+``/catch/external``
+-------------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/catch/external?source=URL``
+
+Generates a HTML page with a link leading to (the decoded version of) ``URL``
+and a warning that following that link will load an external (out of ZIM)
+resource.
+
+**Parameters:**
+
+  ``source``: URL of the external resource (must be URL-encoded).
+
+**Example:**
+
+.. code:: sh
+
+  # Intercept an external link to https://example.com?query=abcd
+  $ curl 'http://localhost:8080/catch/external?source=https%3A%2F%2Fexample.com%3Fquery%3Dabcd'
+
+
+
+``/content``
+------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+ZIM file content is served under the ``/content`` endpoint as described below.
+
+
+``/content/ZIMNAME/PATH/IN/ZIMFILE``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME``.
+
+
+``/content/ZIMNAME``
+^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+``/content/ZIMNAME`` redirects to the main page of the ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME`` (unless that ZIM file contains an entry with an empty
+path or path equal to ``/``, in which case that entry is returned).
+
+
+``/random``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/random?content=ZIMNAME``
+
+Generates a HTTP redirect to a randomly selected article/page from the
+specified ZIM file.
+
+**Parameters:**
+
+  ``content``: :term:`name of the ZIM file <ZIM name>`.
+
+
+.. _raw:
+
+``/raw``
+--------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+The ``/raw`` API provides access to ZIM file data. It consists of two separate
+endpoints for accessing data and metadata.
+
+
+``/raw/ZIMNAME/content/PATH/IN/ZIMFILE``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <raw>`
+===== ===========
+
+Returns the entry with path ``PATH/IN/ZIMFILE`` from the ZIM file with
+:term:`name <ZIM name>` ``ZIMNAME``. Currently, this endpoint almost duplicates
+(with some subtle technical differences) the newer endpoint
+`/content/ZIMNAME/PATH/IN/ZIMFILE`_. The important difference is that the
+``/raw`` endpoint guarantees that no server-side processing will be applied to
+the returned content, whereas content obtained via the ``/content`` endpoint
+may in the future undergo some processing intended to improve the operation of
+the viewer (e.g. compensating for certain bugs in ZIM creation). Also note that
+``/raw`` is :ref:`public <public-api-endpoint>`, whereas ``/content`` is
+:ref:`private <private-api-endpoint>`.
+
+
+``/raw/ZIMNAME/meta/METADATAID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <raw>`
+===== ===========
+
+Returns the metadata item ``METADATAID`` from the ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME``.
+
+
+``/search``
+-----------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+Performs a full text search on one or more ZIM files and returns an HTML page
+with a list of links to matching pages along with snippets of the matching
+portions of those pages.
+
+.. _multi-zim-search-constraints:
+
+A multi-ZIM search request must comply with the following constraints:
+
+* the number of ZIM files participating in the search operation must not exceed
+  the limit imposed by the :option:`--searchLimit` option of ``kiwix-serve``.
+
+* all of the ZIM files participating in the same search operation must be in
+  the same language.
+
+**Parameters:**
+
+
+  ZIM file selection parameters:
+
+    At least one of the following parameters must be provided in order to
+    specify in which ZIM file(s) to search. Parameters appearing earlier in
+    below list take precedence over subsequent ones (the later ones, even if
+    present in the request, are simply ignored).
+
+    ``content``: :term:`name of the ZIM file <ZIM name>` (for a single-ZIM
+    search). This is a :ref:`legacy parameter <deprecation>`. ``books.name``
+    should be used instead.
+
+    ``books.id``: :term:`UUID <ZIM UUID>` of the ZIM file. Can be repeated for
+    a multi-ZIM search, however must respect the :ref:`multi-ZIM search
+    constraints <multi-zim-search-constraints>`.
+
+      .. note::
+
+        If any of the provided ``books.id`` values refers to a book missing
+        from the library then an error is returned instead of running the
+        search on the remaining (valid) entries.
+
+    ``books.name``: :term:`name of the ZIM file <ZIM name>` (not to be confused
+    with ``books.filter.name`` which selects/filters based on the :term:`book
+    name <Book name>`). Can be repeated for a multi-ZIM search, however must
+    respect the :ref:`multi-ZIM search constraints
+    <multi-zim-search-constraints>`.
+
+      .. note::
+
+        If any of the provided ``books.name`` values refers to a book missing
+        from the library then an error is returned instead of running the
+        search on the remaining (valid) entries.
+
+    ``books.filter.{criteria}``: allows to take full advantage of :ref:`library
+    filtering <library-filtering>` functionality of the `/catalog/v2/entries`_
+    endpoint (``{criteria}`` must be replaced with an attribute/filtering
+    criteria name supported by :ref:`library filtering <library-filtering>`).
+
+  Query parameters:
+
+    ``pattern`` (optional; defaults to an empty string): text to search for.
+
+    ``latitude``, ``longitude`` & ``distance`` (optional): geospatial query
+    parameters. If all of these are provided, then the results will be
+    restricted to geotagged pages that are within ``distance`` metres from the
+    location on Earth with coordinates ``latitude`` and ``longitude``.
+
+  Pagination parameters:
+
+    ``pageLength`` (optional, default: 25): maximum number of search results in
+    the response. Capped at 140.
+
+    ``start`` (optional, default: 0): this parameter enables pagination of
+    results. The response will include up to ``pageLength`` results starting
+    with entry # ``start`` from the full list of search results (the first
+    result is assumed to have index 0).
+
+  Other parameters:
+
+    ``format`` (optional, default: html): format of the search results. Allowed
+    values are: html, xml.
+
+Examples:
+
+.. code:: sh
+
+  # Search for 'android' in the book with name 'scifi-library'
+  # Return results ## 51-60.
+  $ curl 'http://localhost:8080/search?pattern=android&books.name=scifi-library&start=51&pageLength=10'
+
+  # Search for 'napoli' in books in Italian
+  $ curl 'http://localhost:8080/search?books.filter.lang=ita&pattern=napoli'
+
+  # Search for 'chateau' in books in French that have a category of 'wikipedia'.
+  # Return the results as XML.
+  $ curl 'http://localhost:8080/search?pattern=chateau&books.filter.lang=fra&books.filter.category=wikipedia&format=xml'
+
+
+``/search/searchdescription.xml``
+---------------------------------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+Describes the `/search`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+
+``/skin``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Static front-end resources (such as CSS, javascript and images) are all grouped
+under ``/skin``.
+
+**Usage:**
+  ``/skin/PATH/TO/RESOURCE[?cacheid=CACHEID]``
+
+`Cache busting
+<https://javascript.plainenglish.io/what-is-cache-busting-55366b3ac022>`_ of
+static resources is supported via the optional param ``cacheid``. By default,
+i.e. when the ``cacheid`` parameter is not specified while accessing the
+``/skin`` endpoint, static resources are served as if they were dynamic (i.e.
+could be different for an immediately repeated request). Specifying the
+``cacheid`` parameter with a correct value (matching the value embedded in the
+``kiwix-serve`` instance), makes the returned resource to be presented as
+immutable. However, if the value of the ``cacheid`` parameter mismatches then
+``kiwix-serve`` responds with a 404 HTTP error.
+
+``kiwix-serve``'s default front-end (the :ref:`welcome page <welcome-page>` and
+the :ref:`ZIM file viewer <zim-file-viewer>`) access all underlying static
+resources by using explicit ``cacheid`` s.
+
+
+``/suggest``
+------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/suggest?content=ZIMNAME[&term=QUERY][&count=N][&start=S]``
+
+Returns suggestions (in JSON format) for a text string that is assumed to be a
+partially typed search for a page inside a particular ZIM file.
+
+Suggestions are obtained as matches of the query text against the page titles
+in the ZIM file using the title index database generated during the creation of
+the ZIM file.
+
+In case of a multi-word query the order of the words matters in two ways:
+
+1. the last word is considered as partially typed, unless followed by a space;
+2. ranking of the matches.
+
+If the ZIM file doesn't contain a title index then suggestions are generated by
+listing page titles starting *exactly* (in a case sensitive manner) with the
+query text. Otherwise, suggestions are case-insensitive.
+
+If the ZIM file contains a full text search index, then an extra suggestion is
+added as an option to perform a full text search in the said ZIM file.
+
+**Parameters:**
+
+  ``content`` (mandatory): :term:`name of the ZIM file <ZIM name>`.
+
+  ``term`` (optional; defaults to an empty string): query text.
+
+  ``count`` (optional, default: 10): maximum number of page suggestions in the
+  response (i.e. the extra option to perform a full text search is not included
+  in this count).
+
+  ``start`` (optional, default: 0): this parameter enables pagination of
+  results. The response will include up to ``count`` entries starting with
+  entry # ``start`` from the full list of page suggestions (the first result is
+  assumed to have index 0).
+
+**Example:**
+
+.. code:: sh
+
+  $ curl 'http://localhost/suggest?content=stackoverflow_en&term=pyth&count=50'
+
+
+.. _zim-file-viewer:
+
+``/viewer``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+ZIM file viewer. The ZIM file and entry therein must be specified via the hash
+component of the URL as ``/viewer#ZIMNAME/PATH/IN/ZIMFILE``.
+
+
+``/viewer_settings.js``
+-----------------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Settings of the ZIM file viewer that are configurable via certain command line
+options of ``kiwix-serve`` (e.g. ``--nolibrarybutton``).
+
+
+/ANYTHING/ELSE
+--------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Any other URL is considered as an attempt to access ZIM file content using the
+legacy URL scheme and is redirected to ``/content/ANYTHING/ELSE``.
+
+
+Glossary
+========
+
+.. glossary::
+
+  Book name
+
+    Name of the book as specified in the ZIM file metadata (for a
+    ``kiwix-serve`` started *WITHOUT* the :option:`--library` option) or the
+    library XML file (for a ``kiwix-serve`` started with the
+    :option:`--library` option).
+
+    .. note::
+
+      Two or more books may have the same name in the library. That's not
+      considered a conflict, because there may be multiple versions of the
+      "same" book (differing by the settings of the scraper, date, etc).
+      :ref:`Library filtering <library-filtering>` by name will return all
+      matching books.
+
+  ZIM filename
+
+    Name of a ZIM file on the server filesystem.
+
+  ZIM name
+
+    Identifier of a ZIM file in the server's library (used for referring to a
+    particular ZIM file in requests).
+
+    ZIM names are derived from the filenames as follows:
+
+    - file extension is removed,
+    - all characters are converted to lowercase,
+    - diacritics are removed,
+    - spaces are replaced with underscores,
+    - ``+`` symbols are replaced with the text ``plus``.
+
+    Presence of the :option:`-z`/:option:`--nodatealiases` option will create
+    additional names (aliases) for filenames with dates.
+
+    ZIM names are expected to be unique across the library. Any name conflicts
+    (including those caused by the usage of the
+    :option:`-z`/:option:`--nodatealiases` option) are reported on STDERR but,
+    otherwise, are ignored (i.e. only one of the entries can be accessed via
+    the conflicting name).
+
+  ZIM title
+
+    Title of a ZIM file. This can be any text (with whitespace). It is never
+    used as a way of referring to a ZIM file.
+
+  ZIM UUID
+
+    This is a unique identifier of a ZIM file designated at its creation time
+    and embedded in the ZIM file. Certain ``kiwix-serve`` operations may
+    require that a ZIM file be referenced through its UUID rather than name.

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+Sphinx==5.3.0
+sphinx-rtd-theme==1.1.1

format_code.sh

@@ -3,7 +3,6 @@
 files=(
 "src/installer/kiwix-install.cpp"
 "src/searcher/kiwix-search.cpp"
-"src/reader/kiwix-read.cpp"
 "src/manager/kiwix-manage.cpp"
 "src/server/kiwix-serve.cpp"
 )

meson.build

@@ -1,23 +1,30 @@
 project('kiwix-tools', 'cpp',
-  version : '3.0.2',
+  version : '3.8.1',
   license : 'GPL',
-  default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
+  default_options: ['c_std=c11', 'cpp_std=c++17', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
 add_global_arguments('-DKIWIX_TOOLS_VERSION="@0@"'.format(meson.project_version()), language : 'cpp')
 
+if host_machine.system() == 'windows'
+  add_project_arguments('-DNOMINMAX', language: 'cpp')
+endif
+
 static_linkage = get_option('static-linkage')
 if static_linkage
-  add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
+  # Static build is not supported on MacOS
+  if host_machine.system() != 'darwin'
+    add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
+  endif
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', version:'>=9.0.0', static:static_linkage)
-microhttpd_dep = dependency('libmicrohttpd', static:static_linkage)
-z_dep = dependency('zlib', static:static_linkage)
+libzim_dep = dependency('libzim', version:['>=9.0.0', '<10.0.0'], static:static_linkage)
+libkiwix_dep = dependency('libkiwix', version:['>=14.1.0', '<15.0.0'], static:static_linkage)
+libdocopt_dep = dependency('docopt', static:static_linkage)
 
-all_deps = [thread_dep, kiwixlib_dep, microhttpd_dep, z_dep]
+all_deps = [thread_dep, libkiwix_dep, libzim_dep, libdocopt_dep]
 
 if static_linkage
   librt = compiler.find_library('rt', required:false)
@@ -27,3 +34,6 @@ if static_linkage
 endif
 
 subdir('src')
+if get_option('doc')
+  subdir('docs')
+endif

meson_options.txt

@@ -1,2 +1,4 @@
 option('static-linkage', type : 'boolean', value : false,
   description : 'Create statically linked binaries.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')

src/man/kiwix-manage.1

@@ -1,62 +1,69 @@
-.TH KIWIX 1 "21 May 2012"
+.TH KIWIX-MANAGE 1 "21 May 2012"
+
 .SH NAME
 kiwix\-manage \- Kiwix Library Manager
+
 .SH SYNOPSIS
 .IX Header SYNOPSIS
-.B kiwix\-manage
-.br
-kiwix\-manage LIBRARY_PATH add ZIM_PATH ...
-.br
-	kiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ...
-.br
-	kiwix\-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]
-.SH DESCRIPTION
-.SS kiwix\-manage
-.PP
-\fBkiwix\-manage\fP tool for managing a kiwix library.
-.PP
-Allows one to manage the content of the kiwix library. The library file is a
-flat XML file listing ZIM files with all necessary information like favicon,
-date, creator, description, indexpath, filepath, title, source/metalink, etc.
-.
-.PP
- Example library file can be found at https://www.kiwix.org/library\.xml
-.br
 .TP
-\fBadd\fR
-Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP.
+\fBkiwix\-manage\fR LIBRARY_PATH \fBadd\fR ZIM_PATH ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBshow\fR [ZIM_ID_1] [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBremove\fR ZIM_ID_1 [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR --version
+.TP
+\fBkiwix\-manage\fR --help
+
+.SH DESCRIPTION
+.PP
+\fBkiwix\-manage\fP is a command line tool for manipulating a Kiwix XML library.
+.PP
+\fBkiwix\-manage\fP allows to manage the entries of the Kiwix
+library. The library file is a flat XML file listing ZIM files with
+all necessary information like id, favicon, date, creator,
+description, filepath, title, url, etc.
+
+.SH ACTIONS
 
 .TP
-\fBshow\fR
-Show given \fBCONTENT_ID\fP from \fBLIBRARY_FILE\fR. If no \fBCONTENT_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+\fBadd\fR
+Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP. Create the library file if necessary.
 
 .TP
 \fBremove\fR
-Remove the given \fBCONTENT_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBCONTENT_ID\fR should be specified.
+Remove the given \fBZIM_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBZIM_ID\fR should be specified.
 
 .TP
-\fB\-\-zimPathToSave=\fR
-Change the location reference of the ZIM file in the library.
+\fBshow\fR
+Show given \fBZIM_ID\fP from \fBLIBRARY_FILE\fR. If no \fBZIM_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+
+.SH OPTIONS
+.TP
+Options to be used with the action \fBadd\fR:
 
 .TP
-\fB\-\-current\fR
-Mark this content as the current (default) content in the library.
+\fB\-\-url=HTTP_URL\fR
+Set the ZIM online HTTP(S) URL
 
 .TP
-\fB\-\-backend=xapian|clucene\fR
-Select an indexing backend.
+\fB\-\-zimPathToSave=OTHER_FS_PATH\fR
+Set an arbitrary ZIM filesystem path (instead of the ZIM_PATH)
 
 .TP
-\fB\-\-indexPath=FULLTEXT_IDX_PATH\fR
-Path to full-text index for that ZIM file.
+Other options (to be used alone):
 
 .TP
-\fBurl\fR
-Set the content location of the ZIM file over the network for in\-kiwix download.
+\fB\-\-help | \-h\fR
+Display the kiwix-manage help
+
+.TP
+\fB\-\-version | \-v\fR
+Display the version of kiwix-manage and all dependences
 
 .SH SEE ALSO
-kiwix(1) kiwix\-serve(1)
-.SH AUTHOR
-Emmanuel Engelhart <kelson@kiwix.org>
-.br
-Vasudev Kamath <kamathvasudev@gmail.com> (Manual)
+kiwix\-serve(1)
+
+.SH AUTHORS
+Kiwix team <contact@kiwix.org>

src/man/kiwix-search.1

@@ -0,0 +1,21 @@
+.TH KIWIX-SEARCH "1" "July 2020" "kiwix-tools" "User Commands"
+.SH NAME
+kiwix-search \- find articles using a fulltext search pattern
+.SH SYNOPSIS
+\fBkiwix-search\fR [OPTIONS] ZIM PATTERN\fR
+.SH DESCRIPTION
+.TP
+ZIM
+ZIM file to search
+.TP
+PATTERN
+Words or parts of words to search for in the ZIM file
+.TP
+\fB\-s\fR, \fB\-\-suggestion\fR
+Suggest article titles based on the PATTERN instead of a fulltext search
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+print software version
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Give details about the search process

src/man/kiwix-serve.1

@@ -1,60 +1,142 @@
-.TH KIWIX 1 "21 May 2012"
+.TH KIWIX 1 "10 July 2023"
+
 .SH NAME
-kiwix\-serve \- Kiwix HTTP Server
+kiwix-serve \- Kiwix HTTP Server
+
 .SH SYNOPSIS
-.IX Header "SYNOPSIS"
+
+.B kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
 .br
-kiwix\-serve [\-\-index=INDEX_PATH] [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [\-\-attachToProcess=PID] ZIM_PATH
-.br
-kiwix\-serve \-\-library [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [\-\-attachToProcess=PID] LIBRARY_PATH
+.B kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
+
 .SH DESCRIPTION
-.PP
-Stand\-alone HTTP server for serving ZIM contents over the network.
+The \fBkiwix-serve\fR command is used to run a stand-alone HTTP server for serving ZIM contents over the network.
+
+.SH ARGUMENTS
+.TP
+\fBLIBRARY_FILE_PATH\fR
+Path of an XML library file listing ZIM files to serve. To be used only with the --library option. Multiple library files can be provided as a semicolon (;) separated list.
 
 .TP
-\fB\-\-index=INDEX_PATH\fR
-Path to index folder created using \fBkiwix-install\fB.
+\fBZIM_FILE_PATH ...\fR
+ZIM file path(s). Multiple arguments are allowed.
+
+.SH OPTIONS
+.TP
+\fB--library\fR
+By default, kiwix-serve expects a list of ZIM files as command line arguments. Providing the --library option tells kiwix-serve that the command line argument is rather a library XML file.
 
 .TP
-\fB\-\-port=PORT\fR
-Port number on which the server should listen.
+\fB-i ADDR, --address=ADDR\fR
+Listen only on this IP address. By default, the server listens on all available IP addresses. Alternatively, you can use special values to define which types of connections to accept:
+
+all : Listen for connections on all IP addresses (IPv4 and IPv6).
 .br
-By default, server listens on port 80.
-
-.TP
-\fB\-\-verbose\fR
-Enable verbose output.
-
-.TP
-\fB\-\-daemon\fR
-Run the server as a daemon.
-
-.TP
-\fB\-\-attachToProcess=PID\fR
-Shutdown the server when process numbered PID dies.
-
-.TP
-\fBZIM_PATH\fR
-ZIM file of the content to serve.
+ipv4 : Listen for connections on all IPv4 addresses.
 .br
-Mandatory if not in library mode.
+ipv6 : Listen for connections on all IPv6 addresses.
 
 .TP
-\fB\-\-library\fR
-Enable library mode.
-.br
-Serves the contents of a library file instead of a single ZIM file.
+\fB-p PORT, --port=PORT\fR
+TCP port on which to listen for HTTP requests (default: 80).
 
 .TP
-\fBLIBRARY_PATH\fR
-Kiwix library file path.
+\fB-r ROOT, --urlRootLocation=ROOT\fR
+URL prefix on which the content should be made available (default: empty).
+
+.TP
+\fB-d, --daemon\fR
+Detach the HTTP server daemon from the main process.
+
+.TP
+\fB-a PID, --attachToProcess=PID\fR
+Exit when the process with id PID stops running.
+
+.TP
+\fB-M, --monitorLibrary\fR
+Monitor the XML library file and reload it automatically when it changes.
+
+Library reloading can be forced anytime by sending a SIGHUP signal to the
+\*(lqkiwix-serve\*(rq process (this works regardless of the presence of the
+\*(lq--monitorLibrary\*(rq/\*(lq-M\*(rq option).
+
+.TP
+\fB-m, --nolibrarybutton\fR
+Disable the library home button in the ZIM viewer toolbar.
+
+.TP
+\fB-n, --nosearchbar\fR
+Disable the search box in the ZIM viewer toolbar.
+
+.TP
+\fB-b, --blockexternal\fR
+Prevent users from directly navigating to external resources via links in ZIM content.
+
+.TP
+\fB-t N, --threads=N\fR
+Number of threads to run in parallel (default: 4).
+
+.TP
+\fB-s N, --searchLimit=N\fR
+Maximum number of ZIM files in a fulltext multizim search (default: No limit).
+
+.TP
+\fB-z, --nodatealiases\fR
+Create URL aliases for each content by removing the date embedded in the file name.
+
+The expected format of the date in the filename is \*(lq_YYYY-MM\*(rq. For example, a ZIM file named \*(lqwikipedia_en_all_2020-08.zim\*(rq will be accessible both as \*(lqwikipedia_en_all_2020-08\*(rq and \*(lqwikipedia_en_all\*(rq.
+
+.TP
+\fB-c PATH, --customIndex=PATH\fR
+Override the welcome page with a custom HTML file.
+
+.TP
+\fB-L N, --ipConnectionLimit=N\fR
+Max number of (concurrent) connections per IP (default: infinite, recommended: >= 6).
+
+.TP
+\fB-k, --skipInvalid\fR
+Startup even when ZIM files are invalid (those will be skipped)
+
+.TP
+\fB-v, --verbose\fR
+Print debug log to STDOUT.
+
+.TP
+\fB-V, --version\fR
+Print the software version.
+
+.TP
+\fB-h, --help\fR
+Print a help message.
+
+.SH EXAMPLES
+Serve a single ZIM file:
+.sp
+.nf
+.B kiwix-serve myzim.zim
+.fi
+
+Serve multiple ZIM files:
+.sp
+.nf
+.B kiwix-serve zim1.zim zim2.zim zim3.zim
+.fi
+
+Serve ZIM files from a library:
+.sp
+.nf
+.B kiwix-serve --library library.xml
+.fi
+
+.SH DOCUMENTATION
+Online documentation: https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html
 .br
-Library is an XML file created using \fBkiwix-manage\fB.
+Source code: https://github.com/kiwix/kiwix-tools
+.br
+More info: https://wiki.kiwix.org/wiki/Kiwix-serve
 
-.SH SEE ALSO
-kiwix(1) kiwix\-manage(1)
-
-.SH AUTHOR
+.SH AUTHORS
 Emmanuel Engelhart <kelson@kiwix.org>
 .br
-Vasudev Kamath <kamathvasudev@gmail.com> (Manual)
+Vasudev Kamath <kamathvasudev@gmail.com>

src/man/meson.build

@@ -1,3 +1,4 @@
 install_man('kiwix-manage.1',
+            'kiwix-search.1',
             'kiwix-serve.1')
 subdir('fr')

src/manager/kiwix-manage.cpp

@@ -17,9 +17,9 @@
  * MA 02110-1301, USA.
  */
 
-#include <getopt.h>
-#include <kiwix/tools/stringTools.h>
+#include <docopt/docopt.h>
 #include <kiwix/manager.h>
+#include <kiwix/tools.h>
 #include <cstdlib>
 #include <iostream>
 
@@ -29,10 +29,10 @@ using namespace std;
 
 enum supportedAction { NONE, ADD, SHOW, REMOVE };
 
-void show(kiwix::Library* library, const std::string& bookId)
+void show(const kiwix::Library& library, const std::string& bookId)
 {
   try {
-    auto& book = library->getBookById(bookId);
+    auto& book = library.getBookById(bookId);
     std::cout << "id:\t\t" << book.getId() << std::endl
               << "path:\t\t" << book.getPath() << std::endl
               << "url:\t\t" << book.getUrl() << std::endl
@@ -51,161 +51,106 @@ void show(kiwix::Library* library, const std::string& bookId)
   std::cout << std::endl;
 }
 
+// Older version of docopt doesn't declare Options. Let's declare it ourself.
+using Options = std::map<std::string, docopt::value>;
+
+
 /* Print correct console usage options */
-void usage()
+static const char USAGE[] =
+R"(Manipulates the Kiwix library XML file
+
+Usage:
+ kiwix-manage LIBRARYPATH add [--zimPathToSave=<custom_zim_path>] [--url=<http_zim_url>] ZIMPATH ...
+ kiwix-manage LIBRARYPATH (delete|remove) ZIMID ...
+ kiwix-manage LIBRARYPATH show [ZIMID ...]
+ kiwix-manage -v | --version
+ kiwix-manage -h | --help
+
+Arguments:
+  LIBRARYPATH    The XML library file path.
+  ZIMID          ZIM file unique ID.
+  ZIMPATH        A path to a ZIM to add.
+
+Options:
+  Custom options for "add" action:
+    --zimPathToSave=<custom_zim_path>  Replace the current ZIM file path
+    --url=<http_zim_url>               Create an "url" attribute for the online version of the ZIM file
+
+  Other options:
+    -h --help                          Print this help
+    -v --version                       Print the software version
+
+Examples:
+ Add ZIM files to library:       kiwix-manage my_library.xml add first.zim second.zim
+ Remove ZIM files from library:  kiwix-manage my_library.xml remove e5c2c003-b49e-2756-5176-5d9c86393dd9
+ Show all library ZIM files:     kiwix-manage my_library.xml show
+
+Documentation:
+  Source code  https://github.com/kiwix/kiwix-tools
+  More info    https://wiki.kiwix.org/wiki/kiwix-manage
+)";
+
+int handle_show(const kiwix::Library& library, const std::string& libraryPath,
+                 const Options& options)
 {
-  std::cout << "Usage:" << std::endl
-            << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH [OPTIONS]" << std::endl
-            << "\tkiwix-manage LIBRARY_PATH remove ZIM_ID [ZIM_ID]..." << std::endl
-            << "\tkiwix-manage LIBRARY_PATH show [ZIM_ID]..." << std::endl
-            << std::endl
-
-            << "Purpose:" << std::endl
-            << "\tManipulates the Kiwix library XML file"
-            << std::endl << std::endl
-
-            << "Arguments:" << std::endl
-            << "\tLIBRARY_PATH\tis the XML library file path."
-            << std::endl << std::endl
-            << "\tACTION\t\tis the pre-defined string to specify the action to run on the XML library file."
-            << std::endl << std::endl
-            << "\t\t\tMust be one of the following values:" << std::endl
-            << "\t\t\t* add: add a ZIM file to the library" << std::endl
-            << "\t\t\t* remove: remove a ZIM file from the library" << std::endl
-            << "\t\t\t* show: show the content of the library"
-            << std::endl << std::endl
-            << "\tZIM_ID\t\tZIM file unique ID"
-            << std::endl << std::endl
-            << "\tOPTIONS\t\tCustom options for \"add\" action:" << std::endl
-            << "\t\t\t--zimPathToSave=CUSTOM_ZIM_PATH to replace the current ZIM file path" << std::endl
-            << "\t\t\t--url=HTTP_ZIM_URL to create an \"url\" attribute for the online version of the ZIM file" << std::endl
-            << std::endl
-            << "\t\t\tOther options:" << std::endl
-            << "\t\t\t-v, --version to print the software version" << std::endl
-            << std::endl
-
-            << "Examples:" << std::endl
-            << "\tAdd ZIM files to library: kiwix-manage my_library.xml add first.zim second.zim" << std::endl
-            << "\tRemove ZIM files from library: kiwix-manage my_library.xml remove e5c2c003-b49e-2756-5176-5d9c86393dd9" << std::endl
-            << "\tShow all library ZIM files: kiwix-manage my_library.xml show" << std::endl
-            << std::endl
-
-            << "Documentation:" << std::endl
-            << "\tSource code\thttps://github.com/kiwix/kiwix-tools" << std::endl
-            << "\tMore info\thttps://wiki.kiwix.org/wiki/Kiwix-manage" << std::endl
-            << std::endl;
-}
-
-int handle_show(kiwix::Library* library, const std::string& libraryPath,
-                 int argc, char* argv[])
-{
-  if (argc > 3 ) {
-    for(auto i=3; i<argc; i++) {
-       std::string bookId = argv[i];
-       show(library, bookId);
-    }
-  } else {
-    auto booksIds = library->getBooksIds();
+  if (options.at("ZIMID").asStringList().empty()) {
+    auto booksIds = library.getBooksIds();
     for(auto& bookId: booksIds) {
       show(library, bookId);
     }
+  } else {
+    auto bookIds = options.at("ZIMID").asStringList();
+    for(auto& bookId: bookIds) {
+       show(library, bookId);
+    }
   }
+
   return(0);
 }
 
-int handle_add(kiwix::Library* library, const std::string& libraryPath,
-                int argc, char* argv[])
+int handle_add(kiwix::LibraryPtr library, const std::string& libraryPath,
+                const Options& options)
 {
-  string zimPath;
-  string zimPathToSave = ".";
+  string zimPathToSave;
   string url;
-  string origID = "";
-  int option_index = 0;
-  int c = 0;
-  int resultCode = 0;
-
-  if (argc <= 3) {
-    std::cerr << "Path to zim file to add is missing in the command line" << std::endl;
-    return (-1);
-  }
-
-  /* Options parsing */
-  optind = 3;
-  static struct option long_options[] = {
-    {"url", required_argument, 0, 'u'},
-    {"origId", required_argument, 0, 'o'},
-    {"zimPathToSave", required_argument, 0, 'z'},
-    {0, 0, 0, 0}
-  };
-
-  bool has_option = false;
-  while (true) {
-    c = getopt_long(argc, argv, "cz:u:", long_options, &option_index);
-    if (c == -1)
-      break;
-
-    has_option = true;
-    switch (c) {
-      case 'u':
-        url = optarg;
-        break;
-      case 'o':
-        origID = optarg;
-        break;
-      case 'z':
-        zimPathToSave = optarg;
-        break;
-    }
-  }
-
-  if (optind >= argc) {
-    std::cerr << "Path to zim file to add is missing in the command line" << std::endl;
-    return (-1);
-  }
-
-  if (has_option && argc-optind > 1) {
-    std::cerr << "You cannot give option and several zim files to add" << std::endl;
-    return (-1);
-  }
 
   kiwix::Manager manager(library);
 
-  for(auto i=optind; i<argc; i++) {
-    std::string zimPath = argv[i];
-    if (!zimPath.empty()) {
-      auto _zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false);
+  auto zimPaths = options.at("ZIMPATH").asStringList();
+  for (auto& zimPath: zimPaths) {
+    if (options.at("--zimPathToSave").isString()) {
+      zimPathToSave = options.at("--zimPathToSave").asString();
     } else {
-      std::cerr << "Invalid zim file path" << std::endl;
-      resultCode = 1;
+      zimPathToSave = zimPath;
+    }
+    if (options.at("--url").isString()) {
+      url = options.at("--url").asString();
+    }
+
+    if (manager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false).empty()) {
+      std::cerr << "Cannot add ZIM " << zimPath << " to the library." << std::endl;
+      return 1;
     }
   }
 
-  return(resultCode);
+  return 0;
 }
 
-int handle_remove(kiwix::Library* library, const std::string& libraryPath,
-                   int argc, char* argv[])
+int handle_remove(kiwix::Library& library, const std::string& libraryPath,
+                   const Options& options)
 {
-  std::string bookId;
-  const unsigned int totalBookCount = library->getBookCount(true, true);
+  const unsigned int totalBookCount = library.getBookCount(true, true);
   int exitCode = 0;
 
-  if (argc <= 3) {
-    std::cerr << "BookId to remove missing in the command line" << std::endl;
-    return (-1);
-  }
-
   if (!totalBookCount) {
     std::cerr << "Library is empty, no book to delete."
               << std::endl;
     return 1;
   }
 
-  for (int i = 3; i<argc; i++) {
-    bookId = argv[i];
-
-    if (!library->removeBookById(bookId)) {
+  auto bookIds = options.at("ZIMID").asStringList();
+  for (auto& bookId: bookIds) {
+    if (!library.removeBookById(bookId)) {
       std::cerr << "Invalid book id '" << bookId << "'." << std::endl;
       exitCode = 1;
     }
@@ -216,76 +161,77 @@ int handle_remove(kiwix::Library* library, const std::string& libraryPath,
 
 int main(int argc, char** argv)
 {
-  string libraryPath = "";
   supportedAction action = NONE;
-  kiwix::Library library;
+  auto library = kiwix::Library::create();
 
-  /* General argument parsing */
-  static struct option long_options[] = {
-    {"version", no_argument, 0, 'v'},
-    {0, 0, 0, 0}
-  };
-
-  int option_index = 0;
-  int c;
-  while (true) {
-    c = getopt_long(argc, argv, "v", long_options, &option_index);
-    if (c == -1)
-      break;
-
-    switch (c) {
-      case 'v':
-        version();
-        return 0;
-    }
+  Options args;
+  try {
+    args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
+  } catch (docopt::DocoptArgumentError const & error ) {
+    std::cerr << error.what() << std::endl;
+    std::cerr << USAGE << std::endl;
+    return -1;
   }
 
-  /* Action related argument parsing */
-  if (argc > 2) {
-    libraryPath = argv[1];
-    string actionString = argv[2];
-
-    if (actionString == "add")
-      action = ADD;
-    else if (actionString == "show")
-      action = SHOW;
-    else if (actionString == "remove" || actionString == "delete")
-      action = REMOVE;
+  if (args["--help"].asBool()) {
+    std::cout << USAGE << std::endl;
+    return 0;
   }
 
-  /* Print usage)) if necessary */
-  if (libraryPath == "" || action == NONE) {
-    usage();
-    exit(1);
+  if (args["--version"].asBool()) {
+    version();
+    return 0;
   }
 
+  std::string libraryPath = args.at("LIBRARYPATH").asString();
+
+  if (args.at("add").asBool())
+    action = ADD;
+  else if (args.at("show").asBool())
+    action = SHOW;
+  else if (args.at("remove").asBool() || args.at("delete").asBool())
+    action = REMOVE;
+
   /* Try to read the file */
-  libraryPath = isRelativePath(libraryPath)
-                    ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
+  libraryPath = kiwix::isRelativePath(libraryPath)
+                    ? kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryPath)
                     : libraryPath;
-  kiwix::Manager manager(&library);
-  manager.readFile(libraryPath, false);
+  kiwix::Manager manager(library);
+  if (!manager.readFile(libraryPath, false)) {
+    if (kiwix::fileExists(libraryPath) || action!=ADD) {
+      std::cerr << "Cannot read the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
 
   /* SHOW */
   int exitCode = 0;
   switch (action) {
     case SHOW:
-      exitCode = handle_show(&library, libraryPath, argc, argv);
+      exitCode = handle_show(*library, libraryPath, args);
       break;
     case ADD:
-      exitCode = handle_add(&library, libraryPath, argc, argv);
+      exitCode = handle_add(library, libraryPath, args);
       break;
     case REMOVE:
-      exitCode = handle_remove(&library, libraryPath, argc, argv);
+      exitCode = handle_remove(*library, libraryPath, args);
       break;
     case NONE:
       break;
   }
 
-  /* Rewrite the library file */
-  if (action == REMOVE || action == ADD) {
-    library.writeToFile(libraryPath);
+  if (exitCode) {
+    return exitCode;
   }
 
-  exit(exitCode);
+  /* Rewrite the library file */
+  if (action == REMOVE || action == ADD) {
+    // writeToFile return true (1) if everything is ok => exitCode is 0
+    if (!library->writeToFile(libraryPath)) {
+      std::cerr << "Cannot write the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
 }

src/meson.build

@@ -1,6 +1,4 @@
-
 subdir('manager')
-subdir('reader')
 subdir('searcher')
 subdir('server')
 subdir('man')

src/reader/kiwix-read.cpp

@@ -1,107 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <getopt.h>
-#include <kiwix/reader.h>
-#include <map>
-#include <string>
-
-#include "../version.h"
-
-void usage()
-{
-  cout << "Usage: kiwix-read [--verbose] [--version] --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
-  exit(1);
-}
-
-int main(int argc, char** argv)
-{
-  /* Init the variables */
-  const char* filePath = NULL;
-  const char* pattern = NULL;
-  int option_index = 0;
-  int c = 0;
-
-  kiwix::Reader* reader = NULL;
-
-  /* Argument parsing */
-  while (42) {
-    static struct option long_options[]
-        = {{"verbose", no_argument, 0, 'v'},
-           {"version", no_argument, 0, 'V'},
-           {"suggest", required_argument, 0, 's'},
-           {0, 0, 0, 0}};
-
-    if (c != -1) {
-      c = getopt_long(argc, argv, "Vvs:", long_options, &option_index);
-
-      switch (c) {
-        case 'v':
-          break;
-        case 'V':
-          version();
-          return 0;
-        case 's':
-          pattern = optarg;
-          break;
-      }
-    } else {
-      if (optind < argc) {
-        if (filePath == NULL) {
-          filePath = argv[optind++];
-        } else {
-          usage();
-        }
-      } else {
-        break;
-      }
-    }
-  }
-
-  /* Check if we have enough arguments */
-  if (filePath == NULL) {
-    usage();
-  }
-
-  /* Instanciate the reader */
-  reader = new kiwix::Reader(filePath);
-
-  /* Start to read an article */
-  if (reader != NULL) {
-    string content;
-    string contentType;
-    string suggestion;
-
-    if (pattern != NULL) {
-      std::cout << "Searching suggestions for: " << pattern << std::endl;
-
-      reader->searchSuggestionsSmart(pattern, 10);
-      while (reader->getNextSuggestion(suggestion)) {
-        std::cout << suggestion << std::endl;
-      }
-    }
-
-    delete reader;
-  } else {
-    cerr << "Unable instanciate the Kiwix reader." << endl;
-    exit(1);
-  }
-
-  exit(0);
-}

src/reader/meson.build

@@ -1,3 +0,0 @@
-executable('kiwix-read', ['kiwix-read.cpp'],
-  dependencies:all_deps,
-  install:true)

src/searcher/kiwix-search.cpp

@@ -17,119 +17,106 @@
  * MA 02110-1301, USA.
  */
 
-#include <getopt.h>
-#include <kiwix/reader.h>
-#include <kiwix/searcher.h>
+#include <docopt/docopt.h>
+
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
+#include <kiwix/spelling_correction.h>
+#include <xapian.h>
+
+#include <iostream>
+#include <filesystem>
 
 #include "../version.h"
 
-void usage()
+using namespace std;
+
+
+// Older version of docopt doesn't declare Options. Let's declare it ourself.
+using Options = std::map<std::string, docopt::value>;
+
+static const char USAGE[] =
+R"(Find articles based on a fulltext search pattern.
+
+Usage:
+  kiwix-search [options] ZIM PATTERN
+  kiwix-search -h | --help
+  kiwix-search -V | --version
+
+Arguments:
+  ZIM       The full path of the ZIM file
+  PATTERN   Word(s) - or part of - to search in the ZIM.
+
+Options:
+  -s --suggestion    Suggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution
+  --spelling         Suggest article titles based on the spelling corrected PATTERN instead of making a fulltext search.
+  -v --verbose       Give details about the search process
+  -V --version       Print software version
+  -h --help          Print this help
+)";
+
+std::filesystem::path getKiwixCachedDataDirPath()
 {
-  cout << "Usage: kiwix-search [OPTIONS] ZIM PATTERN" << endl << endl
-       << "  kiwix-search allows to find articles based on the a fulltext search pattern." << endl << endl
-       << "  ZIM is the full path of the ZIM file." << endl
-       << "  PATTERN is/are word(s) - or part of - to search in the ZIM." << endl << endl
-       << "  -s, --suggestion\tSuggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution." << endl
-       << "  -v, --verbose\t\tGive details about the search process" << endl
-       << "  -V, --version\t\tPrint software version" << endl;
-  exit(1);
+  std::filesystem::path home(getenv("HOME"));
+  std::filesystem::path cacheDirPath = home / ".cache" / "kiwix";
+  std::filesystem::create_directories(cacheDirPath);
+  return cacheDirPath;
 }
 
 int main(int argc, char** argv)
 {
-  /* Init the variables */
-  // const char *indexPath =
-  // "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
-  // const char *indexPath = "/home/itamar/testindex";
-  const char* zimPath = NULL;
-  const char* search = NULL;
-  bool verboseFlag = false;
-  bool suggestionFlag = false;
-  int option_index = 0;
-  int c = 0;
-
-  kiwix::Searcher* searcher = NULL;
-  kiwix::Reader* reader = NULL;
-
-  /* Argument parsing */
-  while (42) {
-    static struct option long_options[]
-        = {{"verbose", no_argument, 0, 'v'},
-           {"suggestion", no_argument, 0, 's'},
-           {"version", no_argument, 0, 'V'},
-           {0, 0, 0, 0}};
-
-    if (c != -1) {
-      c = getopt_long(argc, argv, "Vvsb:", long_options, &option_index);
-
-      switch (c) {
-        case 'v':
-          verboseFlag = true;
-          break;
-        case 'V':
-          version();
-          return 0;
-        case 's':
-          suggestionFlag = true;
-          break;
-      }
-    } else {
-      if (optind < argc) {
-        if (zimPath == NULL) {
-          zimPath = argv[optind++];
-        } else if (search == NULL) {
-          search = argv[optind++];
-        } else {
-          cout << zimPath << endl;
-          usage();
-        }
-      } else {
-        break;
-      }
-    }
+  Options args;
+  try {
+    args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
+  } catch (docopt::DocoptArgumentError const & error ) {
+    std::cerr << error.what() << std::endl;
+    std::cerr << USAGE << std::endl;
+    return -1;
   }
 
-  /* Check if we have enough arguments */
-  if (zimPath == NULL || search == NULL) {
-    usage();
+  if (args.at("--help").asBool()) {
+    std::cout << USAGE << std::endl;
+    return 0;
   }
 
+  if (args.at("--version").asBool()) {
+    version();
+    return 0;
+  }
+
+  auto zimPath = args.at("ZIM").asString();
+  auto pattern = args.at("PATTERN").asString();
+  auto verboseFlag = args.at("--verbose").asBool();
+
   /* Try to prepare the indexing */
   try {
-    reader = new kiwix::Reader(zimPath);
-  } catch (...) {
-    /* Cannot open the zimPath, maybe it is a plain old xapian database
-     * directory */
-  }
+    zim::Archive archive(zimPath);
 
-  if (reader) {
-    searcher = new kiwix::Searcher();
-    searcher->add_reader(reader);
-  } else {
-    cerr << "Unable to search through zim '" << zimPath << "'." << endl;
+    if (args.at("--suggestion").asBool()) {
+      zim::SuggestionSearcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      for (const auto& r:searcher.suggest(pattern).getResults(0, 10)) {
+        cout << r.getTitle() << endl;
+      }
+    }  else if (args.at("--spelling").asBool()) {
+      kiwix::SpellingsDB spellingsDB(archive, getKiwixCachedDataDirPath());
+      for (const auto& r:spellingsDB.getSpellingCorrections(pattern, 1)) {
+        cout << r << endl;
+      }
+    } else {
+      zim::Searcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      const zim::Query query(pattern);
+      for (const auto& r : searcher.search(query).getResults(0, 10) ) {
+        cout << r.getTitle() << endl;
+      }
+    }
+  } catch ( const std::runtime_error& err)  {
+    cerr << err.what() << endl;
     exit(1);
-  }
-
-  /* Start the indexing */
-  if (searcher != NULL) {
-    string searchString(search);
-    if (suggestionFlag) {
-        searcher->suggestions(searchString, verboseFlag);
-    } else  {
-        searcher->search(searchString, 0, 10, verboseFlag);
-    }
-    kiwix::Result* p_result;
-    while ((p_result = searcher->getNextResult())) {
-      cout << p_result->get_title() << endl;
-      delete p_result;
-    }
-
-    delete searcher;
-    delete reader;
-
-    //      kiwix::XapianSearcher::terminate();
-  } else {
-    cerr << "Unable instanciate the Kiwix searcher." << endl;
+  } catch ( const Xapian::Error& err)  {
+    cerr << err.get_msg() << endl;
     exit(1);
   }
 

src/server/kiwix-serve.cpp

@@ -1,3 +1,4 @@
+
 /*
  * Copyright 2009-2019 Emmanuel Engelhart <kelson@kiwix.org>
  *
@@ -17,15 +18,19 @@
  * MA 02110-1301, USA.
  */
 
-#include <getopt.h>
+#include <docopt/docopt.h>
 #include <kiwix/manager.h>
 #include <kiwix/server.h>
 #include <kiwix/name_mapper.h>
-#include <kiwix/tools/otherTools.h>
+#include <kiwix/tools.h>
 
-#ifndef _WIN32
+#ifdef _WIN32
+# include <windows.h>
+#else
 # include <unistd.h>
+# include <signal.h>
 #endif
+#include <sys/stat.h>
 
 #ifdef __APPLE__
 # import <sys/sysctl.h>
@@ -36,191 +41,297 @@
 #include "../version.h"
 
 #define DEFAULT_THREADS 4
+#define LITERAL_AS_STR(A) #A
+#define AS_STR(A) LITERAL_AS_STR(A)
 
-void usage()
-{
-  std::cout << "Usage:" << std::endl
-            << "\tkiwix-serve [OPTIONS] ZIM_PATH+" << std::endl
-            << "\tkiwix-serve --library [OPTIONS] LIBRARY_PATH" << std::endl
-            << std::endl
 
-            << "Purpose:" << std::endl
-            << "\tDeliver ZIM file articles via HTTP"
-            << std::endl << std::endl
+static const char USAGE[] =
+R"(Deliver ZIM file(s) articles via HTTP
 
-            << "Mandatory arguments:" << std::endl
-            << "\tLIBRARY_PATH\t\tis the XML library file path listing ZIM file to serve. To be used only with the --library argument."
-            << std::endl
-            << "\tZIM_PATH\t\tis the path of a ZIM file."
-            << std::endl << std::endl
+Usage:
+ kiwix-serve [options] ZIMPATH ...
+ kiwix-serve [options] (-l | --library) LIBRARYPATH
+ kiwix-serve -h | --help
+ kiwix-serve -V | --version
 
-            << "Optional arguments:" << std::endl
-            << "\t-a, --attachToProcess\texit if given process id is not running anymore" << std::endl
-            << "\t-d, --daemon\t\tdetach the HTTP server daemon from the main process" << std::endl
-            << "\t-i, --address\t\tlisten only on this ip address, all available ones otherwise" << std::endl
-            << "\t-m, --nolibrarybutton\tdo not print the builtin home button in the builtin top bar overlay" << std::endl
-            << "\t-n, --nosearchbar\tdo not print the builtin bar overlay on the top of each served page" << std::endl
-            << "\t-p, --port\t\tTCP port on which to listen to HTTP requests (default: 80)" << std::endl
-            << "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
-            << "\t-t, --threads\t\tnumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
-            << "\t-v, --verbose\t\tprint debug log to STDOUT" << std::endl
-            << "\t-V, --version\t\tprint software version" << std::endl
-            << "\t-z, --nodatealiases\tcreate URL aliases for each content by removing the date" << std::endl
-            << "\t--donottrustlibrary\tRead the metadata from the zim file instead of trusting the library." << std::endl
-            << std::endl
+Mandatory arguments:
+  LIBRARYPATH  XML library file path listing ZIM file to serve. To be used only with the --library argument."
+  ZIMPATH      ZIM file path(s)
 
-            << "Documentation:" << std::endl
-            << "\tSource code\t\thttps://github.com/kiwix/kiwix-tools" << std::endl
-            << "\tMore info\t\thttps://wiki.kiwix.org/wiki/Kiwix-serve" << std::endl
-            << std::endl;
+Options:
+ -h --help                               Print this help
+ -a <pid> --attachToProcess=<pid>        Exit if given process id is not running anymore [default: 0]
+ --catalogOnly                           Serve only the library catalog
+ --contentServerURL=<url>                Root URL of the server serving ZIM content for this library
+ -d --daemon                             Detach the HTTP server daemon from the main process
+ -i <address> --address=<address>        Listen only on the specified IP address. Specify 'ipv4', 'ipv6' or 'all' to listen on all IPv4, IPv6 or both types of addresses, respectively [default: all]
+ -M --monitorLibrary                     Monitor the XML library file and reload it automatically
+ -m --nolibrarybutton                    Don't print the builtin home button in the builtin top bar overlay
+ -n --nosearchbar                        Don't print the builtin bar overlay on the top of each served page
+ -b --blockexternal                      Prevent users from directly accessing external links
+ -p <port> --port=<port>                 Port on which to listen to HTTP requests [default: 80]
+ -r <root> --urlRootLocation=<root>      URL prefix on which the content should be made available [default: /]
+ -s <limit> --searchLimit=<limit>        Maximun number of zim in a fulltext multizim search [default: 0]
+ -t <threads> --threads=<threads>        Number of threads to run in parallel [default: )" AS_STR(DEFAULT_THREADS) R"(]
+ -v --verbose                            Print debug log to STDOUT
+ -V --version                            Print software version
+ -z --nodatealiases                      Create URL aliases for each content by removing the date
+ -c <path> --customIndex=<path>          Add path to custom index.html for welcome page
+ -L <limit> --ipConnectionLimit=<limit>  Max number of (concurrent) connections per IP [default: 0] (recommended: >= 6)
+ -k --skipInvalid                        Startup even when ZIM files are invalid (those will be skipped)
+
+Documentation:
+  Source code   https://github.com/kiwix/kiwix-tools
+  More info     https://wiki.kiwix.org/wiki/Kiwix-serve
+                https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html
+)";
+
+std::string loadCustomTemplate (std::string customIndexPath) {
+  customIndexPath = kiwix::isRelativePath(customIndexPath) ?
+                      kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), customIndexPath) :
+                      customIndexPath;
+  if (!kiwix::fileReadable(customIndexPath)) {
+    throw std::runtime_error("No such file exist (or file is not readable) " + customIndexPath);
+  }
+  if (kiwix::getMimeTypeForFile(customIndexPath) != "text/html") {
+    throw std::runtime_error("Invalid File Mime Type " + kiwix::getMimeTypeForFile(customIndexPath));
+  }
+  std::string indexTemplateString = kiwix::getFileContent(customIndexPath);
+
+  if (indexTemplateString.empty()) {
+    throw std::runtime_error("Unreadable or empty file " + customIndexPath);
+  }
+  return indexTemplateString;
 }
 
+#ifndef _WIN32
+volatile sig_atomic_t waiting = false;
+volatile sig_atomic_t libraryMustBeReloaded = false;
+void handle_sigterm(int signum)
+{
+    if ( waiting == false ) {
+        _exit(signum);
+    }
+    waiting = false;
+}
+
+void handle_sighup(int signum)
+{
+  libraryMustBeReloaded = true;
+}
+
+typedef void (*SignalHandler)(int);
+
+void set_signal_handler(int sig, SignalHandler handler)
+{
+    struct sigaction sa;
+    sigaction(sig, NULL, &sa);
+    sa.sa_handler = handler;
+    sigaction(sig, &sa, NULL);
+}
+
+void setup_sighandlers()
+{
+    set_signal_handler(SIGTERM, &handle_sigterm);
+    set_signal_handler(SIGINT,  &handle_sigterm);
+    set_signal_handler(SIGHUP,  &handle_sighup);
+}
+#else
+bool waiting = false;
+bool libraryMustBeReloaded = false;
+#endif
+
+uint64_t fileModificationTime(const std::string& path)
+{
+#if defined(_WIN32) && !defined(stat)
+#define stat _stat
+#endif
+  struct stat fileStatData;
+  if ( stat(path.c_str(), &fileStatData) == 0 ) {
+    return fileStatData.st_mtime;
+  }
+  return 0;
+#ifdef _WIN32
+#undef stat
+#endif
+}
+
+uint64_t newestFileTimestamp(const std::vector<std::string>& paths)
+{
+  uint64_t t = 0;
+  for ( const auto& p : paths ) {
+    t = std::max(t, fileModificationTime(p));
+  }
+
+  return t;
+}
+
+bool reloadLibrary(kiwix::Manager& mgr, const std::vector<std::string>& paths)
+{
+    try {
+      std::cout << "Loading the library from the following files:\n";
+      for ( const auto& p : paths ) {
+        std::cout << "\t" << p << std::endl;
+      }
+      mgr.reload(paths);
+      std::cout << "The library was successfully loaded." << std::endl;
+      return true;
+    } catch ( const std::runtime_error& err ) {
+      std::cerr << "ERROR: " << err.what() << std::endl;
+      std::cerr << "Errors encountered while loading the library." << std::endl;
+      return false;
+    }
+}
+
+// docopt::value::isLong() is counting repeated values.
+// It doesn't check if the string can be parsed as long.
+// (Contrarly to `asLong` which will try to convert string to long)
+// See https://github.com/docopt/docopt.cpp/issues/62
+// `isLong` is a small helper to get if the value can be parsed as long.
+inline bool isLong(const docopt::value& v) {
+  try {
+    v.asLong();
+    return true;
+  } catch (...) {
+    return false;
+  }
+}
+
+#define FLAG(NAME, VAR) if (arg.first == NAME) { VAR = arg.second.asBool(); continue; }
+#define STRING(NAME, VAR) if (arg.first == NAME && arg.second.isString() ) { VAR = arg.second.asString(); continue; }
+#define STRING_LIST(NAME, VAR, ERRORSTR) if (arg.first == NAME) { if (arg.second.isStringList()) { VAR = arg.second.asStringList(); continue; } else { errorString = ERRORSTR; break; } }
+#define INT(NAME, VAR, ERRORSTR) if (arg.first == NAME ) { if (isLong(arg.second)) { VAR = arg.second.asLong(); continue; } else { errorString = ERRORSTR; break; } }
+
+// Older version of docopt doesn't declare Options. Let's declare it ourself.
+using Options = std::map<std::string, docopt::value>;
+
 int main(int argc, char** argv)
 {
-  std::string rootLocation = "";
-  kiwix::Library library;
+#ifndef _WIN32
+  setup_sighandlers();
+#endif
+
+  std::string rootLocation = "/";
+  auto library = kiwix::Library::create();
   unsigned int nb_threads = DEFAULT_THREADS;
-  vector<string> zimPathes;
-  string libraryPath;
-  string rootPath;
-  string address;
+  std::vector<std::string> zimPathes;
+  std::string libraryPath;
+  std::string rootPath;
+  std::string address;
+  std::string customIndexPath="";
+  std::string indexTemplateString="";
   int serverPort = 80;
-  int daemonFlag [[gnu::unused]] = false;
-  int libraryFlag = false;
+  bool catalogOnlyFlag = false;
+  std::string contentServerURL;
+  bool daemonFlag [[gnu::unused]] = false;
+  bool helpFlag = false;
   bool noLibraryButtonFlag = false;
   bool noSearchBarFlag = false;
   bool noDateAliasesFlag = false;
+  bool blockExternalLinks = false;
   bool isVerboseFlag = false;
-  bool trustlibrary = true;
-  string PPIDString;
+  bool monitorLibrary = false;
+  bool versionFlag = false;
   unsigned int PPID = 0;
+  int ipConnectionLimit = 0;
+  int searchLimit = 0;
+  bool skipInvalid = false;
 
-  static struct option long_options[]
-      = {{"daemon", no_argument, 0, 'd'},
-         {"verbose", no_argument, 0, 'v'},
-         {"version", no_argument, 0, 'V'},
-         {"library", no_argument, 0, 'l'},
-         {"nolibrarybutton", no_argument, 0, 'm'},
-         {"nodatealiases", no_argument, 0, 'z'},
-         {"nosearchbar", no_argument, 0, 'n'},
-         {"attachToProcess", required_argument, 0, 'a'},
-         {"port", required_argument, 0, 'p'},
-         {"address", required_argument, 0, 'i'},
-         {"threads", required_argument, 0, 't'},
-         {"urlRootLocation", required_argument, 0, 'r'},
-         {"donottrustlibrary", no_argument, 0, 'T'},
-         {0, 0, 0, 0}};
+  std::string errorString;
 
-  /* Argument parsing */
-  while (true) {
-    int option_index = 0;
-    int c
-        = getopt_long(argc, argv, "zmndvVla:p:f:t:r:i:", long_options, &option_index);
-
-    if (c != -1) {
-      switch (c) {
-        case 'd':
-          daemonFlag = true;
-          break;
-        case 'v':
-          isVerboseFlag = true;
-          break;
-        case 'V':
-          version();
-          return 0;
-        case 'l':
-          libraryFlag = true;
-          break;
-        case 'n':
-          noSearchBarFlag = true;
-          break;
-        case 'z':
-          noDateAliasesFlag = true;
-          break;
-        case 'm':
-          noLibraryButtonFlag = true;
-          break;
-        case 'T':
-          trustlibrary = false;
-        case 'p':
-          serverPort = atoi(optarg);
-          break;
-        case 'a':
-          PPIDString = string(optarg);
-          PPID = atoi(optarg);
-          break;
-        case 'i':
-          address = string(optarg);
-          break;
-        case 't':
-          nb_threads = atoi(optarg);
-          break;
-        case 'r':
-          rootLocation = string(optarg);
-          break;
-      }
-    } else {
-      if (optind < argc) {
-        if (libraryFlag) {
-          libraryPath = argv[optind++];
-        } else {
-          while (optind < argc)
-            zimPathes.push_back(std::string(argv[optind++]));
-        }
-      }
-      break;
-    }
+  Options args;
+  try {
+    args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
+  } catch (docopt::DocoptArgumentError const & error) {
+    std::cerr << error.what() << std::endl;
+    std::cerr << USAGE << std::endl;
+    return -1;
   }
 
-  /* Print usage)) if necessary */
-  if (zimPathes.empty() && libraryPath.empty()) {
-    usage();
-    exit(1);
-  }
+  for (auto const& arg: args) {
+    FLAG("--help", helpFlag)
+    FLAG("--catalogOnly", catalogOnlyFlag)
+    STRING("--contentServerURL", contentServerURL)
+    FLAG("--daemon", daemonFlag)
+    FLAG("--verbose", isVerboseFlag)
+    FLAG("--nosearchbar", noSearchBarFlag)
+    FLAG("--blockexternal", blockExternalLinks)
+    FLAG("--nodatealiases", noDateAliasesFlag)
+    FLAG("--nolibrarybutton",noLibraryButtonFlag)
+    FLAG("--monitorLibrary", monitorLibrary)
+    FLAG("--skipInvalid", skipInvalid)
+    FLAG("--version", versionFlag)
+    STRING("LIBRARYPATH", libraryPath)
+    INT("--port", serverPort, "Port must be an integer")
+    INT("--attachToProcess", PPID, "Process to attach must be an integer")
+    STRING("--address", address)
+    INT("--threads", nb_threads, "Number of threads must be an integer")
+    STRING("--urlRootLocation", rootLocation)
+    STRING("--customIndex", customIndexPath)
+    INT("--ipConnectionLimit", ipConnectionLimit, "IP connection limit must be an integer")
+    INT("--searchLimit", searchLimit, "Search limit must be an integer")
+    STRING_LIST("ZIMPATH", zimPathes, "ZIMPATH must be a string list")
+ }
+
+ if (!errorString.empty()) {
+   std::cerr << errorString << std::endl;
+   std::cerr << USAGE << std::endl;
+   return -1;
+ }
+
+ if (helpFlag) {
+   std::cout << USAGE << std::endl;
+   return 0;
+ }
+
+ if (versionFlag) {
+   version();
+   return 0;
+ }
 
   /* Setup the library manager and get the list of books */
-  kiwix::Manager manager(&library);
-  if (libraryFlag) {
-    vector<string> libraryPaths = kiwix::split(libraryPath, ";");
-    vector<string>::iterator itr;
-
-    for (itr = libraryPaths.begin(); itr != libraryPaths.end(); ++itr) {
-      if (!itr->empty()) {
-        bool retVal = false;
-
-        try {
-          string libraryPath
-              = isRelativePath(*itr)
-                    ? computeAbsolutePath(getCurrentDirectory(), *itr)
-                    : *itr;
-          retVal = manager.readFile(libraryPath, true, trustlibrary);
-        } catch (...) {
-          retVal = false;
-        }
-
-        if (!retVal) {
-          cerr << "Unable to open the XML library file '" << *itr << "'."
-               << endl;
-          exit(1);
-        }
-      }
+  kiwix::Manager manager(library);
+  std::vector<std::string> libraryPaths;
+  if (!libraryPath.empty()) {
+    libraryPaths = kiwix::split(libraryPath, ";");
+    if ( !reloadLibrary(manager, libraryPaths) ) {
+      exit(1);
     }
 
     /* Check if the library is not empty (or only remote books)*/
-    if (library.getBookCount(true, false) == 0) {
-      cerr << "The XML library file '" << libraryPath
-           << "' is empty (or has only remote books)." << endl;
+    if (library->getBookCount(true, false) == 0) {
+      std::cerr << "The XML library file '" << libraryPath
+           << "' is empty (or has only remote books)." << std::endl;
     }
   } else {
     std::vector<std::string>::iterator it;
     for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
       if (!manager.addBookFromPath(*it, *it, "", false)) {
-        cerr << "Unable to add the ZIM file '" << *it
-             << "' to the internal library." << endl;
-        exit(1);
+        if (skipInvalid) {
+          std::cerr << "Skipping invalid '" << *it << "' ...continuing" << std::endl;
+        } else {
+          std::cerr << "Unable to add the ZIM file '" << *it
+               << "' to the internal library." << std::endl;
+          exit(1);
+        }
       }
     }
   }
+  auto libraryFileTimestamp = newestFileTimestamp(libraryPaths);
+  auto curLibraryFileTimestamp = libraryFileTimestamp;
+
+  kiwix::IpMode ipMode = kiwix::IpMode::AUTO;
+
+  if (address == "all") {
+    address.clear();
+    ipMode = kiwix::IpMode::ALL;
+  } else if (address == "ipv4") {
+    address.clear();
+    ipMode = kiwix::IpMode::IPV4;
+  } else if (address == "ipv6") {
+    address.clear();
+    ipMode = kiwix::IpMode::IPV6;
+  }
 
 #ifndef _WIN32
   /* Fork if necessary */
@@ -241,25 +352,45 @@ int main(int argc, char** argv)
   }
 #endif
 
-  kiwix::HumanReadableNameMapper nameMapper(library, noDateAliasesFlag);
-  kiwix::Server server(&library, &nameMapper);
+  auto nameMapper = std::make_shared<kiwix::UpdatableNameMapper>(library, noDateAliasesFlag);
+  kiwix::Server server(library, nameMapper);
+
+  if (!customIndexPath.empty()) {
+    try {
+      indexTemplateString = loadCustomTemplate(customIndexPath);
+    } catch (std::runtime_error& e) {
+      std::cerr << "ERROR: " << e.what() << std::endl;
+      exit(1);
+    }
+  }
+
   server.setAddress(address);
   server.setRoot(rootLocation);
   server.setPort(serverPort);
   server.setNbThreads(nb_threads);
   server.setVerbose(isVerboseFlag);
   server.setTaskbar(!noSearchBarFlag, !noLibraryButtonFlag);
+  server.setBlockExternalLinks(blockExternalLinks);
+  server.setIndexTemplateString(indexTemplateString);
+  server.setIpConnectionLimit(ipConnectionLimit);
+  server.setMultiZimSearchLimit(searchLimit);
+  server.setIpMode(ipMode);
+  server.setCatalogOnlyMode(catalogOnlyFlag);
+  while ( !contentServerURL.empty() && contentServerURL.back() == '/' )
+    contentServerURL.pop_back();
+  server.setContentServerUrl(contentServerURL);
 
   if (! server.start()) {
-    cerr << "Unable to instantiate the HTTP daemon. The port " << serverPort
-         << " is maybe already occupied or need more permissions to be open. "
-            "Please try as root or with a port number higher or equal to 1024."
-         << endl;
     exit(1);
   }
+  
+  std::cout << "The Kiwix server is running and can be accessed in the local network at: " << std::endl;
+  for (const auto& url : server.getServerAccessUrls()) {
+    std::cout << "  - " << url << std::endl;
+  }
 
   /* Run endless (until PPID dies) */
-  bool waiting = true;
+  waiting = true;
   do {
     if (PPID > 0) {
 #ifdef _WIN32
@@ -280,7 +411,7 @@ int main(int argc, char** argv)
       int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0);
       if (ret != -1 && len > 0) {
 #else /* Linux & co */
-      string procPath = "/proc/" + string(PPIDString);
+      std::string procPath = "/proc/" + std::to_string(PPID);
       if (access(procPath.c_str(), F_OK) != -1) {
 #endif
       } else {
@@ -289,6 +420,20 @@ int main(int argc, char** argv)
     }
 
     kiwix::sleep(1000);
+
+    if ( monitorLibrary ) {
+      curLibraryFileTimestamp = newestFileTimestamp(libraryPaths);
+      if ( !libraryMustBeReloaded ) {
+        libraryMustBeReloaded = curLibraryFileTimestamp > libraryFileTimestamp;
+      }
+    }
+
+    if ( libraryMustBeReloaded && !libraryPaths.empty() ) {
+      libraryFileTimestamp = curLibraryFileTimestamp;
+      reloadLibrary(manager, libraryPaths);
+      nameMapper->update();
+      libraryMustBeReloaded = false;
+    }
   } while (waiting);
 
   /* Stop the daemon */

src/version.h

@@ -24,9 +24,16 @@
   #define KIWIX_TOOLS_VERSION "undefined"
 #endif
 
+#include <kiwix/version.h>
+#include <zim/version.h>
+#include <iostream>
+
 void version()
 {
-  std::cout << KIWIX_TOOLS_VERSION << std::endl;
+  std::cout << "kiwix-tools " << KIWIX_TOOLS_VERSION << std::endl << std::endl;
+  kiwix::printVersions();
+  std::cout << std::endl;
+  zim::printVersions();
 }
 
 #endif //_KIWIX_TOOLs_VERSION_H_