Merge pull request #585 from kiwix/version_3.4.0

* Remove last reference to kiwix-read tool (@legoktm #569)

kiwix-serve
-----------

 * Fix broken indentation in usage (@kelson42 #560)
 * Exit if wrong arguments are passed (@kelson42 #567)
 * Do not allow multiple values for same option (@juuz0 #564)
 * Fix default location of "rootLocation" (@rgaudin #571)
 * [DOCKER] Change default port to 8080 (@neyder #581)
 * [DOCKER] Simplify dockerfile (@rgaudin #582)

kiwix-manage
------------

 * Fix man page (@kelson42 #576)

.github/workflows/docker.yml

@@ -13,9 +13,9 @@ jobs:
     name: Deploy kiwix-tools Docker Image
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3.1.0
       - name: build and publish kiwix-tools
-        uses: openzim/docker-publish-action@v5
+        uses: openzim/docker-publish-action@v9
         with:
           image-name: kiwix/kiwix-tools
           credentials: |
@@ -33,15 +33,17 @@ jobs:
             linux/arm64
           restrict-to: kiwix/kiwix-tools
           manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix command line tools
 
   build-and-push-kiwix-serve:
     name: Deploy kiwix-serve Docker Image
     runs-on: ubuntu-20.04
     needs: build-and-push-kiwix-tools
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3.1.0
       - name: build and publish kiwix-serve
-        uses: openzim/docker-publish-action@v5
+        uses: openzim/docker-publish-action@v9
         with:
           image-name: kiwix/kiwix-serve
           credentials: |
@@ -59,3 +61,5 @@ jobs:
             linux/arm64
           restrict-to: kiwix/kiwix-tools
           manual-tag: ${{ github.event.inputs.version }}
+          repo_description: auto
+          repo_overview: Kiwix web-server

Changelog

@@ -1,3 +1,23 @@
+kiwix-tools 3.4.0
+=================
+
+ * Remove last reference to kiwix-read tool (@legoktm #569)
+
+kiwix-serve
+-----------
+
+ * Fix broken indentation in usage (@kelson42 #560)
+ * Exit if wrong arguments are passed (@kelson42 #567)
+ * Do not allow multiple values for same option (@juuz0 #564)
+ * Fix default location of "rootLocation" (@rgaudin #571)
+ * [DOCKER] Change default port to 8080 (@neyder #581)
+ * [DOCKER] Simplify dockerfile (@rgaudin #582)
+
+kiwix-manage
+------------
+
+ * Fix man page (@kelson42 #576)
+
 kiwix-tools 3.3.0
 =================
 
@@ -6,7 +26,7 @@ kiwix-tools 3.3.0
 kiwix-serve
 -----------
 
- * Add an option to limit the number of connections for a same IP (@nikhil #534)
+ * Add an option to limit the number of connections for a same IP (@juuz0 #534)
  * Add an option to limit the number of zim in a multizim fulltext search (@mgautierfr #558)
 
 kiwix-search
@@ -34,7 +54,7 @@ kiwix-tools 3.2.0
 kiwix-serve
 -----------
 
- * Print the url on which a user can connect to on startup (@nikhil #499 #522)
+ * Print the url on which a user can connect to on startup (@juuz0 #499 #522)
  * Reload library on SIGHUP signal (@veloman-yunkan #497)
  * Add a option `--monitorLibrary` to monitor and automically reload the library
    (@veloman-yunkan #503)

README.md

@@ -4,7 +4,6 @@ Kiwix tools
 The Kiwix tools is a collection of [Kiwix](https://kiwix.org) related
 command line tools:
 * kiwix-manage: Manage XML based library of ZIM files
-* kiwix-read: Read ZIM file content
 * kiwix-search: Fulltext search in ZIM files
 * kiwix-serve: HTTP daemon serving ZIM files
 

debian/control

@@ -21,6 +21,5 @@ Description: collection of Kiwix tools
     over the network.
   * kiwix-manage allows one to manage the content of the Kiwix library (an
     XML file listing available ZIM files).
-  * kiwix-read allows for reading ZIM files from the command-line.
   * kiwix-search allows one to find articles in a ZIM file using fulltext
     search patterns.

docker/Dockerfile

@@ -1,46 +1,18 @@
-# dumb-init to use as entrypoint so it can forward signals to our CMD
-FROM debian:bullseye-slim as builder
-RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends \
-    build-essential ca-certificates musl-tools wget unzip make \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-# compile dumb-init
-RUN wget -nv https://github.com/Yelp/dumb-init/archive/refs/tags/v1.2.5.zip && \
-    unzip v1.2.5.zip && cd dumb-init-1.2.5 && CC=musl-gcc make
-
-# declare build option ARCH if not using buildx
-ARG ARCH=
-# declare version to build image for
-ARG VERSION=
-
-# alpine is a multi-arch image
-FROM alpine:3
+FROM alpine:3.16
 LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
-
-# TARGETARCH is injected by buildx
-ARG TARGETARCH
 ARG VERSION
-ARG ARCH
 
-# copy built dumb-init
-COPY --from=builder /dumb-init-1.2.5/dumb-init /usr/bin/dumb-init
-
-# find and store arch in image, using (in this order):
-# $TARGETARCH || --build-arg ARCH (not using buildx) || amd64
-RUN echo "amd64" > /etc/docker_arch && \
-    if [ ! -z "$ARCH" ] ; then echo "$ARCH" > /etc/docker_arch ; fi && \
-    if [ ! -z "$TARGETARCH" ] ; then echo "$TARGETARCH" > /etc/docker_arch ; fi
-
-# decide which kiwix arch to download later (`armhf` for all arm* and x86_64 otherwise)
-RUN if [ $(cut -c 1-3 /etc/docker_arch) = "arm" ] ; then echo "armhf" > /etc/kiwix_arch ; else echo "x86_64" > /etc/kiwix_arch ; fi
-
-# Install kiwix-tools
-RUN url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$(cat /etc/kiwix_arch)-$VERSION.tar.gz" && \
+RUN apk --no-cache add dumb-init curl && \
+    ARCH=$(cat /etc/apk/arch) && \
+    if [ "$ARCH" = "x86" ]; then ARCH="i586"; \
+    elif [ "$ARCH" = "aarch64" ]; then ARCH="armhf"; \
+    elif [ "$ARCH" = "armv7" ]; then ARCH="armhf"; fi && \
+    # download requested kiwix-tools version
+    url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$ARCH-$VERSION.tar.gz" && \
     echo "URL: $url" && \
-    wget -q -O - $url | tar -xz && \
-    mv kiwix-tools*/kiwix-* /usr/local/bin && \
-    rm -r kiwix-tools*
+    curl -k -L $url | tar -xz -C /usr/local/bin/ --strip-components 1 && \
+    # only needed in dockerfile
+    apk del curl
 
 # expose kiwix-serve default port
 EXPOSE 80

docker/README.md

@@ -11,13 +11,13 @@ Kiwix-tools Docker image
 $ docker run -it kiwix/kiwix-tools:3.1.2
 
 Welcome to kiwix-tools! The following binaries are available:
-kiwix-manage  kiwix-read    kiwix-search  kiwix-serve
+kiwix-manage  kiwix-search  kiwix-serve
 ```
 
 `kiwix-tools` operates on zim files. You shall mount a volume to access the files.
 
 ```sh
-docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-read --suggest="Mali" /data/wikipedia_fr_test.zim
+docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
 ```
 
 ## Building and reusing
@@ -29,4 +29,4 @@ docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-read --suggest="Mali" /da
 **Notes:**
 
 - `wget` in `alpine:3` on `arm/v7` (__inside github action only__) crashes when downloading from HTTPs locations. Keep http-only in Dockerfile.
-- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.
+- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.

docker/server/Dockerfile

@@ -5,7 +5,7 @@ FROM kiwix/kiwix-tools:$VERSION
 LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
 
 # expose kiwix-serve default port and workdir
-EXPOSE 80
+EXPOSE 8080
 VOLUME /data
 WORKDIR /data
 

docker/server/README.md

@@ -9,14 +9,23 @@ With local ZIM file(s)
 * Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following:
 
 ```bash
-docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
+docker run -v /tmp/zim:/data -p 8080:8080 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
 ```
 
 With remote ZIM file
 --------------------
 
 ```bash
-docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:80 kiwix/kiwix-serve
+docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 kiwix/kiwix-serve
+```
+
+Change default port
+-------------------
+
+You can change port to expose with environment variable PORT, useful if running on Podman, K8s or OpenShift
+
+```bash
+podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 kiwix/kiwix-serve
 ```
 
 ARM

docker/server/docker-compose.yml.example

@@ -2,7 +2,7 @@ version: '3.3'
 services:
   kiwix-serve:
       ports:
-        - 8080:80
+        - 8080:8080
       image: kiwix/kiwix-serve
       # uncomment next 4 lines to use it with local zim file in /tmp/zim
       # volumes:

docker/server/start.sh

@@ -13,7 +13,11 @@ then
     fi
 fi
 
-CMD="/usr/local/bin/kiwix-serve --port=80 $@"
+if [ -z "$PORT" ]
+then
+    PORT=8080
+fi
+CMD="/usr/local/bin/kiwix-serve --port=$PORT $@"
 echo $CMD
 $CMD
 

format_code.sh

@@ -3,7 +3,6 @@
 files=(
 "src/installer/kiwix-install.cpp"
 "src/searcher/kiwix-search.cpp"
-"src/reader/kiwix-read.cpp"
 "src/manager/kiwix-manage.cpp"
 "src/server/kiwix-serve.cpp"
 )

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-tools', 'cpp',
-  version : '3.3.0',
+  version : '3.4.0',
   license : 'GPL',
   default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -13,8 +13,8 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', version:'>=11.0.0', static:static_linkage)
-libzim_dep = dependency('libzim', version:'>=7.2.0', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'>=12.0.0', static:static_linkage)
+libzim_dep = dependency('libzim', version:'>=8.1.0', static:static_linkage)
 
 all_deps = [thread_dep, kiwixlib_dep, libzim_dep]
 

src/man/kiwix-manage.1

@@ -1,62 +1,69 @@
-.TH KIWIX 1 "21 May 2012"
+.TH KIWIX-MANAGE 1 "21 May 2012"
+
 .SH NAME
 kiwix\-manage \- Kiwix Library Manager
+
 .SH SYNOPSIS
 .IX Header SYNOPSIS
-.B kiwix\-manage
-.br
-kiwix\-manage LIBRARY_PATH add ZIM_PATH ...
-.br
-	kiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ...
-.br
-	kiwix\-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]
-.SH DESCRIPTION
-.SS kiwix\-manage
-.PP
-\fBkiwix\-manage\fP tool for managing a kiwix library.
-.PP
-Allows one to manage the content of the kiwix library. The library file is a
-flat XML file listing ZIM files with all necessary information like favicon,
-date, creator, description, indexpath, filepath, title, source/metalink, etc.
-.
-.PP
- Example library file can be found at https://www.kiwix.org/library\.xml
-.br
 .TP
-\fBadd\fR
-Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP.
+\fBkiwix\-manage\fR LIBRARY_PATH \fBadd\fR ZIM_PATH ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBshow\fR [ZIM_ID_1] [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR LIBRARY_PATH \fBremove\fR ZIM_ID_1 [ZIM_ID_2] ...
+.TP
+\fBkiwix\-manage\fR --version
+.TP
+\fBkiwix\-manage\fR --help
+
+.SH DESCRIPTION
+.PP
+\fBkiwix\-manage\fP is a command line tool for manipulating a Kiwix XML library.
+.PP
+\fBkiwix\-manage\fP allows to manage the entries of the Kiwix
+library. The library file is a flat XML file listing ZIM files with
+all necessary information like id, favicon, date, creator,
+description, filepath, title, url, etc.
+
+.SH ACTIONS
 
 .TP
-\fBshow\fR
-Show given \fBCONTENT_ID\fP from \fBLIBRARY_FILE\fR. If no \fBCONTENT_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+\fBadd\fR
+Add \fBZIM_FILE\fP to \fBLIBRARY_FILE\fP. Create the library file if necessary.
 
 .TP
 \fBremove\fR
-Remove the given \fBCONTENT_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBCONTENT_ID\fR should be specified.
+Remove the given \fBZIM_ID\fR from \fBLIBRARY_FILE\fR. At least one \fBZIM_ID\fR should be specified.
 
 .TP
-\fB\-\-zimPathToSave=\fR
-Change the location reference of the ZIM file in the library.
+\fBshow\fR
+Show given \fBZIM_ID\fP from \fBLIBRARY_FILE\fR. If no \fBZIM_ID\fP is given then all contents from \fBLIBRARY_FILE\fR are shown.
+
+.SH OPTIONS
+.TP
+Options to be used with the action \fBadd\fR:
 
 .TP
-\fB\-\-current\fR
-Mark this content as the current (default) content in the library.
+\fB\-\-url=HTTP_URL\fR
+Set the ZIM online HTTP(S) URL
 
 .TP
-\fB\-\-backend=xapian|clucene\fR
-Select an indexing backend.
+\fB\-\-zimPathToSave=OTHER_FS_PATH\fR
+Set an arbitrary ZIM filesystem path (instead of the ZIM_PATH)
 
 .TP
-\fB\-\-indexPath=FULLTEXT_IDX_PATH\fR
-Path to full-text index for that ZIM file.
+Other options (to be used alone):
 
 .TP
-\fBurl\fR
-Set the content location of the ZIM file over the network for in\-kiwix download.
+\fB\-\-help | \-h\fR
+Display the kiwix-manage help
+
+.TP
+\fB\-\-version | \-v\fR
+Display the version of kiwix-manage and all dependences
 
 .SH SEE ALSO
-kiwix(1) kiwix\-serve(1)
-.SH AUTHOR
-Emmanuel Engelhart <kelson@kiwix.org>
-.br
-Vasudev Kamath <kamathvasudev@gmail.com> (Manual)
+kiwix\-serve(1)
+
+.SH AUTHORS
+Kiwix team <contact@kiwix.org>

src/man/kiwix-read.1

@@ -1,18 +0,0 @@
-.TH KIWIX-READ "1" "July 2020" "kiwix-tools" "User Commands"
-.SH NAME
-kiwix-read \- Read ZIM file content
-.SH SYNOPSIS
-\fBkiwix-read\fR [\-\-verbose] [\-\-version] \-\-suggest=PATTERN ZIM\fR
-.SH DESCRIPTION
-.TP
-ZIM
-ZIM file to search
-.TP
-\fB\-\-verbose\fR
-Enable extra output
-.TP
-\fB\-\-version\fR
-print software version
-.TP
-\fB\-\-suggest=PATTERN\fR
-Get search suggestions for the provided pattern

src/man/meson.build

@@ -1,5 +1,4 @@
 install_man('kiwix-manage.1',
-            'kiwix-read.1',
             'kiwix-search.1',
             'kiwix-serve.1')
 subdir('fr')

src/server/kiwix-serve.cpp

@@ -1,3 +1,4 @@
+
 /*
  * Copyright 2009-2019 Emmanuel Engelhart <kelson@kiwix.org>
  *
@@ -69,7 +70,7 @@ void usage()
             << "\t-b, --blockexternal\tPrevent users from directly accessing external links" << std::endl
             << "\t-p, --port\t\tTCP port on which to listen to HTTP requests (default: 80)" << std::endl
             << "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
-            << "\t-s, --searchLimit\t\tMaximun number of zim in a fulltext multizim search (default: No limit)" << std::endl
+            << "\t-s, --searchLimit\tMaximun number of zim in a fulltext multizim search (default: No limit)" << std::endl
             << "\t-t, --threads\t\tNumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
             << "\t-v, --verbose\t\tPrint debug log to STDOUT" << std::endl
             << "\t-V, --version\t\tPrint software version" << std::endl
@@ -84,24 +85,34 @@ void usage()
             << std::endl;
 }
 
-string loadCustomTemplate (string customIndexPath) {
+std::string loadCustomTemplate (std::string customIndexPath) {
   customIndexPath = kiwix::isRelativePath(customIndexPath) ?
                       kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), customIndexPath) :
                       customIndexPath;
   if (!kiwix::fileReadable(customIndexPath)) {
-    throw runtime_error("No such file exist (or file is not readable) " + customIndexPath);
+    throw std::runtime_error("No such file exist (or file is not readable) " + customIndexPath);
   }
   if (kiwix::getMimeTypeForFile(customIndexPath) != "text/html") {
-    throw runtime_error("Invalid File Mime Type " + kiwix::getMimeTypeForFile(customIndexPath));
+    throw std::runtime_error("Invalid File Mime Type " + kiwix::getMimeTypeForFile(customIndexPath));
   }
-  string indexTemplateString = kiwix::getFileContent(customIndexPath);
+  std::string indexTemplateString = kiwix::getFileContent(customIndexPath);
 
   if (indexTemplateString.empty()) {
-    throw runtime_error("Unreadable or empty file " + customIndexPath);
+    throw std::runtime_error("Unreadable or empty file " + customIndexPath);
   }
   return indexTemplateString;
 }
 
+inline std::string normalizeRootUrl(std::string rootUrl)
+{
+  while ( !rootUrl.empty() && rootUrl.back() == '/' )
+    rootUrl.pop_back();
+
+  while ( !rootUrl.empty() && rootUrl.front() == '/' )
+    rootUrl = rootUrl.substr(1);
+  return rootUrl.empty() ? rootUrl : "/" + rootUrl;
+}
+
 volatile sig_atomic_t waiting = false;
 volatile sig_atomic_t libraryMustBeReloaded = false;
 
@@ -185,15 +196,15 @@ int main(int argc, char** argv)
   setup_sighandlers();
 #endif
 
-  std::string rootLocation = "";
+  std::string rootLocation = "/";
   kiwix::Library library;
   unsigned int nb_threads = DEFAULT_THREADS;
-  vector<string> zimPathes;
-  string libraryPath;
-  string rootPath;
-  string address;
-  string customIndexPath="";
-  string indexTemplateString="";
+  std::vector<std::string> zimPathes;
+  std::string libraryPath;
+  std::string rootPath;
+  std::string address;
+  std::string customIndexPath="";
+  std::string indexTemplateString="";
   int serverPort = 80;
   int daemonFlag [[gnu::unused]] = false;
   int libraryFlag = false;
@@ -228,6 +239,7 @@ int main(int argc, char** argv)
          {"searchLimit", required_argument, 0, 's'},
          {0, 0, 0, 0}};
 
+  std::set<int> usedOptions;
   /* Argument parsing */
   while (true) {
     int option_index = 0;
@@ -235,6 +247,11 @@ int main(int argc, char** argv)
         = getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:ML:s:", long_options, &option_index);
 
     if (c != -1) {
+      auto insertRes = usedOptions.insert(c);
+      if (!insertRes.second) {
+        std::cerr << "Multiple values of same option are not allowed." << std::endl;
+        exit(1);
+      }
       switch (c) {
         case 'h':
           usage();
@@ -270,16 +287,16 @@ int main(int argc, char** argv)
           PPID = atoi(optarg);
           break;
         case 'i':
-          address = string(optarg);
+          address = std::string(optarg);
           break;
         case 't':
           nb_threads = atoi(optarg);
           break;
         case 'r':
-          rootLocation = string(optarg);
+          rootLocation = std::string(optarg);
           break;
         case 'c':
-          customIndexPath = string(optarg);
+          customIndexPath = std::string(optarg);
           break;
         case 'M':
           monitorLibrary = true;
@@ -290,6 +307,9 @@ int main(int argc, char** argv)
         case 's':
           searchLimit = atoi(optarg);
           break;
+        case '?':
+          usage();
+          return 2;
       }
     } else {
       if (optind < argc) {
@@ -312,7 +332,7 @@ int main(int argc, char** argv)
 
   /* Setup the library manager and get the list of books */
   kiwix::Manager manager(&library);
-  vector<string> libraryPaths;
+  std::vector<std::string> libraryPaths;
   if (libraryFlag) {
     libraryPaths = kiwix::split(libraryPath, ";");
     if ( !reloadLibrary(manager, libraryPaths) ) {
@@ -321,15 +341,15 @@ int main(int argc, char** argv)
 
     /* Check if the library is not empty (or only remote books)*/
     if (library.getBookCount(true, false) == 0) {
-      cerr << "The XML library file '" << libraryPath
-           << "' is empty (or has only remote books)." << endl;
+      std::cerr << "The XML library file '" << libraryPath
+           << "' is empty (or has only remote books)." << std::endl;
     }
   } else {
     std::vector<std::string>::iterator it;
     for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
       if (!manager.addBookFromPath(*it, *it, "", false)) {
-        cerr << "Unable to add the ZIM file '" << *it
-             << "' to the internal library." << endl;
+        std::cerr << "Unable to add the ZIM file '" << *it
+             << "' to the internal library." << std::endl;
         exit(1);
       }
     }
@@ -383,7 +403,7 @@ int main(int argc, char** argv)
     exit(1);
   }
 
-  std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + "/" + rootLocation;
+  std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + normalizeRootUrl(rootLocation);
   std::cout << "The Kiwix server is running and can be accessed in the local network at: "
             << url << std::endl;
 
@@ -409,7 +429,7 @@ int main(int argc, char** argv)
       int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0);
       if (ret != -1 && len > 0) {
 #else /* Linux & co */
-      string procPath = "/proc/" + std::to_string(PPID);
+      std::string procPath = "/proc/" + std::to_string(PPID);
       if (access(procPath.c_str(), F_OK) != -1) {
 #endif
       } else {

src/version.h

@@ -26,6 +26,7 @@
 
 #include <kiwix/version.h>
 #include <zim/version.h>
+#include <iostream>
 
 void version()
 {