Merge pull request #559 from kiwix/version_3.3.0

Add a option to limit the number of zim in a fulltext multizim search.

.github/workflows/ci.yml

@@ -14,16 +14,16 @@ jobs:
           - win32_dyn
         include:
           - target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: native_dyn
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
     env:
       HOME: /home/runner

.github/workflows/package.yml

@@ -10,7 +10,6 @@ jobs:
         distro:
           - ubuntu-jammy
           - ubuntu-impish
-          - ubuntu-hirsute
           - ubuntu-focal
     steps:
       - uses: actions/checkout@v2
@@ -50,14 +49,6 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-hirsute
-        if: matrix.distro == 'ubuntu-hirsute'
-        name: Build package for ubuntu-hirsute
-        id: build-ubuntu-hirsute
-        with:
-          args: --no-sign
-          ppa: ${{ steps.ppa.outputs.ppa }}
-
       - uses: legoktm/gh-action-build-deb@ubuntu-focal
         if: matrix.distro == 'ubuntu-focal'
         name: Build package for ubuntu-focal

Changelog

@@ -1,3 +1,25 @@
+kiwix-tools 3.3.0
+=================
+
+ * Remove kiwix-read tool (@veloman-yunkan #535)
+
+kiwix-serve
+-----------
+
+ * Add an option to limit the number of connections for a same IP (@nikhil #534)
+ * Add an option to limit the number of zim in a multizim fulltext search (@mgautierfr #558)
+
+kiwix-search
+------------
+
+ * Remove usage of libkiwix's deprecated api (@veloman-yunkan #535)
+
+kiwix-manage
+------------
+
+ * Correctly return a value !0 if something went wrong (@mgautierfr #553)
+
+
 kiwix-tools 3.2.0
 =================
 

debian/control

@@ -3,8 +3,8 @@ Section: utils
 Priority: optional
 Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
-               libkiwix-dev (>= 10.0.0),
-               libzim-dev (>= 7.2.0),
+               libkiwix-dev (>= 10.0.0~),
+               libzim-dev (>= 7.2.0~),
                meson,
                pkg-config,
 Standards-Version: 4.5.0

meson.build

@@ -1,7 +1,7 @@
 project('kiwix-tools', 'cpp',
-  version : '3.2.0',
+  version : '3.3.0',
   license : 'GPL',
-  default_options: ['c_std=c11', 'cpp_std=c++11'])
+  default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 
@@ -13,7 +13,7 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', version:'>=10.0.0', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'>=11.0.0', static:static_linkage)
 libzim_dep = dependency('libzim', version:'>=7.2.0', static:static_linkage)
 
 all_deps = [thread_dep, kiwixlib_dep, libzim_dep]

src/manager/kiwix-manage.cpp

@@ -169,7 +169,10 @@ int handle_add(kiwix::Library* library, const std::string& libraryPath,
     std::string zimPath = argv[i];
     if (!zimPath.empty()) {
       auto _zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false);
+      if (manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false).empty()) {
+        std::cerr << "Cannot add zim " << zimPath << " to the library." << std::endl;
+        resultCode = 1;
+      }
     } else {
       std::cerr << "Invalid zim file path" << std::endl;
       resultCode = 1;
@@ -251,7 +254,7 @@ int main(int argc, char** argv)
   /* Print usage)) if necessary */
   if (libraryPath == "" || action == NONE) {
     usage();
-    exit(1);
+    return -1;
   }
 
   /* Try to read the file */
@@ -259,7 +262,12 @@ int main(int argc, char** argv)
                     ? kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryPath)
                     : libraryPath;
   kiwix::Manager manager(&library);
-  manager.readFile(libraryPath, false);
+  if (!manager.readFile(libraryPath, false)) {
+    if (kiwix::fileExists(libraryPath) || action!=ADD) {
+      std::cerr << "Cannot read the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
 
   /* SHOW */
   int exitCode = 0;
@@ -277,10 +285,18 @@ int main(int argc, char** argv)
       break;
   }
 
-  /* Rewrite the library file */
-  if (action == REMOVE || action == ADD) {
-    library.writeToFile(libraryPath);
+  if (exitCode) {
+    return exitCode;
   }
 
-  exit(exitCode);
+  /* Rewrite the library file */
+  if (action == REMOVE || action == ADD) {
+    // writeToFile return true (1) if everything is ok => exitCode is 0
+    if (!library.writeToFile(libraryPath)) {
+      std::cerr << "Cannot write the library " << libraryPath << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
 }

src/meson.build

@@ -1,6 +1,4 @@
-
 subdir('manager')
-subdir('reader')
 subdir('searcher')
 subdir('server')
 subdir('man')

src/reader/kiwix-read.cpp

@@ -1,107 +0,0 @@
-/*
- * Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <getopt.h>
-#include <kiwix/reader.h>
-#include <map>
-#include <string>
-
-#include "../version.h"
-
-void usage()
-{
-  cout << "Usage: kiwix-read [--verbose] [--version] --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
-  exit(1);
-}
-
-int main(int argc, char** argv)
-{
-  /* Init the variables */
-  const char* filePath = NULL;
-  const char* pattern = NULL;
-  int option_index = 0;
-  int c = 0;
-
-  kiwix::Reader* reader = NULL;
-
-  /* Argument parsing */
-  while (42) {
-    static struct option long_options[]
-        = {{"verbose", no_argument, 0, 'v'},
-           {"version", no_argument, 0, 'V'},
-           {"suggest", required_argument, 0, 's'},
-           {0, 0, 0, 0}};
-
-    if (c != -1) {
-      c = getopt_long(argc, argv, "Vvs:", long_options, &option_index);
-
-      switch (c) {
-        case 'v':
-          break;
-        case 'V':
-          version();
-          return 0;
-        case 's':
-          pattern = optarg;
-          break;
-      }
-    } else {
-      if (optind < argc) {
-        if (filePath == NULL) {
-          filePath = argv[optind++];
-        } else {
-          usage();
-        }
-      } else {
-        break;
-      }
-    }
-  }
-
-  /* Check if we have enough arguments */
-  if (filePath == NULL) {
-    usage();
-  }
-
-  /* Instanciate the reader */
-  reader = new kiwix::Reader(filePath);
-
-  /* Start to read an article */
-  if (reader != NULL) {
-    string content;
-    string contentType;
-    string suggestion;
-
-    if (pattern != NULL) {
-      std::cout << "Searching suggestions for: " << pattern << std::endl;
-      kiwix::SuggestionsList_t suggestions;
-      reader->searchSuggestionsSmart(pattern, 10, suggestions);
-      for (auto& suggestion: suggestions) {
-        std::cout << suggestion.getTitle() << std::endl;
-      }
-    }
-
-    delete reader;
-  } else {
-    cerr << "Unable instanciate the Kiwix reader." << endl;
-    exit(1);
-  }
-
-  exit(0);
-}

src/reader/meson.build

@@ -1,3 +0,0 @@
-executable('kiwix-read', ['kiwix-read.cpp'],
-  dependencies:all_deps,
-  install:true)

src/searcher/kiwix-search.cpp

@@ -18,11 +18,16 @@
  */
 
 #include <getopt.h>
-#include <kiwix/reader.h>
-#include <kiwix/searcher.h>
+
+#include <zim/search.h>
+#include <zim/suggestion.h>
+
+#include <iostream>
 
 #include "../version.h"
 
+using namespace std;
+
 void usage()
 {
   cout << "Usage: kiwix-search [OPTIONS] ZIM PATTERN" << endl << endl
@@ -48,9 +53,6 @@ int main(int argc, char** argv)
   int option_index = 0;
   int c = 0;
 
-  kiwix::Searcher* searcher = NULL;
-  kiwix::Reader* reader = NULL;
-
   /* Argument parsing */
   while (42) {
     static struct option long_options[]
@@ -96,47 +98,24 @@ int main(int argc, char** argv)
 
   /* Try to prepare the indexing */
   try {
-    reader = new kiwix::Reader(zimPath);
-  } catch (...) {
-    /* Cannot open the zimPath, maybe it is a plain old xapian database
-     * directory */
-  }
+    zim::Archive archive(zimPath);
 
-  if (reader) {
-    searcher = new kiwix::Searcher();
-    bool contians_FTIndex=searcher->add_reader(reader);
-    if(!contians_FTIndex){
-        std::cerr << "The Zim file does not contain a full-text index." << std::endl;
-        if(suggestionFlag){
-            exit(0);
-        }
-        exit(1);
-      }
-  } else {
-    cerr << "Unable to search through zim '" << zimPath << "'." << endl;
-    exit(1);
-  }
-
-  /* Start the indexing */
-  if (searcher != NULL) {
-    string searchString(search);
     if (suggestionFlag) {
-        searcher->suggestions(searchString, verboseFlag);
-    } else  {
-        searcher->search(searchString, 0, 10, verboseFlag);
+      zim::SuggestionSearcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      for (const auto& r : searcher.suggest(search).getResults(0, 10) ) {
+        cout << r.getTitle() << endl;
+      }
+    } else {
+      zim::Searcher searcher(archive);
+      searcher.setVerbose(verboseFlag);
+      const zim::Query query(search);
+      for (const auto& r : searcher.search(query).getResults(0, 10) ) {
+        cout << r.getTitle() << endl;
+      }
     }
-    kiwix::Result* p_result;
-    while ((p_result = searcher->getNextResult())) {
-      cout << p_result->get_title() << endl;
-      delete p_result;
-    }
-
-    delete searcher;
-    delete reader;
-
-    //      kiwix::XapianSearcher::terminate();
-  } else {
-    cerr << "Unable instanciate the Kiwix searcher." << endl;
+  } catch ( const std::runtime_error& err)  {
+    cerr << err.what() << endl;
     exit(1);
   }
 

src/server/kiwix-serve.cpp

@@ -49,31 +49,33 @@ void usage()
             << std::endl
 
             << "Purpose:" << std::endl
-            << "\tDeliver ZIM file articles via HTTP"
+            << "\tDeliver ZIM file(s) articles via HTTP"
             << std::endl << std::endl
 
             << "Mandatory arguments:" << std::endl
-            << "\tLIBRARY_PATH\t\tis the XML library file path listing ZIM file to serve. To be used only with the --library argument."
+            << "\tLIBRARY_PATH\t\tXML library file path listing ZIM file to serve. To be used only with the --library argument."
             << std::endl
-            << "\tZIM_PATH\t\tis the path of a ZIM file."
+            << "\tZIM_PATH\t\tZIM file path(s)"
             << std::endl << std::endl
 
             << "Optional arguments:" << std::endl << std::endl
-            << "\t-h, --help\t\tprint this help" << std::endl << std::endl
-            << "\t-a, --attachToProcess\texit if given process id is not running anymore" << std::endl
-            << "\t-d, --daemon\t\tdetach the HTTP server daemon from the main process" << std::endl
-            << "\t-i, --address\t\tlisten only on this ip address, all available ones otherwise" << std::endl
-            << "\t-M, --monitorLibrary\t\tmonitor the XML library file and reload it automatically" << std::endl
-            << "\t-m, --nolibrarybutton\tdo not print the builtin home button in the builtin top bar overlay" << std::endl
-            << "\t-n, --nosearchbar\tdo not print the builtin bar overlay on the top of each served page" << std::endl
-            << "\t-b, --blockexternal\tprevent users from directly accessing external links" << std::endl
+            << "\t-h, --help\t\tPrint this help" << std::endl << std::endl
+            << "\t-a, --attachToProcess\tWxit if given process id is not running anymore" << std::endl
+            << "\t-d, --daemon\t\tDetach the HTTP server daemon from the main process" << std::endl
+            << "\t-i, --address\t\tListen only on this ip address, all available ones otherwise" << std::endl
+            << "\t-M, --monitorLibrary\tMonitor the XML library file and reload it automatically" << std::endl
+            << "\t-m, --nolibrarybutton\tDon't print the builtin home button in the builtin top bar overlay" << std::endl
+            << "\t-n, --nosearchbar\tDon't print the builtin bar overlay on the top of each served page" << std::endl
+            << "\t-b, --blockexternal\tPrevent users from directly accessing external links" << std::endl
             << "\t-p, --port\t\tTCP port on which to listen to HTTP requests (default: 80)" << std::endl
             << "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
-            << "\t-t, --threads\t\tnumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
-            << "\t-v, --verbose\t\tprint debug log to STDOUT" << std::endl
-            << "\t-V, --version\t\tprint software version" << std::endl
-            << "\t-z, --nodatealiases\tcreate URL aliases for each content by removing the date" << std::endl
-            << "\t-c, --customIndex\tadd path to custom index.html for welcome page" << std::endl
+            << "\t-s, --searchLimit\t\tMaximun number of zim in a fulltext multizim search (default: No limit)" << std::endl
+            << "\t-t, --threads\t\tNumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
+            << "\t-v, --verbose\t\tPrint debug log to STDOUT" << std::endl
+            << "\t-V, --version\t\tPrint software version" << std::endl
+            << "\t-z, --nodatealiases\tCreate URL aliases for each content by removing the date" << std::endl
+            << "\t-c, --customIndex\tAdd path to custom index.html for welcome page" << std::endl
+            << "\t-L, --ipConnectionLimit\tMax number of (concurrent) connections per IP (default: infinite, recommended: >= 6)" << std::endl
             << std::endl
 
             << "Documentation:" << std::endl
@@ -137,7 +139,7 @@ void setup_sighandlers()
 
 uint64_t fileModificationTime(const std::string& path)
 {
-#if defined(_WIN32)
+#if defined(_WIN32) && !defined(stat)
 #define stat _stat
 #endif
   struct stat fileStatData;
@@ -202,6 +204,8 @@ int main(int argc, char** argv)
   bool isVerboseFlag = false;
   bool monitorLibrary = false;
   unsigned int PPID = 0;
+  int ipConnectionLimit = 0;
+  int searchLimit = 0;
 
   static struct option long_options[]
       = {{"daemon", no_argument, 0, 'd'},
@@ -220,13 +224,15 @@ int main(int argc, char** argv)
          {"urlRootLocation", required_argument, 0, 'r'},
          {"customIndex", required_argument, 0, 'c'},
          {"monitorLibrary", no_argument, 0, 'M'},
+         {"ipConnectionLimit", required_argument, 0, 'L'},
+         {"searchLimit", required_argument, 0, 's'},
          {0, 0, 0, 0}};
 
   /* Argument parsing */
   while (true) {
     int option_index = 0;
     int c
-        = getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:M", long_options, &option_index);
+        = getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:ML:s:", long_options, &option_index);
 
     if (c != -1) {
       switch (c) {
@@ -278,6 +284,12 @@ int main(int argc, char** argv)
         case 'M':
           monitorLibrary = true;
           break;
+        case 'L':
+          ipConnectionLimit = atoi(optarg);
+          break;
+        case 's':
+          searchLimit = atoi(optarg);
+          break;
       }
     } else {
       if (optind < argc) {
@@ -364,15 +376,17 @@ int main(int argc, char** argv)
   server.setTaskbar(!noSearchBarFlag, !noLibraryButtonFlag);
   server.setBlockExternalLinks(blockExternalLinks);
   server.setIndexTemplateString(indexTemplateString);
+  server.setIpConnectionLimit(ipConnectionLimit);
+  server.setMultiZimSearchLimit(searchLimit);
 
   if (! server.start()) {
     exit(1);
   }
-  
+
   std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + "/" + rootLocation;
   std::cout << "The Kiwix server is running and can be accessed in the local network at: "
             << url << std::endl;
-            
+
   /* Run endless (until PPID dies) */
   waiting = true;
   do {