fixup! fixup! fixup! Recursively scan directories for ZIM files

Added the ability to provide a directory path along with files.
If the path provided is a directory, we recursively scan it and either add the files in it or go through the same operation for sub directories.

Changelog

@@ -1,3 +1,13 @@
+kiwix-tools 3.8.1
+=================
+
+ * Kiwix server
+   - Hide port number in URL when server is running on port 80 (@vighnesh-sawant #763)
+   - Better deal with container /data dir permissions (@kelson42 #787)
+
+ * Other
+   - Fix kiwix-manage docopt integration (@kelson42 #783)
+
 kiwix-tools 3.8.0
 =================
 

docs/kiwix-serve.rst

@@ -39,6 +39,8 @@ be provided as a semicolon (``;``) separated list.
 
 ``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed).
 
+``DIRECTORY_PATH``: Directory path containing ZIM files (multiple arguments are allowed).
+
 Options
 -------
 

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-tools', 'cpp',
-  version : '3.8.0',
+  version : '3.8.1',
   license : 'GPL',
   default_options: ['c_std=c11', 'cpp_std=c++17', 'werror=true'])
 

src/man/kiwix-serve.1

@@ -8,6 +8,10 @@ kiwix-serve \- Kiwix HTTP Server
 .B kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
 .br
 .B kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
+.br
+.B kiwix-serve [OPTIONS] DIRECTORY_PATH ...
+.br
+.B kiwix-serve [OPTIONS] ZIM_FILE_PATH DIRECTORY_PATH ...
 
 .SH DESCRIPTION
 The \fBkiwix-serve\fR command is used to run a stand-alone HTTP server for serving ZIM contents over the network.
@@ -21,6 +25,10 @@ Path of an XML library file listing ZIM files to serve. To be used only with the
 \fBZIM_FILE_PATH ...\fR
 ZIM file path(s). Multiple arguments are allowed.
 
+.TP
+\fBDIRECTORY_PATH ...\fR
+Path(s) of directories containing ZIM files. Multiple arguments are allowed.
+
 .SH OPTIONS
 .TP
 \fB--library\fR
@@ -123,6 +131,11 @@ Serve multiple ZIM files:
 .B kiwix-serve zim1.zim zim2.zim zim3.zim
 .fi
 
+Serve ZIM files within a directory or directories:
+.sp
+.nf
+.B kiwix-serve zimDir1 zimDir2
+
 Serve ZIM files from a library:
 .sp
 .nf

src/manager/kiwix-manage.cpp

@@ -60,8 +60,8 @@ static const char USAGE[] =
 R"(Manipulates the Kiwix library XML file
 
 Usage:
- kiwix-manage LIBRARYPATH add [--zimPathToSave=<custom_zim_path>] [--url=<http_zim_url>] ZIMPATH
- kiwix-manage LIBRARYPATH remove|delete ZIMID ...
+ kiwix-manage LIBRARYPATH add [--zimPathToSave=<custom_zim_path>] [--url=<http_zim_url>] ZIMPATH ...
+ kiwix-manage LIBRARYPATH (delete|remove) ZIMID ...
  kiwix-manage LIBRARYPATH show [ZIMID ...]
  kiwix-manage -v | --version
  kiwix-manage -h | --help
@@ -69,7 +69,7 @@ Usage:
 Arguments:
   LIBRARYPATH    The XML library file path.
   ZIMID          ZIM file unique ID.
-  ZIMPATH        A path to a zim to add.
+  ZIMPATH        A path to a ZIM to add.
 
 Options:
   Custom options for "add" action:
@@ -87,23 +87,24 @@ Examples:
 
 Documentation:
   Source code  https://github.com/kiwix/kiwix-tools
-  More info   https://wiki.kiwix.org/wiki/Kiwix-manage
+  More info    https://wiki.kiwix.org/wiki/kiwix-manage
 )";
 
 int handle_show(const kiwix::Library& library, const std::string& libraryPath,
                  const Options& options)
 {
-  if (options.at("ZIMID").isStringList()) {
-    auto bookIds = options.at("ZIMID").asStringList();
-    for(auto& bookId: bookIds) {
-       show(library, bookId);
-    }
-  } else {
+  if (options.at("ZIMID").asStringList().empty()) {
     auto booksIds = library.getBooksIds();
     for(auto& bookId: booksIds) {
       show(library, bookId);
     }
+  } else {
+    auto bookIds = options.at("ZIMID").asStringList();
+    for(auto& bookId: bookIds) {
+       show(library, bookId);
+    }
   }
+
   return(0);
 }
 
@@ -115,19 +116,21 @@ int handle_add(kiwix::LibraryPtr library, const std::string& libraryPath,
 
   kiwix::Manager manager(library);
 
-  std::string zimPath = options.at("ZIMPATH").asString();
-  if (options.at("--zimPathToSave").isString()) {
-    zimPathToSave = options.at("--zimPathToSave").asString();
-  } else {
-    zimPathToSave = zimPath;
-  }
-  if (options.at("--url").isString()) {
-    url = options.at("--url").asString();
-  }
+  auto zimPaths = options.at("ZIMPATH").asStringList();
+  for (auto& zimPath: zimPaths) {
+    if (options.at("--zimPathToSave").isString()) {
+      zimPathToSave = options.at("--zimPathToSave").asString();
+    } else {
+      zimPathToSave = zimPath;
+    }
+    if (options.at("--url").isString()) {
+      url = options.at("--url").asString();
+    }
 
-  if (manager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false).empty()) {
-    std::cerr << "Cannot add zim " << zimPath << " to the library." << std::endl;
-    return 1;
+    if (manager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false).empty()) {
+      std::cerr << "Cannot add ZIM " << zimPath << " to the library." << std::endl;
+      return 1;
+    }
   }
 
   return 0;

src/server/kiwix-serve.cpp

@@ -23,6 +23,7 @@
 #include <kiwix/server.h>
 #include <kiwix/name_mapper.h>
 #include <kiwix/tools.h>
+#include <filesystem>
 
 #ifdef _WIN32
 # include <windows.h>
@@ -44,19 +45,22 @@
 #define LITERAL_AS_STR(A) #A
 #define AS_STR(A) LITERAL_AS_STR(A)
 
+namespace fs = std::filesystem;
 
 static const char USAGE[] =
 R"(Deliver ZIM file(s) articles via HTTP
 
 Usage:
  kiwix-serve [options] ZIMPATH ...
+ kiwix-serve [options] DIRECTORYPATH ...
+ kiwix-serve [options] ZIMPATH DIRECTORYPATH ...
  kiwix-serve [options] (-l | --library) LIBRARYPATH
  kiwix-serve -h | --help
  kiwix-serve -V | --version
 
 Mandatory arguments:
   LIBRARYPATH  XML library file path listing ZIM file to serve. To be used only with the --library argument."
-  ZIMPATH      ZIM file path(s)
+  ZIMPATH      ZIM file/directory path(s)
 
 Options:
  -h --help                               Print this help
@@ -104,16 +108,6 @@ std::string loadCustomTemplate (std::string customIndexPath) {
   return indexTemplateString;
 }
 
-inline std::string normalizeRootUrl(std::string rootUrl)
-{
-  while ( !rootUrl.empty() && rootUrl.back() == '/' )
-    rootUrl.pop_back();
-
-  while ( !rootUrl.empty() && rootUrl.front() == '/' )
-    rootUrl = rootUrl.substr(1);
-  return rootUrl.empty() ? rootUrl : "/" + rootUrl;
-}
-
 #ifndef _WIN32
 volatile sig_atomic_t waiting = false;
 volatile sig_atomic_t libraryMustBeReloaded = false;
@@ -193,6 +187,26 @@ bool reloadLibrary(kiwix::Manager& mgr, const std::vector<std::string>& paths)
     }
 }
 
+void addPathsInManager(kiwix::Manager& manager, const std::vector<std::string>& zimPaths,
+                      bool skipInvalid, bool isVerboseFlag)
+{
+  for (auto it = zimPaths.begin(); it != zimPaths.end(); it++) {
+    if (fs::is_directory(*it)) {
+      manager.addBooksFromDirectory(*it, isVerboseFlag);
+    } else {
+      if (!manager.addBookFromPath(*it, *it, "", false)) {
+        if (skipInvalid) {
+          std::cerr << "Skipping invalid '" << *it << "' ...continuing" << std::endl;
+        } else {
+          std::cerr << "Unable to add the ZIM file '" << *it
+              << "' to the internal library." << std::endl;
+          exit(1);
+        }
+      }
+    }
+  }
+}
+
 // docopt::value::isLong() is counting repeated values.
 // It doesn't check if the string can be parsed as long.
 // (Contrarly to `asLong` which will try to convert string to long)
@@ -224,7 +238,7 @@ int main(int argc, char** argv)
   std::string rootLocation = "/";
   auto library = kiwix::Library::create();
   unsigned int nb_threads = DEFAULT_THREADS;
-  std::vector<std::string> zimPathes;
+  std::vector<std::string> zimPaths;
   std::string libraryPath;
   std::string rootPath;
   std::string address;
@@ -280,7 +294,7 @@ int main(int argc, char** argv)
     STRING("--customIndex", customIndexPath)
     INT("--ipConnectionLimit", ipConnectionLimit, "IP connection limit must be an integer")
     INT("--searchLimit", searchLimit, "Search limit must be an integer")
-    STRING_LIST("ZIMPATH", zimPathes, "ZIMPATH must be a string list")
+    STRING_LIST("ZIMPATH", zimPaths, "ZIMPATH must be a string list")
  }
 
  if (!errorString.empty()) {
@@ -314,18 +328,7 @@ int main(int argc, char** argv)
            << "' is empty (or has only remote books)." << std::endl;
     }
   } else {
-    std::vector<std::string>::iterator it;
-    for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
-      if (!manager.addBookFromPath(*it, *it, "", false)) {
-        if (skipInvalid) {
-          std::cerr << "Skipping invalid '" << *it << "' ...continuing" << std::endl;
-        } else {
-          std::cerr << "Unable to add the ZIM file '" << *it
-               << "' to the internal library." << std::endl;
-          exit(1);
-        }
-      }
-    }
+    addPathsInManager(manager, zimPaths, skipInvalid, isVerboseFlag);
   }
   auto libraryFileTimestamp = newestFileTimestamp(libraryPaths);
   auto curLibraryFileTimestamp = libraryFileTimestamp;
@@ -393,13 +396,11 @@ int main(int argc, char** argv)
   if (! server.start()) {
     exit(1);
   }
-
-  std::string prefix = "http://";
-  kiwix::IpAddress addresses = server.getAddress();
-  std::string suffix = ":" + std::to_string(server.getPort()) + normalizeRootUrl(rootLocation);
+  
   std::cout << "The Kiwix server is running and can be accessed in the local network at: " << std::endl;
-  if(!addresses.addr.empty()) std::cout << "  - " << prefix << addresses.addr << suffix << std::endl;
-  if(!addresses.addr6.empty()) std::cout << "  - " << prefix << "[" << addresses.addr6 << "]" <<  suffix << std::endl;
+  for (const auto& url : server.getServerAccessUrls()) {
+    std::cout << "  - " << url << std::endl;
+  }
 
   /* Run endless (until PPID dies) */
   waiting = true;