Merge pull request #668 from kiwix/prepare-release-3.7.0

Prepare release 3.7.0

.github/workflows/ci.yml

@@ -37,11 +37,10 @@ jobs:
       id: extract_branch
     - name: Checkout code
       uses: actions/checkout@v3
-    - name: Install deps
-      shell: bash
-      run: |
-        ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-tools.tar.xz
-        wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C ${{env.HOME}}
+    - name: Install dependencies
+      uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+      with:
+        target_platform: ${{ matrix.target }}
     - name: Compile
       shell: bash
       run: |

Changelog

@@ -1,3 +1,10 @@
+kiwix-tools 3.7.0
+=================
+
+ * Fixed ZIM name vs Book name confusion in documentation (@veloman-yunkan #663)
+ * Fixes compilation dependencies to rely on appropriate version (@kelson42 #667)
+ * New --skipInvalid Kiwix Server command line option (@schuellerf @kelson42 #666)
+
 kiwix-tools 3.6.0
 =================
 

debian/control

@@ -3,8 +3,8 @@ Section: utils
 Priority: optional
 Maintainer: Kiwix team <kiwix@kiwix.org>
 Build-Depends: debhelper-compat (= 13),
-               libkiwix-dev (>= 10.0.0~),
-               libzim-dev (>= 7.2.0~),
+               libzim-dev (>= 9.0.0), libzim-dev (<< 10.0.0),
+               libkiwix-dev (>= 13.0.0), libkiwix-dev (<< 14.0.0),
                meson,
                pkg-config,
 Standards-Version: 4.5.0

docs/conf.py

@@ -18,7 +18,7 @@ import os
 # -- Project information -----------------------------------------------------
 
 project = 'kiwix-tools'
-copyright = '2022, kiwix-team'
+copyright = '2024, kiwix-team'
 author = 'kiwix-team'
 
 
@@ -30,6 +30,7 @@ on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+        "sphinx_rtd_theme"
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -41,8 +42,7 @@ templates_path = ['_templates']
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 
-if not on_rtd:
-    html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinx_rtd_theme'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

docs/kiwix-serve.rst

@@ -339,8 +339,8 @@ filtering criteria must be specified via the following URL parameters:
 * ``q`` - include in the results only entries that contain the specified text
   in the title or description.
 
-* ``name`` - include in the results only the entry with the specified
-  :term:`name <ZIM name>`.
+* ``name`` - include in the results only entries with a matching
+  :term:`book name <Book name>`.
 
 
 Examples:
@@ -532,7 +532,7 @@ specified ZIM file.
 
 **Parameters:**
 
-  ``content``: :term:`name <ZIM name>` of the ZIM file.
+  ``content``: :term:`name of the ZIM file <ZIM name>`.
 
 
 .. _raw:
@@ -609,7 +609,7 @@ A multi-ZIM search request must comply with the following constraints:
     below list take precedence over subsequent ones (the later ones, even if
     present in the request, are simply ignored).
 
-    ``content``: :term:`name <ZIM name>` of the ZIM file (for a single-ZIM
+    ``content``: :term:`name of the ZIM file <ZIM name>` (for a single-ZIM
     search). This is a :ref:`legacy parameter <deprecation>`. ``books.name``
     should be used instead.
 
@@ -617,9 +617,23 @@ A multi-ZIM search request must comply with the following constraints:
     a multi-ZIM search, however must respect the :ref:`multi-ZIM search
     constraints <multi-zim-search-constraints>`.
 
-    ``books.name``: :term:`name <ZIM name>` of the ZIM file. Can be repeated
-    for a multi-ZIM search, however must respect the :ref:`multi-ZIM search
-    constraints <multi-zim-search-constraints>`.
+      .. note::
+
+        If any of the provided ``books.id`` values refers to a book missing
+        from the library then an error is returned instead of running the
+        search on the remaining (valid) entries.
+
+    ``books.name``: :term:`name of the ZIM file <ZIM name>` (not to be confused
+    with ``books.filter.name`` which selects/filters based on the :term:`book
+    name <Book name>`). Can be repeated for a multi-ZIM search, however must
+    respect the :ref:`multi-ZIM search constraints
+    <multi-zim-search-constraints>`.
+
+      .. note::
+
+        If any of the provided ``books.name`` values refers to a book missing
+        from the library then an error is returned instead of running the
+        search on the remaining (valid) entries.
 
     ``books.filter.{criteria}``: allows to take full advantage of :ref:`library
     filtering <library-filtering>` functionality of the `/catalog/v2/entries`_
@@ -739,7 +753,7 @@ added as an option to perform a full text search in the said ZIM file.
 
 **Parameters:**
 
-  ``content`` (mandatory): :term:`name <ZIM name>` of the ZIM file.
+  ``content`` (mandatory): :term:`name of the ZIM file <ZIM name>`.
 
   ``term`` (optional; defaults to an empty string): query text.
 
@@ -799,6 +813,21 @@ Glossary
 
 .. glossary::
 
+  Book name
+
+    Name of the book as specified in the ZIM file metadata (for a
+    ``kiwix-serve`` started *WITHOUT* the :option:`--library` option) or the
+    library XML file (for a ``kiwix-serve`` started with the
+    :option:`--library` option).
+
+    .. note::
+
+      Two or more books may have the same name in the library. That's not
+      considered a conflict, because there may be multiple versions of the
+      "same" book (differing by the settings of the scraper, date, etc).
+      :ref:`Library filtering <library-filtering>` by name will return all
+      matching books.
+
   ZIM filename
 
     Name of a ZIM file on the server filesystem.
@@ -808,20 +837,22 @@ Glossary
     Identifier of a ZIM file in the server's library (used for referring to a
     particular ZIM file in requests).
 
-    For a ``kiwix-serve`` started with a list of ZIM files, ZIM names are
-    derived from the filename by dropping the extension and replacing certain
-    characters (spaces are replaced with underscores, and ``+`` symbols are
-    replaced with the text ``plus``). Presence of the
-    :option:`-z`/:option:`--nodatealiases` option will create additional names
-    (aliases) for filenames with dates.
+    ZIM names are derived from the filenames as follows:
 
-    For a ``kiwix-serve`` started with the :option:`--library` option, ZIM
-    names come from the library XML file.
+    - file extension is removed,
+    - all characters are converted to lowercase,
+    - diacritics are removed,
+    - spaces are replaced with underscores,
+    - ``+`` symbols are replaced with the text ``plus``.
+
+    Presence of the :option:`-z`/:option:`--nodatealiases` option will create
+    additional names (aliases) for filenames with dates.
 
     ZIM names are expected to be unique across the library. Any name conflicts
     (including those caused by the usage of the
     :option:`-z`/:option:`--nodatealiases` option) are reported on STDERR but,
-    otherwise, are ignored.
+    otherwise, are ignored (i.e. only one of the entries can be accessed via
+    the conflicting name).
 
   ZIM title
 

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-tools', 'cpp',
-  version : '3.6.0',
+  version : '3.7.0',
   license : 'GPL',
   default_options: ['c_std=c11', 'cpp_std=c++17', 'werror=true'])
 
@@ -16,8 +16,10 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
+libzim_dep = dependency('libzim', version:'>=9.0.0', static:static_linkage)
+libzim_dep = dependency('libzim', version:'<10.0.0', static:static_linkage)
 kiwixlib_dep = dependency('kiwix', version:'>=13.0.0', static:static_linkage)
-libzim_dep = dependency('libzim', version:'>=8.1.0', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'<14.0.0', static:static_linkage)
 
 all_deps = [thread_dep, kiwixlib_dep, libzim_dep]
 

src/man/kiwix-serve.1

@@ -88,6 +88,10 @@ Override the welcome page with a custom HTML file.
 \fB-L N, --ipConnectionLimit=N\fR
 Max number of (concurrent) connections per IP (default: infinite, recommended: >= 6).
 
+.TP
+\fB-k, --skipInvalid\fR
+Startup even when ZIM files are invalid (those will be skipped)
+
 .TP
 \fB-v, --verbose\fR
 Print debug log to STDOUT.

src/server/kiwix-serve.cpp

@@ -77,6 +77,7 @@ void usage()
             << "\t-z, --nodatealiases\tCreate URL aliases for each content by removing the date" << std::endl
             << "\t-c, --customIndex\tAdd path to custom index.html for welcome page" << std::endl
             << "\t-L, --ipConnectionLimit\tMax number of (concurrent) connections per IP (default: infinite, recommended: >= 6)" << std::endl
+            << "\t-k, --skipInvalid\tStartup even when ZIM files are invalid (those will be skipped)" << std::endl
             << std::endl
 
             << "Documentation:" << std::endl
@@ -217,6 +218,7 @@ int main(int argc, char** argv)
   unsigned int PPID = 0;
   int ipConnectionLimit = 0;
   int searchLimit = 0;
+  bool skipInvalid = false;
 
   static struct option long_options[]
       = {{"daemon", no_argument, 0, 'd'},
@@ -237,6 +239,7 @@ int main(int argc, char** argv)
          {"monitorLibrary", no_argument, 0, 'M'},
          {"ipConnectionLimit", required_argument, 0, 'L'},
          {"searchLimit", required_argument, 0, 's'},
+         {"skipInvalid", no_argument, 0, 'k'},
          {0, 0, 0, 0}};
 
   std::set<int> usedOptions;
@@ -307,6 +310,9 @@ int main(int argc, char** argv)
         case 's':
           searchLimit = atoi(optarg);
           break;
+        case 'k':
+          skipInvalid = true;
+          break;
         case '?':
           usage();
           return 2;
@@ -348,9 +354,13 @@ int main(int argc, char** argv)
     std::vector<std::string>::iterator it;
     for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
       if (!manager.addBookFromPath(*it, *it, "", false)) {
-        std::cerr << "Unable to add the ZIM file '" << *it
-             << "' to the internal library." << std::endl;
-        exit(1);
+        if (skipInvalid) {
+          std::cerr << "Skipping invalid '" << *it << "' ...continuing" << std::endl;
+        } else {
+          std::cerr << "Unable to add the ZIM file '" << *it
+               << "' to the internal library." << std::endl;
+          exit(1);
+        }
       }
     }
   }