Merge pull request #616 from kiwix/version_3.5.0

removed references to docker.io

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
       HOME: /home/runner
     runs-on: ubuntu-latest
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:36"
     steps:
     - name: Extract branch name
       shell: bash

.github/workflows/docker.yml

@@ -11,16 +11,15 @@ on:
 jobs:
   build-and-push-kiwix-tools:
     name: Deploy kiwix-tools Docker Image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v3.4.0
       - name: build and publish kiwix-tools
-        uses: openzim/docker-publish-action@v9
+        uses: openzim/docker-publish-action@v10
         with:
           image-name: kiwix/kiwix-tools
+          registries: ghcr.io
           credentials: |
-            DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-            DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }}
             GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
             GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
           context: docker
@@ -38,17 +37,16 @@ jobs:
 
   build-and-push-kiwix-serve:
     name: Deploy kiwix-serve Docker Image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: build-and-push-kiwix-tools
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v3.4.0
       - name: build and publish kiwix-serve
-        uses: openzim/docker-publish-action@v9
+        uses: openzim/docker-publish-action@v10
         with:
           image-name: kiwix/kiwix-serve
+          registries: ghcr.io
           credentials: |
-            DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-            DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }}
             GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
             GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
           context: docker/server

.github/workflows/package.yml

@@ -8,11 +8,11 @@ jobs:
       fail-fast: false
       matrix:
         distro:
+          - ubuntu-kinetic
           - ubuntu-jammy
-          - ubuntu-impish
           - ubuntu-focal
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Determine which PPA we should upload to
       - name: PPA
@@ -41,10 +41,10 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: legoktm/gh-action-build-deb@ubuntu-impish
-        if: matrix.distro == 'ubuntu-impish'
-        name: Build package for ubuntu-impish
-        id: build-ubuntu-impish
+      - uses: legoktm/gh-action-build-deb@ubuntu-kinetic
+        if: matrix.distro == 'ubuntu-kinetic'
+        name: Build package for ubuntu-kinetic
+        id: build-ubuntu-kinetic
         with:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
@@ -57,15 +57,15 @@ jobs:
           args: --no-sign
           ppa: ${{ steps.ppa.outputs.ppa }}
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: Packages for ${{ matrix.distro }}
           path: output
 
       - uses: legoktm/gh-action-dput@master
         name: Upload dev package
-        # Only upload on pushes to master
-        if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
+        # Only upload on pushes to git default branch
+        if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
         with:
           gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
           repository: ppa:kiwixteam/dev

Changelog

@@ -1,3 +1,17 @@
+kiwix-tools 3.5.0
+=================
+
+ * Do not use `--static` option when compiling on MacOs (@mgautierfr #615)
+ * Move main branch from `master` to `main`.
+ * Fix docker image (@jacroe #597)
+ * Various CI improvements (@kelson42)
+
+kiwix-serve
+-----------
+
+ * Add documentation about the kiwix-serve API (@veloman-yunkan #586)
+   https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html#http-api
+
 kiwix-tools 3.4.0
 =================
 

README.md

@@ -9,10 +9,11 @@ command line tools:
 
 [![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/)
 [![Repositories](https://img.shields.io/repology/repositories/kiwix-tools?label=repositories)](https://github.com/kiwix/kiwix-tools/wiki/Repology)
-[![Docker](https://img.shields.io/docker/v/kiwix/kiwix-tools?arch=amd64&label=Docker&sort=date)](https://hub.docker.com/r/kiwix/kiwix-tools)
-[![Docker (kiwix-serve)](https://img.shields.io/docker/v/kiwix/kiwix-serve?arch=amd64&label=Docker%20(kiwix-serve)&sort=date)](https://hub.docker.com/r/kiwix/kiwix-serve)
+[![Docker](https://ghcr-badge.deta.dev/kiwix/kiwix-tools/latest_tag?label=docker)](https://ghcr.io/kiwix/kiwix-tools)
+[![Docker](https://ghcr-badge.deta.dev/kiwix/kiwix-tools/latest_tag?label=docker%20(kiwix-serve))](https://ghcr.io/kiwix/kiwix-tools)
 [![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h)
-[![Build Status](https://github.com/kiwix/kiwix-tools/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-tools/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/kiwix/kiwix-tools/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/kiwix-tools/actions?query=branch%3Amain)
+[![Doc](https://readthedocs.org/projects/kiwix-tools/badge/?style=flat)](https://kiwix-tools.readthedocs.org/en/latest/?badge=latest)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-tools/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-tools)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
@@ -112,10 +113,10 @@ Like for the installation, you might need to run the command as `root`
 Docker
 ------
 
-An official Docker image of the Kiwix tools can be found in the
-[Docker Hub](https://hub.docker.com/r/kiwix/kiwix-tools). A
+An official Docker image of the Kiwix tools can be found on 
+[GHCR](https://ghcr.io/kiwix/kiwix-tools). A
 `kiwix-serve` dedicated Docker image [exists
-too](https://hub.docker.com/r/kiwix/kiwix-serve).
+too](https://ghcr.io/kiwix/kiwix-serve).
 
 Troubleshooting
 ---------------

docker/README.md

@@ -1,14 +1,14 @@
 Kiwix-tools Docker image
 ===
 
-- Available on [docker.io](https://hub.docker.com/r/kiwix/kiwix-tools) and [ghcr.io](https://ghcr.io/kiwix/kiwix-tools).
+- Available on [ghcr.io](https://ghcr.io/kiwix/kiwix-tools).
 - multi-arch (`linux/amd64`, `linux/arm64`, `linux/arm/v7`)
 - based on official `kiwix-tools` binaries.
 
 ## Usage
 
 ``` sh
-$ docker run -it kiwix/kiwix-tools:3.1.2
+$ docker run -it ghcr.io/kiwix/kiwix-tools:3.1.2
 
 Welcome to kiwix-tools! The following binaries are available:
 kiwix-manage  kiwix-search  kiwix-serve
@@ -17,7 +17,7 @@ kiwix-manage  kiwix-search  kiwix-serve
 `kiwix-tools` operates on zim files. You shall mount a volume to access the files.
 
 ```sh
-docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
+docker run -v $(pwd):/data -it ghcr.io/kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
 ```
 
 ## Building and reusing

docker/server/Dockerfile

@@ -1,7 +1,7 @@
 ARG VERSION=latest
 
 # kiwix-tools is multi-arch
-FROM kiwix/kiwix-tools:$VERSION
+FROM ghcr.io/kiwix/kiwix-tools:$VERSION
 LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
 
 # expose kiwix-serve default port and workdir

docker/server/README.md

@@ -9,14 +9,14 @@ With local ZIM file(s)
 * Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following:
 
 ```bash
-docker run -v /tmp/zim:/data -p 8080:8080 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
+docker run -v /tmp/zim:/data -p 8080:8080 ghcr.io/kiwix/kiwix-serve wikipedia.zim wiktionary.zim
 ```
 
 With remote ZIM file
 --------------------
 
 ```bash
-docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 kiwix/kiwix-serve
+docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 ghcr.io/kiwix/kiwix-serve
 ```
 
 Change default port
@@ -25,7 +25,7 @@ Change default port
 You can change port to expose with environment variable PORT, useful if running on Podman, K8s or OpenShift
 
 ```bash
-podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 kiwix/kiwix-serve
+podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 ghcr.io/kiwix/kiwix-serve
 ```
 
 ARM
@@ -33,7 +33,7 @@ ARM
 
 Build an image for an ARM based GNU/Linux:
 ```bash
-docker build . -t kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/"
+docker build . -t ghcr.io/kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/"
 ```
 
 You can also deploy kiwix with [`docker-compose`](https://docs.docker.com/compose/). Check out a sample at [docker-compose.yml.example](docker-compose.yml.example)

docker/server/docker-compose.yml.example

@@ -8,7 +8,7 @@ services:
       # volumes:
       #   - /tmp/zim:/data
       # command:
-      #   - *.zim
+      #   - '*.zim'
       # uncomment next 2 lines to use it with remote zim file
       # environment:
       #   - 'DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim'

docs/conf.py

@@ -0,0 +1,50 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'kiwix-tools'
+copyright = '2022, kiwix-team'
+author = 'kiwix-team'
+
+
+# -- General configuration ---------------------------------------------------
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+if not on_rtd:
+    html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []

docs/index.rst

@@ -0,0 +1,8 @@
+Welcome to kiwix-tools' documentation!
+======================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   kiwix-serve

docs/kiwix-serve.rst

@@ -0,0 +1,835 @@
+***********
+kiwix-serve
+***********
+
+Introduction
+============
+
+``kiwix-serve`` is a tool for serving ZIM file content over HTTP. It supports
+serving a library containing multiple ZIM files. In a large library served by a
+``kiwix-serve`` instance clients can look up/filter ZIM files of interest by
+words in their :term:`titles <ZIM title>` and/or descriptions, language, tags, etc.
+
+``kiwix-serve`` provides a ZIM file viewer for displaying inidividual pages
+from a ZIM file inside the user's web browser (without downloading the full ZIM
+file).
+
+Clients can also remotely search inside those ZIM files that contain a full-text
+search database.
+
+
+Usage
+=====
+
+.. code-block:: sh
+
+  kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
+  kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
+
+
+Arguments
+---------
+
+.. _cli-arg-library-file-path:
+
+``LIBRARY_FILE_PATH``: path of an XML library file listing ZIM files to serve.
+To be used only with the :option:`--library` option. Multiple library files can
+be provided as a semicolon (``;``) separated list.
+
+``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed).
+
+Options
+-------
+
+.. option:: --library
+
+  By default, ``kiwix-serve`` expects a list of ZIM files as command line
+  arguments. Providing the :option:`--library` option tells ``kiwix-serve``
+  that the command line argument is rather a :ref:`library XML file
+  <cli-arg-library-file-path>`.
+
+.. option:: -i ADDR, --address=ADDR
+
+  Listen only on this IP address. By default the server listens on all
+  available IP addresses.
+
+
+.. option:: -p PORT, --port=PORT
+
+  TCP port on which to listen for HTTP requests (default: 80).
+
+
+.. option:: -r ROOT, --urlRootLocation=ROOT
+
+  URL prefix on which the content should be made available (default: empty).
+
+
+.. option:: -d, --daemon
+
+  Detach the HTTP server daemon from the main process.
+
+
+.. option:: -a PID, --attachToProcess=PID
+
+  Exit when the process with id PID stops running.
+
+
+.. option:: -M, --monitorLibrary
+
+  Monitor the XML library file and reload it automatically when it changes.
+
+  Library reloading can be forced anytime by sending a SIGHUP signal to the
+  ``kiwix-serve`` process (this works regardless of the presence of the
+  :option:`--monitorLibrary`/:option:`-M` option).
+
+
+.. option:: -m, --nolibrarybutton
+
+  Disable the library home button in the ZIM viewer toolbar.
+
+
+.. option:: -n, --nosearchbar
+
+  Disable the searchbox in the ZIM viewer toolbar.
+
+
+.. option:: -b, --blockexternal
+
+  Prevent the users from directly navigating to external resources via such
+  links in ZIM content.
+
+
+.. option:: -t N, --threads=N
+
+  Number of threads to run in parallel (default: 4).
+
+
+.. option:: -s N, --searchLimit=N
+
+  Maximum number of ZIM files in a fulltext multizim search (default: No limit).
+
+
+.. option:: -z, --nodatealiases
+
+  Create URL aliases for each content by removing the date embedded in the file
+  name. The expected format of the date in the filename is ``_YYYY-MM``. For
+  example, ZIM file ``wikipedia_en_all_2020-08.zim`` will be accessible both as
+  ``wikipedia_en_all_2020-08`` and ``wikipedia_en_all``.
+
+
+.. option:: -c PATH, --customIndex=PATH
+
+  Override the welcome page with a custom HTML file.
+
+
+.. option:: -L N, --ipConnectionLimit=N
+
+  Max number of (concurrent) connections per IP (default: infinite,
+  recommended: >= 6).
+
+
+.. option:: -v, --verbose
+
+  Print debug log to STDOUT.
+
+
+.. option:: -V, --version
+
+  Print the software version.
+
+
+.. option:: -h, --help
+
+  Print the help text.
+
+
+HTTP API
+========
+
+``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where
+``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the
+:option:`--address`/:option:`-i`, :option:`--port`/:option:`-p` and
+:option:`--urlRootLocation`/:option:`-r` options, respectively.
+
+HTTP API endpoints presented below are relative to that location, i.e.
+``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``.
+
+.. note::
+
+  The HTTP API is documented in its entirety in order to facilitate the work of
+  the Kiwix development team. Note, however, that only a subset of the HTTP API
+  constitutes ``kiwix-serves``'s public interface.
+
+  .. _public-api-endpoint:
+
+  **Public API endpoint**
+
+    A public HTTP API endpoint is intended to serve the outside world (in
+    addition to ``kiwix-serve``'s front-end and other Kiwix products). The
+    Kiwix development team will do its best to ensure gratifying experience for
+    clients of public API endpoints at all stages of the endpoint lifecycle.
+
+  .. _private-api-endpoint:
+
+  **Private API endpoint**
+
+    A private API endpoint is intended to be used only by ``kiwix-serve``'s
+    frontend or by other products maintained solely by the Kiwix team. Private
+    API comes without any guaranees. It may change as frequently and as
+    drasticaly as the Kiwix development team sees fit.
+
+  .. _deprecation:
+
+  **Deprecation**
+
+    Public API doesn't stay frozen once and forever. As the API evolves, Kiwix
+    team reserves the right to drop support for certain old functionality. In
+    such events, an advance notice will be issued and the users will be given
+    enough time to prepare for the change.
+
+  Currently, public endpoints are limited to the following list:
+
+    -  :ref:`OPDS API <new-opds-api>`
+    -  ``/raw``
+    -  ``/search`` (with ``/search/searchdescription.xml``)
+
+.. _welcome-page:
+
+``/``
+-----
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Welcome page is served under ``/``. By default this is the library page, where
+books are listed and can be looked up/filtered interactively. However, the
+welcome page can be overriden through the :option:`--customIndex`/:option:`-c`
+command line option of ``kiwix-serve``.
+
+
+.. _new-opds-api:
+
+``/catalog/v2`` (OPDS API)
+------------------------------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+The new OPDS API of ``kiwix-serve`` is based on the `OPDS Catalog specification
+v1.2 <https://specs.opds.io/opds-1.2>`_. All of its endpoints are grouped under
+``/catalog/v2``.
+
+:ref:`Legacy OPDS API <legacy-opds-api>` is preserved for backward
+compatibility.
+
+
+``/catalog/v2/root.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+The OPDS Catalog Root links to the OPDS acquisition and navigation feeds
+accessible through the other endpoints of the OPDS API.
+
+
+``/catalog/v2/searchdescription.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Describes the `/catalog/v2/entries`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+
+``/catalog/v2/categories``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full list of ZIM file categories as an `OPDS Navigation Feed
+<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
+
+
+``/catalog/v2/entries``
+^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns a full or filtered list of ZIM files as a paginated `OPDS acquisition
+feed <https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `complete
+entries
+<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
+
+**Pagination:**
+
+By default, no more than 10 first entries are returned from the library. To
+obtain the remaining entries the URL query parameters ``start`` and/or
+``count`` must be used. The output of ``/catalog/v2/entries?start=s&count=n``
+will contain at most ``n`` (default value: 10) results starting from entry #
+``s`` (default value: 0).  ``count`` with a negative value (e.g.  ``count=-1``)
+removes the limit on the number of results in the output.
+
+
+.. note::
+
+  Previously ``count=0`` also designated an unbounded query (i.e. worked
+  similarly to ``count=-1``). The response to a ``count=0`` query was changed
+  to consist of 0 results, as such a query/response combination is a good way
+  to find out the total number of results (when only that information is
+  needed) with minimal consumption of resources.
+
+Examples:
+
+.. code:: sh
+
+  # Returns the first 10 entries (internally numbered 0 through 9)
+  $ curl 'http://localhost:8080/catalog/v2/entries'
+
+  # Returns the next 10 entries (internally numbered 10 through 19)
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=10'
+
+  # Returns the first 50 entries
+  $ curl 'http://localhost:8080/catalog/v2/entries?count=50'
+
+  # Returns 50 entries starting from entry # 100 (i.e. entries ## 100-149)
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=50'
+
+  # Returns all entries
+  $ curl 'http://localhost:8080/catalog/v2/entries?count=-1'
+
+  # Returns all entries starting from entry # 100
+  $ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=-1'
+
+
+.. _library-filtering:
+
+**Filtering:**
+
+A filtered subset of the library can be requested by providing one or more
+filtering criteria, whereupon only entries matching *all* of the criteria are
+included in the response. Pagination is applied to the filtered list. The
+filtering criteria must be specified via the following URL parameters:
+
+* ``lang`` - filter by language (specified as a 3-letter language code).
+
+* ``category`` - filter by categories associated with the library entries.
+
+* ``tag`` - filter by tags associated with the library entries. Multiple tags
+  can be provided as a semicolon separated list (e.g
+  ``tag=wikipedia;_videos:no``). The result will contain only those entries
+  that contain *all* of the requested tags.
+
+* ``notag`` - filter out (exclude) entries with *any* of the specified tags
+  (example - ``notag=ted;youtube``).
+
+* ``maxsize`` - include in the results only entries whose size (in bytes)
+  doesn't exceed the provided value.
+
+* ``q`` - include in the results only entries that contain the specified text
+  in the title or description.
+
+* ``name`` - include in the results only the entry with the specified
+  :term:`name <ZIM name>`.
+
+
+Examples:
+
+.. code:: sh
+
+  # List only books in Italian (lang=ita) but
+  # return only results ## 100-149 (start=100&count=50)
+  $ curl 'http://localhost:8080/catalog/v2/entries?lang=ita&start=100&count=50'
+
+  # List only books with category of 'wikipedia' AND containing the word
+  # 'science' in the title or description. Return only the first 10 results.
+  $ curl 'http://localhost:8080/catalog/v2/entries?q=science&category=wikipedia'
+
+
+``/catalog/v2/entry/ZIMID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns full info about the library entry with :term:`UUID <ZIM UUID>`
+``ZIMID``.
+
+
+``/catalog/v2/illustration/ZIMID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+**Usage:**
+
+  ``/catalog/v2/illustration/ZIMID?size=N``
+
+Returns the illustration of size ``NxN`` pixels for the library entry with
+:term:`UUID <ZIM UUID>` ``ZIMID``.
+
+If no illustration of requested size is found a HTTP 404 error is returned.
+
+
+``/catalog/v2/languages``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full list of ZIM file languages as an `OPDS Navigation Feed
+<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
+
+
+``/catalog/v2/partial_entries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <new-opds-api>`
+===== ===========
+
+Returns the full or filtered list of ZIM files as an `OPDS acquisition feed
+<https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `partial entries
+<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
+
+Supported filters are the same as for the `/catalog/v2/entries`_ endpoint.
+
+
+.. _legacy-opds-api:
+
+``/catalog`` (Legacy OPDS API)
+------------------------------
+
+===== ===========
+Type: :ref:`deprecated <deprecation>`
+===== ===========
+
+The legacy OPDS API is preserved for backward compatibility and is deprecated.
+:ref:`New OPDS API <new-opds-api>` should be used instead.
+
+
+``/catalog/root.xml``
+^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Full library OPDS catalog (list of all ZIM files).
+
+
+``/catalog/searchdescription.xml``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Describes the `/catalog/search`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+``/catalog/search``
+^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`deprecated API <legacy-opds-api>`
+===== ===========
+
+Returns the list of ZIM files (in OPDS catalog format) matching the
+search/filtering criteria. Supported filters are the same as for the
+`/catalog/v2/entries`_ endpoint.
+
+
+``/catch/external``
+-------------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/catch/external?source=URL``
+
+Generates a HTML page with a link leading to (the decoded version of) ``URL``
+and a warning that following that link will load an external (out of ZIM)
+resource.
+
+**Parameters:**
+
+  ``source``: URL of the external resource (must be URL-encoded).
+
+**Example:**
+
+.. code:: sh
+
+  # Intercept an external link to https://example.com?query=abcd
+  $ curl 'http://localhost:8080/catch/external?source=https%3A%2F%2Fexample.com%3Fquery%3Dabcd'
+
+
+
+``/content``
+------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+ZIM file content is served under the ``/content`` endpoint as described below.
+
+
+``/content/ZIMNAME/PATH/IN/ZIMFILE``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME``.
+
+
+``/content/ZIMNAME``
+^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+``/content/ZIMNAME`` redirects to the main page of the ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME`` (unless that ZIM file contains an entry with an empty
+path or path equal to ``/``, in which case that entry is returned).
+
+
+``/random``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/random?content=ZIMNAME``
+
+Generates a HTTP redirect to a randomly selected article/page from the
+specified ZIM file.
+
+**Parameters:**
+
+  ``content``: :term:`name <ZIM name>` of the ZIM file.
+
+
+.. _raw:
+
+``/raw``
+--------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+The ``/raw`` API provides access to ZIM file data. It consists of two separate
+endpoints for accessing data and metadata.
+
+
+``/raw/ZIMNAME/content/PATH/IN/ZIMFILE``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <raw>`
+===== ===========
+
+Returns the entry with path ``PATH/IN/ZIMFILE`` from the ZIM file with
+:term:`name <ZIM name>` ``ZIMNAME``. Currently, this endpoint almost duplicates
+(with some subtle technical differences) the newer endpoint
+`/content/ZIMNAME/PATH/IN/ZIMFILE`_. The important difference is that the
+``/raw`` endpoint guarantees that no server-side processing will be applied to
+the returned content, whereas content obtained via the ``/content`` endpoint
+may in the future undergo some processing intended to improve the operation of
+the viewer (e.g. compensating for certain bugs in ZIM creation). Also note that
+``/raw`` is :ref:`public <public-api-endpoint>`, whereas ``/content`` is
+:ref:`private <private-api-endpoint>`.
+
+
+``/raw/ZIMNAME/meta/METADATAID``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+===== ===========
+Type: member of a :ref:`public API <raw>`
+===== ===========
+
+Returns the metadata item ``METADATAID`` from the ZIM file with :term:`name
+<ZIM name>` ``ZIMNAME``.
+
+
+``/search``
+-----------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+Performs a full text search on one or more ZIM files and returns an HTML page
+with a list of links to matching pages along with snippets of the matching
+portions of those pages.
+
+.. _multi-zim-search-constraints:
+
+A multi-ZIM search request must comply with the following constraints:
+
+* the number of ZIM files participating in the search operation must not exceed
+  the limit imposed by the :option:`--searchLimit` option of ``kiwix-serve``.
+
+* all of the ZIM files participating in the same search operation must be in
+  the same language.
+
+**Parameters:**
+
+
+  ZIM file selection parameters:
+
+    At least one of the following parameters must be provided in order to
+    specify in which ZIM file(s) to search. Parameters appearing earlier in
+    below list take precedence over subsequent ones (the later ones, even if
+    present in the request, are simply ignored).
+
+    ``content``: :term:`name <ZIM name>` of the ZIM file (for a single-ZIM
+    search). This is a :ref:`legacy parameter <deprecation>`. ``books.name``
+    should be used instead.
+
+    ``books.id``: :term:`UUID <ZIM UUID>` of the ZIM file. Can be repeated for
+    a multi-ZIM search, however must respect the :ref:`multi-ZIM search
+    constraints <multi-zim-search-constraints>`.
+
+    ``books.name``: :term:`name <ZIM name>` of the ZIM file. Can be repeated
+    for a multi-ZIM search, however must respect the :ref:`multi-ZIM search
+    constraints <multi-zim-search-constraints>`.
+
+    ``books.filter.{criteria}``: allows to take full advantage of :ref:`library
+    filtering <library-filtering>` functionality of the `/catalog/v2/entries`_
+    endpoint (``{criteria}`` must be replaced with an attribute/filtering
+    criteria name supported by :ref:`library filtering <library-filtering>`).
+
+  Query parameters:
+
+    ``pattern`` (optional; defaults to an empty string): text to search for.
+
+    ``latitude``, ``longitude`` & ``distance`` (optional): geospatial query
+    parameters. If all of these are provided, then the results will be
+    restricted to geotagged pages that are within ``distance`` metres from the
+    location on Earth with coordinates ``latitude`` and ``longitude``.
+
+  Pagination parameters:
+
+    ``pageLength`` (optional, default: 25): maximum number of search results in
+    the response. Capped at 140.
+
+    ``start`` (optional, default: 1): this parameter enables pagination of
+    results. The response will include up to ``pageLength`` results starting
+    with entry # ``start`` from the full list of search results (the first
+    result is assumed to have index 1).
+
+  Other parameters:
+
+    ``format`` (optional, default: html): format of the search results. Allowed
+    values are: html, xml.
+
+Examples:
+
+.. code:: sh
+
+  # Search for 'android' in the book with name 'scifi-library'
+  # Return results ## 51-60.
+  $ curl 'http://localhost:8080/search?pattern=android&books.name=scifi-library&start=51&pageLength=10'
+
+  # Search for 'napoli' in books in Italian
+  $ curl 'http://localhost:8080/search?books.filter.lang=ita&pattern=napoli'
+
+  # Search for 'chateau' in books in French that have a category of 'wikipedia'.
+  # Return the results as XML.
+  $ curl 'http://localhost:8080/search?pattern=chateau&books.filter.lang=fra&books.filter.category=wikipedia&format=xml'
+
+
+``/search/searchdescription.xml``
+---------------------------------
+
+===== ===========
+Type: :ref:`public <public-api-endpoint>`
+===== ===========
+
+Describes the `/search`_ endpoint in `OpenSearch description format
+<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
+
+
+
+``/skin``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Static front-end resources (such as CSS, javascript and images) are all grouped
+under ``/skin``.
+
+**Usage:**
+  ``/skin/PATH/TO/RESOURCE[?cacheid=CACHEID]``
+
+`Cache busting
+<https://javascript.plainenglish.io/what-is-cache-busting-55366b3ac022>`_ of
+static resources is supported via the optional param ``cacheid``. By default,
+i.e. when the ``cacheid`` parameter is not specified while accessing the
+``/skin`` endpoint, static resources are served as if they were dynamic (i.e.
+could be different for an immediately repeated request). Specifying the
+``cacheid`` parameter with a correct value (matching the value embedded in the
+``kiwix-serve`` instance), makes the returned resource to be presented as
+immutable. However, if the value of the ``cacheid`` parameter mismatches then
+``kiwix-serve`` responds with a 404 HTTP error.
+
+``kiwix-serve``'s default front-end (the :ref:`welcome page <welcome-page>` and
+the :ref:`ZIM file viewer <zim-file-viewer>`) access all underlying static
+resources by using explicit ``cacheid`` s.
+
+
+``/suggest``
+------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+**Usage:**
+
+  ``/suggest?content=ZIMNAME[&term=QUERY][&count=N][&start=S]``
+
+Returns suggestions (in JSON format) for a text string that is assumed to be a
+partially typed search for a page inside a particular ZIM file.
+
+Suggestions are obtained as matches of the query text against the page titles
+in the ZIM file using the title index database generated during the creation of
+the ZIM file.
+
+In case of a multi-word query the order of the words matters in two ways:
+
+1. the last word is considered as partially typed, unless followed by a space;
+2. ranking of the matches.
+
+If the ZIM file doesn't contain a title index then suggestions are generated by
+listing page titles starting *exactly* (in a case sensitive manner) with the
+query text. Otherwise, suggestions are case-insensitive.
+
+If the ZIM file contains a full text search index, then an extra suggestion is
+added as an option to perform a full text search in the said ZIM file.
+
+**Parameters:**
+
+  ``content`` (mandatory): :term:`name <ZIM name>` of the ZIM file.
+
+  ``term`` (optional; defaults to an empty string): query text.
+
+  ``count`` (optional, default: 10): maximum number of page suggestions in the
+  response (i.e. the extra option to perform a full text search is not included
+  in this count).
+
+  ``start`` (optional, default: 0): this parameter enables pagination of
+  results. The response will include up to ``count`` entries starting with
+  entry # ``start`` from the full list of page suggestions (the first result is
+  assumed to have index 0).
+
+**Example:**
+
+.. code:: sh
+
+  $ curl 'http://localhost/suggest?content=stackoverflow_en&term=pyth&count=50'
+
+
+.. _zim-file-viewer:
+
+``/viewer``
+-----------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+ZIM file viewer. The ZIM file and entry therein must be specified via the hash
+component of the URL as ``/viewer#ZIMNAME/PATH/IN/ZIMFILE``.
+
+
+``/viewer_settings.js``
+-----------------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Settings of the ZIM file viewer that are configurable via certain command line
+options of ``kiwix-serve`` (e.g. ``--nolibrarybutton``).
+
+
+/ANYTHING/ELSE
+--------------
+
+===== ===========
+Type: :ref:`private <private-api-endpoint>`
+===== ===========
+
+Any other URL is considered as an attempt to access ZIM file content using the
+legacy URL scheme and is redirected to ``/content/ANYTHING/ELSE``.
+
+
+Glossary
+========
+
+.. glossary::
+
+  ZIM filename
+
+    Name of a ZIM file on the server filesystem.
+
+  ZIM name
+
+    Identifier of a ZIM file in the server's library (used for referring to a
+    particular ZIM file in requests).
+
+    For a ``kiwix-serve`` started with a list of ZIM files, ZIM names are
+    derived from the filename by dropping the extension and replacing certain
+    characters (spaces are replaced with underscores, and ``+`` symbols are
+    replaced with the text ``plus``). Presence of the
+    :option:`-z`/:option:`--nodatealiases` option will create additional names
+    (aliases) for filenames with dates.
+
+    For a ``kiwix-serve`` started with the :option:`--library` option, ZIM
+    names come from the library XML file.
+
+    ZIM names are expected to be unique across the library. Any name conflicts
+    (including those caused by the usage of the
+    :option:`-z`/:option:`--nodatealiases` option) are reported on STDERR but,
+    otherwise, are ignored.
+
+  ZIM title
+
+    Title of a ZIM file. This can be any text (with whitespace). It is never
+    used as a way of referring to a ZIM file.
+
+  ZIM UUID
+
+    This is a unique identifier of a ZIM file designated at its creation time
+    and embedded in the ZIM file. Certain ``kiwix-serve`` operations may
+    require that a ZIM file be referenced through its UUID rather than name.

docs/meson.build

@@ -0,0 +1,7 @@
+
+sphinx = find_program('sphinx-build', native:true)
+
+sphinx_target = run_target('doc',
+    command: [sphinx, '-bhtml',
+              meson.current_source_dir(),
+              meson.current_build_dir()])

docs/requirements.txt

@@ -0,0 +1,2 @@
+Sphinx==5.3.0
+sphinx-rtd-theme==1.1.1

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-tools', 'cpp',
-  version : '3.4.0',
+  version : '3.5.0',
   license : 'GPL',
   default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -9,7 +9,10 @@ add_global_arguments('-DKIWIX_TOOLS_VERSION="@0@"'.format(meson.project_version(
 
 static_linkage = get_option('static-linkage')
 if static_linkage
-  add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
+  # Static build is not supported on MacOS
+  if host_machine.system() != 'darwin'
+    add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
+  endif
 endif
 
 thread_dep = dependency('threads')
@@ -26,3 +29,6 @@ if static_linkage
 endif
 
 subdir('src')
+if get_option('doc')
+  subdir('docs')
+endif

meson_options.txt

@@ -1,2 +1,4 @@
 option('static-linkage', type : 'boolean', value : false,
   description : 'Create statically linked binaries.')
+option('doc', type : 'boolean', value : false,
+  description : 'Build the documentations.')