186 Commits
3.4.0 ... main

Author SHA1 Message Date
Kelson
2308b02504 Merge pull request #787 from kiwix/better-deal-with-data-dir-permissions
Better deal with container /data dir permissions
2025-12-01 10:51:36 +01:00
rgaudin
81d9a6cc02 Use explicit UID/GID and show them in error message 2025-12-01 09:38:29 +00:00
Emmanuel Engelhart
0232f0a95f Check if /data is writable 2025-11-30 17:48:40 +01:00
Emmanuel Engelhart
309d4d01f1 /data per default writable 2025-11-30 17:48:40 +01:00
Kelson
abcbe0c9df Merge pull request #788 from kiwix/release-3.8.1
Release 3.8.1
2025-11-30 17:48:26 +01:00
Emmanuel Engelhart
d08638429e Changelog for 3.8.1 2025-11-30 17:47:37 +01:00
Emmanuel Engelhart
5348495c2b Bump-up version to 3.8.1 2025-11-30 17:39:14 +01:00
Kelson
9b91ff1529 Merge pull request #783 from kiwix/fix-kiwix-mange-docopt-integration
Fix kiwix-manage docopt integration
2025-11-17 17:20:39 +01:00
Emmanuel Engelhart
6f599589f9 Fix regression with kiwix-mange 'show' without arguments 2025-11-15 15:33:21 +01:00
Emmanuel Engelhart
6272c6d11e Small kiwix-mange usage() fix 2025-11-15 15:21:44 +01:00
Emmanuel Engelhart
0bcb3957e5 Fix kiwix-manage 'add' docopt definition 2025-11-15 14:58:53 +01:00
Emmanuel Engelhart
16f2734699 'ZIM' not 'zim' 2025-11-15 14:58:53 +01:00
tim-moody
45f0b0601e Fix kiwix-manage 'remove' docopt definition 2025-11-15 14:58:46 +01:00
Kelson
d61eafe6a2 Merge pull request #763 from vighnesh-sawant/standard-port-enhancment
Improve message when server is running on standard port
2025-11-01 17:09:53 +01:00
Vighnesh
ae21ba42da Display better message when using standard port 2025-11-01 07:09:25 +01:00
Kelson
830a9e5e29 Merge pull request #773 from kiwix/release-3.8.0
Release 3.8.0
2025-10-31 17:41:12 +01:00
Emmanuel Engelhart
b22c10f6ff 3.8.0 Changelog 2025-10-31 17:37:31 +01:00
Emmanuel Engelhart
863fe0f973 Use libkiwix 14.1.0 2025-10-31 16:45:34 +01:00
Kelson
769182731c Merge pull request #772 from kiwix/yashgoyal0110-alpine-version-update
Update Kiwix Server base image from alpine:3.18 to alpine:3.22
2025-10-24 12:40:38 +02:00
yashgoyal0110
b679e5d159 Update Kiwix Server base image from alpine:3.18 to alpine:3.22 2025-10-24 12:39:15 +02:00
Kelson
a6af7707e4 Merge pull request #747 from kiwix/doc-fix-search-pagination-offset
Pagination default index is '0'
2025-10-24 12:36:04 +02:00
Emmanuel Engelhart
3ebd070d20 Pagination default index is '0' 2025-10-24 12:25:43 +02:00
Kelson
6ed59398e4 Merge pull request #768 from kiwix/spelling_correction
kiwix-search --spelling ZIMFILE PATTERN
2025-10-17 16:10:42 +02:00
Veloman Yunkan
80dcf95b7f kiwix-search --spelling ZIMFILE PATTERN
Added spelling correction functionality to kiwix-search. The spellings
database is created under ~/.cache/kiwix.
2025-10-06 16:57:12 +04:00
Kelson
90f4562408 Merge pull request #755 from Sedetius/rootles-docker
[Docker] kiwix-serve image to use unprivileged user
2025-09-28 17:30:36 +02:00
rgaudin
869b08cf71 Minor comment wording 2025-09-28 17:25:04 +02:00
Sedetius
ae1640736c Moved user creation in kiwix-serve dockerfile before COPY start.sh to automatically use the new users file permissions. 2025-09-28 17:25:04 +02:00
Sedetius
40d32cfc32 Switched from legacy dockerfile LABEL key value format to new one. 2025-09-28 17:25:04 +02:00
Sedetius
463015da45 Changed default user name in kiwix-serve Dockerfile to avoid confusion with other kiwix services. 2025-09-28 17:25:04 +02:00
Sedetius
0ba30b5525 Adapted Kiwix Serve Dockerfile for rootless Docker support 2025-09-28 17:25:04 +02:00
Kelson
e22cfea9d9 Merge pull request #759 from kiwix/catalog_only_mode
kiwix-serve --catalogOnly --contentServerURL=<URL>
2025-09-12 15:19:15 +02:00
Veloman Yunkan
fcc8877cb0 Upgraded CI runner to jammy 2025-09-12 16:20:57 +04:00
Veloman Yunkan
5fdea94c05 Strip trailing slashes from --contentServerURL option 2025-09-08 16:25:51 +04:00
Veloman Yunkan
597223196d kiwix-serve --contentServerURL=<URL> 2025-09-08 16:25:51 +04:00
Veloman Yunkan
824eddaf94 kiwix-serve --catalogOnly 2025-09-08 16:25:51 +04:00
Kelson
7413682060 Fix Build Status badge link in README.md 2025-09-08 11:55:15 +00:00
Kelson
0125188bc0 Update Build Status badge in README.md 2025-09-08 11:54:14 +00:00
Kelson
acad8a85ab Update Docker badges README.md
Because old domain name has been deprecated, see https://github.com/eggplants/ghcr-badge
2025-06-13 10:01:27 +00:00
Kelson
2e4cae72ef Merge pull request #748 from kiwix/remove-ubuntu-20.04-ci-cd
Stop publishing on Ubuntu 20.04 PPA
2025-06-13 10:58:20 +02:00
Emmanuel Engelhart
5c2a483929 Stop publishing on Ubuntu 20.04 PPA 2025-06-13 10:42:28 +02:00
benoit74
0d38d071a8 Fix funding-manifest-urls content 2025-04-20 17:24:10 +02:00
benoit74
0091ffa1ca Add file for funding.json 2025-04-19 07:36:31 +00:00
Kelson
43b00419dd Merge pull request #722 from kiwix/stop-windows-ci-debug
Compile in release mode for Windows
2024-12-21 12:20:28 +01:00
Emmanuel Engelhart
f2c78ca03b Compile in release mode for Windows 2024-12-21 12:14:30 +01:00
Kelson
73f357a627 Merge pull request #720 from kiwix/few-workflow-fixes
Few workflow fixes
2024-12-20 17:10:08 +01:00
Emmanuel Engelhart
0fd4abeeae Remove a CI step which is now useless 2024-12-20 17:05:40 +01:00
Emmanuel Engelhart
998bbc6b0e Add a few spaces in workflow 2024-12-20 17:01:10 +01:00
Emmanuel Engelhart
e12687b1ec Use fix OS dependency 2024-12-20 16:58:07 +01:00
Emmanuel Engelhart
e6ff088ffa Bump-up to action/checkout@v4 2024-12-20 16:54:58 +01:00
Emmanuel Engelhart
87c6eb837d Bump-up to actions/upload-artifact@v4 2024-12-20 16:53:47 +01:00
Emmanuel Engelhart
88ceebab8a Deprecate set-output fron package workflow 2024-12-20 16:53:46 +01:00
Kelson
dac84999bc Merge pull request #719 from kiwix/fix-deb-control-dep-versions
Remove useless patch part of dep. versions
2024-12-20 16:53:31 +01:00
Emmanuel Engelhart
5b8d72de5d Remove useless patch part of dep. versions 2024-12-20 16:41:34 +01:00
Kelson
e65354a486 Merge pull request #704 from kiwix/feature/serve-all-ips
kiwix-serve displays both protocol attached ips
2024-10-02 19:32:11 +00:00
sgourdas
55b42f27b6 Code cleanup 2024-10-02 21:36:44 +03:00
sgourdas
24c0b7220e Fix usage 2024-10-02 21:36:44 +03:00
sgourdas
b1dd681c59 kiwix-serve displays both protocol attached ips 2024-10-02 21:36:42 +03:00
Kelson
c6b72e9362 Merge pull request #706 from kiwix/kiwix-serve-supported-browsers
Clear support list of kiwix-serve supported browsers
2024-09-20 19:07:31 +00:00
Emmanuel Engelhart
362db1094d Clear support list of kiwix-serve supported browsers 2024-09-20 21:06:52 +02:00
Kelson
7512611146 Merge pull request #707 from kiwix/kiwix2libkiwix_transition
libkiwix now identifies itself as libkiwix
2024-09-19 19:24:19 +00:00
Veloman Yunkan
3599e8476d libkiwix now identifies itself as libkiwix 2024-09-16 17:50:20 +04:00
Kelson
5e222b3f0b Merge pull request #705 from kiwix/better-libzim-version-check
Better check libzim and libkiwix deps
2024-09-15 19:38:01 +02:00
Emmanuel Engelhart
ce3c3bd153 Better check libzim and libkiwix deps 2024-09-15 19:01:25 +02:00
Kelson
da22757a14 Merge pull request #702 from kiwix/remove-ci-windows-crosscompilation
Remove Windows binaries cross-compile from CI
2024-09-07 13:13:47 +00:00
Emmanuel Engelhart
f0a8643603 Remove Windows binaries cross-compile from CI 2024-09-07 13:03:44 +00:00
Kelson
84faf9c09e Merge pull request #698 from kiwix/refresh-deb-packages-ci-cd
Refresh deb packages CI/CD
2024-09-07 13:01:09 +00:00
Emmanuel Engelhart
f912519647 Use pkgconf package in palce of pkg-config 2024-09-02 18:23:51 +00:00
Emmanuel Engelhart
e609c1c826 Use fix build OS 2024-09-02 18:23:51 +00:00
Emmanuel Engelhart
fdcea9889f Proper workflow triggers 2024-09-02 18:23:51 +00:00
Emmanuel Engelhart
4ec05bf98e Refresh deb packages CI/CD 2024-09-02 18:23:51 +00:00
Kelson
2bab9a210e Merge pull request #700 from kiwix/update_libkiwix_dep
Update libkiwix dependency version
2024-09-02 09:31:15 +00:00
Matthieu Gautier
28008efb28 Update libkiwix dependency version
Libkiwix has move to major version 14 https://github.com/kiwix/libkiwix/pull/1116
2024-09-02 11:13:45 +02:00
Kelson
f6baad903f Merge pull request #697 from kiwix/bump-up-version-to-3.8.0
Bump up version to 3.8.0
2024-09-01 13:03:01 +00:00
Emmanuel Engelhart
984741de03 deb files requires libkiwix14 2024-09-01 14:57:16 +02:00
Emmanuel Engelhart
b557d0a1dc Bump-up version to 3.8.0 2024-09-01 14:56:21 +02:00
Kelson
98483170c9 Merge pull request #695 from kiwix/windows_ci
Add a Windows CI.
2024-08-31 12:24:07 +00:00
Emmanuel Engelhart
c94c1bae6c Add cmake to deb control file (needed by docopt) 2024-08-31 14:23:20 +02:00
Matthieu Gautier
b45426640e Add kiwix-serve documentation link in USAGE 2024-08-30 16:03:29 +02:00
Matthieu Gautier
504affb481 Add docopt dependencies to ubuntu packages 2024-08-30 16:03:29 +02:00
Matthieu Gautier
6a4b7508f0 Ask windows to not define min and max. 2024-08-30 16:03:29 +02:00
Matthieu Gautier
cebe079c11 Don't use sig_atomic_t on Windows.
We don't setup signal handler on Windows, so we can replace
`volatil sig_atomic_t` by simple `bool`.
2024-08-30 16:03:29 +02:00
Matthieu Gautier
945c4d61e6 Port kiwix-search to docopt 2024-08-30 15:49:10 +02:00
Matthieu Gautier
504a7a1ab7 Port kiwix-manage to docopt. 2024-08-30 15:48:51 +02:00
Matthieu Gautier
76b617817a Port kiwix-serve to docopt. 2024-08-30 15:48:37 +02:00
Matthieu Gautier
9eab766ff9 Add a Windows CI.
Compilation of kiwix-tools is broken on Windows.
But let's setup the CI to validate the PR.
2024-08-27 15:49:17 +02:00
Kelson
4b3d6202e7 Merge pull request #681 from kiwix/sandstorm-badge-removal
Remove Sandstorm badge in README.md (unmaintained)
2024-06-02 09:15:05 +02:00
Kelson
e18ffe19dc Remove Sandstorm badge in README.md (unmaintained) 2024-06-02 09:12:41 +02:00
Kelson
353d20bf83 Merge pull request #673 from aryanA101a/main
Add IPv6 support to Kiwix Server
2024-06-01 17:49:06 +02:00
Aryan Arora
5f69a41259 add ipv6 support & update docs 2024-06-01 14:59:43 +05:30
Matthieu Gautier
2cc1ca4017 Merge pull request #679 from kiwix/fix_ci_pr 2024-05-30 11:43:57 +02:00
Matthieu Gautier
dc1067b8c4 Run the CI on pull_request.
Running the CI on push only trigger the CI on a push on the current repository.
It works on PR based on local branch but not for PR from remote repository.
2024-05-30 11:38:02 +02:00
Matthieu Gautier
5e7166cb07 Merge pull request #672 from kiwix/build_dir 2024-04-09 12:56:35 +02:00
Matthieu Gautier
24d81f9507 Build kiwix-tools with new archive from kiwix-build 2024-04-08 16:45:54 +02:00
Kelson
fcbb4a56bc Merge pull request #668 from kiwix/prepare-release-3.7.0
Prepare release 3.7.0
2024-03-13 15:17:32 +01:00
Emmanuel Engelhart
e37af51dd4 new 3.7.0 Changelog 2024-03-13 15:15:09 +01:00
Emmanuel Engelhart
43704c6154 Proper deb lizim/libkiwix dependencies 2024-03-13 15:15:09 +01:00
Emmanuel Engelhart
d25fda82be Request libzim9 2024-03-13 15:15:03 +01:00
Emmanuel Engelhart
279cdcda0b Bump-up version to 3.7.0 2024-03-13 15:14:11 +01:00
Kelson
331743ea29 Merge pull request #661 from kiwix/max-dependence-version
Check max version of libzim/libkiwix
2024-03-13 15:12:23 +01:00
Emmanuel Engelhart
19f11d8777 Check max version of libzim/libkiwix 2024-03-13 15:10:07 +01:00
Kelson
3377f9e8bb Merge pull request #666 from kiwix/implement_skip_option_2
kiwix-serve: Skip invalid ZIM files and continue startup #659
2024-03-13 15:09:43 +01:00
Emmanuel Engelhart
930c8321ca Add --skipInvalid option to man page 2024-03-10 16:53:15 +01:00
Emmanuel Engelhart
a3239ea8bc Rename --skipBroken to --skipInvalid 2024-03-10 16:50:34 +01:00
Florian Schüller
456c77d8fd kiwix-serve: Skip broken ZIM files and continue startup 2024-03-10 16:47:14 +01:00
Matthieu Gautier
7913b6a4f0 Merge pull request #665 from kiwix/documentation 2024-03-08 09:28:06 +01:00
Veloman Yunkan
027f7d6620 Reinstated the RTD Sphinx Theme for documentation
Sometime in 2023 the default theme on Read-the-Docs was changed to
Alabaster. This change enforces the old theme (sphinx_rtd_theme).
2024-03-07 18:53:37 +04:00
Matthieu Gautier
18b532a923 Merge pull request #664 from kiwix/documentation 2024-03-06 18:19:40 +01:00
Veloman Yunkan
21a145d999 Fixed the copyright date 2024-03-06 21:15:09 +04:00
Veloman Yunkan
be89de2bbe More conspicuous "ZIM name" links 2024-03-06 21:15:09 +04:00
Veloman Yunkan
433010a900 Clarified handling of naming conflicts 2024-03-06 21:15:09 +04:00
Veloman Yunkan
dd14e02741 Documented a couple of /search API's caveats 2024-03-06 21:15:09 +04:00
Veloman Yunkan
8ea12b2ce5 Fixed the "book name"-vs-"ZIM name" bug in docs 2024-03-06 21:15:09 +04:00
Matthieu Gautier
67c0ebd6ad Merge pull request #652 from kiwix/ghaction
Use kiwix-build's github action to download dependencies.
2023-12-20 11:57:05 +01:00
Matthieu Gautier
c5197ceb51 Use kiwix-build's github action to download dependencies. 2023-12-19 17:17:05 +01:00
Kelson
98b4fb3cdf Merge pull request #649 from kiwix/need-libkiwix-13
Requires libkiwix13+
2023-11-19 14:19:31 +01:00
Emmanuel Engelhart
33bdf56bdd Requires libkiwix13+ 2023-11-19 14:16:05 +01:00
Kelson
4556262c87 Merge pull request #648 from kiwix/release-3.6.0
Release 3.6.0
2023-11-19 14:13:57 +01:00
Emmanuel Engelhart
cb2159b114 Changelog for 3.6.0 2023-11-19 14:11:59 +01:00
Emmanuel Engelhart
5b45e1898b Bump-up version to 3.7.0 2023-11-19 13:59:52 +01:00
Kelson
f6128b9079 Merge pull request #643 from computerscienceiscool/patch-2 2023-10-20 10:20:47 +02:00
JJ
ba64c1637e Update README.md
Single spelling error fixed.
2023-10-20 01:02:02 -07:00
Matthieu Gautier
3a0e87d8ea Merge pull request #633 from kiwix/adapt_libkiwix_991 2023-10-12 18:05:52 +02:00
Kelson
9ca2b9fc16 Merge pull request #639 from computerscienceiscool/patch-1
Update README.md
2023-10-08 18:38:47 +02:00
JJ
f5932bf5d3 Update README.md
Fixed a few spelling and grammar errors.
2023-10-08 18:37:52 +02:00
Matthieu Gautier
018df0d563 Adapt to new api introduced in kiwix/libkiwix#991 2023-10-06 15:47:30 +02:00
Matthieu Gautier
76ac5ce93f Remove Ubuntu Kinetic from CI/CD (deprecated) 2023-10-06 15:47:30 +02:00
Matthieu Gautier
d0847525c6 Time to switch the CI to focal. 2023-10-06 15:47:30 +02:00
Matthieu Gautier
7f0c1d8a3f Merge pull request #642 from kiwix/fix_docs 2023-10-06 15:47:10 +02:00
Matthieu Gautier
bd9e67f48e Add readthedocs configuration 2023-10-06 15:33:40 +02:00
Kelson
a4cc4ba55b Merge pull request #636 from kiwix/cpp17
Move to c++17.
2023-09-22 17:38:55 +02:00
Matthieu Gautier
1a1274012f Move to c++17.
All our compilers should handle c++17. Let's move on.
2023-08-30 17:58:40 +02:00
Kelson
74712ff022 Merge pull request #631 from kiwix/kiwix-serve-docker-readme
Dedicated Docker Compose section
2023-08-13 09:39:44 +02:00
Emmanuel Engelhart
3c6971fced Dedicated Docker Compose section 2023-08-13 09:38:59 +02:00
Kelson
7a7eaaba0e Merge pull request #626 from iArchitSharma/patch-1
Added missing parameters to kiwix-serve.1
2023-07-12 15:20:43 +02:00
Archit Sharma
d166947447 Added missing parameters to kiwix-serve.1 2023-07-12 13:38:11 +05:30
Kelson
92ee72a8f2 Merge pull request #622 from kiwix/new-archs
Docker images for new architectures
2023-06-16 18:03:35 +02:00
renaud gaudin
d0f8226f50 Docker images for new architectures
- Fixes the release filename for arm64 and armv7
- Add new architectures for i386 and armv6
2023-06-16 17:59:33 +02:00
Kelson
221055f49c Merge pull request #621 from kiwix/docker-readme-improvements
Docker readme improvements
2023-05-24 10:53:31 +03:00
Emmanuel Engelhart
a463c31a0c Add example with globbing 2023-05-24 09:51:42 +02:00
Emmanuel Engelhart
6e239d5cb6 Add link to Kiwix server container image 2023-05-24 09:51:11 +02:00
Emmanuel Engelhart
191dc95266 Fix small typo in the usage() 2023-05-23 17:05:03 +02:00
Matthieu Gautier
840f92c677 Merge pull request #618 from kiwix/fix_clone_ci 2023-05-03 16:31:10 +02:00
Matthieu Gautier
f03dc46342 Remove custom git clone
Now possible because of new Docker images and proper user/permission mgmt
within the container.
2023-05-03 15:34:41 +02:00
Kelson
a8a62aef74 Merge pull request #617 from jgmoss/main
Update docker-compose.yml to use ghcr.io image
2023-04-29 13:42:28 +02:00
Graham Moss
7f9bf58742 Update docker-compose.yml to use ghcr.io image 2023-04-28 15:01:05 -04:00
Matthieu Gautier
4cf2f4f2c9 Merge pull request #616 from kiwix/version_3.5.0 2023-04-28 11:36:55 +02:00
Matthieu Gautier
b6e7ed3443 New version 3.5.0 2023-04-28 11:07:22 +02:00
Matthieu Gautier
806747320c Merge pull request #615 from kiwix/no_static_on_macos 2023-04-28 08:20:38 +02:00
Matthieu Gautier
6765239341 Do not build with --static on maco_os. 2023-04-28 08:16:31 +02:00
Kelson
4e992a18e8 Merge pull request #608 from kiwix/ghcr
removed references to docker.io
2023-03-22 16:30:32 +01:00
renaud gaudin
3d9348092b removed references to docker.io 2023-03-22 15:23:31 +00:00
Kelson
1024360598 Merge pull request #607 from kiwix/update-ci-base-container-images
Bump-up CI container images for r36
2023-03-16 13:52:33 +01:00
Emmanuel Engelhart
cfc249b9cf Bump-up CI container images for r36 2023-03-16 13:48:43 +01:00
Kelson
688bb356ea Merge pull request #603 from kiwix/documentation
Documentation updates
2023-02-11 12:47:10 +01:00
Veloman Yunkan
d398ebdb3e Added missing information about /search endpoint 2023-02-11 12:04:18 +01:00
Veloman Yunkan
6ab6abc6d3 Noted the HTTP API type (public/private) in the docs 2023-02-11 11:20:32 +01:00
Matthieu Gautier
0059c6a213 Merge pull request #602 from kiwix/zerocount_catalog_query_doc_update 2023-02-10 19:46:51 +01:00
Veloman Yunkan
eac5898ca8 Doc update for /catalog/v2/entries?count=0 2023-02-10 19:33:10 +01:00
Kelson
edb97fb055 Merge pull request #601 from kiwix/update-ubuntu-packages
Remove PPA Impish, add Kinetic
2023-02-10 18:59:24 +01:00
Emmanuel Engelhart
e1b314267b Remove PPA Impish, add Kinetic 2023-02-10 18:56:21 +01:00
Kelson
0df6a1624b Merge pull request #599 from kiwix/documentation_of_catalog_pagination
Fixed & elaborated documentation of OPDS catalog pagination
2023-02-10 18:52:23 +01:00
Veloman Yunkan
07bc21754b Fixed & elaborated documentation of catalog pagination 2023-02-10 16:46:14 +01:00
Kelson
6f2e6ce866 Merge pull request #598 from kiwix/update-workflow-actions
Update GitHub workflow actions
2023-02-05 13:28:13 +01:00
Emmanuel Engelhart
4e8d3097b9 Update GitHub workflow actions 2023-02-05 13:25:19 +01:00
Kelson
ad58d74699 Merge pull request #597 from jacroe/docker-compose-fix-wildcard
Docker compose example: Surround wildcard statement in quotes
2023-02-02 07:13:20 +01:00
Jacob Roeland
0f25fa69d3 Surround wildcard statement in quotes 2023-02-02 00:26:46 -05:00
Matthieu Gautier
4949accdf3 Merge pull request #586 from kiwix/documentation 2023-01-04 17:56:43 +01:00
Veloman Yunkan
7c7cd42e83 Added docs badge to README 2023-01-04 19:44:49 +04:00
Veloman Yunkan
996ea031fc Documented the /search endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
7179244c47 Documented the /raw endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
68d3c9d81d Documented the /suggest endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
7f86317537 Documented the /catch/external endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
436f1b83f6 Documented library reloading via SIGHUP 2023-01-04 19:43:20 +04:00
Veloman Yunkan
490ba0f3f2 Documented the OPDS API 2023-01-04 19:43:20 +04:00
Veloman Yunkan
591a347647 Added glossary to kiwix-serve's documentation 2023-01-04 19:43:20 +04:00
Veloman Yunkan
ecf5e67759 Documented the --library option 2023-01-04 19:43:20 +04:00
Veloman Yunkan
ec2effe257 Cross-referenced a few options 2023-01-04 19:43:20 +04:00
Veloman Yunkan
244009f3b1 Marked options with directives
This will enable cross-referencing them.
2023-01-04 19:43:20 +04:00
Veloman Yunkan
11ef036174 Documented the /random endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
7b5b0cd7f7 Documented the /skin endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
2aef2d4a92 Expanded the /catalog endpoint 2023-01-04 19:43:20 +04:00
Veloman Yunkan
76023e9231 Started documenting kiwix-serve HTTP API 2023-01-04 19:43:20 +04:00
Veloman Yunkan
614f15913f Added introduction to kiwix-serve's documentation 2023-01-04 19:43:20 +04:00
Veloman Yunkan
6957a8cb1e Enter docs/
This commit introduces an embryo of sphinx-based documentation. The
content of documentation in this commit is limited to the command line
options of kiwix-serve.
2023-01-04 19:43:20 +04:00
Kelson
c6b6a47711 "main" is the new git default branch 2022-12-27 14:16:58 +01:00
Kelson
9f93677b66 "main" is the new git default branch 2022-12-27 14:10:56 +01:00
25 changed files with 1737 additions and 577 deletions

View File

@@ -1,72 +1,106 @@
name: CI
on: [push]
on:
pull_request:
push:
branches:
- main
jobs:
Windows:
runs-on: windows-2022
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install packages
run:
choco install pkgconfiglite ninja
- name: Install python modules
run: pip3 install meson
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: Install dependencies
uses: kiwix/kiwix-build/actions/dl_deps_archive@main
with:
target_platform: win-x86_64-static
- name: Compile
shell: cmd
run: |
set PKG_CONFIG_PATH=%cd%\BUILD_win-amd64\INSTALL\lib\pkgconfig
set CPPFLAGS=-I%cd%\BUILD_win-amd64\INSTALL\include
meson.exe setup . build -Dstatic-linkage=true --buildtype=release
cd build
ninja.exe
- name: Test
shell: cmd
run: |
cd build
meson.exe test --verbose
env:
WAIT_TIME_FACTOR_TEST: 10
Linux:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
target:
- native_static
- native_dyn
- win32_static
- win32_dyn
- linux-x86_64-static
- linux-x86_64-dyn
include:
- target: native_static
image_variant: bionic
- target: linux-x86_64-static
image_variant: jammy
lib_postfix: '/x86_64-linux-gnu'
- target: native_dyn
image_variant: bionic
arch_name: linux-x86_64
- target: linux-x86_64-dyn
image_variant: jammy
lib_postfix: '/x86_64-linux-gnu'
- target: win32_static
image_variant: f35
lib_postfix: '64'
- target: win32_dyn
image_variant: f35
lib_postfix: '64'
arch_name: linux-x86_64
env:
HOME: /home/runner
runs-on: ubuntu-latest
container:
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2025-06-07"
steps:
- name: Extract branch name
shell: bash
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch
- name: Checkout code
shell: python
run: |
from subprocess import check_call
from os import environ
command = [
'git', 'clone',
'https://github.com/${{github.repository}}',
'--depth=1',
'--branch', '${{steps.extract_branch.outputs.branch}}'
]
check_call(command, cwd=environ['HOME'])
- name: Install deps
shell: bash
run: |
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-tools.tar.xz
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
uses: actions/checkout@v4
- name: Install dependencies
uses: kiwix/kiwix-build/actions/dl_deps_archive@main
with:
target_platform: ${{ matrix.target }}
- name: Compile
shell: bash
run: |
if [[ "${{matrix.target}}" =~ .*_static ]]; then
if [[ "${{matrix.target}}" =~ .*-static ]]; then
MESON_OPTION="-Dstatic-linkage=true"
else
MESON_OPTION=""
fi
if [[ ! "${{matrix.target}}" =~ native_.* ]]; then
MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
if [ -e "$HOME/BUILD_${{matrix.arch_name}}/meson_cross_file.txt" ]; then
MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.arch_name}}/meson_cross_file.txt"
fi
cd $HOME/kiwix-tools
meson . build ${MESON_OPTION}
cd build
ninja
env:
PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
PKG_CONFIG_PATH: "${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib/pkgconfig:${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
CPPFLAGS: "-I${{env.HOME}}/BUILD_${{matrix.arch_name}}/INSTALL/include"

View File

@@ -11,16 +11,15 @@ on:
jobs:
build-and-push-kiwix-tools:
name: Deploy kiwix-tools Docker Image
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3.1.0
- uses: actions/checkout@v4
- name: build and publish kiwix-tools
uses: openzim/docker-publish-action@v9
uses: openzim/docker-publish-action@v10
with:
image-name: kiwix/kiwix-tools
registries: ghcr.io
credentials: |
DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }}
GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
context: docker
@@ -29,8 +28,10 @@ jobs:
VERSION={tag}
platforms: |
linux/amd64
linux/arm/v7
linux/arm64
linux/arm/v7
linux/arm/v6
linux/386
restrict-to: kiwix/kiwix-tools
manual-tag: ${{ github.event.inputs.version }}
repo_description: auto
@@ -38,17 +39,16 @@ jobs:
build-and-push-kiwix-serve:
name: Deploy kiwix-serve Docker Image
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs: build-and-push-kiwix-tools
steps:
- uses: actions/checkout@v3.1.0
- uses: actions/checkout@v4
- name: build and publish kiwix-serve
uses: openzim/docker-publish-action@v9
uses: openzim/docker-publish-action@v10
with:
image-name: kiwix/kiwix-serve
registries: ghcr.io
credentials: |
DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }}
GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
context: docker/server
@@ -57,8 +57,10 @@ jobs:
VERSION={tag}
platforms: |
linux/amd64
linux/arm/v7
linux/arm64
linux/arm/v7
linux/arm/v6
linux/386
restrict-to: kiwix/kiwix-tools
manual-tag: ${{ github.event.inputs.version }}
repo_description: auto

View File

@@ -1,18 +1,28 @@
name: Packages
on: [push, pull_request]
on:
pull_request:
push:
branches:
- main
release:
types: [published]
jobs:
build-deb:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
distro:
# - debian-unstable
# - debian-trixie
# - debian-bookworm
# - debian-bullseye
- ubuntu-noble
- ubuntu-jammy
- ubuntu-impish
- ubuntu-focal
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
# Determine which PPA we should upload to
- name: PPA
@@ -20,19 +30,55 @@ jobs:
run: |
if [[ $REF == refs/tags* ]]
then
echo "::set-output name=ppa::kiwixteam/release"
echo "ppa=kiwixteam/release" >> $GITHUB_OUTPUT
else
echo "::set-output name=ppa::kiwixteam/dev"
echo "ppa=kiwixteam/dev" >> $GITHUB_OUTPUT
fi
env:
REF: ${{ github.ref }}
- uses: legoktm/gh-action-auto-dch@master
- uses: legoktm/gh-action-auto-dch@main
with:
fullname: Kiwix builder
email: release+launchpad@kiwix.org
distro: ${{ matrix.distro }}
# - uses: legoktm/gh-action-build-deb@debian-unstable
# if: matrix.distro == 'debian-unstable'
# name: Build package for debian-unstable
# id: build-debian-unstable
# with:
# args: --no-sign
#
# - uses: legoktm/gh-action-build-deb@b47978ba8498dc8b8153cc3b5f99a5fc1afa5de1 # pin@debian-trixie
# if: matrix.distro == 'debian-trixie'
# name: Build package for debian-trixie
# id: build-debian-trixie
# with:
# args: --no-sign
#
# - uses: legoktm/gh-action-build-deb@1f4e86a6bb34aaad388167eaf5eb85d553935336 # pin@debian-bookworm
# if: matrix.distro == 'debian-bookworm'
# name: Build package for debian-bookworm
# id: build-debian-bookworm
# with:
# args: --no-sign
#
# - uses: legoktm/gh-action-build-deb@084b4263209252ec80a75d2c78a586192c17f18d # pin@debian-bullseye
# if: matrix.distro == 'debian-bullseye'
# name: Build package for debian-bullseye
# id: build-debian-bullseye
# with:
# args: --no-sign
- uses: legoktm/gh-action-build-deb@9114a536498b65c40b932209b9833aa942bf108d # pin@ubuntu-noble
if: matrix.distro == 'ubuntu-noble'
name: Build package for ubuntu-noble
id: build-ubuntu-noble
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-jammy
if: matrix.distro == 'ubuntu-jammy'
name: Build package for ubuntu-jammy
@@ -41,39 +87,23 @@ jobs:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-impish
if: matrix.distro == 'ubuntu-impish'
name: Build package for ubuntu-impish
id: build-ubuntu-impish
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-focal
if: matrix.distro == 'ubuntu-focal'
name: Build package for ubuntu-focal
id: build-ubuntu-focal
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v4
with:
name: Packages for ${{ matrix.distro }}
path: output
- uses: legoktm/gh-action-dput@master
- uses: legoktm/gh-action-dput@main
name: Upload dev package
# Only upload on pushes to master
if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
# Only upload on pushes to git default branch
if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
with:
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
repository: ppa:kiwixteam/dev
packages: output/*_source.changes
- uses: legoktm/gh-action-dput@master
- uses: legoktm/gh-action-dput@main
name: Upload release package
# Only upload on pushes to master or tag
# Only upload on pushes to main or tag
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
with:
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}

21
.readthedocs.yaml Normal file
View File

@@ -0,0 +1,21 @@
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Set the version of Python and other tools you might need
build:
os: ubuntu-22.04
tools:
python: "3.11"
# Build documentation in the docs/ directory with Sphinx
sphinx:
configuration: docs/conf.py
# We recommend specifying your dependencies to enable reproducible builds:
# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
python:
install:
- requirements: docs/requirements.txt

View File

@@ -0,0 +1 @@
https://kiwix.org/funding.json

View File

@@ -1,3 +1,61 @@
kiwix-tools 3.8.1
=================
* Kiwix server
- Hide port number in URL when server is running on port 80 (@vighnesh-sawant #763)
- Better deal with container /data dir permissions (@kelson42 #787)
* Other
- Fix kiwix-manage docopt integration (@kelson42 #783)
kiwix-tools 3.8.0
=================
* Kiwix server
- Improve message when server is running on standard port (@vighnesh-sawant #763)
- Update container base image to latest Alpline version (@yashgoyal0110 #771)
- Container image to use unprivileged user (@Sedetisu #755)
- Empty urlRootLocation doesn't disable book preview links anymore (@veloman-yunkan #1224)
- Add support of IPv6 (@sgourdas #673 #704)
- Popups are allowed to escape the browser sandbox (@veloman-yunkan #1208)
* Other
- Stop publishing on Ubuntu 20.04 PPA (@kelson42 #748)
- Use Docopt for command line argument parsing (@mgautierfr #695)
* Compilation & Packaging
- Based on libkiwix 14.1.0 (@kelson42 #773)
- Improve CI/CD (@kelson42 #681 #698 #702 #705 #720 #722, @mgautierfr #695)
kiwix-tools 3.7.0
=================
* Fixed ZIM name vs Book name confusion in documentation (@veloman-yunkan #663)
* Fixes compilation dependencies to rely on appropriate version (@kelson42 #667)
* New --skipInvalid Kiwix Server command line option (@schuellerf @kelson42 #666)
kiwix-tools 3.6.0
=================
* Improved kiwix-serve man page (@iArchitSharma #626)
* C++17 compliant code base (@mgautierfr #636)
* Support of libkiwix13 (@mgautierfr #633)
* Additional docker images archs for armv6 and i386 (@rgaudin #622)
kiwix-tools 3.5.0
=================
* Do not use `--static` option when compiling on MacOs (@mgautierfr #615)
* Move main branch from `master` to `main`.
* Fix docker image (@jacroe #597)
* Various CI improvements (@kelson42)
kiwix-serve
-----------
* Add documentation about the kiwix-serve API (@veloman-yunkan #586)
https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html#http-api
kiwix-tools 3.4.0
=================

View File

@@ -4,15 +4,15 @@ Kiwix tools
The Kiwix tools is a collection of [Kiwix](https://kiwix.org) related
command line tools:
* kiwix-manage: Manage XML based library of ZIM files
* kiwix-search: Fulltext search in ZIM files
* kiwix-search: Full text search in ZIM files
* kiwix-serve: HTTP daemon serving ZIM files
[![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/)
[![Repositories](https://img.shields.io/repology/repositories/kiwix-tools?label=repositories)](https://github.com/kiwix/kiwix-tools/wiki/Repology)
[![Docker](https://img.shields.io/docker/v/kiwix/kiwix-tools?arch=amd64&label=Docker&sort=date)](https://hub.docker.com/r/kiwix/kiwix-tools)
[![Docker (kiwix-serve)](https://img.shields.io/docker/v/kiwix/kiwix-serve?arch=amd64&label=Docker%20(kiwix-serve)&sort=date)](https://hub.docker.com/r/kiwix/kiwix-serve)
[![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h)
[![Build Status](https://github.com/kiwix/kiwix-tools/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-tools/actions?query=branch%3Amaster)
[![Docker](https://ghcr-badge.egpl.dev/kiwix/kiwix-tools/latest_tag?label=docker)](https://ghcr.io/kiwix/kiwix-tools)
[![Docker](https://ghcr-badge.egpl.dev/kiwix/kiwix-tools/latest_tag?label=docker%20(kiwix-serve))](https://ghcr.io/kiwix/kiwix-tools)
[![Build Status](https://github.com/kiwix/kiwix-tools/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-tools/actions/workflows/ci.yml?query=branch%3Amain)
[![Doc](https://readthedocs.org/projects/kiwix-tools/badge/?style=flat)](https://kiwix-tools.readthedocs.org/en/latest/?badge=latest)
[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-tools/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-tools)
[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
@@ -28,8 +28,8 @@ Preamble
--------
Although the Kiwix tools can be compiled/cross-compiled on/for many
sytems, the following documentation explains how to do it on POSIX
ones. It is primarly thought for GNU/Linux systems and has been tested
systems, the following documentation explains how to do it on POSIX
ones. It is primarily thought for GNU/Linux systems and has been tested
on recent releases of
[Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) and
[Fedora](https://getfedora.org).
@@ -61,7 +61,7 @@ Environment
-------------
The Kiwix tools build using [Meson](http://mesonbuild.com/) version
0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
0.43 or higher. Meson relies itself on Ninja, pkg-config and a few other
compilation tools. Install them first:
* [Meson](http://mesonbuild.com/)
* [Ninja](https://ninja-build.org/)
@@ -95,7 +95,7 @@ ninja -C build install
```
You might need to run the command as `root` (or using `sudo`),
depending where you want to install the Kiwix tools. After the
depending on where you want to install the Kiwix tools. After the
installation succeeded, you may need to run ldconfig (as `root`).
Uninstallation
@@ -112,10 +112,10 @@ Like for the installation, you might need to run the command as `root`
Docker
------
An official Docker image of the Kiwix tools can be found in the
[Docker Hub](https://hub.docker.com/r/kiwix/kiwix-tools). A
An official Docker image of the Kiwix tools can be found on
[GHCR](https://ghcr.io/kiwix/kiwix-tools). A
`kiwix-serve` dedicated Docker image [exists
too](https://hub.docker.com/r/kiwix/kiwix-serve).
too](https://ghcr.io/kiwix/kiwix-serve).
Troubleshooting
---------------

10
debian/control vendored
View File

@@ -3,11 +3,13 @@ Section: utils
Priority: optional
Maintainer: Kiwix team <kiwix@kiwix.org>
Build-Depends: debhelper-compat (= 13),
libkiwix-dev (>= 10.0.0~),
libzim-dev (>= 7.2.0~),
libzim-dev (>= 9.0), libzim-dev (<< 10.0),
libkiwix-dev (>= 14.0), libkiwix-dev (<< 15.0),
cmake,
libdocopt-dev,
meson,
pkg-config,
Standards-Version: 4.5.0
pkgconf,
Standards-Version: 4.6.2
Homepage: https://github.com/kiwix/kiwix-tools
Rules-Requires-Root: no

View File

@@ -1,12 +1,25 @@
FROM alpine:3.16
LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
FROM alpine:3.22
LABEL org.opencontainers.image.source=https://github.com/openzim/kiwix-tools
# TARGETPLATFORM is injected by docker build
ARG TARGETPLATFORM
ARG VERSION
RUN apk --no-cache add dumb-init curl && \
ARCH=$(cat /etc/apk/arch) && \
if [ "$ARCH" = "x86" ]; then ARCH="i586"; \
elif [ "$ARCH" = "aarch64" ]; then ARCH="armhf"; \
elif [ "$ARCH" = "armv7" ]; then ARCH="armhf"; fi && \
RUN set -e && \
apk --no-cache add dumb-init curl && \
echo "TARGETPLATFORM: $TARGETPLATFORM" && \
if [ "$TARGETPLATFORM" = "linux/386" ]; then ARCH="i586"; \
# linux/arm64/v8 points to linux/arm64
elif [ "$TARGETPLATFORM" = "linux/arm64/v8" \
-o "$TARGETPLATFORM" = "linux/arm64" ]; then ARCH="aarch64"; \
# linux/arm translates to linux/arm/v7
elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then ARCH="armv8"; \
elif [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then ARCH="armv6"; \
elif [ "$TARGETPLATFORM" = "linux/amd64/v3" \
-o "$TARGETPLATFORM" = "linux/amd64/v2" \
-o "$TARGETPLATFORM" = "linux/amd64" ]; then ARCH="x86_64"; \
# we dont suppot any other arch so let it fail
else ARCH="unknown"; fi && \
# download requested kiwix-tools version
url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$ARCH-$VERSION.tar.gz" && \
echo "URL: $url" && \

View File

@@ -1,14 +1,14 @@
Kiwix-tools Docker image
===
- Available on [docker.io](https://hub.docker.com/r/kiwix/kiwix-tools) and [ghcr.io](https://ghcr.io/kiwix/kiwix-tools).
- Available on [ghcr.io](https://ghcr.io/kiwix/kiwix-tools).
- multi-arch (`linux/amd64`, `linux/arm64`, `linux/arm/v7`)
- based on official `kiwix-tools` binaries.
## Usage
``` sh
$ docker run -it kiwix/kiwix-tools:3.1.2
$ docker run -it ghcr.io/kiwix/kiwix-tools:3.1.2
Welcome to kiwix-tools! The following binaries are available:
kiwix-manage kiwix-search kiwix-serve
@@ -17,7 +17,7 @@ kiwix-manage kiwix-search kiwix-serve
`kiwix-tools` operates on zim files. You shall mount a volume to access the files.
```sh
docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
docker run -v $(pwd):/data -it ghcr.io/kiwix/kiwix-tools kiwix-search /data/wikipedia_fr_test.zim "Mali"
```
## Building and reusing
@@ -30,3 +30,7 @@ docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-search /data/wikipedia_fr
- `wget` in `alpine:3` on `arm/v7` (__inside github action only__) crashes when downloading from HTTPs locations. Keep http-only in Dockerfile.
- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose.
## See also
If you are interested by a Kiwix server only container image, [here it is](server/README.md).

View File

@@ -1,14 +1,19 @@
ARG VERSION=latest
# kiwix-tools is multi-arch
FROM kiwix/kiwix-tools:$VERSION
LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools
FROM ghcr.io/kiwix/kiwix-tools:$VERSION
LABEL org.opencontainers.image.source=https://github.com/openzim/kiwix-tools
# expose kiwix-serve default port and workdir
EXPOSE 8080
VOLUME /data
WORKDIR /data
# running as a named unprivileged user
RUN addgroup -S -g 1001 user && adduser -S -u 1001 user -G user
RUN chown user:user /data
USER user
COPY ./start.sh /usr/local/bin/
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/start.sh"]

View File

@@ -9,14 +9,20 @@ With local ZIM file(s)
* Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following:
```bash
docker run -v /tmp/zim:/data -p 8080:8080 kiwix/kiwix-serve wikipedia.zim wiktionary.zim
docker run -v /tmp/zim:/data -p 8080:8080 ghcr.io/kiwix/kiwix-serve wikipedia.zim wiktionary.zim
```
or, if you want to load all ZIM files within a directory, then use globbing:
```bash
docker run -v /tmp/zim:/data -p 8080:8080 ghcr.io/kiwix/kiwix-serve '*.zim'
```
With remote ZIM file
--------------------
```bash
docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 kiwix/kiwix-serve
docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:8080 ghcr.io/kiwix/kiwix-serve
```
Change default port
@@ -25,7 +31,7 @@ Change default port
You can change port to expose with environment variable PORT, useful if running on Podman, K8s or OpenShift
```bash
podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 kiwix/kiwix-serve
podman run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -e PORT=8888 -p 8080:8888 ghcr.io/kiwix/kiwix-serve
```
ARM
@@ -33,10 +39,15 @@ ARM
Build an image for an ARM based GNU/Linux:
```bash
docker build . -t kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/"
docker build . -t ghcr.io/kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/"
```
You can also deploy kiwix with [`docker-compose`](https://docs.docker.com/compose/). Check out a sample at [docker-compose.yml.example](docker-compose.yml.example)
Docker Compose
--------------
You can also deploy kiwix with
[`docker-compose`](https://docs.docker.com/compose/). Check out a
sample at [docker-compose.yml.example](docker-compose.yml.example).
Screenshots
-----------

View File

@@ -3,12 +3,12 @@ services:
kiwix-serve:
ports:
- 8080:8080
image: kiwix/kiwix-serve
image: ghcr.io/kiwix/kiwix-serve:latest
# uncomment next 4 lines to use it with local zim file in /tmp/zim
# volumes:
# - /tmp/zim:/data
# command:
# - *.zim
# - '*.zim'
# uncomment next 2 lines to use it with remote zim file
# environment:
# - 'DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim'

View File

@@ -3,6 +3,14 @@
# Download if necessary a file
if [ ! -z "$DOWNLOAD" ]
then
# Check if /data is writable
if [ ! -w /data ]
then
echo "'/data' directory is not writable by '$(id -n -u):$(id -n -g)' ($(id -u):$(id -g)). ZIM file(s) can not be written."
exit 1
fi
# Dwonload ZIM file
ZIM=`basename $DOWNLOAD`
wget $DOWNLOAD -O "$ZIM"

50
docs/conf.py Normal file
View File

@@ -0,0 +1,50 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'kiwix-tools'
copyright = '2024, kiwix-team'
author = 'kiwix-team'
# -- General configuration ---------------------------------------------------
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"sphinx_rtd_theme"
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

8
docs/index.rst Normal file
View File

@@ -0,0 +1,8 @@
Welcome to kiwix-tools' documentation!
======================================
.. toctree::
:maxdepth: 2
:caption: Contents:
kiwix-serve

889
docs/kiwix-serve.rst Normal file
View File

@@ -0,0 +1,889 @@
***********
kiwix-serve
***********
Introduction
============
``kiwix-serve`` is a tool for serving ZIM file content over HTTP. It supports
serving a library containing multiple ZIM files. In a large library served by a
``kiwix-serve`` instance clients can look up/filter ZIM files of interest by
words in their :term:`titles <ZIM title>` and/or descriptions, language, tags, etc.
``kiwix-serve`` provides a ZIM file viewer for displaying inidividual pages
from a ZIM file inside the user's web browser (without downloading the full ZIM
file).
Clients can also remotely search inside those ZIM files that contain a full-text
search database.
``kiwix-serve`` supports Web browsers `Firefox >= 70, Chrome >= 80, Edge >= 80, ChromeAndroid >= 80, Safari >= 14, iOS >= 14 <https://browsersl.ist/#q=Firefox+%3E%3D+70%2C+Chrome+%3E%3D+80%2C+Edge+%3E%3D+80%2C+ChromeAndroid+%3E%3D+80%2C+Safari+%3E%3D+14%2C+iOS+%3E%3D+14>`_.
Usage
=====
.. code-block:: sh
kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
Arguments
---------
.. _cli-arg-library-file-path:
``LIBRARY_FILE_PATH``: path of an XML library file listing ZIM files to serve.
To be used only with the :option:`--library` option. Multiple library files can
be provided as a semicolon (``;``) separated list.
``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed).
Options
-------
.. option:: --library
By default, ``kiwix-serve`` expects a list of ZIM files as command line
arguments. Providing the :option:`--library` option tells ``kiwix-serve``
that the command line argument is rather a :ref:`library XML file
<cli-arg-library-file-path>`.
.. option:: --catalogOnly
In this mode ``kiwix-serve`` only serves the welcome (library) page and the
OPDS catalog. ZIM files referred by the :ref:`library XML file
<cli-arg-library-file-path>` need not be accessible.
This option may be combined with the :option:`--contentServerURL` option.
.. option:: --contentServerURL=URL
In :option:`--catalogOnly` mode book content is not served by this instance
of `kiwix-serve`. If a separate instance of `kiwix-serve` is running for the
same library without that option and thus serves book content, then the root
URL of that server can be passed to this instance so that books can still be
previewed.
This option must be combined with the :option:`--catalogOnly` option.
.. option:: -i ADDR, --address=ADDR
Listen only on this IP address. By default the server listens on all
available IP addresses. Alternatively, you can use special values to define which types of connections to accept:
- all : Listen for connections on all IP addresses (IPv4 and IPv6).
- ipv4 : Listen for connections on all IPv4 addresses.
- ipv6 : Listen for connections on all IPv6 addresses.
.. option:: -p PORT, --port=PORT
TCP port on which to listen for HTTP requests (default: 80).
.. option:: -r ROOT, --urlRootLocation=ROOT
URL prefix on which the content should be made available (default: empty).
.. option:: -d, --daemon
Detach the HTTP server daemon from the main process.
.. option:: -a PID, --attachToProcess=PID
Exit when the process with id PID stops running.
.. option:: -M, --monitorLibrary
Monitor the XML library file and reload it automatically when it changes.
Library reloading can be forced anytime by sending a SIGHUP signal to the
``kiwix-serve`` process (this works regardless of the presence of the
:option:`--monitorLibrary`/:option:`-M` option).
.. option:: -m, --nolibrarybutton
Disable the library home button in the ZIM viewer toolbar.
.. option:: -n, --nosearchbar
Disable the searchbox in the ZIM viewer toolbar.
.. option:: -b, --blockexternal
Prevent the users from directly navigating to external resources via such
links in ZIM content.
.. option:: -t N, --threads=N
Number of threads to run in parallel (default: 4).
.. option:: -s N, --searchLimit=N
Maximum number of ZIM files in a fulltext multizim search (default: No limit).
.. option:: -z, --nodatealiases
Create URL aliases for each content by removing the date embedded in the file
name. The expected format of the date in the filename is ``_YYYY-MM``. For
example, ZIM file ``wikipedia_en_all_2020-08.zim`` will be accessible both as
``wikipedia_en_all_2020-08`` and ``wikipedia_en_all``.
.. option:: -c PATH, --customIndex=PATH
Override the welcome page with a custom HTML file.
.. option:: -L N, --ipConnectionLimit=N
Max number of (concurrent) connections per IP (default: infinite,
recommended: >= 6).
.. option:: -v, --verbose
Print debug log to STDOUT.
.. option:: -V, --version
Print the software version.
.. option:: -h, --help
Print the help text.
HTTP API
========
``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where
``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the
:option:`--address`/:option:`-i`, :option:`--port`/:option:`-p` and
:option:`--urlRootLocation`/:option:`-r` options, respectively.
HTTP API endpoints presented below are relative to that location, i.e.
``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``.
.. note::
The HTTP API is documented in its entirety in order to facilitate the work of
the Kiwix development team. Note, however, that only a subset of the HTTP API
constitutes ``kiwix-serves``'s public interface.
.. _public-api-endpoint:
**Public API endpoint**
A public HTTP API endpoint is intended to serve the outside world (in
addition to ``kiwix-serve``'s front-end and other Kiwix products). The
Kiwix development team will do its best to ensure gratifying experience for
clients of public API endpoints at all stages of the endpoint lifecycle.
.. _private-api-endpoint:
**Private API endpoint**
A private API endpoint is intended to be used only by ``kiwix-serve``'s
frontend or by other products maintained solely by the Kiwix team. Private
API comes without any guaranees. It may change as frequently and as
drasticaly as the Kiwix development team sees fit.
.. _deprecation:
**Deprecation**
Public API doesn't stay frozen once and forever. As the API evolves, Kiwix
team reserves the right to drop support for certain old functionality. In
such events, an advance notice will be issued and the users will be given
enough time to prepare for the change.
Currently, public endpoints are limited to the following list:
- :ref:`OPDS API <new-opds-api>`
- ``/raw``
- ``/search`` (with ``/search/searchdescription.xml``)
.. _welcome-page:
``/``
-----
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
Welcome page is served under ``/``. By default this is the library page, where
books are listed and can be looked up/filtered interactively. However, the
welcome page can be overriden through the :option:`--customIndex`/:option:`-c`
command line option of ``kiwix-serve``.
.. _new-opds-api:
``/catalog/v2`` (OPDS API)
------------------------------
===== ===========
Type: :ref:`public <public-api-endpoint>`
===== ===========
The new OPDS API of ``kiwix-serve`` is based on the `OPDS Catalog specification
v1.2 <https://specs.opds.io/opds-1.2>`_. All of its endpoints are grouped under
``/catalog/v2``.
:ref:`Legacy OPDS API <legacy-opds-api>` is preserved for backward
compatibility.
``/catalog/v2/root.xml``
^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
The OPDS Catalog Root links to the OPDS acquisition and navigation feeds
accessible through the other endpoints of the OPDS API.
``/catalog/v2/searchdescription.xml``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Describes the `/catalog/v2/entries`_ endpoint in `OpenSearch description format
<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
``/catalog/v2/categories``
^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Returns the full list of ZIM file categories as an `OPDS Navigation Feed
<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
``/catalog/v2/entries``
^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Returns a full or filtered list of ZIM files as a paginated `OPDS acquisition
feed <https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `complete
entries
<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
**Pagination:**
By default, no more than 10 first entries are returned from the library. To
obtain the remaining entries the URL query parameters ``start`` and/or
``count`` must be used. The output of ``/catalog/v2/entries?start=s&count=n``
will contain at most ``n`` (default value: 10) results starting from entry #
``s`` (default value: 0). ``count`` with a negative value (e.g. ``count=-1``)
removes the limit on the number of results in the output.
.. note::
Previously ``count=0`` also designated an unbounded query (i.e. worked
similarly to ``count=-1``). The response to a ``count=0`` query was changed
to consist of 0 results, as such a query/response combination is a good way
to find out the total number of results (when only that information is
needed) with minimal consumption of resources.
Examples:
.. code:: sh
# Returns the first 10 entries (internally numbered 0 through 9)
$ curl 'http://localhost:8080/catalog/v2/entries'
# Returns the next 10 entries (internally numbered 10 through 19)
$ curl 'http://localhost:8080/catalog/v2/entries?start=10'
# Returns the first 50 entries
$ curl 'http://localhost:8080/catalog/v2/entries?count=50'
# Returns 50 entries starting from entry # 100 (i.e. entries ## 100-149)
$ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=50'
# Returns all entries
$ curl 'http://localhost:8080/catalog/v2/entries?count=-1'
# Returns all entries starting from entry # 100
$ curl 'http://localhost:8080/catalog/v2/entries?start=100&count=-1'
.. _library-filtering:
**Filtering:**
A filtered subset of the library can be requested by providing one or more
filtering criteria, whereupon only entries matching *all* of the criteria are
included in the response. Pagination is applied to the filtered list. The
filtering criteria must be specified via the following URL parameters:
* ``lang`` - filter by language (specified as a 3-letter language code).
* ``category`` - filter by categories associated with the library entries.
* ``tag`` - filter by tags associated with the library entries. Multiple tags
can be provided as a semicolon separated list (e.g
``tag=wikipedia;_videos:no``). The result will contain only those entries
that contain *all* of the requested tags.
* ``notag`` - filter out (exclude) entries with *any* of the specified tags
(example - ``notag=ted;youtube``).
* ``maxsize`` - include in the results only entries whose size (in bytes)
doesn't exceed the provided value.
* ``q`` - include in the results only entries that contain the specified text
in the title or description.
* ``name`` - include in the results only entries with a matching
:term:`book name <Book name>`.
Examples:
.. code:: sh
# List only books in Italian (lang=ita) but
# return only results ## 100-149 (start=100&count=50)
$ curl 'http://localhost:8080/catalog/v2/entries?lang=ita&start=100&count=50'
# List only books with category of 'wikipedia' AND containing the word
# 'science' in the title or description. Return only the first 10 results.
$ curl 'http://localhost:8080/catalog/v2/entries?q=science&category=wikipedia'
``/catalog/v2/entry/ZIMID``
^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Returns full info about the library entry with :term:`UUID <ZIM UUID>`
``ZIMID``.
``/catalog/v2/illustration/ZIMID``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
**Usage:**
``/catalog/v2/illustration/ZIMID?size=N``
Returns the illustration of size ``NxN`` pixels for the library entry with
:term:`UUID <ZIM UUID>` ``ZIMID``.
If no illustration of requested size is found a HTTP 404 error is returned.
``/catalog/v2/languages``
^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Returns the full list of ZIM file languages as an `OPDS Navigation Feed
<https://specs.opds.io/opds-1.2#22-navigation-feeds>`_.
``/catalog/v2/partial_entries``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <new-opds-api>`
===== ===========
Returns the full or filtered list of ZIM files as an `OPDS acquisition feed
<https://specs.opds.io/opds-1.2#23-acquisition-feeds>`_ with `partial entries
<https://specs.opds.io/opds-1.2#512-partial-and-complete-catalog-entries>`_.
Supported filters are the same as for the `/catalog/v2/entries`_ endpoint.
.. _legacy-opds-api:
``/catalog`` (Legacy OPDS API)
------------------------------
===== ===========
Type: :ref:`deprecated <deprecation>`
===== ===========
The legacy OPDS API is preserved for backward compatibility and is deprecated.
:ref:`New OPDS API <new-opds-api>` should be used instead.
``/catalog/root.xml``
^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`deprecated API <legacy-opds-api>`
===== ===========
Full library OPDS catalog (list of all ZIM files).
``/catalog/searchdescription.xml``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`deprecated API <legacy-opds-api>`
===== ===========
Describes the `/catalog/search`_ endpoint in `OpenSearch description format
<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
``/catalog/search``
^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`deprecated API <legacy-opds-api>`
===== ===========
Returns the list of ZIM files (in OPDS catalog format) matching the
search/filtering criteria. Supported filters are the same as for the
`/catalog/v2/entries`_ endpoint.
``/catch/external``
-------------------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
**Usage:**
``/catch/external?source=URL``
Generates a HTML page with a link leading to (the decoded version of) ``URL``
and a warning that following that link will load an external (out of ZIM)
resource.
**Parameters:**
``source``: URL of the external resource (must be URL-encoded).
**Example:**
.. code:: sh
# Intercept an external link to https://example.com?query=abcd
$ curl 'http://localhost:8080/catch/external?source=https%3A%2F%2Fexample.com%3Fquery%3Dabcd'
``/content``
------------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
ZIM file content is served under the ``/content`` endpoint as described below.
``/content/ZIMNAME/PATH/IN/ZIMFILE``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with :term:`name
<ZIM name>` ``ZIMNAME``.
``/content/ZIMNAME``
^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
``/content/ZIMNAME`` redirects to the main page of the ZIM file with :term:`name
<ZIM name>` ``ZIMNAME`` (unless that ZIM file contains an entry with an empty
path or path equal to ``/``, in which case that entry is returned).
``/random``
-----------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
**Usage:**
``/random?content=ZIMNAME``
Generates a HTTP redirect to a randomly selected article/page from the
specified ZIM file.
**Parameters:**
``content``: :term:`name of the ZIM file <ZIM name>`.
.. _raw:
``/raw``
--------
===== ===========
Type: :ref:`public <public-api-endpoint>`
===== ===========
The ``/raw`` API provides access to ZIM file data. It consists of two separate
endpoints for accessing data and metadata.
``/raw/ZIMNAME/content/PATH/IN/ZIMFILE``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <raw>`
===== ===========
Returns the entry with path ``PATH/IN/ZIMFILE`` from the ZIM file with
:term:`name <ZIM name>` ``ZIMNAME``. Currently, this endpoint almost duplicates
(with some subtle technical differences) the newer endpoint
`/content/ZIMNAME/PATH/IN/ZIMFILE`_. The important difference is that the
``/raw`` endpoint guarantees that no server-side processing will be applied to
the returned content, whereas content obtained via the ``/content`` endpoint
may in the future undergo some processing intended to improve the operation of
the viewer (e.g. compensating for certain bugs in ZIM creation). Also note that
``/raw`` is :ref:`public <public-api-endpoint>`, whereas ``/content`` is
:ref:`private <private-api-endpoint>`.
``/raw/ZIMNAME/meta/METADATAID``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
===== ===========
Type: member of a :ref:`public API <raw>`
===== ===========
Returns the metadata item ``METADATAID`` from the ZIM file with :term:`name
<ZIM name>` ``ZIMNAME``.
``/search``
-----------
===== ===========
Type: :ref:`public <public-api-endpoint>`
===== ===========
Performs a full text search on one or more ZIM files and returns an HTML page
with a list of links to matching pages along with snippets of the matching
portions of those pages.
.. _multi-zim-search-constraints:
A multi-ZIM search request must comply with the following constraints:
* the number of ZIM files participating in the search operation must not exceed
the limit imposed by the :option:`--searchLimit` option of ``kiwix-serve``.
* all of the ZIM files participating in the same search operation must be in
the same language.
**Parameters:**
ZIM file selection parameters:
At least one of the following parameters must be provided in order to
specify in which ZIM file(s) to search. Parameters appearing earlier in
below list take precedence over subsequent ones (the later ones, even if
present in the request, are simply ignored).
``content``: :term:`name of the ZIM file <ZIM name>` (for a single-ZIM
search). This is a :ref:`legacy parameter <deprecation>`. ``books.name``
should be used instead.
``books.id``: :term:`UUID <ZIM UUID>` of the ZIM file. Can be repeated for
a multi-ZIM search, however must respect the :ref:`multi-ZIM search
constraints <multi-zim-search-constraints>`.
.. note::
If any of the provided ``books.id`` values refers to a book missing
from the library then an error is returned instead of running the
search on the remaining (valid) entries.
``books.name``: :term:`name of the ZIM file <ZIM name>` (not to be confused
with ``books.filter.name`` which selects/filters based on the :term:`book
name <Book name>`). Can be repeated for a multi-ZIM search, however must
respect the :ref:`multi-ZIM search constraints
<multi-zim-search-constraints>`.
.. note::
If any of the provided ``books.name`` values refers to a book missing
from the library then an error is returned instead of running the
search on the remaining (valid) entries.
``books.filter.{criteria}``: allows to take full advantage of :ref:`library
filtering <library-filtering>` functionality of the `/catalog/v2/entries`_
endpoint (``{criteria}`` must be replaced with an attribute/filtering
criteria name supported by :ref:`library filtering <library-filtering>`).
Query parameters:
``pattern`` (optional; defaults to an empty string): text to search for.
``latitude``, ``longitude`` & ``distance`` (optional): geospatial query
parameters. If all of these are provided, then the results will be
restricted to geotagged pages that are within ``distance`` metres from the
location on Earth with coordinates ``latitude`` and ``longitude``.
Pagination parameters:
``pageLength`` (optional, default: 25): maximum number of search results in
the response. Capped at 140.
``start`` (optional, default: 0): this parameter enables pagination of
results. The response will include up to ``pageLength`` results starting
with entry # ``start`` from the full list of search results (the first
result is assumed to have index 0).
Other parameters:
``format`` (optional, default: html): format of the search results. Allowed
values are: html, xml.
Examples:
.. code:: sh
# Search for 'android' in the book with name 'scifi-library'
# Return results ## 51-60.
$ curl 'http://localhost:8080/search?pattern=android&books.name=scifi-library&start=51&pageLength=10'
# Search for 'napoli' in books in Italian
$ curl 'http://localhost:8080/search?books.filter.lang=ita&pattern=napoli'
# Search for 'chateau' in books in French that have a category of 'wikipedia'.
# Return the results as XML.
$ curl 'http://localhost:8080/search?pattern=chateau&books.filter.lang=fra&books.filter.category=wikipedia&format=xml'
``/search/searchdescription.xml``
---------------------------------
===== ===========
Type: :ref:`public <public-api-endpoint>`
===== ===========
Describes the `/search`_ endpoint in `OpenSearch description format
<https://developer.mozilla.org/en-US/docs/Web/OpenSearch>`_.
``/skin``
-----------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
Static front-end resources (such as CSS, javascript and images) are all grouped
under ``/skin``.
**Usage:**
``/skin/PATH/TO/RESOURCE[?cacheid=CACHEID]``
`Cache busting
<https://javascript.plainenglish.io/what-is-cache-busting-55366b3ac022>`_ of
static resources is supported via the optional param ``cacheid``. By default,
i.e. when the ``cacheid`` parameter is not specified while accessing the
``/skin`` endpoint, static resources are served as if they were dynamic (i.e.
could be different for an immediately repeated request). Specifying the
``cacheid`` parameter with a correct value (matching the value embedded in the
``kiwix-serve`` instance), makes the returned resource to be presented as
immutable. However, if the value of the ``cacheid`` parameter mismatches then
``kiwix-serve`` responds with a 404 HTTP error.
``kiwix-serve``'s default front-end (the :ref:`welcome page <welcome-page>` and
the :ref:`ZIM file viewer <zim-file-viewer>`) access all underlying static
resources by using explicit ``cacheid`` s.
``/suggest``
------------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
**Usage:**
``/suggest?content=ZIMNAME[&term=QUERY][&count=N][&start=S]``
Returns suggestions (in JSON format) for a text string that is assumed to be a
partially typed search for a page inside a particular ZIM file.
Suggestions are obtained as matches of the query text against the page titles
in the ZIM file using the title index database generated during the creation of
the ZIM file.
In case of a multi-word query the order of the words matters in two ways:
1. the last word is considered as partially typed, unless followed by a space;
2. ranking of the matches.
If the ZIM file doesn't contain a title index then suggestions are generated by
listing page titles starting *exactly* (in a case sensitive manner) with the
query text. Otherwise, suggestions are case-insensitive.
If the ZIM file contains a full text search index, then an extra suggestion is
added as an option to perform a full text search in the said ZIM file.
**Parameters:**
``content`` (mandatory): :term:`name of the ZIM file <ZIM name>`.
``term`` (optional; defaults to an empty string): query text.
``count`` (optional, default: 10): maximum number of page suggestions in the
response (i.e. the extra option to perform a full text search is not included
in this count).
``start`` (optional, default: 0): this parameter enables pagination of
results. The response will include up to ``count`` entries starting with
entry # ``start`` from the full list of page suggestions (the first result is
assumed to have index 0).
**Example:**
.. code:: sh
$ curl 'http://localhost/suggest?content=stackoverflow_en&term=pyth&count=50'
.. _zim-file-viewer:
``/viewer``
-----------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
ZIM file viewer. The ZIM file and entry therein must be specified via the hash
component of the URL as ``/viewer#ZIMNAME/PATH/IN/ZIMFILE``.
``/viewer_settings.js``
-----------------------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
Settings of the ZIM file viewer that are configurable via certain command line
options of ``kiwix-serve`` (e.g. ``--nolibrarybutton``).
/ANYTHING/ELSE
--------------
===== ===========
Type: :ref:`private <private-api-endpoint>`
===== ===========
Any other URL is considered as an attempt to access ZIM file content using the
legacy URL scheme and is redirected to ``/content/ANYTHING/ELSE``.
Glossary
========
.. glossary::
Book name
Name of the book as specified in the ZIM file metadata (for a
``kiwix-serve`` started *WITHOUT* the :option:`--library` option) or the
library XML file (for a ``kiwix-serve`` started with the
:option:`--library` option).
.. note::
Two or more books may have the same name in the library. That's not
considered a conflict, because there may be multiple versions of the
"same" book (differing by the settings of the scraper, date, etc).
:ref:`Library filtering <library-filtering>` by name will return all
matching books.
ZIM filename
Name of a ZIM file on the server filesystem.
ZIM name
Identifier of a ZIM file in the server's library (used for referring to a
particular ZIM file in requests).
ZIM names are derived from the filenames as follows:
- file extension is removed,
- all characters are converted to lowercase,
- diacritics are removed,
- spaces are replaced with underscores,
- ``+`` symbols are replaced with the text ``plus``.
Presence of the :option:`-z`/:option:`--nodatealiases` option will create
additional names (aliases) for filenames with dates.
ZIM names are expected to be unique across the library. Any name conflicts
(including those caused by the usage of the
:option:`-z`/:option:`--nodatealiases` option) are reported on STDERR but,
otherwise, are ignored (i.e. only one of the entries can be accessed via
the conflicting name).
ZIM title
Title of a ZIM file. This can be any text (with whitespace). It is never
used as a way of referring to a ZIM file.
ZIM UUID
This is a unique identifier of a ZIM file designated at its creation time
and embedded in the ZIM file. Certain ``kiwix-serve`` operations may
require that a ZIM file be referenced through its UUID rather than name.

7
docs/meson.build Normal file
View File

@@ -0,0 +1,7 @@
sphinx = find_program('sphinx-build', native:true)
sphinx_target = run_target('doc',
command: [sphinx, '-bhtml',
meson.current_source_dir(),
meson.current_build_dir()])

2
docs/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
Sphinx==5.3.0
sphinx-rtd-theme==1.1.1

View File

@@ -1,22 +1,30 @@
project('kiwix-tools', 'cpp',
version : '3.4.0',
version : '3.8.1',
license : 'GPL',
default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
default_options: ['c_std=c11', 'cpp_std=c++17', 'werror=true'])
compiler = meson.get_compiler('cpp')
add_global_arguments('-DKIWIX_TOOLS_VERSION="@0@"'.format(meson.project_version()), language : 'cpp')
if host_machine.system() == 'windows'
add_project_arguments('-DNOMINMAX', language: 'cpp')
endif
static_linkage = get_option('static-linkage')
if static_linkage
add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
# Static build is not supported on MacOS
if host_machine.system() != 'darwin'
add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
endif
endif
thread_dep = dependency('threads')
kiwixlib_dep = dependency('kiwix', version:'>=12.0.0', static:static_linkage)
libzim_dep = dependency('libzim', version:'>=8.1.0', static:static_linkage)
libzim_dep = dependency('libzim', version:['>=9.0.0', '<10.0.0'], static:static_linkage)
libkiwix_dep = dependency('libkiwix', version:['>=14.1.0', '<15.0.0'], static:static_linkage)
libdocopt_dep = dependency('docopt', static:static_linkage)
all_deps = [thread_dep, kiwixlib_dep, libzim_dep]
all_deps = [thread_dep, libkiwix_dep, libzim_dep, libdocopt_dep]
if static_linkage
librt = compiler.find_library('rt', required:false)
@@ -26,3 +34,6 @@ if static_linkage
endif
subdir('src')
if get_option('doc')
subdir('docs')
endif

View File

@@ -1,2 +1,4 @@
option('static-linkage', type : 'boolean', value : false,
description : 'Create statically linked binaries.')
option('doc', type : 'boolean', value : false,
description : 'Build the documentations.')

View File

@@ -1,60 +1,142 @@
.TH KIWIX 1 "21 May 2012"
.TH KIWIX 1 "10 July 2023"
.SH NAME
kiwix\-serve \- Kiwix HTTP Server
kiwix-serve \- Kiwix HTTP Server
.SH SYNOPSIS
.IX Header "SYNOPSIS"
.B kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH
.br
kiwix\-serve [\-\-index=INDEX_PATH] [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [\-\-attachToProcess=PID] ZIM_PATH
.br
kiwix\-serve \-\-library [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [\-\-attachToProcess=PID] LIBRARY_PATH
.B kiwix-serve [OPTIONS] ZIM_FILE_PATH ...
.SH DESCRIPTION
.PP
Stand\-alone HTTP server for serving ZIM contents over the network.
The \fBkiwix-serve\fR command is used to run a stand-alone HTTP server for serving ZIM contents over the network.
.SH ARGUMENTS
.TP
\fBLIBRARY_FILE_PATH\fR
Path of an XML library file listing ZIM files to serve. To be used only with the --library option. Multiple library files can be provided as a semicolon (;) separated list.
.TP
\fB\-\-index=INDEX_PATH\fR
Path to index folder created using \fBkiwix-install\fB.
\fBZIM_FILE_PATH ...\fR
ZIM file path(s). Multiple arguments are allowed.
.SH OPTIONS
.TP
\fB--library\fR
By default, kiwix-serve expects a list of ZIM files as command line arguments. Providing the --library option tells kiwix-serve that the command line argument is rather a library XML file.
.TP
\fB\-\-port=PORT\fR
Port number on which the server should listen.
\fB-i ADDR, --address=ADDR\fR
Listen only on this IP address. By default, the server listens on all available IP addresses. Alternatively, you can use special values to define which types of connections to accept:
all : Listen for connections on all IP addresses (IPv4 and IPv6).
.br
By default, server listens on port 80.
.TP
\fB\-\-verbose\fR
Enable verbose output.
.TP
\fB\-\-daemon\fR
Run the server as a daemon.
.TP
\fB\-\-attachToProcess=PID\fR
Shutdown the server when process numbered PID dies.
.TP
\fBZIM_PATH\fR
ZIM file of the content to serve.
ipv4 : Listen for connections on all IPv4 addresses.
.br
Mandatory if not in library mode.
ipv6 : Listen for connections on all IPv6 addresses.
.TP
\fB\-\-library\fR
Enable library mode.
.br
Serves the contents of a library file instead of a single ZIM file.
\fB-p PORT, --port=PORT\fR
TCP port on which to listen for HTTP requests (default: 80).
.TP
\fBLIBRARY_PATH\fR
Kiwix library file path.
\fB-r ROOT, --urlRootLocation=ROOT\fR
URL prefix on which the content should be made available (default: empty).
.TP
\fB-d, --daemon\fR
Detach the HTTP server daemon from the main process.
.TP
\fB-a PID, --attachToProcess=PID\fR
Exit when the process with id PID stops running.
.TP
\fB-M, --monitorLibrary\fR
Monitor the XML library file and reload it automatically when it changes.
Library reloading can be forced anytime by sending a SIGHUP signal to the
\*(lqkiwix-serve\*(rq process (this works regardless of the presence of the
\*(lq--monitorLibrary\*(rq/\*(lq-M\*(rq option).
.TP
\fB-m, --nolibrarybutton\fR
Disable the library home button in the ZIM viewer toolbar.
.TP
\fB-n, --nosearchbar\fR
Disable the search box in the ZIM viewer toolbar.
.TP
\fB-b, --blockexternal\fR
Prevent users from directly navigating to external resources via links in ZIM content.
.TP
\fB-t N, --threads=N\fR
Number of threads to run in parallel (default: 4).
.TP
\fB-s N, --searchLimit=N\fR
Maximum number of ZIM files in a fulltext multizim search (default: No limit).
.TP
\fB-z, --nodatealiases\fR
Create URL aliases for each content by removing the date embedded in the file name.
The expected format of the date in the filename is \*(lq_YYYY-MM\*(rq. For example, a ZIM file named \*(lqwikipedia_en_all_2020-08.zim\*(rq will be accessible both as \*(lqwikipedia_en_all_2020-08\*(rq and \*(lqwikipedia_en_all\*(rq.
.TP
\fB-c PATH, --customIndex=PATH\fR
Override the welcome page with a custom HTML file.
.TP
\fB-L N, --ipConnectionLimit=N\fR
Max number of (concurrent) connections per IP (default: infinite, recommended: >= 6).
.TP
\fB-k, --skipInvalid\fR
Startup even when ZIM files are invalid (those will be skipped)
.TP
\fB-v, --verbose\fR
Print debug log to STDOUT.
.TP
\fB-V, --version\fR
Print the software version.
.TP
\fB-h, --help\fR
Print a help message.
.SH EXAMPLES
Serve a single ZIM file:
.sp
.nf
.B kiwix-serve myzim.zim
.fi
Serve multiple ZIM files:
.sp
.nf
.B kiwix-serve zim1.zim zim2.zim zim3.zim
.fi
Serve ZIM files from a library:
.sp
.nf
.B kiwix-serve --library library.xml
.fi
.SH DOCUMENTATION
Online documentation: https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html
.br
Library is an XML file created using \fBkiwix-manage\fB.
Source code: https://github.com/kiwix/kiwix-tools
.br
More info: https://wiki.kiwix.org/wiki/Kiwix-serve
.SH SEE ALSO
kiwix(1) kiwix\-manage(1)
.SH AUTHOR
.SH AUTHORS
Emmanuel Engelhart <kelson@kiwix.org>
.br
Vasudev Kamath <kamathvasudev@gmail.com> (Manual)
Vasudev Kamath <kamathvasudev@gmail.com>

View File

@@ -17,7 +17,7 @@
* MA 02110-1301, USA.
*/
#include <getopt.h>
#include <docopt/docopt.h>
#include <kiwix/manager.h>
#include <kiwix/tools.h>
#include <cstdlib>
@@ -29,10 +29,10 @@ using namespace std;
enum supportedAction { NONE, ADD, SHOW, REMOVE };
void show(kiwix::Library* library, const std::string& bookId)
void show(const kiwix::Library& library, const std::string& bookId)
{
try {
auto& book = library->getBookById(bookId);
auto& book = library.getBookById(bookId);
std::cout << "id:\t\t" << book.getId() << std::endl
<< "path:\t\t" << book.getPath() << std::endl
<< "url:\t\t" << book.getUrl() << std::endl
@@ -51,159 +51,106 @@ void show(kiwix::Library* library, const std::string& bookId)
std::cout << std::endl;
}
// Older version of docopt doesn't declare Options. Let's declare it ourself.
using Options = std::map<std::string, docopt::value>;
/* Print correct console usage options */
void usage()
static const char USAGE[] =
R"(Manipulates the Kiwix library XML file
Usage:
kiwix-manage LIBRARYPATH add [--zimPathToSave=<custom_zim_path>] [--url=<http_zim_url>] ZIMPATH ...
kiwix-manage LIBRARYPATH (delete|remove) ZIMID ...
kiwix-manage LIBRARYPATH show [ZIMID ...]
kiwix-manage -v | --version
kiwix-manage -h | --help
Arguments:
LIBRARYPATH The XML library file path.
ZIMID ZIM file unique ID.
ZIMPATH A path to a ZIM to add.
Options:
Custom options for "add" action:
--zimPathToSave=<custom_zim_path> Replace the current ZIM file path
--url=<http_zim_url> Create an "url" attribute for the online version of the ZIM file
Other options:
-h --help Print this help
-v --version Print the software version
Examples:
Add ZIM files to library: kiwix-manage my_library.xml add first.zim second.zim
Remove ZIM files from library: kiwix-manage my_library.xml remove e5c2c003-b49e-2756-5176-5d9c86393dd9
Show all library ZIM files: kiwix-manage my_library.xml show
Documentation:
Source code https://github.com/kiwix/kiwix-tools
More info https://wiki.kiwix.org/wiki/kiwix-manage
)";
int handle_show(const kiwix::Library& library, const std::string& libraryPath,
const Options& options)
{
std::cout << "Usage:" << std::endl
<< "\tkiwix-manage LIBRARY_PATH add ZIM_PATH [OPTIONS]" << std::endl
<< "\tkiwix-manage LIBRARY_PATH remove ZIM_ID [ZIM_ID]..." << std::endl
<< "\tkiwix-manage LIBRARY_PATH show [ZIM_ID]..." << std::endl
<< std::endl
<< "Purpose:" << std::endl
<< "\tManipulates the Kiwix library XML file"
<< std::endl << std::endl
<< "Arguments:" << std::endl
<< "\tLIBRARY_PATH\tis the XML library file path."
<< std::endl << std::endl
<< "\tACTION\t\tis the pre-defined string to specify the action to run on the XML library file."
<< std::endl << std::endl
<< "\t\t\tMust be one of the following values:" << std::endl
<< "\t\t\t* add: add a ZIM file to the library" << std::endl
<< "\t\t\t* remove: remove a ZIM file from the library" << std::endl
<< "\t\t\t* show: show the content of the library"
<< std::endl << std::endl
<< "\tZIM_ID\t\tZIM file unique ID"
<< std::endl << std::endl
<< "\tOPTIONS\t\tCustom options for \"add\" action:" << std::endl
<< "\t\t\t--zimPathToSave=CUSTOM_ZIM_PATH to replace the current ZIM file path" << std::endl
<< "\t\t\t--url=HTTP_ZIM_URL to create an \"url\" attribute for the online version of the ZIM file" << std::endl
<< std::endl
<< "\t\t\tOther options:" << std::endl
<< "\t\t\t-v, --version to print the software version" << std::endl
<< std::endl
<< "Examples:" << std::endl
<< "\tAdd ZIM files to library: kiwix-manage my_library.xml add first.zim second.zim" << std::endl
<< "\tRemove ZIM files from library: kiwix-manage my_library.xml remove e5c2c003-b49e-2756-5176-5d9c86393dd9" << std::endl
<< "\tShow all library ZIM files: kiwix-manage my_library.xml show" << std::endl
<< std::endl
<< "Documentation:" << std::endl
<< "\tSource code\thttps://github.com/kiwix/kiwix-tools" << std::endl
<< "\tMore info\thttps://wiki.kiwix.org/wiki/Kiwix-manage" << std::endl
<< std::endl;
}
int handle_show(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
{
if (argc > 3 ) {
for(auto i=3; i<argc; i++) {
std::string bookId = argv[i];
show(library, bookId);
}
} else {
auto booksIds = library->getBooksIds();
if (options.at("ZIMID").asStringList().empty()) {
auto booksIds = library.getBooksIds();
for(auto& bookId: booksIds) {
show(library, bookId);
}
} else {
auto bookIds = options.at("ZIMID").asStringList();
for(auto& bookId: bookIds) {
show(library, bookId);
}
}
return(0);
}
int handle_add(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
int handle_add(kiwix::LibraryPtr library, const std::string& libraryPath,
const Options& options)
{
string zimPath;
string zimPathToSave = ".";
string zimPathToSave;
string url;
int option_index = 0;
int c = 0;
int resultCode = 0;
if (argc <= 3) {
std::cerr << "Path to zim file to add is missing in the command line" << std::endl;
return (-1);
}
/* Options parsing */
optind = 3;
static struct option long_options[] = {
{"url", required_argument, 0, 'u'},
{"zimPathToSave", required_argument, 0, 'z'},
{0, 0, 0, 0}
};
bool has_option = false;
while (true) {
c = getopt_long(argc, argv, "cz:u:", long_options, &option_index);
if (c == -1)
break;
has_option = true;
switch (c) {
case 'u':
url = optarg;
break;
case 'z':
zimPathToSave = optarg;
break;
}
}
if (optind >= argc) {
std::cerr << "Path to zim file to add is missing in the command line" << std::endl;
return (-1);
}
if (has_option && argc-optind > 1) {
std::cerr << "You cannot give option and several zim files to add" << std::endl;
return (-1);
}
kiwix::Manager manager(library);
for(auto i=optind; i<argc; i++) {
std::string zimPath = argv[i];
if (!zimPath.empty()) {
auto _zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
if (manager.addBookFromPathAndGetId(zimPath, _zimPathToSave, url, false).empty()) {
std::cerr << "Cannot add zim " << zimPath << " to the library." << std::endl;
resultCode = 1;
}
auto zimPaths = options.at("ZIMPATH").asStringList();
for (auto& zimPath: zimPaths) {
if (options.at("--zimPathToSave").isString()) {
zimPathToSave = options.at("--zimPathToSave").asString();
} else {
std::cerr << "Invalid zim file path" << std::endl;
resultCode = 1;
zimPathToSave = zimPath;
}
if (options.at("--url").isString()) {
url = options.at("--url").asString();
}
if (manager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false).empty()) {
std::cerr << "Cannot add ZIM " << zimPath << " to the library." << std::endl;
return 1;
}
}
return(resultCode);
return 0;
}
int handle_remove(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
int handle_remove(kiwix::Library& library, const std::string& libraryPath,
const Options& options)
{
std::string bookId;
const unsigned int totalBookCount = library->getBookCount(true, true);
const unsigned int totalBookCount = library.getBookCount(true, true);
int exitCode = 0;
if (argc <= 3) {
std::cerr << "BookId to remove missing in the command line" << std::endl;
return (-1);
}
if (!totalBookCount) {
std::cerr << "Library is empty, no book to delete."
<< std::endl;
return 1;
}
for (int i = 3; i<argc; i++) {
bookId = argv[i];
if (!library->removeBookById(bookId)) {
auto bookIds = options.at("ZIMID").asStringList();
for (auto& bookId: bookIds) {
if (!library.removeBookById(bookId)) {
std::cerr << "Invalid book id '" << bookId << "'." << std::endl;
exitCode = 1;
}
@@ -214,54 +161,42 @@ int handle_remove(kiwix::Library* library, const std::string& libraryPath,
int main(int argc, char** argv)
{
string libraryPath = "";
supportedAction action = NONE;
kiwix::Library library;
auto library = kiwix::Library::create();
/* General argument parsing */
static struct option long_options[] = {
{"version", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
int option_index = 0;
int c;
while (true && argc == 2) {
c = getopt_long(argc, argv, "v", long_options, &option_index);
if (c == -1)
break;
switch (c) {
case 'v':
version();
return 0;
}
}
/* Action related argument parsing */
if (argc > 2) {
libraryPath = argv[1];
string actionString = argv[2];
if (actionString == "add")
action = ADD;
else if (actionString == "show")
action = SHOW;
else if (actionString == "remove" || actionString == "delete")
action = REMOVE;
}
/* Print usage)) if necessary */
if (libraryPath == "" || action == NONE) {
usage();
Options args;
try {
args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
} catch (docopt::DocoptArgumentError const & error ) {
std::cerr << error.what() << std::endl;
std::cerr << USAGE << std::endl;
return -1;
}
if (args["--help"].asBool()) {
std::cout << USAGE << std::endl;
return 0;
}
if (args["--version"].asBool()) {
version();
return 0;
}
std::string libraryPath = args.at("LIBRARYPATH").asString();
if (args.at("add").asBool())
action = ADD;
else if (args.at("show").asBool())
action = SHOW;
else if (args.at("remove").asBool() || args.at("delete").asBool())
action = REMOVE;
/* Try to read the file */
libraryPath = kiwix::isRelativePath(libraryPath)
? kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryPath)
: libraryPath;
kiwix::Manager manager(&library);
kiwix::Manager manager(library);
if (!manager.readFile(libraryPath, false)) {
if (kiwix::fileExists(libraryPath) || action!=ADD) {
std::cerr << "Cannot read the library " << libraryPath << std::endl;
@@ -273,13 +208,13 @@ int main(int argc, char** argv)
int exitCode = 0;
switch (action) {
case SHOW:
exitCode = handle_show(&library, libraryPath, argc, argv);
exitCode = handle_show(*library, libraryPath, args);
break;
case ADD:
exitCode = handle_add(&library, libraryPath, argc, argv);
exitCode = handle_add(library, libraryPath, args);
break;
case REMOVE:
exitCode = handle_remove(&library, libraryPath, argc, argv);
exitCode = handle_remove(*library, libraryPath, args);
break;
case NONE:
break;
@@ -292,7 +227,7 @@ int main(int argc, char** argv)
/* Rewrite the library file */
if (action == REMOVE || action == ADD) {
// writeToFile return true (1) if everything is ok => exitCode is 0
if (!library.writeToFile(libraryPath)) {
if (!library->writeToFile(libraryPath)) {
std::cerr << "Cannot write the library " << libraryPath << std::endl;
return 1;
}

View File

@@ -17,99 +17,97 @@
* MA 02110-1301, USA.
*/
#include <getopt.h>
#include <docopt/docopt.h>
#include <zim/search.h>
#include <zim/suggestion.h>
#include <kiwix/spelling_correction.h>
#include <xapian.h>
#include <iostream>
#include <filesystem>
#include "../version.h"
using namespace std;
void usage()
// Older version of docopt doesn't declare Options. Let's declare it ourself.
using Options = std::map<std::string, docopt::value>;
static const char USAGE[] =
R"(Find articles based on a fulltext search pattern.
Usage:
kiwix-search [options] ZIM PATTERN
kiwix-search -h | --help
kiwix-search -V | --version
Arguments:
ZIM The full path of the ZIM file
PATTERN Word(s) - or part of - to search in the ZIM.
Options:
-s --suggestion Suggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution
--spelling Suggest article titles based on the spelling corrected PATTERN instead of making a fulltext search.
-v --verbose Give details about the search process
-V --version Print software version
-h --help Print this help
)";
std::filesystem::path getKiwixCachedDataDirPath()
{
cout << "Usage: kiwix-search [OPTIONS] ZIM PATTERN" << endl << endl
<< " kiwix-search allows one to find articles based on the a fulltext search pattern." << endl << endl
<< " ZIM is the full path of the ZIM file." << endl
<< " PATTERN is/are word(s) - or part of - to search in the ZIM." << endl << endl
<< " -s, --suggestion\tSuggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution." << endl
<< " -v, --verbose\t\tGive details about the search process" << endl
<< " -V, --version\t\tPrint software version" << endl;
exit(1);
std::filesystem::path home(getenv("HOME"));
std::filesystem::path cacheDirPath = home / ".cache" / "kiwix";
std::filesystem::create_directories(cacheDirPath);
return cacheDirPath;
}
int main(int argc, char** argv)
{
/* Init the variables */
// const char *indexPath =
// "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
// const char *indexPath = "/home/itamar/testindex";
const char* zimPath = NULL;
const char* search = NULL;
bool verboseFlag = false;
bool suggestionFlag = false;
int option_index = 0;
int c = 0;
/* Argument parsing */
while (42) {
static struct option long_options[]
= {{"verbose", no_argument, 0, 'v'},
{"suggestion", no_argument, 0, 's'},
{"version", no_argument, 0, 'V'},
{0, 0, 0, 0}};
if (c != -1) {
c = getopt_long(argc, argv, "Vvsb:", long_options, &option_index);
switch (c) {
case 'v':
verboseFlag = true;
break;
case 'V':
version();
return 0;
case 's':
suggestionFlag = true;
break;
}
} else {
if (optind < argc) {
if (zimPath == NULL) {
zimPath = argv[optind++];
} else if (search == NULL) {
search = argv[optind++];
} else {
cout << zimPath << endl;
usage();
}
} else {
break;
}
}
Options args;
try {
args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
} catch (docopt::DocoptArgumentError const & error ) {
std::cerr << error.what() << std::endl;
std::cerr << USAGE << std::endl;
return -1;
}
/* Check if we have enough arguments */
if (zimPath == NULL || search == NULL) {
usage();
if (args.at("--help").asBool()) {
std::cout << USAGE << std::endl;
return 0;
}
if (args.at("--version").asBool()) {
version();
return 0;
}
auto zimPath = args.at("ZIM").asString();
auto pattern = args.at("PATTERN").asString();
auto verboseFlag = args.at("--verbose").asBool();
/* Try to prepare the indexing */
try {
zim::Archive archive(zimPath);
if (suggestionFlag) {
if (args.at("--suggestion").asBool()) {
zim::SuggestionSearcher searcher(archive);
searcher.setVerbose(verboseFlag);
for (const auto& r : searcher.suggest(search).getResults(0, 10) ) {
for (const auto& r:searcher.suggest(pattern).getResults(0, 10)) {
cout << r.getTitle() << endl;
}
} else if (args.at("--spelling").asBool()) {
kiwix::SpellingsDB spellingsDB(archive, getKiwixCachedDataDirPath());
for (const auto& r:spellingsDB.getSpellingCorrections(pattern, 1)) {
cout << r << endl;
}
} else {
zim::Searcher searcher(archive);
searcher.setVerbose(verboseFlag);
const zim::Query query(search);
const zim::Query query(pattern);
for (const auto& r : searcher.search(query).getResults(0, 10) ) {
cout << r.getTitle() << endl;
}
@@ -117,6 +115,9 @@ int main(int argc, char** argv)
} catch ( const std::runtime_error& err) {
cerr << err.what() << endl;
exit(1);
} catch ( const Xapian::Error& err) {
cerr << err.get_msg() << endl;
exit(1);
}
exit(0);

View File

@@ -18,7 +18,7 @@
* MA 02110-1301, USA.
*/
#include <getopt.h>
#include <docopt/docopt.h>
#include <kiwix/manager.h>
#include <kiwix/server.h>
#include <kiwix/name_mapper.h>
@@ -41,49 +41,50 @@
#include "../version.h"
#define DEFAULT_THREADS 4
#define LITERAL_AS_STR(A) #A
#define AS_STR(A) LITERAL_AS_STR(A)
void usage()
{
std::cout << "Usage:" << std::endl
<< "\tkiwix-serve [OPTIONS] ZIM_PATH+" << std::endl
<< "\tkiwix-serve --library [OPTIONS] LIBRARY_PATH" << std::endl
<< std::endl
<< "Purpose:" << std::endl
<< "\tDeliver ZIM file(s) articles via HTTP"
<< std::endl << std::endl
static const char USAGE[] =
R"(Deliver ZIM file(s) articles via HTTP
<< "Mandatory arguments:" << std::endl
<< "\tLIBRARY_PATH\t\tXML library file path listing ZIM file to serve. To be used only with the --library argument."
<< std::endl
<< "\tZIM_PATH\t\tZIM file path(s)"
<< std::endl << std::endl
Usage:
kiwix-serve [options] ZIMPATH ...
kiwix-serve [options] (-l | --library) LIBRARYPATH
kiwix-serve -h | --help
kiwix-serve -V | --version
<< "Optional arguments:" << std::endl << std::endl
<< "\t-h, --help\t\tPrint this help" << std::endl << std::endl
<< "\t-a, --attachToProcess\tWxit if given process id is not running anymore" << std::endl
<< "\t-d, --daemon\t\tDetach the HTTP server daemon from the main process" << std::endl
<< "\t-i, --address\t\tListen only on this ip address, all available ones otherwise" << std::endl
<< "\t-M, --monitorLibrary\tMonitor the XML library file and reload it automatically" << std::endl
<< "\t-m, --nolibrarybutton\tDon't print the builtin home button in the builtin top bar overlay" << std::endl
<< "\t-n, --nosearchbar\tDon't print the builtin bar overlay on the top of each served page" << std::endl
<< "\t-b, --blockexternal\tPrevent users from directly accessing external links" << std::endl
<< "\t-p, --port\t\tTCP port on which to listen to HTTP requests (default: 80)" << std::endl
<< "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
<< "\t-s, --searchLimit\tMaximun number of zim in a fulltext multizim search (default: No limit)" << std::endl
<< "\t-t, --threads\t\tNumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
<< "\t-v, --verbose\t\tPrint debug log to STDOUT" << std::endl
<< "\t-V, --version\t\tPrint software version" << std::endl
<< "\t-z, --nodatealiases\tCreate URL aliases for each content by removing the date" << std::endl
<< "\t-c, --customIndex\tAdd path to custom index.html for welcome page" << std::endl
<< "\t-L, --ipConnectionLimit\tMax number of (concurrent) connections per IP (default: infinite, recommended: >= 6)" << std::endl
<< std::endl
Mandatory arguments:
LIBRARYPATH XML library file path listing ZIM file to serve. To be used only with the --library argument."
ZIMPATH ZIM file path(s)
<< "Documentation:" << std::endl
<< "\tSource code\t\thttps://github.com/kiwix/kiwix-tools" << std::endl
<< "\tMore info\t\thttps://wiki.kiwix.org/wiki/Kiwix-serve" << std::endl
<< std::endl;
}
Options:
-h --help Print this help
-a <pid> --attachToProcess=<pid> Exit if given process id is not running anymore [default: 0]
--catalogOnly Serve only the library catalog
--contentServerURL=<url> Root URL of the server serving ZIM content for this library
-d --daemon Detach the HTTP server daemon from the main process
-i <address> --address=<address> Listen only on the specified IP address. Specify 'ipv4', 'ipv6' or 'all' to listen on all IPv4, IPv6 or both types of addresses, respectively [default: all]
-M --monitorLibrary Monitor the XML library file and reload it automatically
-m --nolibrarybutton Don't print the builtin home button in the builtin top bar overlay
-n --nosearchbar Don't print the builtin bar overlay on the top of each served page
-b --blockexternal Prevent users from directly accessing external links
-p <port> --port=<port> Port on which to listen to HTTP requests [default: 80]
-r <root> --urlRootLocation=<root> URL prefix on which the content should be made available [default: /]
-s <limit> --searchLimit=<limit> Maximun number of zim in a fulltext multizim search [default: 0]
-t <threads> --threads=<threads> Number of threads to run in parallel [default: )" AS_STR(DEFAULT_THREADS) R"(]
-v --verbose Print debug log to STDOUT
-V --version Print software version
-z --nodatealiases Create URL aliases for each content by removing the date
-c <path> --customIndex=<path> Add path to custom index.html for welcome page
-L <limit> --ipConnectionLimit=<limit> Max number of (concurrent) connections per IP [default: 0] (recommended: >= 6)
-k --skipInvalid Startup even when ZIM files are invalid (those will be skipped)
Documentation:
Source code https://github.com/kiwix/kiwix-tools
More info https://wiki.kiwix.org/wiki/Kiwix-serve
https://kiwix-tools.readthedocs.io/en/latest/kiwix-serve.html
)";
std::string loadCustomTemplate (std::string customIndexPath) {
customIndexPath = kiwix::isRelativePath(customIndexPath) ?
@@ -103,20 +104,9 @@ std::string loadCustomTemplate (std::string customIndexPath) {
return indexTemplateString;
}
inline std::string normalizeRootUrl(std::string rootUrl)
{
while ( !rootUrl.empty() && rootUrl.back() == '/' )
rootUrl.pop_back();
while ( !rootUrl.empty() && rootUrl.front() == '/' )
rootUrl = rootUrl.substr(1);
return rootUrl.empty() ? rootUrl : "/" + rootUrl;
}
#ifndef _WIN32
volatile sig_atomic_t waiting = false;
volatile sig_atomic_t libraryMustBeReloaded = false;
#ifndef _WIN32
void handle_sigterm(int signum)
{
if ( waiting == false ) {
@@ -146,6 +136,9 @@ void setup_sighandlers()
set_signal_handler(SIGINT, &handle_sigterm);
set_signal_handler(SIGHUP, &handle_sighup);
}
#else
bool waiting = false;
bool libraryMustBeReloaded = false;
#endif
uint64_t fileModificationTime(const std::string& path)
@@ -190,6 +183,28 @@ bool reloadLibrary(kiwix::Manager& mgr, const std::vector<std::string>& paths)
}
}
// docopt::value::isLong() is counting repeated values.
// It doesn't check if the string can be parsed as long.
// (Contrarly to `asLong` which will try to convert string to long)
// See https://github.com/docopt/docopt.cpp/issues/62
// `isLong` is a small helper to get if the value can be parsed as long.
inline bool isLong(const docopt::value& v) {
try {
v.asLong();
return true;
} catch (...) {
return false;
}
}
#define FLAG(NAME, VAR) if (arg.first == NAME) { VAR = arg.second.asBool(); continue; }
#define STRING(NAME, VAR) if (arg.first == NAME && arg.second.isString() ) { VAR = arg.second.asString(); continue; }
#define STRING_LIST(NAME, VAR, ERRORSTR) if (arg.first == NAME) { if (arg.second.isStringList()) { VAR = arg.second.asStringList(); continue; } else { errorString = ERRORSTR; break; } }
#define INT(NAME, VAR, ERRORSTR) if (arg.first == NAME ) { if (isLong(arg.second)) { VAR = arg.second.asLong(); continue; } else { errorString = ERRORSTR; break; } }
// Older version of docopt doesn't declare Options. Let's declare it ourself.
using Options = std::map<std::string, docopt::value>;
int main(int argc, char** argv)
{
#ifndef _WIN32
@@ -197,7 +212,7 @@ int main(int argc, char** argv)
#endif
std::string rootLocation = "/";
kiwix::Library library;
auto library = kiwix::Library::create();
unsigned int nb_threads = DEFAULT_THREADS;
std::vector<std::string> zimPathes;
std::string libraryPath;
@@ -206,141 +221,85 @@ int main(int argc, char** argv)
std::string customIndexPath="";
std::string indexTemplateString="";
int serverPort = 80;
int daemonFlag [[gnu::unused]] = false;
int libraryFlag = false;
bool catalogOnlyFlag = false;
std::string contentServerURL;
bool daemonFlag [[gnu::unused]] = false;
bool helpFlag = false;
bool noLibraryButtonFlag = false;
bool noSearchBarFlag = false;
bool noDateAliasesFlag = false;
bool blockExternalLinks = false;
bool isVerboseFlag = false;
bool monitorLibrary = false;
bool versionFlag = false;
unsigned int PPID = 0;
int ipConnectionLimit = 0;
int searchLimit = 0;
bool skipInvalid = false;
static struct option long_options[]
= {{"daemon", no_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{"verbose", no_argument, 0, 'v'},
{"version", no_argument, 0, 'V'},
{"library", no_argument, 0, 'l'},
{"nolibrarybutton", no_argument, 0, 'm'},
{"nodatealiases", no_argument, 0, 'z'},
{"nosearchbar", no_argument, 0, 'n'},
{"blockexternallinks", no_argument, 0, 'b'},
{"attachToProcess", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'},
{"address", required_argument, 0, 'i'},
{"threads", required_argument, 0, 't'},
{"urlRootLocation", required_argument, 0, 'r'},
{"customIndex", required_argument, 0, 'c'},
{"monitorLibrary", no_argument, 0, 'M'},
{"ipConnectionLimit", required_argument, 0, 'L'},
{"searchLimit", required_argument, 0, 's'},
{0, 0, 0, 0}};
std::string errorString;
std::set<int> usedOptions;
/* Argument parsing */
while (true) {
int option_index = 0;
int c
= getopt_long(argc, argv, "hzmnbdvVla:p:f:t:r:i:c:ML:s:", long_options, &option_index);
if (c != -1) {
auto insertRes = usedOptions.insert(c);
if (!insertRes.second) {
std::cerr << "Multiple values of same option are not allowed." << std::endl;
exit(1);
}
switch (c) {
case 'h':
usage();
return 0;
case 'd':
daemonFlag = true;
break;
case 'v':
isVerboseFlag = true;
break;
case 'V':
version();
return 0;
case 'l':
libraryFlag = true;
break;
case 'n':
noSearchBarFlag = true;
break;
case 'b':
blockExternalLinks = true;
break;
case 'z':
noDateAliasesFlag = true;
break;
case 'm':
noLibraryButtonFlag = true;
break;
case 'p':
serverPort = atoi(optarg);
break;
case 'a':
PPID = atoi(optarg);
break;
case 'i':
address = std::string(optarg);
break;
case 't':
nb_threads = atoi(optarg);
break;
case 'r':
rootLocation = std::string(optarg);
break;
case 'c':
customIndexPath = std::string(optarg);
break;
case 'M':
monitorLibrary = true;
break;
case 'L':
ipConnectionLimit = atoi(optarg);
break;
case 's':
searchLimit = atoi(optarg);
break;
case '?':
usage();
return 2;
}
} else {
if (optind < argc) {
if (libraryFlag) {
libraryPath = argv[optind++];
} else {
while (optind < argc)
zimPathes.push_back(std::string(argv[optind++]));
}
}
break;
}
Options args;
try {
args = docopt::docopt_parse(USAGE, {argv+1, argv+argc}, false, false);
} catch (docopt::DocoptArgumentError const & error) {
std::cerr << error.what() << std::endl;
std::cerr << USAGE << std::endl;
return -1;
}
/* Print usage)) if necessary */
if (zimPathes.empty() && libraryPath.empty()) {
usage();
exit(1);
}
for (auto const& arg: args) {
FLAG("--help", helpFlag)
FLAG("--catalogOnly", catalogOnlyFlag)
STRING("--contentServerURL", contentServerURL)
FLAG("--daemon", daemonFlag)
FLAG("--verbose", isVerboseFlag)
FLAG("--nosearchbar", noSearchBarFlag)
FLAG("--blockexternal", blockExternalLinks)
FLAG("--nodatealiases", noDateAliasesFlag)
FLAG("--nolibrarybutton",noLibraryButtonFlag)
FLAG("--monitorLibrary", monitorLibrary)
FLAG("--skipInvalid", skipInvalid)
FLAG("--version", versionFlag)
STRING("LIBRARYPATH", libraryPath)
INT("--port", serverPort, "Port must be an integer")
INT("--attachToProcess", PPID, "Process to attach must be an integer")
STRING("--address", address)
INT("--threads", nb_threads, "Number of threads must be an integer")
STRING("--urlRootLocation", rootLocation)
STRING("--customIndex", customIndexPath)
INT("--ipConnectionLimit", ipConnectionLimit, "IP connection limit must be an integer")
INT("--searchLimit", searchLimit, "Search limit must be an integer")
STRING_LIST("ZIMPATH", zimPathes, "ZIMPATH must be a string list")
}
if (!errorString.empty()) {
std::cerr << errorString << std::endl;
std::cerr << USAGE << std::endl;
return -1;
}
if (helpFlag) {
std::cout << USAGE << std::endl;
return 0;
}
if (versionFlag) {
version();
return 0;
}
/* Setup the library manager and get the list of books */
kiwix::Manager manager(&library);
kiwix::Manager manager(library);
std::vector<std::string> libraryPaths;
if (libraryFlag) {
if (!libraryPath.empty()) {
libraryPaths = kiwix::split(libraryPath, ";");
if ( !reloadLibrary(manager, libraryPaths) ) {
exit(1);
}
/* Check if the library is not empty (or only remote books)*/
if (library.getBookCount(true, false) == 0) {
if (library->getBookCount(true, false) == 0) {
std::cerr << "The XML library file '" << libraryPath
<< "' is empty (or has only remote books)." << std::endl;
}
@@ -348,15 +307,32 @@ int main(int argc, char** argv)
std::vector<std::string>::iterator it;
for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
if (!manager.addBookFromPath(*it, *it, "", false)) {
std::cerr << "Unable to add the ZIM file '" << *it
<< "' to the internal library." << std::endl;
exit(1);
if (skipInvalid) {
std::cerr << "Skipping invalid '" << *it << "' ...continuing" << std::endl;
} else {
std::cerr << "Unable to add the ZIM file '" << *it
<< "' to the internal library." << std::endl;
exit(1);
}
}
}
}
auto libraryFileTimestamp = newestFileTimestamp(libraryPaths);
auto curLibraryFileTimestamp = libraryFileTimestamp;
kiwix::IpMode ipMode = kiwix::IpMode::AUTO;
if (address == "all") {
address.clear();
ipMode = kiwix::IpMode::ALL;
} else if (address == "ipv4") {
address.clear();
ipMode = kiwix::IpMode::IPV4;
} else if (address == "ipv6") {
address.clear();
ipMode = kiwix::IpMode::IPV6;
}
#ifndef _WIN32
/* Fork if necessary */
if (daemonFlag) {
@@ -376,8 +352,8 @@ int main(int argc, char** argv)
}
#endif
kiwix::UpdatableNameMapper nameMapper(library, noDateAliasesFlag);
kiwix::Server server(&library, &nameMapper);
auto nameMapper = std::make_shared<kiwix::UpdatableNameMapper>(library, noDateAliasesFlag);
kiwix::Server server(library, nameMapper);
if (!customIndexPath.empty()) {
try {
@@ -398,14 +374,20 @@ int main(int argc, char** argv)
server.setIndexTemplateString(indexTemplateString);
server.setIpConnectionLimit(ipConnectionLimit);
server.setMultiZimSearchLimit(searchLimit);
server.setIpMode(ipMode);
server.setCatalogOnlyMode(catalogOnlyFlag);
while ( !contentServerURL.empty() && contentServerURL.back() == '/' )
contentServerURL.pop_back();
server.setContentServerUrl(contentServerURL);
if (! server.start()) {
exit(1);
}
std::string url = "http://" + server.getAddress() + ":" + std::to_string(server.getPort()) + normalizeRootUrl(rootLocation);
std::cout << "The Kiwix server is running and can be accessed in the local network at: "
<< url << std::endl;
std::cout << "The Kiwix server is running and can be accessed in the local network at: " << std::endl;
for (const auto& url : server.getServerAccessUrls()) {
std::cout << " - " << url << std::endl;
}
/* Run endless (until PPID dies) */
waiting = true;
@@ -441,13 +423,15 @@ int main(int argc, char** argv)
if ( monitorLibrary ) {
curLibraryFileTimestamp = newestFileTimestamp(libraryPaths);
libraryMustBeReloaded += curLibraryFileTimestamp > libraryFileTimestamp;
if ( !libraryMustBeReloaded ) {
libraryMustBeReloaded = curLibraryFileTimestamp > libraryFileTimestamp;
}
}
if ( libraryMustBeReloaded && !libraryPaths.empty() ) {
libraryFileTimestamp = curLibraryFileTimestamp;
reloadLibrary(manager, libraryPaths);
nameMapper.update();
nameMapper->update();
libraryMustBeReloaded = false;
}
} while (waiting);