mirror of
https://github.com/kiwix/libkiwix.git
synced 2025-12-24 06:57:59 -05:00
Compare commits
1 Commits
can-revert
...
httplib
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2e42ac65c |
27
.github/move.yml
vendored
Normal file
27
.github/move.yml
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
# Configuration for Move Issues - https://github.com/dessant/move-issues
|
||||
|
||||
# Delete the command comment when it contains no other content
|
||||
deleteCommand: true
|
||||
|
||||
# Close the source issue after moving
|
||||
closeSourceIssue: true
|
||||
|
||||
# Lock the source issue after moving
|
||||
lockSourceIssue: false
|
||||
|
||||
# Mention issue and comment authors
|
||||
mentionAuthors: true
|
||||
|
||||
# Preserve mentions in the issue content
|
||||
keepContentMentions: true
|
||||
|
||||
# Move labels that also exist on the target repository
|
||||
moveLabels: true
|
||||
|
||||
# Set custom aliases for targets
|
||||
# aliases:
|
||||
# r: repo
|
||||
# or: owner/repo
|
||||
|
||||
# Repository to extend settings from
|
||||
# _extends: repo
|
||||
178
.github/workflows/ci.yml
vendored
178
.github/workflows/ci.yml
vendored
@@ -1,41 +1,163 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- can-revert
|
||||
pull_request:
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
macOS:
|
||||
runs-on: macos-12
|
||||
env:
|
||||
HOME: /Users/runner
|
||||
Macos:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- name: Retrieve source code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v1
|
||||
- name: Setup python 3.5
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.5'
|
||||
- name: Install packages
|
||||
run: |
|
||||
brew update
|
||||
brew install pkg-config ninja meson
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
ARCHIVE_NAME: deps2_osx_native_dyn_libkiwix.tar.xz
|
||||
brew install gcovr pkg-config ninja
|
||||
- name: Install python modules
|
||||
run: pip3 install meson==0.49.2 pytest
|
||||
- name: Install deps
|
||||
shell: bash
|
||||
run: |
|
||||
wget -O- https://tmp.kiwix.org/ci/${{env.ARCHIVE_NAME}} | tar -xJ -C ${{env.HOME}}
|
||||
|
||||
- name: Compile source code
|
||||
env:
|
||||
PKG_CONFIG_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib/pkgconfig
|
||||
CPPFLAGS: -I${{env.HOME}}/BUILD_native_dyn/INSTALL/include
|
||||
ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
|
||||
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
|
||||
- name: Compile
|
||||
shell: bash
|
||||
run: |
|
||||
export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
|
||||
export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
|
||||
meson . build --default-library=shared -Db_coverage=true
|
||||
ninja -C build
|
||||
|
||||
- name: Test libkiwix
|
||||
cd build
|
||||
ninja
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
|
||||
cd build
|
||||
meson test --verbose
|
||||
ninja coverage
|
||||
env:
|
||||
SKIP_BIG_MEMORY_TEST: 1
|
||||
LD_LIBRARY_PATH: ${{env.HOME}}/BUILD_native_dyn/INSTALL/lib:${{env.HOME}}/BUILD_native_dyn/INSTALL/lib64
|
||||
run: meson test -C build --verbose
|
||||
- name: Publish coverage
|
||||
shell: bash
|
||||
run: |
|
||||
curl https://codecov.io/bash -o codecov.sh
|
||||
bash codecov.sh -n osx_native_dyn -Z
|
||||
rm codecov.sh
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
Linux:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
name:
|
||||
- native_static
|
||||
- native_dyn
|
||||
- native_dyn_bionic
|
||||
- android_arm
|
||||
- android_arm64
|
||||
- win32_static
|
||||
- win32_dyn
|
||||
include:
|
||||
- name: native_static
|
||||
target: native_static
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: native_dyn
|
||||
target: native_dyn
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: native_dyn_bionic
|
||||
target: native_dyn
|
||||
image_variant: bionic
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: android_arm
|
||||
target: android_arm
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: android_arm64
|
||||
target: android_arm64
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: win32_static
|
||||
target: win32_static
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
- name: win32_dyn
|
||||
target: win32_dyn
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
env:
|
||||
HOME: /home/runner
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
|
||||
steps:
|
||||
- name: Extract branch name
|
||||
shell: bash
|
||||
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
|
||||
id: extract_branch
|
||||
- name: Checkout code
|
||||
shell: python
|
||||
run: |
|
||||
from subprocess import check_call
|
||||
from os import environ
|
||||
command = [
|
||||
'git', 'clone',
|
||||
'https://github.com/${{github.repository}}',
|
||||
'--depth=1',
|
||||
'--branch', '${{steps.extract_branch.outputs.branch}}'
|
||||
]
|
||||
check_call(command, cwd=environ['HOME'])
|
||||
- name: Install deps
|
||||
shell: bash
|
||||
run: |
|
||||
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
|
||||
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
|
||||
- name: Compile
|
||||
shell: bash
|
||||
run: |
|
||||
meson --version
|
||||
if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
|
||||
MESON_OPTION="--default-library=shared"
|
||||
else
|
||||
MESON_OPTION="--default-library=static"
|
||||
fi
|
||||
if [[ "${{matrix.target}}" =~ native_.* ]]; then
|
||||
MESON_OPTION="$MESON_OPTION -Db_coverage=true"
|
||||
else
|
||||
MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
|
||||
fi
|
||||
if [[ "${{matrix.target}}" =~ android_.* ]]; then
|
||||
MESON_OPTION="$MESON_OPTION -Dandroid=true"
|
||||
fi
|
||||
cd $HOME/libkiwix
|
||||
meson . build ${MESON_OPTION}
|
||||
cd build
|
||||
ninja
|
||||
env:
|
||||
PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
|
||||
CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
|
||||
- name: Test
|
||||
if: startsWith(matrix.target, 'native_')
|
||||
shell: bash
|
||||
run: |
|
||||
cd $HOME/libkiwix/build
|
||||
meson test --verbose
|
||||
ninja coverage
|
||||
env:
|
||||
LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
|
||||
SKIP_BIG_MEMORY_TEST: 1
|
||||
- name: Publish coverage
|
||||
shell: bash
|
||||
run: |
|
||||
cd $HOME/libkiwix
|
||||
curl https://codecov.io/bash -o codecov.sh
|
||||
bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
|
||||
rm codecov.sh
|
||||
if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
37
.github/workflows/package.yml
vendored
37
.github/workflows/package.yml
vendored
@@ -1,10 +1,5 @@
|
||||
name: Packages
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build-deb:
|
||||
@@ -12,13 +7,9 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
distro:
|
||||
- ubuntu-kinetic
|
||||
- ubuntu-jammy
|
||||
- ubuntu-focal
|
||||
- ubuntu-bionic
|
||||
distro: [ubuntu-hirsute, ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# Determine which PPA we should upload to
|
||||
- name: PPA
|
||||
@@ -39,18 +30,18 @@ jobs:
|
||||
email: release+launchpad@kiwix.org
|
||||
distro: ${{ matrix.distro }}
|
||||
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-kinetic
|
||||
if: matrix.distro == 'ubuntu-kinetic'
|
||||
name: Build package for ubuntu-kinetic
|
||||
id: build-ubuntu-kinetic
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-hirsute
|
||||
if: matrix.distro == 'ubuntu-hirsute'
|
||||
name: Build package for ubuntu-hirsute
|
||||
id: build-ubuntu-hirsute
|
||||
with:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-jammy
|
||||
if: matrix.distro == 'ubuntu-jammy'
|
||||
name: Build package for ubuntu-jammy
|
||||
id: build-ubuntu-jammy
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-groovy
|
||||
if: matrix.distro == 'ubuntu-groovy'
|
||||
name: Build package for ubuntu-groovy
|
||||
id: build-ubuntu-groovy
|
||||
with:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
@@ -71,15 +62,15 @@ jobs:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: Packages for ${{ matrix.distro }}
|
||||
path: output
|
||||
|
||||
- uses: legoktm/gh-action-dput@master
|
||||
name: Upload dev package
|
||||
# Only upload on pushes to git default branch
|
||||
if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
|
||||
# Only upload on pushes to master
|
||||
if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
|
||||
with:
|
||||
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
|
||||
repository: ppa:kiwixteam/dev
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -5,5 +5,3 @@ subprojects/googletest-release*
|
||||
build/
|
||||
.vscode/
|
||||
builddir/
|
||||
.cache/
|
||||
.clangd/
|
||||
|
||||
162
ChangeLog
162
ChangeLog
@@ -1,167 +1,7 @@
|
||||
libkiwix 12.0.0
|
||||
===============
|
||||
|
||||
* [API Break] Remove wrapper around libzim (@mgautierfr #789)
|
||||
* Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
|
||||
* Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
|
||||
* Prevent search on multi language content (@veloman-yunkan #838)
|
||||
* Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
|
||||
* Catalog:
|
||||
- Include tags in free text catalog search (@veloman-yunkan #802)
|
||||
- Illustration's url is based on book's uuid (@veloman-yunkan #804)
|
||||
- Cleanup of the opds-dumper (@veloman-yunkan #829)
|
||||
- Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
|
||||
- Make opds-dumper respect the namemapper (@mgautierfr #837)
|
||||
* Server:
|
||||
- Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
|
||||
- Better http caching (@veloman-yunkan #833)
|
||||
- Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
|
||||
- Better redirection of main page (@veloman-yunkan #827)
|
||||
- Remove jquery (@mgautierfr @juuz0 #796)
|
||||
- Better Viewer of zim content :
|
||||
. Introduce `/content` endpoints (@veloman-yunkan #806)
|
||||
. Switch to iframe based content viewer (@veloman-yunkan #716)
|
||||
- Optimised design of the welcome page:
|
||||
. Alignement (@juuz0 @kelson42 #786)
|
||||
. Exit download modal on pressing escape key (@juzz0 #800)
|
||||
. Add favicon for different devices (@juzz0 #805)
|
||||
. Fix auto hidding of the toolbar (@veloman-yunkan #821)
|
||||
. Allow user to filter books by tags in the front page (@juuz0 #711)
|
||||
* CI :
|
||||
- Trigger CI on pull_request (@kelson42 #791)
|
||||
- Drop Ubuntu Impish packaging (@legoktm #825)
|
||||
- Add Ubuntu Kinetic packaging (@legoktm #801)
|
||||
* Testing:
|
||||
- Test ICULanguageInfo (@veloman-yunkan #795)
|
||||
- Introduce fake `test` language to test i18n (@veloman-yunkan #848)
|
||||
* Fix documentation (@kelson42 #816)
|
||||
* Udpate translation (#787 #839 #847)
|
||||
|
||||
|
||||
libkiwix 11.0.0
|
||||
===============
|
||||
|
||||
* [server] Add support for internationalization (@veloman-yunkan #679)
|
||||
* [server] Use gzip compression instead of deflat (mgautierfr #757)
|
||||
* [server] Version the static resources. This allow better invalidating
|
||||
browser cache when resources are changed (@veloman-yunkan #712)
|
||||
* [server|front] Use integer to query the host for page length (@juuz0 #772)
|
||||
* [server] Improve multizim search API:
|
||||
- Improvement of the cache system
|
||||
- Better API to select on which books to search in.
|
||||
- SysAdmin is now able to limit the number of book we search in for a multizim search
|
||||
* [server] Introduce a opensearch API for multizim fulltext search
|
||||
* [wrapper] Remove java wrapper
|
||||
* Testing:
|
||||
- Testing of search result pages content (@veloman-yunkan #765)
|
||||
- Better testing structure of xml search result (@veloman-yunkan #780)
|
||||
|
||||
libkiwix 10.1.1
|
||||
===============
|
||||
|
||||
* Correctly detect the number of article for older zims (<=6) (@mgautier #743)
|
||||
* [server] Fix fulltext search (@mgautierfr #724)
|
||||
* [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
|
||||
* Fix CI (@mgautierfr #736)
|
||||
|
||||
libkiwix 10.1.0
|
||||
===============
|
||||
|
||||
This release is an important one as it fixes a Xss vulnerability introduced
|
||||
in libkiwix 10.0.0
|
||||
|
||||
* [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
|
||||
* [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
|
||||
* [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
|
||||
* [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
|
||||
* Add `dc:issued` to opds output stream (@veloman-yunkan #715)
|
||||
* Add handling of several languages not provided by ICU (@juuz0 #701)
|
||||
* [server] Add a caching system for search and suggestion (@maneeshpm #620)
|
||||
* Fix cross-compilation (@kelson42 #703)
|
||||
* Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
|
||||
* Better testing system of html response (@veloman-yunkan #725)
|
||||
|
||||
libkiwix 10.0.1
|
||||
===============
|
||||
|
||||
* [server] The catalog search interpret `count=0` as no limit.
|
||||
This was the case for a long time. This was changed unintentionally
|
||||
(@veloman-yunkan #686)
|
||||
* [server] Correctly generere a human friendly title in the server frontend.
|
||||
(@juuz0 #687, @kelson42 #689)
|
||||
* [server] Fix download button if there is no url do download from.
|
||||
(@juuz0 #691)
|
||||
* Add non-minified isotope.pkdg.js
|
||||
Needed for debian packaging as we need the source and minified version is
|
||||
not the source (@legoktm #693)
|
||||
* [server] Add a tooltip with the full language for the lang tag.
|
||||
* CI fixes (@kelson42 @legoktm)
|
||||
|
||||
libkiwix 10.0.0
|
||||
===============
|
||||
|
||||
This release is huge release.
|
||||
The project has been renamed to libkiwix, it is more coherent with the library name.
|
||||
|
||||
* Server front page :
|
||||
- Use js in the front page to display the available book,
|
||||
using the OPDS stream as source. The front page is now populated only with
|
||||
the visible books and user can search for books. (@MananJethwany #530, #541, #534)
|
||||
(@kelson42 #628)
|
||||
- Revamp css (@MananJethwany #559)
|
||||
- Correctly Convert 3iso language code to 2iso (@juuz0 #672)
|
||||
|
||||
* Server suggestions search :
|
||||
- Add pagination for suggestion search (@maneeshpm #591)
|
||||
- Fix suggestion system (@MananJethwany #498)
|
||||
- Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
|
||||
- The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
|
||||
- Properly handle html encoding of suggestions (@veloman-yunkan #458)
|
||||
|
||||
* Server improvements :
|
||||
- Remove meta endpoints (@mgautier #669)
|
||||
- Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
|
||||
- Add details on 404 error pages (@soumyankar #490)
|
||||
- Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
|
||||
- Better headbar insertion (after charset definition) (@kelson42 #442)
|
||||
|
||||
* New OPDS Stream v2 :
|
||||
- Add a list of categories (@veloman-yunkan)
|
||||
- Support for partial entries (@veloman-yunkan #602)
|
||||
- Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
|
||||
- Add language endpoint to catalog (@veloman-yunkan #553)
|
||||
- Add illustration API to get the illustration of a book (@mgautierfr #645)
|
||||
- OPDS search can now filter books by category (@veloman-yunkan #459)
|
||||
|
||||
* Library improvements :
|
||||
- Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
|
||||
- Properly handle removing of book from the library (@veloman-yunkan #485)
|
||||
- Use xapian to search for books in the library (@veloman-yunkan #460, #488)
|
||||
|
||||
* Added methods/functions :
|
||||
- Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
|
||||
- Add `getVersions` and `printVersions` functions (@kelson42 #665)
|
||||
- Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
|
||||
- Add `get_zimid()` method to the search result (@maneeshpm #510)
|
||||
|
||||
* Various improvements :
|
||||
- Better secret value for aria2c rpc (@juuz0 #666)
|
||||
- Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
|
||||
- Add basic documentation (@mgautierfr #640)
|
||||
- Do not use Reader internally (@maneeshpm #536 #576)
|
||||
- Remove dependency headers from our public headers (@mgautierfr #574)
|
||||
- Downloader now don't write metalink on the filesystem (@kelson42 #502)
|
||||
- Support opening a zim file using a fd (@veloman-yukan #429)
|
||||
- Use C++11 std::thread instead of pthread (@mgautierfr #445)
|
||||
- [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
|
||||
- Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
|
||||
- Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
|
||||
- Better parsing of the counterMap (@kelson42 #437)
|
||||
- Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
|
||||
- Remove deprecated methods (@mgautierfr)
|
||||
- CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
|
||||
- Fix compilation for FreeBSD (@swills g#432)
|
||||
- Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
|
||||
* ...
|
||||
|
||||
kiwix-lib 9.4.1
|
||||
===============
|
||||
|
||||
102
README.md
102
README.md
@@ -5,12 +5,10 @@ The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
|
||||
core. It contains the code shared by all Kiwix ports (Windows,
|
||||
GNU/Linux, macOS, Android, iOS, ...).
|
||||
|
||||
[](https://download.kiwix.org/release/libkiwix/)
|
||||
[](https://github.com/kiwix/libkiwix/wiki/Repology)
|
||||
[](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
|
||||
[](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
|
||||
[](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
|
||||
[](https://www.codefactor.io/repository/github/kiwix/libkiwix)
|
||||
[](https://codecov.io/gh/kiwix/libkiwix)
|
||||
[](https://codecov.io/gh/kiwix/libkiwix)
|
||||
[](https://www.gnu.org/licenses/gpl-3.0)
|
||||
|
||||
Disclaimer
|
||||
@@ -93,41 +91,6 @@ Meson. If you want statically linked libraries, you can add
|
||||
|
||||
Depending of you system, `ninja` may be called `ninja-build`.
|
||||
|
||||
The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
|
||||
with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
|
||||
|
||||
```bash
|
||||
meson . build -Dwrapper=android -Dwerror=false
|
||||
ninja -C build
|
||||
```
|
||||
|
||||
Static files compilation
|
||||
------------------------
|
||||
|
||||
Libkiwix has a few static files 'compiled' within the binary
|
||||
code. This is mostly Javascript/HTML/pictures necessary for the HTTP
|
||||
daemon.
|
||||
|
||||
These static files are available in the `static` directory and are
|
||||
compiled by custom Python code available in this repository `scripts`
|
||||
directory. This happens automatically at compilation time without any
|
||||
additional command to run.
|
||||
|
||||
To avoid HTTP caching issues, the URLs (to the static content) are
|
||||
appended with a `cacheid` parameter (this is called "cache
|
||||
busting"). This `cacheid` value derived from the
|
||||
[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
|
||||
static file. As a consequence, each time you change a static file, the
|
||||
corresponding `cacheid` value will change.
|
||||
|
||||
To properly test this feature, this `cacheid` needs to be added
|
||||
manually to the automated tests and has to be commited. After
|
||||
modifying the needed static file, [run the automated
|
||||
tests](#Testing). They will fail, but the inspection of the testing
|
||||
log will give you the new `cacheid` value(s). Finally update
|
||||
`test/server.cpp` with the appropriate `cacheid` value(s) which have
|
||||
changed.
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
@@ -151,7 +114,7 @@ where you want to install the libraries. After the installation
|
||||
succeeded, you may need to run `ldconfig` (as `root`).
|
||||
|
||||
Uninstallation
|
||||
--------------
|
||||
------------
|
||||
|
||||
If you want to uninstall the Kiwix library:
|
||||
```bash
|
||||
@@ -161,55 +124,6 @@ ninja -C build uninstall
|
||||
Like for the installation, you might need to run the command as `root`
|
||||
(or using `sudo`).
|
||||
|
||||
Custom Index Page
|
||||
-----------------
|
||||
|
||||
to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
|
||||
(note - while using custom html file please mention all external links as absolute path.)
|
||||
|
||||
to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
|
||||
|
||||
To use JS provided by kiwix-serve you can use the following template to start with ->
|
||||
|
||||
```
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||||
<title><-- Custom Tittle --></title>
|
||||
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
|
||||
<script src="{{root}}/skin/iso6391To3.js"></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
|
||||
- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
|
||||
- To add language select box add - `<select id="languageFilter"></select>` under body tag.
|
||||
- To add category select box add - `<select id="categoryFilter"></select>` under body tag.
|
||||
- To add search box for books use following form -
|
||||
```
|
||||
<form id='kiwixSearchForm'>
|
||||
<input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
|
||||
<input type="submit" class="kiwixButton" value="Search"/>
|
||||
</form>
|
||||
```
|
||||
|
||||
|
||||
If you compile manually Libmicrohttpd, you might need to compile it
|
||||
without GNU TLS, a bug here will empeach further compilation
|
||||
otherwise.
|
||||
|
||||
If the compilation still fails, you might need to get a more recent
|
||||
version of a dependency than the one packaged by your Linux
|
||||
distribution. Try then with a source tarball distributed by the
|
||||
problematic upstream project or even directly from the source code
|
||||
repository.
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
@@ -232,6 +146,16 @@ cp ninja ../bin
|
||||
cd ..
|
||||
```
|
||||
|
||||
If you compile manually Libmicrohttpd, you might need to compile it
|
||||
without GNU TLS, a bug here will empeach further compilation
|
||||
otherwise.
|
||||
|
||||
If the compilation still fails, you might need to get a more recent
|
||||
version of a dependency than the one packaged by your Linux
|
||||
distribution. Try then with a source tarball distributed by the
|
||||
problematic upstream project or even directly from the source code
|
||||
repository.
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
|
||||
13
android-kiwix-lib-publisher/.gitignore
vendored
Normal file
13
android-kiwix-lib-publisher/.gitignore
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
*.iml
|
||||
.gradle
|
||||
/local.properties
|
||||
/.idea/caches
|
||||
/.idea/libraries
|
||||
/.idea/modules.xml
|
||||
/.idea/workspace.xml
|
||||
/.idea/navEditor.xml
|
||||
/.idea/assetWizardSettings.xml
|
||||
.DS_Store
|
||||
/build
|
||||
/captures
|
||||
.externalNativeBuild
|
||||
25
android-kiwix-lib-publisher/build.gradle
Normal file
25
android-kiwix-lib-publisher/build.gradle
Normal file
@@ -0,0 +1,25 @@
|
||||
// Top-level build file where you can add configuration options common to all sub-projects/modules.
|
||||
|
||||
buildscript {
|
||||
repositories {
|
||||
google()
|
||||
jcenter()
|
||||
|
||||
}
|
||||
dependencies {
|
||||
classpath 'com.android.tools.build:gradle:3.4.1'
|
||||
// NOTE: Do not place your application dependencies here; they belong
|
||||
// in the individual module build.gradle files
|
||||
}
|
||||
}
|
||||
|
||||
allprojects {
|
||||
repositories {
|
||||
google()
|
||||
jcenter()
|
||||
}
|
||||
}
|
||||
|
||||
task clean(type: Delete) {
|
||||
delete rootProject.buildDir
|
||||
}
|
||||
15
android-kiwix-lib-publisher/gradle.properties
Normal file
15
android-kiwix-lib-publisher/gradle.properties
Normal file
@@ -0,0 +1,15 @@
|
||||
# Project-wide Gradle settings.
|
||||
# IDE (e.g. Android Studio) users:
|
||||
# Gradle settings configured through the IDE *will override*
|
||||
# any settings specified in this file.
|
||||
# For more details on how to configure your build environment visit
|
||||
# http://www.gradle.org/docs/current/userguide/build_environment.html
|
||||
# Specifies the JVM arguments used for the daemon process.
|
||||
# The setting is particularly useful for tweaking memory settings.
|
||||
org.gradle.jvmargs=-Xmx1536m
|
||||
# When configured, Gradle will run in incubating parallel mode.
|
||||
# This option should only be used with decoupled projects. More details, visit
|
||||
# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
|
||||
# org.gradle.parallel=true
|
||||
# Kotlin code style for this project: "official" or "obsolete":
|
||||
kotlin.code.style=official
|
||||
BIN
android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
6
android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
6
android-kiwix-lib-publisher/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
#Wed Jun 19 15:28:39 BST 2019
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip
|
||||
172
android-kiwix-lib-publisher/gradlew
vendored
Executable file
172
android-kiwix-lib-publisher/gradlew
vendored
Executable file
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
##############################################################################
|
||||
##
|
||||
## Gradle start up script for UN*X
|
||||
##
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
# Resolve links: $0 may be a link
|
||||
PRG="$0"
|
||||
# Need this for relative symlinks.
|
||||
while [ -h "$PRG" ] ; do
|
||||
ls=`ls -ld "$PRG"`
|
||||
link=`expr "$ls" : '.*-> \(.*\)$'`
|
||||
if expr "$link" : '/.*' > /dev/null; then
|
||||
PRG="$link"
|
||||
else
|
||||
PRG=`dirname "$PRG"`"/$link"
|
||||
fi
|
||||
done
|
||||
SAVED="`pwd`"
|
||||
cd "`dirname \"$PRG\"`/" >/dev/null
|
||||
APP_HOME="`pwd -P`"
|
||||
cd "$SAVED" >/dev/null
|
||||
|
||||
APP_NAME="Gradle"
|
||||
APP_BASE_NAME=`basename "$0"`
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS=""
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD="maximum"
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "`uname`" in
|
||||
CYGWIN* )
|
||||
cygwin=true
|
||||
;;
|
||||
Darwin* )
|
||||
darwin=true
|
||||
;;
|
||||
MINGW* )
|
||||
msys=true
|
||||
;;
|
||||
NONSTOP* )
|
||||
nonstop=true
|
||||
;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD="java"
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
||||
MAX_FD_LIMIT=`ulimit -H -n`
|
||||
if [ $? -eq 0 ] ; then
|
||||
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
||||
MAX_FD="$MAX_FD_LIMIT"
|
||||
fi
|
||||
ulimit -n $MAX_FD
|
||||
if [ $? -ne 0 ] ; then
|
||||
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
||||
fi
|
||||
else
|
||||
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# For Darwin, add options to specify how the application appears in the dock
|
||||
if $darwin; then
|
||||
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
||||
fi
|
||||
|
||||
# For Cygwin, switch paths to Windows format before running java
|
||||
if $cygwin ; then
|
||||
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
||||
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
||||
JAVACMD=`cygpath --unix "$JAVACMD"`
|
||||
|
||||
# We build the pattern for arguments to be converted via cygpath
|
||||
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
||||
SEP=""
|
||||
for dir in $ROOTDIRSRAW ; do
|
||||
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
||||
SEP="|"
|
||||
done
|
||||
OURCYGPATTERN="(^($ROOTDIRS))"
|
||||
# Add a user-defined pattern to the cygpath arguments
|
||||
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
||||
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
||||
fi
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
i=0
|
||||
for arg in "$@" ; do
|
||||
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
||||
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
||||
|
||||
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
||||
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
||||
else
|
||||
eval `echo args$i`="\"$arg\""
|
||||
fi
|
||||
i=$((i+1))
|
||||
done
|
||||
case $i in
|
||||
(0) set -- ;;
|
||||
(1) set -- "$args0" ;;
|
||||
(2) set -- "$args0" "$args1" ;;
|
||||
(3) set -- "$args0" "$args1" "$args2" ;;
|
||||
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
||||
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
||||
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
||||
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
||||
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
||||
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Escape application args
|
||||
save () {
|
||||
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
||||
echo " "
|
||||
}
|
||||
APP_ARGS=$(save "$@")
|
||||
|
||||
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
||||
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
||||
|
||||
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
||||
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
||||
cd "$(dirname "$0")"
|
||||
fi
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
84
android-kiwix-lib-publisher/gradlew.bat
vendored
Executable file
84
android-kiwix-lib-publisher/gradlew.bat
vendored
Executable file
@@ -0,0 +1,84 @@
|
||||
@if "%DEBUG%" == "" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%" == "" set DIRNAME=.
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS=
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if "%ERRORLEVEL%" == "0" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto init
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:init
|
||||
@rem Get command-line arguments, handling Windows variants
|
||||
|
||||
if not "%OS%" == "Windows_NT" goto win9xME_args
|
||||
|
||||
:win9xME_args
|
||||
@rem Slurp the command line arguments.
|
||||
set CMD_LINE_ARGS=
|
||||
set _SKIP=2
|
||||
|
||||
:win9xME_args_slurp
|
||||
if "x%~1" == "x" goto execute
|
||||
|
||||
set CMD_LINE_ARGS=%*
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if "%ERRORLEVEL%"=="0" goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
||||
exit /b 1
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
1
android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore
vendored
Normal file
1
android-kiwix-lib-publisher/kiwixLibAndroid/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
/build
|
||||
64
android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
Normal file
64
android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
Normal file
@@ -0,0 +1,64 @@
|
||||
apply plugin: 'com.android.library'
|
||||
apply plugin: 'maven'
|
||||
|
||||
android {
|
||||
compileSdkVersion 28
|
||||
defaultConfig {
|
||||
minSdkVersion 15
|
||||
targetSdkVersion 28
|
||||
versionCode 1
|
||||
versionName "1.0"
|
||||
}
|
||||
buildTypes {
|
||||
release {
|
||||
minifyEnabled false
|
||||
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
|
||||
}
|
||||
|
||||
task writePom {
|
||||
pom {
|
||||
project {
|
||||
groupId 'org.kiwix.kiwixlib'
|
||||
artifactId 'kiwixlib'
|
||||
version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
|
||||
packaging 'aar'
|
||||
name 'kiwixlib'
|
||||
url 'https://github.com/kiwix/libkiwix'
|
||||
licenses {
|
||||
license {
|
||||
name 'GPLv3'
|
||||
url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
|
||||
}
|
||||
}
|
||||
developers {
|
||||
developer {
|
||||
id 'kiwix'
|
||||
name 'kiwix'
|
||||
email 'contact@kiwix.org'
|
||||
}
|
||||
}
|
||||
scm {
|
||||
connection 'https://github.com/kiwix/libkiwix.git'
|
||||
developerConnection 'https://github.com/kiwix/libkiwix.git'
|
||||
url 'https://github.com/kiwix/libkiwix'
|
||||
}
|
||||
}
|
||||
}.withXml {
|
||||
def dependenciesNode = asNode().appendNode('dependencies')
|
||||
|
||||
//Iterate over the implementation dependencies, adding a <dependency> node for each
|
||||
configurations.implementation.allDependencies.each {
|
||||
def dependencyNode = dependenciesNode.appendNode('dependency')
|
||||
dependencyNode.appendNode('groupId', it.group)
|
||||
dependencyNode.appendNode('artifactId', it.name)
|
||||
dependencyNode.appendNode('version', it.version)
|
||||
}
|
||||
}.writeTo("$buildDir/pom.xml")
|
||||
}
|
||||
|
||||
21
android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro
vendored
Normal file
21
android-kiwix-lib-publisher/kiwixLibAndroid/proguard-rules.pro
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
# Add project specific ProGuard rules here.
|
||||
# You can control the set of applied configuration files using the
|
||||
# proguardFiles setting in build.gradle.
|
||||
#
|
||||
# For more details, see
|
||||
# http://developer.android.com/guide/developing/tools/proguard.html
|
||||
|
||||
# If your project uses WebView with JS, uncomment the following
|
||||
# and specify the fully qualified class name to the JavaScript interface
|
||||
# class:
|
||||
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
|
||||
# public *;
|
||||
#}
|
||||
|
||||
# Uncomment this to preserve the line number information for
|
||||
# debugging stack traces.
|
||||
#-keepattributes SourceFile,LineNumberTable
|
||||
|
||||
# If you keep the line number information, uncomment this to
|
||||
# hide the original source file name.
|
||||
#-renamesourcefileattribute SourceFile
|
||||
@@ -0,0 +1,10 @@
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="org.kiwix.kiwixlib">
|
||||
|
||||
<application
|
||||
android:allowBackup="true"
|
||||
android:supportsRtl="true">
|
||||
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
1
android-kiwix-lib-publisher/settings.gradle
Normal file
1
android-kiwix-lib-publisher/settings.gradle
Normal file
@@ -0,0 +1 @@
|
||||
include ':kiwixLibAndroid'
|
||||
4
debian/control
vendored
4
debian/control
vendored
@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
|
||||
Build-Depends: debhelper-compat (= 13),
|
||||
meson,
|
||||
pkg-config,
|
||||
libzim-dev (>= 7.2.0~),
|
||||
libzim-dev (>= 6.1.8),
|
||||
libcurl4-gnutls-dev,
|
||||
libicu-dev,
|
||||
libgtest-dev,
|
||||
@@ -23,7 +23,7 @@ Section: libdevel
|
||||
Architecture: any
|
||||
Multi-Arch: same
|
||||
Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
|
||||
libzim-dev (>= 7.2.0~),
|
||||
libzim-dev (>= 6.0.0),
|
||||
libicu-dev,
|
||||
libpugixml-dev,
|
||||
libcurl4-gnutls-dev,
|
||||
|
||||
1
debian/libkiwix-dev.manpages
vendored
1
debian/libkiwix-dev.manpages
vendored
@@ -1,2 +1 @@
|
||||
usr/share/man/man1/kiwix-compile-resources.1*
|
||||
usr/share/man/man1/kiwix-compile-i18n.1*
|
||||
|
||||
2
docs/.gitignore
vendored
2
docs/.gitignore
vendored
@@ -1,2 +0,0 @@
|
||||
api
|
||||
xml
|
||||
72
docs/conf.py
72
docs/conf.py
@@ -1,72 +0,0 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# This file only contains a selection of the most common options. For a full
|
||||
# list see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Path setup --------------------------------------------------------------
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
# import sys
|
||||
# sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
||||
project = 'libkiwix'
|
||||
copyright = '2022, libkiwix-team'
|
||||
author = 'libkiwix-team'
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
|
||||
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
'breathe',
|
||||
'exhale'
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
|
||||
if not on_rtd:
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
breathe_projects = {
|
||||
"libkiwix": "./xml"
|
||||
}
|
||||
breathe_default_project = 'libkiwix'
|
||||
|
||||
exhale_args = {
|
||||
"containmentFolder": "./api",
|
||||
"rootFileName": "ref_api.rst",
|
||||
"rootFileTitle": "Reference API",
|
||||
"doxygenStripFromPath":"..",
|
||||
"treeViewIsBootstrap": True,
|
||||
"createTreeView" : True,
|
||||
"exhaleExecutesDoxygen": True,
|
||||
"exhaleDoxygenStdin": "INPUT = ../include"
|
||||
}
|
||||
|
||||
primary_domain = 'cpp'
|
||||
|
||||
highlight_language = 'cpp'
|
||||
@@ -1,14 +0,0 @@
|
||||
.. libkiwix documentation master file, created by
|
||||
sphinx-quickstart on Fri Jul 24 15:40:50 2020.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to libkiwix's documentation!
|
||||
==================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents:
|
||||
|
||||
usage
|
||||
api/ref_api
|
||||
@@ -1,7 +0,0 @@
|
||||
|
||||
sphinx = find_program('sphinx-build', native:true)
|
||||
|
||||
sphinx_target = run_target('doc',
|
||||
command: [sphinx, '-bhtml',
|
||||
meson.current_source_dir(),
|
||||
meson.current_build_dir()])
|
||||
@@ -1,2 +0,0 @@
|
||||
breathe
|
||||
exhale
|
||||
@@ -1,15 +0,0 @@
|
||||
Libkiwix programming
|
||||
====================
|
||||
|
||||
Introduction
|
||||
------------
|
||||
|
||||
libkiwix is written in C++. To use the library, you need the include files of libkiwix have
|
||||
to link against libzim.
|
||||
|
||||
Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
|
||||
which is always derived from std::exception.
|
||||
|
||||
All classes are defined in the namespace kiwix.
|
||||
|
||||
libkiwix is a set of tools to manage zim files and provide some common functionnality.
|
||||
36
format_code.sh
Executable file
36
format_code.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
files=(
|
||||
"include/library.h"
|
||||
"include/common/stringTools.h"
|
||||
"include/common/pathTools.h"
|
||||
"include/common/otherTools.h"
|
||||
"include/common/regexTools.h"
|
||||
"include/common/networkTools.h"
|
||||
"include/manager.h"
|
||||
"include/reader.h"
|
||||
"include/kiwix.h"
|
||||
"include/xapianSearcher.h"
|
||||
"include/searcher.h"
|
||||
"src/library.cpp"
|
||||
"src/android/kiwix.cpp"
|
||||
"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
|
||||
"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
|
||||
"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
|
||||
"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
|
||||
"src/searcher.cpp"
|
||||
"src/common/pathTools.cpp"
|
||||
"src/common/regexTools.cpp"
|
||||
"src/common/otherTools.cpp"
|
||||
"src/common/networkTools.cpp"
|
||||
"src/common/stringTools.cpp"
|
||||
"src/xapianSearcher.cpp"
|
||||
"src/manager.cpp"
|
||||
"src/reader.cpp"
|
||||
)
|
||||
|
||||
for i in "${files[@]}"
|
||||
do
|
||||
echo $i
|
||||
clang-format -i -style=file $i
|
||||
done
|
||||
@@ -21,54 +21,28 @@
|
||||
#define KIWIX_BOOK_H
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include "common.h"
|
||||
|
||||
namespace pugi {
|
||||
class xml_node;
|
||||
}
|
||||
|
||||
namespace zim {
|
||||
class Archive;
|
||||
}
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
class OPDSDumper;
|
||||
class Reader;
|
||||
|
||||
/**
|
||||
* A class to store information about a book (a zim file)
|
||||
*/
|
||||
class Book
|
||||
{
|
||||
public: // types
|
||||
class Illustration
|
||||
{
|
||||
friend class Book;
|
||||
public:
|
||||
uint16_t width = 48;
|
||||
uint16_t height = 48;
|
||||
std::string mimeType;
|
||||
std::string url;
|
||||
|
||||
const std::string& getData() const;
|
||||
|
||||
private:
|
||||
mutable std::string data;
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
|
||||
|
||||
public: // functions
|
||||
public:
|
||||
Book();
|
||||
~Book();
|
||||
|
||||
bool update(const Book& other);
|
||||
void update(const zim::Archive& archive);
|
||||
void update(const Reader& reader);
|
||||
void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
|
||||
void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
|
||||
std::string getHumanReadableIdFromPath() const;
|
||||
@@ -79,9 +53,7 @@ class Book
|
||||
bool isPathValid() const { return m_pathValid; }
|
||||
const std::string& getTitle() const { return m_title; }
|
||||
const std::string& getDescription() const { return m_description; }
|
||||
DEPRECATED const std::string& getLanguage() const { return m_language; }
|
||||
const std::string& getCommaSeparatedLanguages() const { return m_language; }
|
||||
const std::vector<std::string> getLanguages() const;
|
||||
const std::string& getLanguage() const { return m_language; }
|
||||
const std::string& getCreator() const { return m_creator; }
|
||||
const std::string& getPublisher() const { return m_publisher; }
|
||||
const std::string& getDate() const { return m_date; }
|
||||
@@ -96,13 +68,9 @@ class Book
|
||||
const uint64_t& getArticleCount() const { return m_articleCount; }
|
||||
const uint64_t& getMediaCount() const { return m_mediaCount; }
|
||||
const uint64_t& getSize() const { return m_size; }
|
||||
DEPRECATED const std::string& getFavicon() const;
|
||||
DEPRECATED const std::string& getFaviconUrl() const;
|
||||
DEPRECATED const std::string& getFaviconMimeType() const;
|
||||
|
||||
Illustrations getIllustrations() const;
|
||||
std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
|
||||
|
||||
const std::string& getFavicon() const;
|
||||
const std::string& getFaviconUrl() const { return m_faviconUrl; }
|
||||
const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
|
||||
const std::string& getDownloadId() const { return m_downloadId; }
|
||||
|
||||
void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
|
||||
@@ -123,13 +91,14 @@ class Book
|
||||
void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
|
||||
void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
|
||||
void setSize(uint64_t size) { m_size = size; }
|
||||
void setFavicon(const std::string& favicon) { m_favicon = favicon; }
|
||||
void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
|
||||
void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
|
||||
|
||||
private: // functions
|
||||
private:
|
||||
std::string getCategoryFromTags() const;
|
||||
const Illustration& getDefaultIllustration() const;
|
||||
|
||||
protected: // data
|
||||
protected:
|
||||
std::string m_id;
|
||||
std::string m_downloadId;
|
||||
std::string m_path;
|
||||
@@ -150,11 +119,9 @@ class Book
|
||||
uint64_t m_mediaCount = 0;
|
||||
bool m_readOnly = false;
|
||||
uint64_t m_size = 0;
|
||||
Illustrations m_illustrations;
|
||||
|
||||
// Used as the return value of getDefaultIllustration() when no default
|
||||
// illustration is found in the book
|
||||
static const Illustration missingDefaultIllustration;
|
||||
mutable std::string m_favicon;
|
||||
std::string m_faviconUrl;
|
||||
std::string m_faviconMimeType;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <mutex>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
@@ -44,14 +43,6 @@ class AriaError : public std::runtime_error {
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* A representation of a current download.
|
||||
*
|
||||
* `Download` is not thread safe. User must care to not call method on a
|
||||
* same download from different threads.
|
||||
* However, it is safe to use different `Download`s from different threads.
|
||||
*/
|
||||
|
||||
class Download {
|
||||
public:
|
||||
typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
|
||||
@@ -62,89 +53,19 @@ class Download {
|
||||
: mp_aria(p_aria),
|
||||
m_status(K_UNKNOWN),
|
||||
m_did(did) {};
|
||||
|
||||
/**
|
||||
* Update the status of the download.
|
||||
*
|
||||
* This call make an aria rpc call and is blocking.
|
||||
* Some download (started with a metalink) are in fact several downloads.
|
||||
* - A first one to download the metadlink.
|
||||
* - A second one to download the real file.
|
||||
*
|
||||
* If `follow` is true, updateStatus tries to detect that and tracks
|
||||
* the second download when the first one is finished.
|
||||
* By passing false to `follow`, `Download` will only track the first download.
|
||||
*
|
||||
* `getFoo` methods are based on the last statusUpdate.
|
||||
*
|
||||
* @param follow: Do we have to follow following downloads.
|
||||
*/
|
||||
void updateStatus(bool follow);
|
||||
|
||||
/**
|
||||
* Pause the download (and call updateStatus)
|
||||
*/
|
||||
void updateStatus(bool follow=false);
|
||||
void pauseDownload();
|
||||
|
||||
/**
|
||||
* Resume the download (and call updateStatus)
|
||||
*/
|
||||
void resumeDownload();
|
||||
|
||||
/**
|
||||
* Cancel the download.
|
||||
*
|
||||
* A canceled downlod cannot be resume and updateStatus does nothing.
|
||||
* However, you can still get information based on the last known information.
|
||||
*/
|
||||
void cancelDownload();
|
||||
|
||||
/*
|
||||
* Get the status of the download.
|
||||
*/
|
||||
StatusResult getStatus() const { return m_status; }
|
||||
|
||||
/*
|
||||
* Get the id of the download.
|
||||
*/
|
||||
const std::string& getDid() const { return m_did; }
|
||||
|
||||
/*
|
||||
* Get the id of the "second" download.
|
||||
*
|
||||
* Set only if the "first" download is a metalink and is complete.
|
||||
*/
|
||||
const std::string& getFollowedBy() const { return m_followedBy; }
|
||||
|
||||
/*
|
||||
* Get the total length of the download.
|
||||
*/
|
||||
uint64_t getTotalLength() const { return m_totalLength; }
|
||||
|
||||
/*
|
||||
* Get the completed length of the download.
|
||||
*/
|
||||
uint64_t getCompletedLength() const { return m_completedLength; }
|
||||
|
||||
/*
|
||||
* Get the download speed of the download.
|
||||
*/
|
||||
uint64_t getDownloadSpeed() const { return m_downloadSpeed; }
|
||||
|
||||
/*
|
||||
* Get the verified length of the download.
|
||||
*/
|
||||
uint64_t getVerifiedLength() const { return m_verifiedLength; }
|
||||
|
||||
/*
|
||||
* Get the path (local file) of the download.
|
||||
*/
|
||||
const std::string& getPath() const { return m_path; }
|
||||
|
||||
/*
|
||||
* Get the download uris of the download.
|
||||
*/
|
||||
const std::vector<std::string>& getUris() const { return m_uris; }
|
||||
StatusResult getStatus() { return m_status; }
|
||||
std::string getDid() { return m_did; }
|
||||
std::string getFollowedBy() { return m_followedBy; }
|
||||
uint64_t getTotalLength() { return m_totalLength; }
|
||||
uint64_t getCompletedLength() { return m_completedLength; }
|
||||
uint64_t getDownloadSpeed() { return m_downloadSpeed; }
|
||||
uint64_t getVerifiedLength() { return m_verifiedLength; }
|
||||
std::string getPath() { return m_path; }
|
||||
std::vector<std::string>& getUris() { return m_uris; }
|
||||
|
||||
protected:
|
||||
std::shared_ptr<Aria2> mp_aria;
|
||||
@@ -162,9 +83,6 @@ class Download {
|
||||
/**
|
||||
* A tool to download things.
|
||||
*
|
||||
* A Downloader manages `Download` using aria2 in the background.
|
||||
* `Downloader` is threadsafe.
|
||||
* However, the returned `Download`s are NOT threadsafe.
|
||||
*/
|
||||
class Downloader
|
||||
{
|
||||
@@ -174,41 +92,14 @@ class Downloader
|
||||
|
||||
void close();
|
||||
|
||||
/**
|
||||
* Start a new download.
|
||||
*
|
||||
* This method is thread safe and return a pointer to a newly created `Download`.
|
||||
* User should call `update` on the returned `Download` to have an accurate status.
|
||||
*
|
||||
* @param uri: The uri of the thing to download.
|
||||
* @param options: A series of pair <option_name, option_value> to pass to aria.
|
||||
* @return: The newly created Download.
|
||||
*/
|
||||
std::shared_ptr<Download> startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
|
||||
Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
|
||||
Download* getDownload(const std::string& did);
|
||||
|
||||
/**
|
||||
* Get a download corrsponding to a download id (did)
|
||||
* User should call `update` on the returned `Download` to have an accurate status.
|
||||
*
|
||||
* @param did: The download id to search for.
|
||||
* @return: The Download corresponding to did.
|
||||
* @throw: Throw std::out_of_range if did is not found.
|
||||
*/
|
||||
std::shared_ptr<Download> getDownload(const std::string& did);
|
||||
|
||||
/**
|
||||
* Get the number of downloads currently managed.
|
||||
*/
|
||||
size_t getNbDownload() const;
|
||||
|
||||
/**
|
||||
* Get the ids of the managed downloads.
|
||||
*/
|
||||
std::vector<std::string> getDownloadIds() const;
|
||||
size_t getNbDownload() { return m_knownDownloads.size(); }
|
||||
std::vector<std::string> getDownloadIds();
|
||||
|
||||
private:
|
||||
mutable std::mutex m_lock;
|
||||
std::map<std::string, std::shared_ptr<Download>> m_knownDownloads;
|
||||
std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
|
||||
std::shared_ptr<Aria2> mp_aria;
|
||||
};
|
||||
}
|
||||
|
||||
183
include/entry.h
Normal file
183
include/entry.h
Normal file
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_ENTRY_H
|
||||
#define KIWIX_ENTRY_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <zim/entry.h>
|
||||
#include <zim/item.h>
|
||||
#include <exception>
|
||||
#include <string>
|
||||
|
||||
#include "common.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
|
||||
class NoEntry : public std::exception {};
|
||||
|
||||
/**
|
||||
* A entry represent an.. entry in a zim file.
|
||||
*/
|
||||
class Entry
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct an entry making reference to an zim article.
|
||||
*
|
||||
* @param article a zim::Article object
|
||||
*/
|
||||
Entry(zim::Entry entry);
|
||||
virtual ~Entry() = default;
|
||||
|
||||
/**
|
||||
* Get the path of the entry.
|
||||
*
|
||||
* The path is the "key" of an entry.
|
||||
*
|
||||
* @return the path of the entry.
|
||||
*/
|
||||
std::string getPath() const { return entry.getPath(); }
|
||||
|
||||
/**
|
||||
* Get the title of the entry.
|
||||
*
|
||||
* @return the title of the entry.
|
||||
*/
|
||||
std::string getTitle() const { return entry.getTitle(); }
|
||||
|
||||
/**
|
||||
* Get the content of the entry.
|
||||
*
|
||||
* The string is a copy of the content.
|
||||
* If you don't want to do a copy, use get_blob.
|
||||
*
|
||||
* @return the content of the entry.
|
||||
*/
|
||||
std::string getContent() const { return entry.getItem().getData(); }
|
||||
|
||||
/**
|
||||
* Get the blob of the entry.
|
||||
*
|
||||
* A blob make reference to the content without copying it.
|
||||
*
|
||||
* @param offset The starting offset of the blob.
|
||||
* @return the blob of the entry.
|
||||
*/
|
||||
zim::Blob getBlob(offset_type offset = 0) const { return entry.getItem().getData(offset); }
|
||||
|
||||
/**
|
||||
* Get the blob of the entry.
|
||||
*
|
||||
* A blob make reference to the content without copying it.
|
||||
*
|
||||
* @param offset The starting offset of the blob.
|
||||
* @param size The size of the blob.
|
||||
* @return the blob of the entry.
|
||||
*/
|
||||
zim::Blob getBlob(offset_type offset, size_type size) const { return entry.getItem().getData(offset, size); }
|
||||
|
||||
/**
|
||||
* Get the info for direct access to the content of the entry.
|
||||
*
|
||||
* Some entry (ie binary ones) have their content plain stored
|
||||
* in the zim file. Knowing the offset where the content is stored
|
||||
* an user can directly read the content in the zim file bypassing the
|
||||
* libkiwix/libzim.
|
||||
*
|
||||
* @return A pair specifying where to read the content.
|
||||
* The string is the real file to read (may be different that .zim
|
||||
* file if zim is cut).
|
||||
* The offset is the offset to read in the file.
|
||||
* Return <"",0> if is not possible to read directly.
|
||||
*/
|
||||
zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); }
|
||||
|
||||
/**
|
||||
* Get the size of the entry.
|
||||
*
|
||||
* @return the size of the entry.
|
||||
*/
|
||||
size_type getSize() const;
|
||||
|
||||
/**
|
||||
* Get the mime_type of the entry.
|
||||
*
|
||||
* @return the mime_type of the entry.
|
||||
*/
|
||||
std::string getMimetype() const;
|
||||
|
||||
|
||||
/**
|
||||
* Get if the entry is a redirect entry.
|
||||
*
|
||||
* @return True if the entry is a redirect.
|
||||
*/
|
||||
bool isRedirect() const;
|
||||
|
||||
/**
|
||||
* Get if the entry is a link target entry.
|
||||
*
|
||||
* @return True if the entry is a link target.
|
||||
*/
|
||||
bool isLinkTarget() const;
|
||||
|
||||
/**
|
||||
* Get if the entry is a deleted entry.
|
||||
*
|
||||
* @return True if the entry is a deleted entry.
|
||||
*/
|
||||
bool isDeleted() const;
|
||||
|
||||
/**
|
||||
* Get the entry pointed by this entry.
|
||||
*
|
||||
* @return the entry pointed.
|
||||
* @throw NoEntry if the entry is not a redirected entry.
|
||||
*/
|
||||
Entry getRedirectEntry() const;
|
||||
|
||||
/**
|
||||
* Get the final entry pointed by this entry.
|
||||
*
|
||||
* Follow the redirection until a "not redirecting" entry is found.
|
||||
* If the entry is not a redirected entry, return the entry itself.
|
||||
*
|
||||
* @return the final entry.
|
||||
*/
|
||||
Entry getFinalEntry() const;
|
||||
|
||||
/**
|
||||
* Get the zim entry wrapped by this (kiwix) entry
|
||||
*
|
||||
* @return the zim entry
|
||||
*/
|
||||
const zim::Entry& getZimEntry() const { return entry; }
|
||||
|
||||
private:
|
||||
zim::Entry entry;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // KIWIX_ENTRY_H
|
||||
@@ -22,4 +22,4 @@
|
||||
|
||||
#include "library.h"
|
||||
|
||||
#endif
|
||||
#endif
|
||||
@@ -24,9 +24,6 @@
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <zim/archive.h>
|
||||
#include <zim/search.h>
|
||||
|
||||
#include "book.h"
|
||||
#include "bookmark.h"
|
||||
@@ -106,22 +103,12 @@ class Filter {
|
||||
Filter& rejectTags(const Tags& tags);
|
||||
|
||||
Filter& category(std::string category);
|
||||
|
||||
/**
|
||||
* Set the filter to only accept books in the specified language.
|
||||
*
|
||||
* Multiple languages can be specified as a comma-separated list (in
|
||||
* which case a book in any of those languages will match).
|
||||
*/
|
||||
Filter& lang(std::string lang);
|
||||
|
||||
Filter& publisher(std::string publisher);
|
||||
Filter& creator(std::string creator);
|
||||
Filter& maxSize(size_t size);
|
||||
Filter& query(std::string query, bool partial=true);
|
||||
Filter& name(std::string name);
|
||||
Filter& clearLang();
|
||||
Filter& clearCategory();
|
||||
|
||||
bool hasQuery() const;
|
||||
const std::string& getQuery() const { return _query; }
|
||||
@@ -152,34 +139,19 @@ private: // functions
|
||||
};
|
||||
|
||||
|
||||
class ZimSearcher : public zim::Searcher
|
||||
{
|
||||
public:
|
||||
explicit ZimSearcher(zim::Searcher&& searcher)
|
||||
: zim::Searcher(searcher)
|
||||
{}
|
||||
|
||||
std::unique_lock<std::mutex> getLock() {
|
||||
return std::unique_lock<std::mutex>(m_mutex);
|
||||
}
|
||||
virtual ~ZimSearcher() = default;
|
||||
private:
|
||||
std::mutex m_mutex;
|
||||
};
|
||||
|
||||
/**
|
||||
* A Library store several books.
|
||||
*/
|
||||
class Library
|
||||
{
|
||||
// all data fields must be added in LibraryBase
|
||||
mutable std::mutex m_mutex;
|
||||
std::map<std::string, kiwix::Book> m_books;
|
||||
std::map<std::string, std::shared_ptr<Reader>> m_readers;
|
||||
std::vector<kiwix::Bookmark> m_bookmarks;
|
||||
class BookDB;
|
||||
std::unique_ptr<BookDB> m_bookDB;
|
||||
|
||||
public:
|
||||
typedef uint64_t Revision;
|
||||
typedef std::vector<std::string> BookIdCollection;
|
||||
typedef std::map<std::string, int> AttributeCounts;
|
||||
typedef std::set<std::string> BookIdSet;
|
||||
|
||||
public:
|
||||
Library();
|
||||
@@ -205,11 +177,6 @@ class Library
|
||||
*/
|
||||
bool addBook(const Book& book);
|
||||
|
||||
/**
|
||||
* A self-explanatory alias for addBook()
|
||||
*/
|
||||
bool addOrUpdateBook(const Book& book) { return addBook(book); }
|
||||
|
||||
/**
|
||||
* Add a bookmark to the library.
|
||||
*
|
||||
@@ -226,18 +193,11 @@ class Library
|
||||
*/
|
||||
bool removeBookmark(const std::string& zimId, const std::string& url);
|
||||
|
||||
// XXX: This is a non-thread-safe operation
|
||||
const Book& getBookById(const std::string& id) const;
|
||||
// XXX: This is a non-thread-safe operation
|
||||
Book& getBookById(const std::string& id);
|
||||
const Book& getBookByPath(const std::string& path) const;
|
||||
|
||||
Book getBookByIdThreadSafe(const std::string& id) const;
|
||||
|
||||
std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
|
||||
std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
|
||||
return getSearcherByIds(BookIdSet{id});
|
||||
}
|
||||
std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
|
||||
Book& getBookByPath(const std::string& path);
|
||||
std::shared_ptr<Reader> getReaderById(const std::string& id);
|
||||
|
||||
/**
|
||||
* Remove a book from the library.
|
||||
@@ -279,13 +239,6 @@ class Library
|
||||
*/
|
||||
std::vector<std::string> getBooksLanguages() const;
|
||||
|
||||
/**
|
||||
* Get all languagues of the books in the library with counts.
|
||||
*
|
||||
* @return A list of languages with the count of books in each language.
|
||||
*/
|
||||
AttributeCounts getBooksLanguagesWithCounts() const;
|
||||
|
||||
/**
|
||||
* Get all categories of the books in the library.
|
||||
*
|
||||
@@ -321,6 +274,17 @@ class Library
|
||||
*/
|
||||
BookIdCollection getBooksIds() const;
|
||||
|
||||
/**
|
||||
* Filter the library and generate a new one with the keep elements.
|
||||
*
|
||||
* This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
|
||||
*
|
||||
* @param search List only books with search in the title or description.
|
||||
* @return The list of bookIds corresponding to the query.
|
||||
*/
|
||||
DEPRECATED BookIdCollection filter(const std::string& search) const;
|
||||
|
||||
|
||||
/**
|
||||
* Filter the library and return the id of the keep elements.
|
||||
*
|
||||
@@ -340,39 +304,43 @@ class Library
|
||||
void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
|
||||
|
||||
/**
|
||||
* Return the current revision of the library.
|
||||
* List books in the library.
|
||||
*
|
||||
* The revision of the library is updated (incremented by one) by
|
||||
* the addBook() and removeBookById() operations.
|
||||
*
|
||||
* @return Current revision of the library.
|
||||
* @param mode The mode of listing :
|
||||
* - LOCAL : list only local books (with a path).
|
||||
* - REMOTE : list only remote books (with an url).
|
||||
* - VALID : list only valid books (without a path or with a
|
||||
* path pointing to a valid zim file).
|
||||
* - NOLOCAL : list only books without valid path.
|
||||
* - NOREMOTE : list only books without url.
|
||||
* - NOVALID : list only books not valid.
|
||||
* - ALL : Do not do any filter (LOCAL or REMOTE)
|
||||
* - Flags can be combined.
|
||||
* @param sortBy Attribute to sort by the book list.
|
||||
* @param search List only books with search in the title, description.
|
||||
* @param language List only books in this language.
|
||||
* @param creator List only books of this creator.
|
||||
* @param publisher List only books of this publisher.
|
||||
* @param maxSize Do not list book bigger than maxSize.
|
||||
* Set to 0 to cancel this filter.
|
||||
* @return The list of bookIds corresponding to the query.
|
||||
*/
|
||||
Revision getRevision() const;
|
||||
|
||||
/**
|
||||
* Remove books that have not been updated since the specified revision.
|
||||
*
|
||||
* @param rev the library revision to use
|
||||
* @return Count of books that were removed by this operation.
|
||||
*/
|
||||
uint32_t removeBooksNotUpdatedSince(Revision rev);
|
||||
DEPRECATED BookIdCollection listBooksIds(
|
||||
int supportedListMode = ALL,
|
||||
supportedListSortBy sortBy = UNSORTED,
|
||||
const std::string& search = "",
|
||||
const std::string& language = "",
|
||||
const std::string& creator = "",
|
||||
const std::string& publisher = "",
|
||||
const std::vector<std::string>& tags = {},
|
||||
size_t maxSize = 0) const;
|
||||
|
||||
friend class OPDSDumper;
|
||||
friend class libXMLDumper;
|
||||
|
||||
private: // types
|
||||
typedef const std::string& (Book::*BookStrPropMemFn)() const;
|
||||
struct Impl;
|
||||
|
||||
private: // functions
|
||||
AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
|
||||
std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
|
||||
BookIdCollection filterViaBookDB(const Filter& filter) const;
|
||||
void updateBookDB(const Book& book);
|
||||
void dropCache(const std::string& bookId);
|
||||
|
||||
private: //data
|
||||
std::unique_ptr<Impl> mp_impl;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
|
||||
* Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_LIBRARY_DUMPER_H
|
||||
#define KIWIX_LIBRARY_DUMPER_H
|
||||
|
||||
#include <string>
|
||||
|
||||
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
#include <mustache.hpp>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/**
|
||||
* A base class to dump Library in various formats.
|
||||
*
|
||||
*/
|
||||
class LibraryDumper
|
||||
{
|
||||
public:
|
||||
LibraryDumper(const Library* library, const NameMapper* NameMapper);
|
||||
~LibraryDumper();
|
||||
|
||||
void setLibraryId(const std::string& id) { this->libraryId = id;}
|
||||
|
||||
/**
|
||||
* Set the root location used when generating url.
|
||||
*
|
||||
* @param rootLocation the root location to use.
|
||||
*/
|
||||
void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
|
||||
|
||||
/**
|
||||
* Set some informations about the search results.
|
||||
*
|
||||
* @param totalResult the total number of results of the search.
|
||||
* @param startIndex the start index of the result.
|
||||
* @param count the number of result of the current set (or page).
|
||||
*/
|
||||
void setOpenSearchInfo(int totalResult, int startIndex, int count);
|
||||
|
||||
/**
|
||||
* Sets user default language
|
||||
*
|
||||
* @param userLang the user language to be set
|
||||
*/
|
||||
void setUserLanguage(std::string userLang) { this->m_userLang = userLang; }
|
||||
|
||||
/**
|
||||
* Get the data of categories
|
||||
*/
|
||||
kainjow::mustache::list getCategoryData() const;
|
||||
|
||||
/**
|
||||
* Get the data of languages
|
||||
*/
|
||||
kainjow::mustache::list getLanguageData() const;
|
||||
|
||||
protected:
|
||||
const kiwix::Library* const library;
|
||||
const kiwix::NameMapper* const nameMapper;
|
||||
std::string libraryId;
|
||||
std::string rootLocation;
|
||||
std::string m_userLang;
|
||||
int m_totalResults;
|
||||
int m_startIndex;
|
||||
int m_count;
|
||||
};
|
||||
}
|
||||
|
||||
#endif // KIWIX_LIBRARY_DUMPER_H
|
||||
@@ -22,10 +22,10 @@
|
||||
|
||||
#include "book.h"
|
||||
#include "library.h"
|
||||
#include "reader.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
namespace pugi {
|
||||
class xml_document;
|
||||
@@ -34,25 +34,26 @@ class xml_document;
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
class LibraryManipulator
|
||||
{
|
||||
public: // functions
|
||||
explicit LibraryManipulator(Library* library);
|
||||
virtual ~LibraryManipulator();
|
||||
class LibraryManipulator {
|
||||
public:
|
||||
virtual ~LibraryManipulator() {}
|
||||
virtual bool addBookToLibrary(Book book) = 0;
|
||||
virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
|
||||
};
|
||||
|
||||
Library& getLibrary() const { return library; }
|
||||
|
||||
bool addBookToLibrary(const Book& book);
|
||||
void addBookmarkToLibrary(const Bookmark& bookmark);
|
||||
uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
|
||||
|
||||
protected: // overrides
|
||||
virtual void bookWasAddedToLibrary(const Book& book);
|
||||
virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
|
||||
virtual void booksWereRemovedFromLibrary();
|
||||
|
||||
private: // data
|
||||
kiwix::Library& library;
|
||||
class DefaultLibraryManipulator : public LibraryManipulator {
|
||||
public:
|
||||
DefaultLibraryManipulator(Library* library) :
|
||||
library(library) {}
|
||||
virtual ~DefaultLibraryManipulator() {}
|
||||
bool addBookToLibrary(Book book) {
|
||||
return library->addBook(book);
|
||||
}
|
||||
void addBookmarkToLibrary(Bookmark bookmark) {
|
||||
library->addBookmark(bookmark);
|
||||
}
|
||||
private:
|
||||
kiwix::Library* library;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -60,12 +61,10 @@ class LibraryManipulator
|
||||
*/
|
||||
class Manager
|
||||
{
|
||||
public: // types
|
||||
typedef std::vector<std::string> Paths;
|
||||
|
||||
public: // functions
|
||||
explicit Manager(LibraryManipulator* manipulator);
|
||||
explicit Manager(Library* library);
|
||||
public:
|
||||
Manager(LibraryManipulator* manipulator);
|
||||
Manager(Library* library);
|
||||
~Manager();
|
||||
|
||||
/**
|
||||
* Read a `library.xml` and add book in the file to the library.
|
||||
@@ -73,22 +72,10 @@ class Manager
|
||||
* @param path The (utf8) path to the `library.xml`.
|
||||
* @param readOnly Set if the libray path could be overwritten latter with
|
||||
* updated content.
|
||||
* @param trustLibrary use book metadata coming from XML.
|
||||
* @return True if file has been properly parsed.
|
||||
*/
|
||||
bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
|
||||
|
||||
/**
|
||||
* Sync the contents of the library with one or more `library.xml` files.
|
||||
*
|
||||
* The metadata of the library files is trusted unconditionally.
|
||||
* Any books not present in the input library.xml files are removed
|
||||
* from the library.
|
||||
*
|
||||
* @param paths The (utf8) paths to the `library.xml` files.
|
||||
*/
|
||||
void reload(const Paths& paths);
|
||||
|
||||
/**
|
||||
* Load a library content store in the string.
|
||||
*
|
||||
@@ -163,7 +150,8 @@ class Manager
|
||||
uint64_t m_itemsPerPage = 0;
|
||||
|
||||
protected:
|
||||
std::shared_ptr<kiwix::LibraryManipulator> manipulator;
|
||||
kiwix::LibraryManipulator* manipulator;
|
||||
bool mustDeleteManipulator;
|
||||
|
||||
bool readBookFromPath(const std::string& path, Book* book);
|
||||
bool parseXmlDom(const pugi::xml_document& doc,
|
||||
|
||||
@@ -7,12 +7,24 @@ headers = [
|
||||
'libxml_dumper.h',
|
||||
'opds_dumper.h',
|
||||
'downloader.h',
|
||||
'reader.h',
|
||||
'entry.h',
|
||||
'searcher.h',
|
||||
'search_renderer.h',
|
||||
'server.h',
|
||||
'kiwixserve.h',
|
||||
'name_mapper.h',
|
||||
'tools.h',
|
||||
'version.h'
|
||||
'name_mapper.h'
|
||||
]
|
||||
|
||||
install_headers(headers, subdir:'kiwix')
|
||||
|
||||
install_headers(
|
||||
'tools/base64.h',
|
||||
'tools/networkTools.h',
|
||||
'tools/otherTools.h',
|
||||
'tools/pathTools.h',
|
||||
'tools/regexTools.h',
|
||||
'tools/stringTools.h',
|
||||
subdir:'kiwix/tools'
|
||||
)
|
||||
|
||||
|
||||
@@ -22,8 +22,6 @@
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
@@ -33,15 +31,15 @@ class Library;
|
||||
class NameMapper {
|
||||
public:
|
||||
virtual ~NameMapper() = default;
|
||||
virtual std::string getNameForId(const std::string& id) const = 0;
|
||||
virtual std::string getIdForName(const std::string& name) const = 0;
|
||||
virtual std::string getNameForId(const std::string& id) = 0;
|
||||
virtual std::string getIdForName(const std::string& name) = 0;
|
||||
};
|
||||
|
||||
|
||||
class IdNameMapper : public NameMapper {
|
||||
public:
|
||||
virtual std::string getNameForId(const std::string& id) const { return id; };
|
||||
virtual std::string getIdForName(const std::string& name) const { return name; };
|
||||
virtual std::string getNameForId(const std::string& id) { return id; };
|
||||
virtual std::string getIdForName(const std::string& name) { return name; };
|
||||
};
|
||||
|
||||
class HumanReadableNameMapper : public NameMapper {
|
||||
@@ -52,29 +50,11 @@ class HumanReadableNameMapper : public NameMapper {
|
||||
public:
|
||||
HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
|
||||
virtual ~HumanReadableNameMapper() = default;
|
||||
virtual std::string getNameForId(const std::string& id) const;
|
||||
virtual std::string getIdForName(const std::string& name) const;
|
||||
virtual std::string getNameForId(const std::string& id);
|
||||
virtual std::string getIdForName(const std::string& name);
|
||||
};
|
||||
|
||||
class UpdatableNameMapper : public NameMapper {
|
||||
typedef std::shared_ptr<NameMapper> NameMapperHandle;
|
||||
public:
|
||||
UpdatableNameMapper(Library& library, bool withAlias);
|
||||
|
||||
virtual std::string getNameForId(const std::string& id) const;
|
||||
virtual std::string getIdForName(const std::string& name) const;
|
||||
|
||||
void update();
|
||||
|
||||
private:
|
||||
NameMapperHandle currentNameMapper() const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mutex;
|
||||
Library& library;
|
||||
NameMapperHandle nameMapper;
|
||||
const bool withAlias;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -26,9 +26,11 @@
|
||||
|
||||
#include <pugixml.hpp>
|
||||
|
||||
#include "tools/base64.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
#include "library_dumper.h"
|
||||
#include "reader.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@@ -39,10 +41,11 @@ namespace kiwix
|
||||
* A tool to dump a `Library` into a opds stream.
|
||||
*
|
||||
*/
|
||||
class OPDSDumper : public LibraryDumper
|
||||
class OPDSDumper
|
||||
{
|
||||
public:
|
||||
OPDSDumper(const Library* library, const NameMapper* NameMapper);
|
||||
OPDSDumper() = default;
|
||||
OPDSDumper(Library* library);
|
||||
~OPDSDumper();
|
||||
|
||||
/**
|
||||
@@ -59,32 +62,48 @@ class OPDSDumper : public LibraryDumper
|
||||
*
|
||||
* @param bookIds the ids of the books to include in the feed
|
||||
* @param query the query used to obtain the list of book ids
|
||||
* @param partial whether the feed should include partial or complete entries
|
||||
* @return The OPDS feed.
|
||||
*/
|
||||
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
|
||||
|
||||
/**
|
||||
* Dump the OPDS complete entry document.
|
||||
*
|
||||
* @param bookId the id of the book
|
||||
* @return The OPDS complete entry document.
|
||||
*/
|
||||
std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
|
||||
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const;
|
||||
|
||||
/**
|
||||
* Dump the categories OPDS feed.
|
||||
*
|
||||
* @param categories list of category names
|
||||
* @return The OPDS feed.
|
||||
*/
|
||||
std::string categoriesOPDSFeed() const;
|
||||
std::string categoriesOPDSFeed(const std::vector<std::string>& categories) const;
|
||||
|
||||
/**
|
||||
* Dump the languages OPDS feed.
|
||||
* Set the id of the library.
|
||||
*
|
||||
* @return The OPDS feed.
|
||||
* @param id the id to use.
|
||||
*/
|
||||
std::string languagesOPDSFeed() const;
|
||||
void setLibraryId(const std::string& id) { this->libraryId = id;}
|
||||
|
||||
/**
|
||||
* Set the root location used when generating url.
|
||||
*
|
||||
* @param rootLocation the root location to use.
|
||||
*/
|
||||
void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
|
||||
|
||||
/**
|
||||
* Set some informations about the search results.
|
||||
*
|
||||
* @param totalResult the total number of results of the search.
|
||||
* @param startIndex the start index of the result.
|
||||
* @param count the number of result of the current set (or page).
|
||||
*/
|
||||
void setOpenSearchInfo(int totalResult, int startIndex, int count);
|
||||
|
||||
protected:
|
||||
kiwix::Library* library;
|
||||
std::string libraryId;
|
||||
std::string rootLocation;
|
||||
int m_totalResults;
|
||||
int m_startIndex;
|
||||
int m_count;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
502
include/reader.h
Normal file
502
include/reader.h
Normal file
@@ -0,0 +1,502 @@
|
||||
/*
|
||||
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_READER_H
|
||||
#define KIWIX_READER_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <zim/zim.h>
|
||||
#include <zim/archive.h>
|
||||
#include <exception>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include "common.h"
|
||||
#include "entry.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/**
|
||||
* The SuggestionItem is a helper class that contains the info about a single
|
||||
* suggestion item.
|
||||
*/
|
||||
|
||||
class SuggestionItem
|
||||
{
|
||||
// Functions
|
||||
private:
|
||||
// Create a sugggestion item.
|
||||
explicit SuggestionItem(std::string title, std::string normalizedTitle,
|
||||
std::string path, std::string snippet = "") :
|
||||
title(title),
|
||||
normalizedTitle(normalizedTitle),
|
||||
path(path),
|
||||
snippet(snippet) {}
|
||||
|
||||
public:
|
||||
const std::string getTitle() {return title;}
|
||||
const std::string getNormalizedTitle() {return normalizedTitle;}
|
||||
const std::string getPath() {return path;}
|
||||
const std::string getSnippet() {return snippet;}
|
||||
|
||||
const bool hasSnippet() {return !snippet.empty();}
|
||||
|
||||
// Data
|
||||
private:
|
||||
std::string title;
|
||||
std::string normalizedTitle;
|
||||
std::string path;
|
||||
std::string snippet;
|
||||
|
||||
friend class Reader;
|
||||
};
|
||||
|
||||
/**
|
||||
* The Reader class is the class who allow to get an entry content from a zim
|
||||
* file.
|
||||
*/
|
||||
|
||||
using SuggestionsList_t = std::vector<SuggestionItem>;
|
||||
class Reader
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Create a Reader to read a zim file specified by zimFilePath.
|
||||
*
|
||||
* @param zimFilePath The path to the zim file to read.
|
||||
* The zim file can be splitted (.zimaa, .zimab, ...).
|
||||
* In this case, the file path must still point to the
|
||||
* unsplitted path as if the file were not splitted
|
||||
* (.zim extesion).
|
||||
*/
|
||||
explicit Reader(const string zimFilePath);
|
||||
#ifndef _WIN32
|
||||
explicit Reader(int fd);
|
||||
Reader(int fd, zim::offset_type offset, zim::size_type size);
|
||||
#endif
|
||||
~Reader() = default;
|
||||
|
||||
/**
|
||||
* Get the number of "displayable" entries in the zim file.
|
||||
*
|
||||
* @return If the zim file has a /M/Counter metadata, return the number of
|
||||
* entries with the 'text/html' MIMEtype specified in the metadata.
|
||||
* Else return the number of entries in the 'A' namespace.
|
||||
*/
|
||||
unsigned int getArticleCount() const;
|
||||
|
||||
/**
|
||||
* Get the number of media in the zim file.
|
||||
*
|
||||
* @return If the zim file has a /M/Counter metadata, return the number of
|
||||
* entries with the 'image/jpeg', 'image/gif' and 'image/png' in
|
||||
* the metadata.
|
||||
* Else return the number of entries in the 'I' namespace.
|
||||
*/
|
||||
unsigned int getMediaCount() const;
|
||||
|
||||
/**
|
||||
* Get the number of all entries in the zim file.
|
||||
*
|
||||
* @return Return the number of all the entries, whatever their MIMEtype or
|
||||
* their namespace.
|
||||
*/
|
||||
unsigned int getGlobalCount() const;
|
||||
|
||||
/**
|
||||
* Get the path of the zim file.
|
||||
*
|
||||
* @return the path of the zim file as given in the constructor.
|
||||
*/
|
||||
string getZimFilePath() const;
|
||||
|
||||
/**
|
||||
* Get the Id of the zim file.
|
||||
*
|
||||
* @return The uuid stored in the zim file.
|
||||
*/
|
||||
string getId() const;
|
||||
|
||||
/**
|
||||
* Get a random page.
|
||||
*
|
||||
* @return A random Entry. The entry is picked from all entries in
|
||||
* the 'A' namespace.
|
||||
* The main entry is excluded from the potential results.
|
||||
*/
|
||||
Entry getRandomPage() const;
|
||||
|
||||
/**
|
||||
* Get the entry of the main page.
|
||||
*
|
||||
* @return Entry of the main page as specified in the zim file.
|
||||
*/
|
||||
Entry getMainPage() const;
|
||||
|
||||
/**
|
||||
* Get the content of a metadata.
|
||||
*
|
||||
* @param[in] name The name of the metadata.
|
||||
* @param[out] value The value will be set to the content of the metadata.
|
||||
* @return True if it was possible to get the content of the metadata.
|
||||
*/
|
||||
bool getMetadata(const string& name, string& value) const;
|
||||
|
||||
/**
|
||||
* Get the name of the zim file.
|
||||
*
|
||||
* @return The name of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getName() const;
|
||||
|
||||
/**
|
||||
* Get the title of the zim file.
|
||||
*
|
||||
* @return The title of zim file as specified in the zim metadata.
|
||||
* If no title has been set, return a title computed from the
|
||||
* file path.
|
||||
*/
|
||||
string getTitle() const;
|
||||
|
||||
/**
|
||||
* Get the creator of the zim file.
|
||||
*
|
||||
* @return The creator of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getCreator() const;
|
||||
|
||||
/**
|
||||
* Get the publisher of the zim file.
|
||||
*
|
||||
* @return The publisher of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getPublisher() const;
|
||||
|
||||
/**
|
||||
* Get the date of the zim file.
|
||||
*
|
||||
* @return The date of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getDate() const;
|
||||
|
||||
/**
|
||||
* Get the description of the zim file.
|
||||
*
|
||||
* @return The description of the zim file as specified in the zim metadata.
|
||||
* If no description has been set, return the subtitle.
|
||||
*/
|
||||
string getDescription() const;
|
||||
|
||||
/**
|
||||
* Get the long description of the zim file.
|
||||
*
|
||||
* @return The long description of the zim file as specifed in the zim metadata.
|
||||
*/
|
||||
string getLongDescription() const;
|
||||
|
||||
/**
|
||||
* Get the language of the zim file.
|
||||
*
|
||||
* @return The language of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getLanguage() const;
|
||||
|
||||
/**
|
||||
* Get the license of the zim file.
|
||||
*
|
||||
* @return The license of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getLicense() const;
|
||||
|
||||
/**
|
||||
* Get the tags of the zim file.
|
||||
*
|
||||
* @param original If true, return the original tags as specified in the zim metadata.
|
||||
* Else, try to convert it to the new 'normalized' format.
|
||||
* @return The tags of the zim file.
|
||||
*/
|
||||
string getTags(bool original=false) const;
|
||||
|
||||
/**
|
||||
* Get the value (as a string) of a specific tag.
|
||||
*
|
||||
* According to https://wiki.openzim.org/wiki/Tags
|
||||
*
|
||||
* @return The value of the specified tag.
|
||||
* @throw std::out_of_range if the specified tag is not found.
|
||||
*/
|
||||
string getTagStr(const std::string& tagName) const;
|
||||
|
||||
/**
|
||||
* Get the boolean value of a specific tag.
|
||||
*
|
||||
* According to https://wiki.openzim.org/wiki/Tags
|
||||
*
|
||||
* @return The boolean value of the specified tag.
|
||||
* @throw std::out_of_range if the specified tag is not found.
|
||||
* std::domain_error if the value of the tag cannot be convert to bool.
|
||||
*/
|
||||
bool getTagBool(const std::string& tagName) const;
|
||||
|
||||
/**
|
||||
* Get the relations of the zim file.
|
||||
*
|
||||
* @return The relation of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getRelation() const;
|
||||
|
||||
/**
|
||||
* Get the flavour of the zim file.
|
||||
*
|
||||
* @return The flavour of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getFlavour() const;
|
||||
|
||||
/**
|
||||
* Get the source of the zim file.
|
||||
*
|
||||
* @return The source of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getSource() const;
|
||||
|
||||
/**
|
||||
* Get the scraper of the zim file.
|
||||
*
|
||||
* @return The scraper of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getScraper() const;
|
||||
|
||||
/**
|
||||
* Get the origId of the zim file.
|
||||
*
|
||||
* The origId is only used in the case of patch zim file and is the Id
|
||||
* of the original zim file.
|
||||
*
|
||||
* @return The origId of the zim file as specified in the zim metadata.
|
||||
*/
|
||||
string getOrigId() const;
|
||||
|
||||
/**
|
||||
* Get the favicon of the zim file.
|
||||
*
|
||||
* @param[out] content The content of the favicon.
|
||||
* @param[out] mimeType The mimeType of the favicon.
|
||||
* @return True if a favicon has been found.
|
||||
*/
|
||||
bool getFavicon(string& content, string& mimeType) const;
|
||||
|
||||
/**
|
||||
* Get an entry associated to an path.
|
||||
*
|
||||
* @param path The path of the entry.
|
||||
* @return The entry.
|
||||
* @throw NoEntry If no entry correspond to the path.
|
||||
*/
|
||||
Entry getEntryFromPath(const std::string& path) const;
|
||||
|
||||
/**
|
||||
* Get an entry associated to an url encoded path.
|
||||
*
|
||||
* Equivalent to `getEntryFromPath(urlDecode(path));`
|
||||
*
|
||||
* @param path The url encoded path.
|
||||
* @return The entry.
|
||||
* @throw NoEntry If no entry correspond to the path.
|
||||
*/
|
||||
Entry getEntryFromEncodedPath(const std::string& path) const;
|
||||
|
||||
/**
|
||||
* Get un entry associated to a title.
|
||||
*
|
||||
* @param title The title.
|
||||
* @return The entry
|
||||
* throw NoEntry If no entry correspond to the url.
|
||||
*/
|
||||
Entry getEntryFromTitle(const std::string& title) const;
|
||||
|
||||
/**
|
||||
* Search for entries with title starting with prefix (case sensitive).
|
||||
*
|
||||
* Suggestions are stored in an internal vector and can be retrieved using
|
||||
* `getNextSuggestion` method.
|
||||
* This method is not thread safe and is deprecated. Use :
|
||||
* bool searchSuggestions(const string& prefix,
|
||||
* unsigned int suggestionsCount,
|
||||
* SuggestionsList_t& results);
|
||||
*
|
||||
* @param prefix The prefix to search.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
* @param reset If true, remove previous suggestions in the internal vector.
|
||||
* If false, add suggestions to the internal vector
|
||||
* (until internal vector size is suggestionCount (or no more
|
||||
* suggestion))
|
||||
* @return True if some suggestions have been added to the internal vector.
|
||||
*/
|
||||
DEPRECATED bool searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
const bool reset = true);
|
||||
|
||||
/**
|
||||
* Search for entries with title starting with prefix (case sensitive).
|
||||
*
|
||||
* Suggestions are added to the `result` vector.
|
||||
*
|
||||
* @param prefix The prefix to search.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
* @param result The vector where to store the suggestions.
|
||||
* @return True if some suggestions have been added to the vector.
|
||||
*/
|
||||
|
||||
bool searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& resuls);
|
||||
|
||||
/**
|
||||
* Search for entries for the given prefix.
|
||||
*
|
||||
* If the zim file has a internal fulltext index, the suggestions will be
|
||||
* searched using it.
|
||||
* Else the suggestions will be search using `searchSuggestions` while trying
|
||||
* to be smart about case sensitivity (using `getTitleVariants`).
|
||||
*
|
||||
* In any case, suggestions are stored in an internal vector and can be
|
||||
* retrieved using `getNextSuggestion` method.
|
||||
* The internal vector will be reset.
|
||||
* This method is not thread safe and is deprecated. Use :
|
||||
* bool searchSuggestionsSmart(const string& prefix,
|
||||
* unsigned int suggestionsCount,
|
||||
* SuggestionsList_t& results);
|
||||
*
|
||||
* @param prefix The prefix to search for.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
*/
|
||||
DEPRECATED bool searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount);
|
||||
|
||||
/**
|
||||
* Search for entries for the given prefix.
|
||||
*
|
||||
* If the zim file has a internal fulltext index, the suggestions will be
|
||||
* searched using it.
|
||||
* Else the suggestions will be search using `searchSuggestions` while trying
|
||||
* to be smart about case sensitivity (using `getTitleVariants`).
|
||||
*
|
||||
* In any case, suggestions are stored in an internal vector and can be
|
||||
* retrieved using `getNextSuggestion` method.
|
||||
* The internal vector will be reset.
|
||||
*
|
||||
* @param prefix The prefix to search for.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
* @param results The vector where to store the suggestions
|
||||
* @return True if some suggestions have been added to the results.
|
||||
*/
|
||||
bool searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results);
|
||||
|
||||
|
||||
/**
|
||||
* Check if the path exists in the zim file.
|
||||
*
|
||||
* @param path the path to check.
|
||||
* @return True if the path exists in the zim file.
|
||||
*/
|
||||
bool pathExists(const string& path) const;
|
||||
|
||||
/**
|
||||
* Check if the zim file has a embedded fulltext index.
|
||||
*
|
||||
* @return True if the zim file has a embedded fulltext index
|
||||
* and is not split (else the fulltext is not accessible).
|
||||
*/
|
||||
bool hasFulltextIndex() const;
|
||||
|
||||
/**
|
||||
* Get potential case title variations for a title.
|
||||
*
|
||||
* @param title a title.
|
||||
* @return the list of variantions.
|
||||
*/
|
||||
std::vector<std::string> getTitleVariants(const std::string& title) const;
|
||||
|
||||
/**
|
||||
* Get the next suggestion title.
|
||||
*
|
||||
* @param[out] title the title of the suggestion.
|
||||
* @return True if title has been set.
|
||||
*/
|
||||
DEPRECATED bool getNextSuggestion(string& title);
|
||||
|
||||
/**
|
||||
* Get the next suggestion title and url.
|
||||
*
|
||||
* @param[out] title the title of the suggestion.
|
||||
* @param[out] url the url of the suggestion.
|
||||
* @return True if title and url have been set.
|
||||
*/
|
||||
DEPRECATED bool getNextSuggestion(string& title, string& url);
|
||||
|
||||
/**
|
||||
* Get if we can check zim file integrity (has a checksum).
|
||||
*
|
||||
* @return True if zim file have a checksum.
|
||||
*/
|
||||
bool canCheckIntegrity() const;
|
||||
|
||||
/**
|
||||
* Check is zim file is corrupted.
|
||||
*
|
||||
* @return True if zim file is corrupted.
|
||||
*/
|
||||
bool isCorrupted() const;
|
||||
|
||||
/**
|
||||
* Return the total size of the zim file.
|
||||
*
|
||||
* If zim file is split, return the sum of all parts' size.
|
||||
*
|
||||
* @return Size of the size file is KiB.
|
||||
*/
|
||||
unsigned int getFileSize() const;
|
||||
|
||||
/**
|
||||
* Get the zim file handler.
|
||||
*
|
||||
* @return The libzim file handler.
|
||||
*/
|
||||
zim::Archive* getZimArchive() const;
|
||||
|
||||
protected:
|
||||
std::unique_ptr<zim::Archive> zimArchive;
|
||||
std::string zimFilePath;
|
||||
|
||||
SuggestionsList_t suggestions;
|
||||
SuggestionsList_t::iterator suggestionsOffset;
|
||||
|
||||
private:
|
||||
std::map<const std::string, unsigned int> parseCounterMetadata() const;
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -21,12 +21,11 @@
|
||||
#define KIWIX_SEARCH_RENDERER_H
|
||||
|
||||
#include <string>
|
||||
#include <zim/search.h>
|
||||
#include "library.h"
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
class Searcher;
|
||||
class NameMapper;
|
||||
/**
|
||||
* The SearcherRenderer class is used to render a search result to a html page.
|
||||
@@ -35,43 +34,21 @@ class SearchRenderer
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct a SearchRenderer from a SearchResultSet.
|
||||
* The default constructor.
|
||||
*
|
||||
* The constructed version of the SearchRenderer will not introduce
|
||||
* the book name for each result. It is better to use the other constructor
|
||||
* with a Library pointer to have a better html page.
|
||||
*
|
||||
* @param srs The `SearchResultSet` to render.
|
||||
* @param mapper The `NameMapper` to use to do the rendering.
|
||||
* @param start The start offset used for the srs.
|
||||
* @param estimatedResultCount The estimatedResultCount of the whole search
|
||||
* @param humanReadableName The global zim's humanReadableName.
|
||||
* Used to generate pagination links.
|
||||
*/
|
||||
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
|
||||
unsigned int start, unsigned int estimatedResultCount);
|
||||
|
||||
/**
|
||||
* Construct a SearchRenderer from a SearchResultSet.
|
||||
*
|
||||
* @param srs The `SearchResultSet` to render.
|
||||
* @param mapper The `NameMapper` to use to do the rendering.
|
||||
* @param library The `Library` to use to look up book details for search results.
|
||||
* @param start The start offset used for the srs.
|
||||
* @param estimatedResultCount The estimatedResultCount of the whole search
|
||||
*/
|
||||
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
|
||||
unsigned int start, unsigned int estimatedResultCount);
|
||||
SearchRenderer(Searcher* searcher, NameMapper* mapper);
|
||||
|
||||
~SearchRenderer();
|
||||
|
||||
/**
|
||||
* Set the search pattern used to do the search
|
||||
*/
|
||||
void setSearchPattern(const std::string& pattern);
|
||||
|
||||
/**
|
||||
* Set the querystring used to select books
|
||||
* Set the search content id.
|
||||
*/
|
||||
void setSearchBookQuery(const std::string& bookQuery);
|
||||
void setSearchContent(const std::string& name);
|
||||
|
||||
/**
|
||||
* Set protocol prefix.
|
||||
@@ -90,25 +67,16 @@ class SearchRenderer
|
||||
this->pageLength = pageLength;
|
||||
}
|
||||
|
||||
std::string renderTemplate(const std::string& tmpl_str);
|
||||
|
||||
/**
|
||||
* Generate the html page with the resutls of the search.
|
||||
*/
|
||||
std::string getHtml();
|
||||
|
||||
/**
|
||||
* Generate the xml page with the resutls of the search.
|
||||
*/
|
||||
std::string getXml();
|
||||
|
||||
|
||||
protected:
|
||||
std::string beautifyInteger(const unsigned int number);
|
||||
zim::SearchResultSet m_srs;
|
||||
Searcher* mp_searcher;
|
||||
NameMapper* mp_nameMapper;
|
||||
Library* mp_library;
|
||||
std::string searchBookQuery;
|
||||
std::string searchContent;
|
||||
std::string searchPattern;
|
||||
std::string protocolPrefix;
|
||||
std::string searchProtocolPrefix;
|
||||
|
||||
171
include/searcher.h
Normal file
171
include/searcher.h
Normal file
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_SEARCHER_H
|
||||
#define KIWIX_SEARCHER_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unicode/putil.h>
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <locale>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
class Reader;
|
||||
class Result
|
||||
{
|
||||
public:
|
||||
virtual ~Result(){};
|
||||
virtual std::string get_url() = 0;
|
||||
virtual std::string get_title() = 0;
|
||||
virtual int get_score() = 0;
|
||||
virtual std::string get_snippet() = 0;
|
||||
virtual std::string get_content() = 0;
|
||||
virtual int get_wordCount() = 0;
|
||||
virtual int get_size() = 0;
|
||||
virtual std::string get_zimId() = 0;
|
||||
};
|
||||
|
||||
struct SearcherInternal;
|
||||
/**
|
||||
* The Searcher class is reponsible to do different kind of search using the
|
||||
* fulltext index.
|
||||
*/
|
||||
class Searcher
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* The default constructor.
|
||||
*/
|
||||
Searcher();
|
||||
|
||||
~Searcher();
|
||||
|
||||
/**
|
||||
* Add a reader (containing embedded fulltext index) to the search.
|
||||
*
|
||||
* @param reader The Reader for the zim containing the fulltext index.
|
||||
* @return true if the reader has been added.
|
||||
* false if the reader cannot be added (no embedded fulltext index present)
|
||||
*/
|
||||
bool add_reader(Reader* reader);
|
||||
|
||||
|
||||
Reader* get_reader(int index);
|
||||
|
||||
/**
|
||||
* Start a search on the zim associated to the Searcher.
|
||||
*
|
||||
* Search results should be retrived using the getNextResult method.
|
||||
*
|
||||
* @param search The search query.
|
||||
* @param resultStart the start offset of the search results (used for pagination).
|
||||
* @param resultEnd the end offset of the search results (used for pagination).
|
||||
* @param verbose print some info on stdout if true.
|
||||
*/
|
||||
void search(const std::string& search,
|
||||
unsigned int resultStart,
|
||||
unsigned int resultEnd,
|
||||
const bool verbose = false);
|
||||
|
||||
/**
|
||||
* Start a geographique search.
|
||||
* The search return result for entry in a disc of center latitude/longitude
|
||||
* and radius distance.
|
||||
*
|
||||
* Search results should be retrived using the getNextResult method.
|
||||
*
|
||||
* @param latitude The latitude of the center point.
|
||||
* @param longitude The longitude of the center point.
|
||||
* @param distance The radius of the disc.
|
||||
* @param resultStart the start offset of the search results (used for pagination).
|
||||
* @param resultEnd the end offset of the search results (used for pagination).
|
||||
* @param verbose print some info on stdout if true.
|
||||
*/
|
||||
void geo_search(float latitude, float longitude, float distance,
|
||||
unsigned int resultStart,
|
||||
unsigned int resultEnd,
|
||||
const bool verbose = false);
|
||||
|
||||
/**
|
||||
* Start a suggestion search.
|
||||
* The search made depend of the "version" of the embedded index.
|
||||
* - If the index is newer enough and have a title namespace, the search is
|
||||
* made in the titles only.
|
||||
* - Else the search is made on the whole article content.
|
||||
* In any case, the search is made "partial" (as adding '*' at the end of the query)
|
||||
*
|
||||
* @param search The search query.
|
||||
* @param verbose print some info on stdout if true.
|
||||
*/
|
||||
void suggestions(std::string& search, const bool verbose = false);
|
||||
|
||||
/**
|
||||
* Get the next result of a started search.
|
||||
* This is the method to use to loop hover the search results.
|
||||
*/
|
||||
Result* getNextResult();
|
||||
|
||||
/**
|
||||
* Restart the previous search.
|
||||
* Next call to getNextResult will return the first result.
|
||||
*/
|
||||
void restart_search();
|
||||
|
||||
/**
|
||||
* Get a estimation of the result count.
|
||||
*/
|
||||
unsigned int getEstimatedResultCount();
|
||||
|
||||
unsigned int getResultStart() { return resultStart; }
|
||||
unsigned int getResultEnd() { return resultEnd; }
|
||||
|
||||
protected:
|
||||
std::string beautifyInteger(const unsigned int number);
|
||||
void closeIndex();
|
||||
void searchInIndex(string& search,
|
||||
const unsigned int resultStart,
|
||||
const unsigned int resultEnd,
|
||||
const bool verbose = false);
|
||||
|
||||
std::vector<Reader*> readers;
|
||||
std::unique_ptr<SearcherInternal> internal;
|
||||
std::string searchPattern;
|
||||
unsigned int estimatedResultCount;
|
||||
unsigned int resultStart;
|
||||
unsigned int resultEnd;
|
||||
|
||||
private:
|
||||
void reset();
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -54,31 +54,23 @@ namespace kiwix
|
||||
void setAddress(const std::string& addr) { m_addr = addr; }
|
||||
void setPort(int port) { m_port = port; }
|
||||
void setNbThreads(int threads) { m_nbThreads = threads; }
|
||||
void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
|
||||
void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
|
||||
void setVerbose(bool verbose) { m_verbose = verbose; }
|
||||
void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
|
||||
void setTaskbar(bool withTaskbar, bool withLibraryButton)
|
||||
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
|
||||
void setBlockExternalLinks(bool blockExternalLinks)
|
||||
{ m_blockExternalLinks = blockExternalLinks; }
|
||||
int getPort();
|
||||
std::string getAddress();
|
||||
|
||||
protected:
|
||||
Library* mp_library;
|
||||
NameMapper* mp_nameMapper;
|
||||
std::string m_root = "";
|
||||
std::string m_addr = "";
|
||||
std::string m_indexTemplateString = "";
|
||||
int m_port = 80;
|
||||
int m_nbThreads = 1;
|
||||
unsigned int m_multizimSearchLimit = 0;
|
||||
bool m_verbose = false;
|
||||
bool m_withTaskbar = true;
|
||||
bool m_withLibraryButton = true;
|
||||
bool m_blockExternalLinks = false;
|
||||
int m_ipConnectionLimit = 0;
|
||||
std::unique_ptr<InternalServer> mp_server;
|
||||
};
|
||||
}
|
||||
|
||||
220
include/tools.h
220
include/tools.h
@@ -1,220 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_TOOLS_H
|
||||
#define KIWIX_TOOLS_H
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
/**
|
||||
* Return the current directory.
|
||||
*
|
||||
* @return the current directory (utf8 encoded)
|
||||
*/
|
||||
std::string getCurrentDirectory();
|
||||
|
||||
/**
|
||||
* Return the data directory.
|
||||
*
|
||||
* The data directory is a directory where to put data (zim files, ...)
|
||||
* It depends of the platform and it may be changed by user using environment variable.
|
||||
*
|
||||
* The resolution order is :
|
||||
* - `KIWIX_DATA_DIR` env variable (if set).
|
||||
* - On Windows :
|
||||
* . `$APPDATA/kiwix` if $APPDATA is set
|
||||
* . `$USERPROFILE/kiwix` if $USERPROFILE is set
|
||||
* - Else :
|
||||
* . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
|
||||
* . `$HOME/.local/share/kiwx` if $HOWE is set
|
||||
* - current directory
|
||||
*
|
||||
* @return the path of the data directory (utf8 encoded)
|
||||
*/
|
||||
std::string getDataDirectory();
|
||||
|
||||
/** Return the path of the executable
|
||||
*
|
||||
* Some application may be packaged in auto extractible archive (Appimage) and the
|
||||
* real executable is different of the path of the archive.
|
||||
* If `realPathOnly` is true, return the path of the real executable instead of the
|
||||
* archive launched by the user.
|
||||
*
|
||||
* @param realPathOnly If we must return the real path of the executable.
|
||||
* @return the path of the executable (utf8 encoded)
|
||||
*/
|
||||
std::string getExecutablePath(bool realPathOnly = false);
|
||||
|
||||
/** Tell if the path is a relative path.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param path A utf8 encoded path.
|
||||
* @return true if the path is relative.
|
||||
*/
|
||||
bool isRelativePath(const std::string& path);
|
||||
|
||||
/** Append a path to another one.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param basePath the base path.
|
||||
* @param relativePath a path to add to the base path, must be a relative path.
|
||||
* @return The concatenation of the paths, using the right separator.
|
||||
*/
|
||||
std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
|
||||
|
||||
/** Remove the last element of a path.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param path a path.
|
||||
* @return The parent directory (or empty string if none).
|
||||
*/
|
||||
std::string removeLastPathElement(const std::string& path);
|
||||
|
||||
/** Get the last element of a path.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param path a path.
|
||||
* @return The base name of the path or empty string if none (ending with a separator).
|
||||
*/
|
||||
std::string getLastPathElement(const std::string& path);
|
||||
|
||||
/** Compute the absolute path of a relative path based on another one
|
||||
*
|
||||
* Equivalent to appendToDirectory followed by a normalization of the path.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param path the base path (if empty, current directory is taken).
|
||||
* @param relativePath the relative path.
|
||||
* @return a absolute path.
|
||||
*/
|
||||
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
|
||||
|
||||
/** Compute the relative path of a path relative to another one
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate paths.
|
||||
*
|
||||
* @param path the base path.
|
||||
* @param absolutePath the absolute path to find the relative path for.
|
||||
* @return a relative path (pointing to absolutePath, relative to path).
|
||||
*/
|
||||
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
|
||||
|
||||
/** Sleep the current thread.
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools.
|
||||
*
|
||||
* @param milliseconds The number of milliseconds to wait for.
|
||||
*/
|
||||
void sleep(unsigned int milliseconds);
|
||||
|
||||
/** Split a string
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools.
|
||||
*
|
||||
* Assuming text = "foo:;bar;baz,oups;"
|
||||
*
|
||||
* split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
|
||||
* split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
|
||||
* split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
|
||||
* split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
|
||||
*
|
||||
* @param str The string to split.
|
||||
* @param delims A string of potential delimiters.
|
||||
* Each charater in the string can be a individual delimiters.
|
||||
* @param dropEmpty true if empty part must be dropped from the result.
|
||||
* @param keepDelim true if delimiter must be included from the result.
|
||||
* @return a list of part (potentially containing delimiters)
|
||||
*/
|
||||
std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
|
||||
|
||||
/** Convert language code from iso2 code to iso3
|
||||
*
|
||||
* This function is provided as a small helper. It is probably better to use native tools
|
||||
* to manipulate locales.
|
||||
*
|
||||
* @param a2code a iso2 code string.
|
||||
* @return the corresponding iso3 code.
|
||||
* @throw std::out_of_range if iso2 code is not known.
|
||||
*/
|
||||
std::string converta2toa3(const std::string& a2code);
|
||||
|
||||
/** Extracts content from given file.
|
||||
*
|
||||
* This function provides content of a file provided it's path.
|
||||
*
|
||||
* @param path The absolute path provided in string format.
|
||||
* @return Content of corresponding file in string format.
|
||||
*/
|
||||
std::string getFileContent(const std::string& path);
|
||||
|
||||
/** Checks if file exists.
|
||||
*
|
||||
* This function returns boolean stating if file exists.
|
||||
*
|
||||
* @param path The absolute path provided in string format.
|
||||
* @return Boolean representing if file exists or not.
|
||||
*/
|
||||
bool fileExists(const std::string& path);
|
||||
|
||||
/** Checks if file is readable.
|
||||
*
|
||||
* This function returns boolean stating if file is readable.
|
||||
*
|
||||
* @param path The absolute path provided in string format.
|
||||
* @return Boolean representing if file is readale or not.
|
||||
*/
|
||||
bool fileReadable(const std::string& path);
|
||||
|
||||
/** Provides mimetype from filename.
|
||||
*
|
||||
* This function provides mimetype from file-name.
|
||||
*
|
||||
* @param filename string containing filename.
|
||||
* @return mimetype from filename in string format.
|
||||
*/
|
||||
std::string getMimeTypeForFile(const std::string& filename);
|
||||
|
||||
/** Provides all available network interfaces
|
||||
*
|
||||
* This function provides the available IPv4 network interfaces
|
||||
*/
|
||||
std::map<std::string, std::string> getNetworkInterfaces();
|
||||
|
||||
/** Provides the best IP address
|
||||
* This function provides the best IP address from the list given by getNetworkInterfaces
|
||||
*/
|
||||
std::string getBestPublicIp();
|
||||
|
||||
}
|
||||
#endif // KIWIX_TOOLS_H
|
||||
@@ -23,23 +23,18 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <zim/zim.h>
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include "stringTools.h"
|
||||
|
||||
namespace pugi {
|
||||
class xml_node;
|
||||
}
|
||||
|
||||
namespace zim {
|
||||
class SuggestionItem;
|
||||
}
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
void sleep(unsigned int milliseconds);
|
||||
std::string nodeToString(const pugi::xml_node& node);
|
||||
std::string converta2toa3(const std::string& a2code);
|
||||
|
||||
/*
|
||||
* Convert all format tag string to new format
|
||||
@@ -49,44 +44,13 @@ namespace kiwix
|
||||
const std::string& tagName);
|
||||
bool convertStrToBool(const std::string& value);
|
||||
|
||||
using MimeCounterType = std::map<const std::string, zim::entry_index_type>;
|
||||
MimeCounterType parseMimetypeCounter(const std::string& counterData);
|
||||
|
||||
std::string gen_date_str();
|
||||
std::string gen_uuid(const std::string& s);
|
||||
|
||||
// if s is empty then returns kainjow::mustache::data(false)
|
||||
// otherwise kainjow::mustache::data(value)
|
||||
kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
|
||||
|
||||
std::string render_template(const std::string& template_str, kainjow::mustache::data data);
|
||||
|
||||
template<typename T>
|
||||
T getEnvVar(const char* name, const T& defaultValue)
|
||||
{
|
||||
try {
|
||||
const char* envString = std::getenv(name);
|
||||
if (envString == nullptr) {
|
||||
throw std::runtime_error("Environment variable not set");
|
||||
}
|
||||
return extractFromString<T>(envString);
|
||||
} catch (...) {}
|
||||
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
class Suggestions
|
||||
{
|
||||
public:
|
||||
Suggestions();
|
||||
|
||||
void add(const zim::SuggestionItem& suggestion);
|
||||
|
||||
void addFTSearchSuggestion(const std::string& uiLang,
|
||||
const std::string& query);
|
||||
|
||||
std::string getJSON() const;
|
||||
|
||||
private:
|
||||
kainjow::mustache::data m_data;
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -26,13 +26,23 @@
|
||||
std::string WideToUtf8(const std::wstring& wstr);
|
||||
std::wstring Utf8ToWide(const std::string& str);
|
||||
#endif
|
||||
bool isRelativePath(const std::string& path);
|
||||
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
|
||||
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
|
||||
std::string removeLastPathElement(const std::string& path);
|
||||
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
|
||||
|
||||
unsigned int getFileSize(const std::string& path);
|
||||
std::string getFileSizeAsString(const std::string& path);
|
||||
std::string getFileContent(const std::string& path);
|
||||
bool fileExists(const std::string& path);
|
||||
bool makeDirectory(const std::string& path);
|
||||
std::string makeTmpDirectory();
|
||||
bool copyFile(const std::string& sourcePath, const std::string& destPath);
|
||||
std::string getLastPathElement(const std::string& path);
|
||||
std::string getExecutablePath(bool realPathOnly = false);
|
||||
std::string getCurrentDirectory();
|
||||
std::string getDataDirectory();
|
||||
bool writeTextFile(const std::string& path, const std::string& content);
|
||||
|
||||
std::string getMimeTypeForFile(const std::string& filename);
|
||||
#endif
|
||||
|
||||
@@ -26,5 +26,11 @@ bool matchRegex(const std::string& content, const std::string& regex);
|
||||
std::string replaceRegex(const std::string& content,
|
||||
const std::string& replacement,
|
||||
const std::string& regex);
|
||||
std::string appendToFirstOccurence(const std::string& content,
|
||||
const std::string& regex,
|
||||
const std::string& replacement);
|
||||
std::string prependToFirstOccurence(const std::string& content,
|
||||
const std::string& regex,
|
||||
const std::string& replacement);
|
||||
|
||||
#endif
|
||||
@@ -21,12 +21,10 @@
|
||||
#define KIWIX_STRINGTOOLS_H
|
||||
|
||||
#include <unicode/unistr.h>
|
||||
#include <unicode/locid.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
@@ -42,24 +40,10 @@ std::string encodeDiples(const std::string& str);
|
||||
std::string removeAccents(const std::string& text);
|
||||
void loadICUExternalTables();
|
||||
|
||||
class ICULanguageInfo
|
||||
{
|
||||
public:
|
||||
explicit ICULanguageInfo(const std::string& langCode);
|
||||
|
||||
std::string iso3Code() const;
|
||||
std::string selfName() const;
|
||||
|
||||
private:
|
||||
const icu::Locale locale;
|
||||
};
|
||||
|
||||
|
||||
/* urlEncode() is the equivalent of JS encodeURIComponent(), with the only
|
||||
* difference that the slash (/) symbol is NOT encoded. */
|
||||
std::string urlEncode(const std::string& value);
|
||||
std::string urlEncode(const std::string& value, bool encodeReserved = false);
|
||||
std::string urlDecode(const std::string& value, bool component = false);
|
||||
|
||||
std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
|
||||
std::string join(const std::vector<std::string>& list, const std::string& sep);
|
||||
|
||||
std::string ucAll(const std::string& word);
|
||||
@@ -82,19 +66,9 @@ T extractFromString(const std::string& str) {
|
||||
std::istringstream iss(str);
|
||||
T ret;
|
||||
iss >> ret;
|
||||
if(iss.fail() || !iss.eof()) {
|
||||
throw std::invalid_argument("no conversion");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string extractFromString(const std::string& str);
|
||||
|
||||
bool startsWith(const std::string& base, const std::string& start);
|
||||
|
||||
std::string stripSuffix(const std::string& str, const std::string& suffix);
|
||||
|
||||
std::vector<std::string> getTitleVariants(const std::string& title);
|
||||
} //namespace kiwix
|
||||
#endif
|
||||
32
meson.build
32
meson.build
@@ -1,20 +1,29 @@
|
||||
project('libkiwix', 'cpp',
|
||||
version : '12.0.0',
|
||||
version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
|
||||
license : 'GPLv3+',
|
||||
default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
|
||||
|
||||
compiler = meson.get_compiler('cpp')
|
||||
|
||||
static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
|
||||
wrapper = get_option('wrapper')
|
||||
|
||||
# See https://github.com/kiwix/libkiwix/issues/371
|
||||
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
|
||||
extra_libs = ['-latomic']
|
||||
static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
|
||||
if wrapper.contains('android')
|
||||
extra_libs = ['-llog']
|
||||
else
|
||||
extra_libs = []
|
||||
endif
|
||||
|
||||
if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
|
||||
if wrapper.contains('java')
|
||||
add_languages('java')
|
||||
endif
|
||||
|
||||
# See https://github.com/kiwix/libkiwix/issues/371
|
||||
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
|
||||
extra_libs += '-latomic'
|
||||
endif
|
||||
|
||||
if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
|
||||
# C++ std::thread is implemented using pthread on linux by gcc
|
||||
thread_dep = dependency('threads')
|
||||
else
|
||||
@@ -35,19 +44,19 @@ else
|
||||
error('Cannot found header mustache.hpp')
|
||||
endif
|
||||
|
||||
libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
|
||||
libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
|
||||
if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
|
||||
error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
|
||||
endif
|
||||
|
||||
|
||||
extra_cflags = ''
|
||||
if host_machine.system() == 'windows' and static_deps
|
||||
if target_machine.system() == 'windows' and static_deps
|
||||
add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
|
||||
extra_cflags += '-DCURL_STATICLIB'
|
||||
endif
|
||||
|
||||
if host_machine.system() == 'windows'
|
||||
if target_machine.system() == 'windows'
|
||||
add_project_arguments('-DNOMINMAX', language: 'cpp')
|
||||
endif
|
||||
|
||||
@@ -56,7 +65,7 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microh
|
||||
inc = include_directories('include', extra_include)
|
||||
|
||||
conf = configuration_data()
|
||||
conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
|
||||
conf.set('VERSION', '"@0@"'.format(meson.project_version()))
|
||||
|
||||
if build_machine.system() == 'windows'
|
||||
extra_link_args = ['-lshlwapi', '-lwinmm']
|
||||
@@ -69,9 +78,6 @@ subdir('scripts')
|
||||
subdir('static')
|
||||
subdir('src')
|
||||
subdir('test')
|
||||
if get_option('doc')
|
||||
subdir('docs')
|
||||
endif
|
||||
|
||||
pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
|
||||
|
||||
|
||||
@@ -1,4 +1,2 @@
|
||||
option('static-linkage', type : 'boolean', value : false,
|
||||
description : 'Link statically with the dependencies.')
|
||||
option('doc', type : 'boolean', value : false,
|
||||
description : 'Build the documentations.')
|
||||
option('wrapper', type:'array', choices:['java', 'android'], value:[],
|
||||
description: 'The wrapper to generate.')
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
# Compute 'src' path
|
||||
SCRIPT_DIR=$(dirname "$0")
|
||||
REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
|
||||
DIRS="src include"
|
||||
|
||||
# Apply formating to all *.cpp and *.h files
|
||||
cd "$REPO_DIR"
|
||||
for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
|
||||
do
|
||||
echo $FILE
|
||||
clang-format -i -style=file "$FILE"
|
||||
done
|
||||
@@ -1,148 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
'''
|
||||
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or any
|
||||
later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful, but
|
||||
WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
02110-1301, USA.
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
import json
|
||||
|
||||
def to_identifier(name):
|
||||
ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
|
||||
if ident[0].isnumeric():
|
||||
return "_"+ident
|
||||
return ident
|
||||
|
||||
def lang_code(filename):
|
||||
filename = os.path.basename(filename)
|
||||
lang = to_identifier(os.path.splitext(filename)[0])
|
||||
print(filename, '->', lang)
|
||||
return lang
|
||||
|
||||
from string import Template
|
||||
|
||||
def expand_cxx_template(t, **kwargs):
|
||||
return Template(t).substitute(**kwargs)
|
||||
|
||||
def cxx_string_literal(s):
|
||||
# Taking advantage of the fact the JSON string escape rules match
|
||||
# those of C++
|
||||
return 'u8' + json.dumps(s)
|
||||
|
||||
string_table_cxx_template = '''
|
||||
const I18nString $TABLE_NAME[] = {
|
||||
$TABLE_ENTRIES
|
||||
};
|
||||
'''
|
||||
|
||||
lang_table_entry_cxx_template = '''
|
||||
{
|
||||
$LANG_STRING_LITERAL,
|
||||
ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
|
||||
$STRING_TABLE_NAME
|
||||
}'''
|
||||
|
||||
cxxfile_template = '''// This file is automatically generated. Do not modify it.
|
||||
|
||||
#include "server/i18n.h"
|
||||
|
||||
namespace kiwix {
|
||||
namespace i18n {
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
$STRING_DATA
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
|
||||
|
||||
extern const I18nStringTable stringTables[] = {
|
||||
$LANG_TABLE
|
||||
};
|
||||
|
||||
extern const size_t langCount = $LANG_COUNT;
|
||||
|
||||
} // namespace i18n
|
||||
} // namespace kiwix
|
||||
'''
|
||||
|
||||
class Resource:
|
||||
def __init__(self, filename):
|
||||
filename = filename.strip()
|
||||
self.filename = filename
|
||||
self.lang_code = lang_code(filename)
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
self.data = f.read()
|
||||
|
||||
def get_string_table_name(self):
|
||||
return "string_table_for_" + self.lang_code
|
||||
|
||||
def get_string_table(self):
|
||||
table_entries = ",\n ".join(self.get_string_table_entries())
|
||||
return expand_cxx_template(string_table_cxx_template,
|
||||
TABLE_NAME=self.get_string_table_name(),
|
||||
TABLE_ENTRIES=table_entries)
|
||||
|
||||
def get_string_table_entries(self):
|
||||
d = json.loads(self.data)
|
||||
for k in sorted(d.keys()):
|
||||
if k != "@metadata":
|
||||
key_string = cxx_string_literal(k)
|
||||
value_string = cxx_string_literal(d[k])
|
||||
yield '{ ' + key_string + ', ' + value_string + ' }'
|
||||
|
||||
def get_lang_table_entry(self):
|
||||
return expand_cxx_template(lang_table_entry_cxx_template,
|
||||
LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
|
||||
STRING_TABLE_NAME=self.get_string_table_name())
|
||||
|
||||
|
||||
|
||||
def gen_c_file(resources):
|
||||
string_data = []
|
||||
lang_table = []
|
||||
for r in resources:
|
||||
string_data.append(r.get_string_table())
|
||||
lang_table.append(r.get_lang_table_entry())
|
||||
|
||||
return expand_cxx_template(cxxfile_template,
|
||||
STRING_DATA="\n".join(string_data),
|
||||
LANG_TABLE=",\n ".join(lang_table),
|
||||
LANG_COUNT=len(resources)
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--cxxfile',
|
||||
required=True,
|
||||
help='The Cpp file name to generate')
|
||||
parser.add_argument('i18n_resource_files', nargs='+',
|
||||
help='The list of resources to compile.')
|
||||
args = parser.parse_args()
|
||||
|
||||
resources = [Resource(filename) for filename in args.i18n_resource_files]
|
||||
|
||||
with open(args.cxxfile, 'w') as f:
|
||||
f.write(gen_c_file(resources))
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
|
||||
.SH NAME
|
||||
kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
|
||||
.SH SYNOPSIS
|
||||
\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
|
||||
.SH DESCRIPTION
|
||||
.TP
|
||||
i18n_resource_files ...
|
||||
The list of i18n resources to compile.
|
||||
.TP
|
||||
\fB\-h\fR, \fB\-\-help\fR
|
||||
show a help message and exit
|
||||
.TP
|
||||
\fB\-\-cxxfile\fR CXXFILE
|
||||
The Cpp file name to generate
|
||||
.TP
|
||||
.SH AUTHOR
|
||||
Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
@@ -52,21 +52,15 @@ resource_getter_template = """
|
||||
return RESOURCE::{identifier};
|
||||
"""
|
||||
|
||||
resource_cacheid_getter_template = """
|
||||
if (name == "{common_name}")
|
||||
return "{cacheid}";
|
||||
"""
|
||||
|
||||
resource_decl_template = """{namespaces_open}
|
||||
extern const std::string {identifier};
|
||||
{namespaces_close}"""
|
||||
|
||||
class Resource:
|
||||
def __init__(self, base_dirs, filename, cacheid=None):
|
||||
filename = filename
|
||||
def __init__(self, base_dirs, filename):
|
||||
filename = filename.strip()
|
||||
self.filename = filename
|
||||
self.identifier = full_identifier(filename)
|
||||
self.cacheid = cacheid
|
||||
found = False
|
||||
for base_dir in base_dirs:
|
||||
try:
|
||||
@@ -77,7 +71,7 @@ class Resource:
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not found:
|
||||
raise Exception("Resource not found: {}".format(filename))
|
||||
raise Exception("Impossible to found {}".format(filename))
|
||||
|
||||
def dump_impl(self):
|
||||
nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
|
||||
@@ -99,12 +93,6 @@ class Resource:
|
||||
identifier="::".join(self.identifier)
|
||||
)
|
||||
|
||||
def dump_cacheid_getter(self):
|
||||
return resource_cacheid_getter_template.format(
|
||||
common_name=self.filename,
|
||||
cacheid=self.cacheid
|
||||
)
|
||||
|
||||
def dump_decl(self):
|
||||
return resource_decl_template.format(
|
||||
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
|
||||
@@ -114,7 +102,7 @@ class Resource:
|
||||
|
||||
|
||||
|
||||
master_c_template = """//This file is automatically generated. Do not modify it.
|
||||
master_c_template = """//This file is automaically generated. Do not modify it.
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <fstream>
|
||||
@@ -135,12 +123,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
|
||||
|
||||
const std::string& getResource_{basename}(const std::string& name) {{
|
||||
{RESOURCES_GETTER}
|
||||
throw ResourceNotFound("Resource not found: " + name);
|
||||
}}
|
||||
|
||||
const char* getResourceCacheId_{basename}(const std::string& name) {{
|
||||
{RESOURCE_CACHEID_GETTER}
|
||||
return nullptr;
|
||||
throw ResourceNotFound("Resource not found.");
|
||||
}}
|
||||
|
||||
{RESOURCES}
|
||||
@@ -151,7 +134,6 @@ def gen_c_file(resources, basename):
|
||||
return master_c_template.format(
|
||||
RESOURCES="\n\n".join(r.dump_impl() for r in resources),
|
||||
RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
|
||||
RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
|
||||
include_file=basename,
|
||||
basename=to_identifier(basename)
|
||||
)
|
||||
@@ -177,10 +159,8 @@ class ResourceNotFound : public std::runtime_error {{
|
||||
}};
|
||||
|
||||
const std::string& getResource_{basename}(const std::string& name);
|
||||
const char* getResourceCacheId_{basename}(const std::string& name);
|
||||
|
||||
#define getResource(a) (getResource_{basename}(a))
|
||||
#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
|
||||
|
||||
#endif // KIWIX_{BASENAME}
|
||||
|
||||
@@ -202,17 +182,15 @@ if __name__ == "__main__":
|
||||
parser.add_argument('--source_dir',
|
||||
help="Additional directory where to look for resources.",
|
||||
action='append')
|
||||
parser.add_argument('resource_files', nargs='+',
|
||||
parser.add_argument('resource_file',
|
||||
help='The list of resources to compile.')
|
||||
args = parser.parse_args()
|
||||
|
||||
base_dir = os.path.dirname(os.path.realpath(args.resource_file))
|
||||
source_dir = args.source_dir or []
|
||||
resources = []
|
||||
for resfile in args.resource_files:
|
||||
base_dir = os.path.dirname(os.path.realpath(resfile))
|
||||
with open(resfile, 'r') as f:
|
||||
resources += [Resource([base_dir]+source_dir, *line.strip().split())
|
||||
for line in f.readlines()]
|
||||
with open(args.resource_file, 'r') as f:
|
||||
resources = [Resource([base_dir]+source_dir, filename)
|
||||
for filename in f.readlines()]
|
||||
|
||||
h_identifier = to_identifier(os.path.basename(args.hfile))
|
||||
with open(args.hfile, 'w') as f:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
.SH NAME
|
||||
kiwix-compile-resources \- helper to compile and generate some Kiwix resources
|
||||
.SH SYNOPSIS
|
||||
\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file ...\fR
|
||||
\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
|
||||
.SH DESCRIPTION
|
||||
.TP
|
||||
resource_file
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
'''
|
||||
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or any
|
||||
later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful, but
|
||||
WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
02110-1301, USA.
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os.path
|
||||
import re
|
||||
|
||||
def read_resource_file(resource_file_path):
|
||||
with open(resource_file_path, 'r') as f:
|
||||
return [line.strip() for line in f]
|
||||
|
||||
def list_resources(resource_file_path):
|
||||
for resource_path in read_resource_file(resource_file_path):
|
||||
print(resource_path)
|
||||
|
||||
def compute_resource_revision(resource_path):
|
||||
with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
|
||||
return hashlib.sha1(f.read()).hexdigest()[:8]
|
||||
|
||||
resource_revisions = {}
|
||||
|
||||
def get_resource_revision(res):
|
||||
if not res in resource_revisions:
|
||||
preprocess_resource(res)
|
||||
resource_revisions[res] = compute_resource_revision(res)
|
||||
return resource_revisions[res]
|
||||
|
||||
RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
|
||||
|
||||
def set_cacheid(resource_matchobj):
|
||||
pre = resource_matchobj.group('pre')
|
||||
resource = resource_matchobj.group('resource')
|
||||
post = resource_matchobj.group('post')
|
||||
cacheid = 'cacheid=' + get_resource_revision(resource)
|
||||
return pre + cacheid + post
|
||||
|
||||
def preprocess_text(s):
|
||||
if 'KIWIXCACHEID' in s:
|
||||
s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
|
||||
assert not 'KIWIXCACHEID' in s
|
||||
return s
|
||||
|
||||
def get_preprocessed_resource(srcpath):
|
||||
"""Get the transformed content of a resource
|
||||
|
||||
If the resource at srcpath is modified by preprocessing then this function
|
||||
returns the transformed content of the resource. Otherwise it returns None.
|
||||
"""
|
||||
try:
|
||||
with open(srcpath, 'r') as resource_file:
|
||||
content = resource_file.read()
|
||||
preprocessed_content = preprocess_text(content)
|
||||
return preprocessed_content if preprocessed_content != content else None
|
||||
except UnicodeDecodeError:
|
||||
# It was a binary resource
|
||||
return None
|
||||
|
||||
|
||||
def symlink_resource(src, resource_path):
|
||||
if os.path.exists(resource_path):
|
||||
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
|
||||
return
|
||||
os.remove(resource_path)
|
||||
os.symlink(src, resource_path)
|
||||
|
||||
def preprocess_resource(resource_path):
|
||||
print('Preprocessing', resource_path, '...')
|
||||
resource_dir = os.path.dirname(resource_path)
|
||||
if resource_dir != '':
|
||||
os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
|
||||
srcpath = os.path.join(BASE_DIR, resource_path)
|
||||
outpath = os.path.join(OUT_DIR, resource_path)
|
||||
if os.path.exists(outpath):
|
||||
os.remove(outpath)
|
||||
preprocessed_content = get_preprocessed_resource(srcpath)
|
||||
if preprocessed_content is None:
|
||||
symlink_resource(srcpath, outpath)
|
||||
else:
|
||||
with open(outpath, 'w') as target:
|
||||
print(preprocessed_content, end='', file=target)
|
||||
|
||||
|
||||
def copy_resource_list_file(src_path, dst_path):
|
||||
with open(src_path, 'r') as src:
|
||||
with open(dst_path, 'w') as dst:
|
||||
for line in src:
|
||||
res = line.strip()
|
||||
if line.startswith("skin/") and res in resource_revisions:
|
||||
dst.write(res + " " + resource_revisions[res] + "\n")
|
||||
else:
|
||||
dst.write(line)
|
||||
|
||||
def preprocess_resources(resource_file_path):
|
||||
resource_filename = os.path.basename(resource_file_path)
|
||||
for resource in read_resource_file(resource_file_path):
|
||||
if resource.startswith('skin/'):
|
||||
get_resource_revision(resource)
|
||||
else:
|
||||
preprocess_resource(resource)
|
||||
copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_mutually_exclusive_group()
|
||||
commands.add_argument('--list-all', action='store_true')
|
||||
commands.add_argument('--preprocess', action='store_true')
|
||||
parser.add_argument('--outdir')
|
||||
parser.add_argument('resource_file')
|
||||
args = parser.parse_args()
|
||||
BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
|
||||
OUT_DIR = args.outdir
|
||||
|
||||
if args.list_all:
|
||||
list_resources(args.resource_file)
|
||||
elif args.preprocess:
|
||||
preprocess_resources(args.resource_file)
|
||||
@@ -1,13 +1,6 @@
|
||||
|
||||
res_manager = find_program('kiwix-resources')
|
||||
res_compiler = find_program('kiwix-compile-resources')
|
||||
|
||||
install_data(res_compiler.path(), install_dir:get_option('bindir'))
|
||||
|
||||
install_man('kiwix-compile-resources.1')
|
||||
|
||||
i18n_compiler = find_program('kiwix-compile-i18n')
|
||||
|
||||
install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
|
||||
|
||||
install_man('kiwix-compile-i18n.1')
|
||||
|
||||
@@ -3,15 +3,13 @@
|
||||
#include "aria2.h"
|
||||
#include "xmlrpc.h"
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
#include "tools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "downloader.h" // For AriaError
|
||||
#include <tools/otherTools.h>
|
||||
#include <tools/pathTools.h>
|
||||
#include <tools/stringTools.h>
|
||||
#include <downloader.h> // For AriaError
|
||||
|
||||
#ifdef _WIN32
|
||||
# define ARIA2_CMD "aria2c.exe"
|
||||
@@ -24,7 +22,7 @@
|
||||
#define LOG_ARIA_ERROR() \
|
||||
{ \
|
||||
std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
|
||||
std::cerr << (curlErrorBuffer[0] ? curlErrorBuffer : curl_easy_strerror(res)) << std::endl; \
|
||||
std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
|
||||
}
|
||||
|
||||
namespace kiwix {
|
||||
@@ -32,7 +30,9 @@ namespace kiwix {
|
||||
Aria2::Aria2():
|
||||
mp_aria(nullptr),
|
||||
m_port(42042),
|
||||
m_secret(getNewRpcSecret())
|
||||
m_secret("kiwixariarpc"),
|
||||
m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
|
||||
mp_curl(nullptr)
|
||||
{
|
||||
m_downloadDir = getDataDirectory();
|
||||
makeDirectory(m_downloadDir);
|
||||
@@ -68,7 +68,7 @@ Aria2::Aria2():
|
||||
callCmd.push_back(rpc_secret.c_str());
|
||||
callCmd.push_back(rpc_port.c_str());
|
||||
callCmd.push_back(download_dir.c_str());
|
||||
if (fileReadable(session_file)) {
|
||||
if (fileExists(session_file)) {
|
||||
callCmd.push_back(inputFile.c_str());
|
||||
}
|
||||
callCmd.push_back(session.c_str());
|
||||
@@ -89,32 +89,36 @@ Aria2::Aria2():
|
||||
launchCmd.append(cmd).append(" ");
|
||||
}
|
||||
mp_aria = Subprocess::run(callCmd);
|
||||
mp_curl = curl_easy_init();
|
||||
|
||||
CURL* p_curl = curl_easy_init();
|
||||
char curlErrorBuffer[CURL_ERROR_SIZE];
|
||||
|
||||
curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
|
||||
curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
|
||||
curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
|
||||
curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
|
||||
|
||||
int watchdog = 50;
|
||||
while(--watchdog) {
|
||||
sleep(10);
|
||||
curlErrorBuffer[0] = 0;
|
||||
auto res = curl_easy_perform(p_curl);
|
||||
m_curlErrorBuffer[0] = 0;
|
||||
auto res = curl_easy_perform(mp_curl);
|
||||
if (res == CURLE_OK) {
|
||||
break;
|
||||
} else if (watchdog == 1) {
|
||||
LOG_ARIA_ERROR();
|
||||
}
|
||||
}
|
||||
curl_easy_cleanup(p_curl);
|
||||
if (!watchdog) {
|
||||
curl_easy_cleanup(mp_curl);
|
||||
throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
|
||||
}
|
||||
}
|
||||
|
||||
Aria2::~Aria2()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
curl_easy_cleanup(mp_curl);
|
||||
}
|
||||
|
||||
void Aria2::close()
|
||||
{
|
||||
saveSession();
|
||||
@@ -134,25 +138,20 @@ std::string Aria2::doRequest(const MethodCall& methodCall)
|
||||
std::stringstream outStream;
|
||||
CURLcode res;
|
||||
long response_code;
|
||||
char curlErrorBuffer[CURL_ERROR_SIZE];
|
||||
CURL* p_curl = curl_easy_init();
|
||||
curl_easy_setopt(p_curl, CURLOPT_URL, "http://localhost/rpc");
|
||||
curl_easy_setopt(p_curl, CURLOPT_PORT, m_port);
|
||||
curl_easy_setopt(p_curl, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(p_curl, CURLOPT_ERRORBUFFER, curlErrorBuffer);
|
||||
curl_easy_setopt(p_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
|
||||
curl_easy_setopt(p_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
|
||||
curl_easy_setopt(p_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
|
||||
curl_easy_setopt(p_curl, CURLOPT_WRITEDATA, &outStream);
|
||||
curlErrorBuffer[0] = 0;
|
||||
res = curl_easy_perform(p_curl);
|
||||
if (res != CURLE_OK) {
|
||||
LOG_ARIA_ERROR();
|
||||
curl_easy_cleanup(p_curl);
|
||||
throw std::runtime_error("Cannot perform request");
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
|
||||
m_curlErrorBuffer[0] = 0;
|
||||
res = curl_easy_perform(mp_curl);
|
||||
if (res != CURLE_OK) {
|
||||
LOG_ARIA_ERROR();
|
||||
throw std::runtime_error("Cannot perform request");
|
||||
}
|
||||
curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
}
|
||||
curl_easy_getinfo(p_curl, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_cleanup(p_curl);
|
||||
|
||||
auto responseContent = outStream.str();
|
||||
if (response_code != 200) {
|
||||
@@ -196,13 +195,6 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
|
||||
return doRequest(methodCall);
|
||||
}
|
||||
|
||||
std::string Aria2::getNewRpcSecret()
|
||||
{
|
||||
std::string uuid = gen_uuid("");
|
||||
uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
|
||||
return uuid.substr(0, 9);
|
||||
}
|
||||
|
||||
std::vector<std::string> Aria2::tellActive()
|
||||
{
|
||||
MethodCall methodCall("aria2.tellActive", m_secret);
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "xmlrpc.h"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <curl/curl.h>
|
||||
|
||||
namespace kiwix {
|
||||
@@ -23,16 +24,19 @@ class Aria2
|
||||
int m_port;
|
||||
std::string m_secret;
|
||||
std::string m_downloadDir;
|
||||
std::unique_ptr<char[]> m_curlErrorBuffer;
|
||||
CURL* mp_curl;
|
||||
std::mutex m_lock;
|
||||
|
||||
std::string doRequest(const MethodCall& methodCall);
|
||||
|
||||
public:
|
||||
Aria2();
|
||||
virtual ~Aria2() = default;
|
||||
virtual ~Aria2();
|
||||
void close();
|
||||
|
||||
std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
|
||||
std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
|
||||
static std::string getNewRpcSecret();
|
||||
std::vector<std::string> tellActive();
|
||||
std::vector<std::string> tellWaiting();
|
||||
void saveSession();
|
||||
|
||||
168
src/book.cpp
168
src/book.cpp
@@ -18,18 +18,13 @@
|
||||
*/
|
||||
|
||||
#include "book.h"
|
||||
#include "reader.h"
|
||||
|
||||
#include "tools.h"
|
||||
#include "tools/base64.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "tools/networkTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/archiveTools.h"
|
||||
|
||||
#include <zim/archive.h>
|
||||
#include <zim/item.h>
|
||||
#include <pugixml.hpp>
|
||||
|
||||
namespace kiwix
|
||||
@@ -40,17 +35,11 @@ Book::Book() :
|
||||
m_readOnly(false)
|
||||
{
|
||||
}
|
||||
|
||||
/* Destructor */
|
||||
Book::~Book()
|
||||
{
|
||||
}
|
||||
|
||||
Book::Illustrations Book::getIllustrations() const
|
||||
{
|
||||
return m_illustrations;
|
||||
}
|
||||
|
||||
bool Book::update(const kiwix::Book& other)
|
||||
{
|
||||
if (m_readOnly)
|
||||
@@ -59,38 +48,55 @@ bool Book::update(const kiwix::Book& other)
|
||||
if (m_id != other.m_id)
|
||||
return false;
|
||||
|
||||
*this = other;
|
||||
m_readOnly = other.m_readOnly;
|
||||
m_path = other.m_path;
|
||||
m_pathValid = other.m_pathValid;
|
||||
m_title = other.m_title;
|
||||
m_description = other.m_description;
|
||||
m_language = other.m_language;
|
||||
m_creator = other.m_creator;
|
||||
m_publisher = other.m_publisher;
|
||||
m_date = other.m_date;
|
||||
m_url = other.m_url;
|
||||
m_name = other.m_name;
|
||||
m_flavour = other.m_flavour;
|
||||
m_tags = other.m_tags;
|
||||
m_category = other.m_category;
|
||||
m_origId = other.m_origId;
|
||||
m_articleCount = other.m_articleCount;
|
||||
m_mediaCount = other.m_mediaCount;
|
||||
m_size = other.m_size;
|
||||
m_favicon = other.m_favicon;
|
||||
m_faviconMimeType = other.m_faviconMimeType;
|
||||
m_faviconUrl = other.m_faviconUrl;
|
||||
|
||||
m_downloadId = other.m_downloadId;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Book::update(const zim::Archive& archive) {
|
||||
m_path = archive.getFilename();
|
||||
void Book::update(const kiwix::Reader& reader)
|
||||
{
|
||||
m_path = reader.getZimFilePath();
|
||||
m_pathValid = true;
|
||||
m_id = std::string(archive.getUuid());
|
||||
m_title = getArchiveTitle(archive);
|
||||
m_description = getMetaDescription(archive);
|
||||
m_language = getMetaLanguage(archive);
|
||||
m_creator = getMetaCreator(archive);
|
||||
m_publisher = getMetaPublisher(archive);
|
||||
m_date = getMetaDate(archive);
|
||||
m_name = getMetaName(archive);
|
||||
m_flavour = getMetaFlavour(archive);
|
||||
m_tags = getMetaTags(archive);
|
||||
m_id = reader.getId();
|
||||
m_title = reader.getTitle();
|
||||
m_description = reader.getDescription();
|
||||
m_language = reader.getLanguage();
|
||||
m_creator = reader.getCreator();
|
||||
m_publisher = reader.getPublisher();
|
||||
m_date = reader.getDate();
|
||||
m_name = reader.getName();
|
||||
m_flavour = reader.getFlavour();
|
||||
m_tags = reader.getTags();
|
||||
m_category = getCategoryFromTags();
|
||||
m_articleCount = archive.getArticleCount();
|
||||
m_mediaCount = archive.getMediaCount();
|
||||
m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
|
||||
m_origId = reader.getOrigId();
|
||||
m_articleCount = reader.getArticleCount();
|
||||
m_mediaCount = reader.getMediaCount();
|
||||
m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
|
||||
m_pathValid = true;
|
||||
|
||||
m_illustrations.clear();
|
||||
for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
|
||||
const auto illustration = std::make_shared<Illustration>();
|
||||
const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
|
||||
illustration->width = illustration->height = illustrationSize;
|
||||
illustration->mimeType = illustrationItem.getMimetype();
|
||||
illustration->data = illustrationItem.getData();
|
||||
// NOTE: illustration->url is left uninitialized
|
||||
m_illustrations.push_back(illustration);
|
||||
}
|
||||
reader.getFavicon(m_favicon, m_faviconMimeType);
|
||||
}
|
||||
|
||||
#define ATTR(name) node.attribute(name).value()
|
||||
@@ -102,7 +108,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
|
||||
path = computeAbsolutePath(baseDir, path);
|
||||
}
|
||||
m_path = path;
|
||||
m_pathValid = fileReadable(path);
|
||||
m_pathValid = fileExists(path);
|
||||
m_title = ATTR("title");
|
||||
m_description = ATTR("description");
|
||||
m_language = ATTR("language");
|
||||
@@ -117,14 +123,9 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
|
||||
m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
|
||||
m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
|
||||
m_size = strtoull(ATTR("size"), 0, 0) << 10;
|
||||
std::string favicon_mimetype = ATTR("faviconMimeType");
|
||||
if (! favicon_mimetype.empty()) {
|
||||
const auto favicon = std::make_shared<Illustration>();
|
||||
favicon->data = base64_decode(ATTR("favicon"));
|
||||
favicon->mimeType = favicon_mimetype;
|
||||
favicon->url = ATTR("faviconUrl");
|
||||
m_illustrations.assign(1, favicon);
|
||||
}
|
||||
m_favicon = base64_decode(ATTR("favicon"));
|
||||
m_faviconMimeType = ATTR("faviconMimeType");
|
||||
m_faviconUrl = ATTR("faviconUrl");
|
||||
try {
|
||||
m_downloadId = ATTR("downloadId");
|
||||
} catch(...) {}
|
||||
@@ -155,9 +156,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
|
||||
m_language = VALUE("language");
|
||||
m_creator = node.child("author").child("name").child_value();
|
||||
m_publisher = node.child("publisher").child("name").child_value();
|
||||
const std::string dcIssuedDate = VALUE("dc:issued");
|
||||
m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
|
||||
m_date = fromOpdsDate(m_date);
|
||||
m_date = fromOpdsDate(VALUE("updated"));
|
||||
m_name = VALUE("name");
|
||||
m_flavour = VALUE("flavour");
|
||||
m_tags = VALUE("tags");
|
||||
@@ -174,11 +173,8 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
|
||||
m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
|
||||
}
|
||||
if (rel == "http://opds-spec.org/image/thumbnail") {
|
||||
const auto favicon = std::make_shared<Illustration>();
|
||||
favicon->data.clear();
|
||||
favicon->url = urlHost + linkNode.attribute("href").value();
|
||||
favicon->mimeType = linkNode.attribute("type").value();
|
||||
m_illustrations.assign(1, favicon);
|
||||
m_faviconUrl = urlHost + linkNode.attribute("href").value();
|
||||
m_faviconMimeType = linkNode.attribute("type").value();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,7 +185,7 @@ std::string Book::getHumanReadableIdFromPath() const
|
||||
{
|
||||
std::string id = m_path;
|
||||
if (!id.empty()) {
|
||||
id = kiwix::removeAccents(id);
|
||||
kiwix::removeAccents(id);
|
||||
|
||||
#ifdef _WIN32
|
||||
id = replaceRegex(id, "", "^.*\\\\");
|
||||
@@ -211,54 +207,15 @@ void Book::setPath(const std::string& path)
|
||||
: path;
|
||||
}
|
||||
|
||||
const Book::Illustration Book::missingDefaultIllustration;
|
||||
|
||||
std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
|
||||
{
|
||||
for ( const auto& ilPtr : m_illustrations ) {
|
||||
if (ilPtr->width == size && ilPtr->height == size) {
|
||||
return ilPtr;
|
||||
}
|
||||
}
|
||||
throw std::runtime_error("Cannot find illustration");
|
||||
}
|
||||
|
||||
const Book::Illustration& Book::getDefaultIllustration() const
|
||||
{
|
||||
try {
|
||||
return *getIllustration(48);
|
||||
} catch (...) {
|
||||
return missingDefaultIllustration;
|
||||
}
|
||||
}
|
||||
|
||||
const std::string& Book::Illustration::getData() const
|
||||
{
|
||||
if (data.empty() && !url.empty()) {
|
||||
const std::lock_guard<std::mutex> l(mutex);
|
||||
if ( data.empty() ) {
|
||||
try {
|
||||
data = download(url);
|
||||
} catch(...) {
|
||||
std::cerr << "Cannot download favicon from " << url;
|
||||
}
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
const std::string& Book::getFavicon() const {
|
||||
return getDefaultIllustration().getData();
|
||||
}
|
||||
|
||||
const std::string& Book::getFaviconUrl() const
|
||||
{
|
||||
return getDefaultIllustration().url;
|
||||
}
|
||||
|
||||
const std::string& Book::getFaviconMimeType() const
|
||||
{
|
||||
return getDefaultIllustration().mimeType;
|
||||
if (m_favicon.empty() && !m_faviconUrl.empty()) {
|
||||
try {
|
||||
m_favicon = download(m_faviconUrl);
|
||||
} catch(...) {
|
||||
std::cerr << "Cannot download favicon from " << m_faviconUrl;
|
||||
}
|
||||
}
|
||||
return m_favicon;
|
||||
}
|
||||
|
||||
std::string Book::getTagStr(const std::string& tagName) const {
|
||||
@@ -286,9 +243,4 @@ std::string Book::getCategoryFromTags() const
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<std::string> Book::getLanguages() const
|
||||
{
|
||||
return kiwix::split(m_language, ",");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
|
||||
#mesondefine LIBKIWIX_VERSION
|
||||
#mesondefine VERSION
|
||||
|
||||
|
||||
@@ -127,24 +127,22 @@ void Download::cancelDownload()
|
||||
Downloader::Downloader() :
|
||||
mp_aria(new Aria2())
|
||||
{
|
||||
try {
|
||||
for (auto gid : mp_aria->tellWaiting()) {
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads[gid]->updateStatus(false);
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
|
||||
}
|
||||
try {
|
||||
for (auto gid : mp_aria->tellActive()) {
|
||||
if( m_knownDownloads.find(gid) == m_knownDownloads.end()) {
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads[gid]->updateStatus(false);
|
||||
}
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads[gid]->updateStatus();
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
|
||||
}
|
||||
try {
|
||||
for (auto gid : mp_aria->tellWaiting()) {
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads[gid]->updateStatus();
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
/* Destructor */
|
||||
@@ -157,8 +155,7 @@ void Downloader::close()
|
||||
mp_aria->close();
|
||||
}
|
||||
|
||||
std::vector<std::string> Downloader::getDownloadIds() const {
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
std::vector<std::string> Downloader::getDownloadIds() {
|
||||
std::vector<std::string> ret;
|
||||
for(auto& p:m_knownDownloads) {
|
||||
ret.push_back(p.first);
|
||||
@@ -166,46 +163,42 @@ std::vector<std::string> Downloader::getDownloadIds() const {
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::shared_ptr<Download> Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
|
||||
Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
for (auto& p: m_knownDownloads) {
|
||||
auto& d = p.second;
|
||||
auto& uris = d->getUris();
|
||||
if (std::find(uris.begin(), uris.end(), uri) != uris.end())
|
||||
return d;
|
||||
return d.get();
|
||||
}
|
||||
std::vector<std::string> uris = {uri};
|
||||
auto gid = mp_aria->addUri(uris, options);
|
||||
m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
|
||||
return m_knownDownloads[gid];
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
return m_knownDownloads[gid].get();
|
||||
}
|
||||
|
||||
std::shared_ptr<Download> Downloader::getDownload(const std::string& did)
|
||||
Download* Downloader::getDownload(const std::string& did)
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
try {
|
||||
return m_knownDownloads.at(did);
|
||||
m_knownDownloads.at(did).get()->updateStatus(true);
|
||||
return m_knownDownloads.at(did).get();
|
||||
} catch(std::exception& e) {
|
||||
for (auto gid : mp_aria->tellWaiting()) {
|
||||
if (gid == did) {
|
||||
m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
|
||||
return m_knownDownloads[gid];
|
||||
}
|
||||
}
|
||||
for (auto gid : mp_aria->tellActive()) {
|
||||
if (gid == did) {
|
||||
m_knownDownloads[gid] = std::make_shared<Download>(mp_aria, gid);
|
||||
return m_knownDownloads[gid];
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads.at(gid).get()->updateStatus(true);
|
||||
return m_knownDownloads[gid].get();
|
||||
}
|
||||
}
|
||||
for (auto gid : mp_aria->tellWaiting()) {
|
||||
if (gid == did) {
|
||||
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
|
||||
m_knownDownloads.at(gid).get()->updateStatus(true);
|
||||
return m_knownDownloads[gid].get();
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
size_t Downloader::getNbDownload() const {
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
return m_knownDownloads.size();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
73
src/entry.cpp
Normal file
73
src/entry.cpp
Normal file
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2018-2020 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "reader.h"
|
||||
#include <time.h>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
Entry::Entry(zim::Entry entry)
|
||||
: entry(entry)
|
||||
{
|
||||
}
|
||||
|
||||
size_type Entry::getSize() const
|
||||
{
|
||||
if (entry.isRedirect()) {
|
||||
return 0;
|
||||
} else {
|
||||
return entry.getItem().getSize();
|
||||
}
|
||||
}
|
||||
|
||||
std::string Entry::getMimetype() const
|
||||
{
|
||||
return entry.getItem(true).getMimetype();
|
||||
}
|
||||
|
||||
bool Entry::isRedirect() const
|
||||
{
|
||||
return entry.isRedirect();
|
||||
}
|
||||
|
||||
Entry Entry::getRedirectEntry() const
|
||||
{
|
||||
if ( !entry.isRedirect() ) {
|
||||
throw NoEntry();
|
||||
}
|
||||
|
||||
return entry.getRedirectEntry();
|
||||
}
|
||||
|
||||
Entry Entry::getFinalEntry() const
|
||||
{
|
||||
int loopCounter = 42;
|
||||
auto final_entry = entry;
|
||||
while (final_entry.isRedirect() && loopCounter--) {
|
||||
final_entry = final_entry.getRedirectEntry();
|
||||
}
|
||||
// Prevent infinite loops.
|
||||
if (final_entry.isRedirect()) {
|
||||
throw NoEntry();
|
||||
}
|
||||
return final_entry;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
#include "html_dumper.h"
|
||||
#include "libkiwix-resources.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "server/i18n.h"
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/* Constructor */
|
||||
HTMLDumper::HTMLDumper(const Library* library, const NameMapper* nameMapper)
|
||||
: LibraryDumper(library, nameMapper)
|
||||
{
|
||||
}
|
||||
/* Destructor */
|
||||
HTMLDumper::~HTMLDumper()
|
||||
{
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
std::string humanFriendlyTitle(std::string title)
|
||||
{
|
||||
std::string humanFriendlyString = replaceRegex(title, "_", " ");
|
||||
humanFriendlyString[0] = toupper(humanFriendlyString[0]);
|
||||
return humanFriendlyString;
|
||||
}
|
||||
|
||||
kainjow::mustache::list getTagList(std::string tags)
|
||||
{
|
||||
const auto tagsList = kiwix::split(tags, ";", true, false);
|
||||
kainjow::mustache::list finalTagList;
|
||||
for (auto tag : tagsList) {
|
||||
if (tag[0] != '_')
|
||||
finalTagList.push_back(kainjow::mustache::object{
|
||||
{"tag", tag}
|
||||
});
|
||||
}
|
||||
return finalTagList;
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::string HTMLDumper::dumpPlainHTML(kiwix::Filter filter) const
|
||||
{
|
||||
kainjow::mustache::list booksData;
|
||||
const auto filteredBooks = library->filter(filter);
|
||||
const auto searchQuery = filter.getQuery();
|
||||
auto languages = getLanguageData();
|
||||
auto categories = getCategoryData();
|
||||
|
||||
for (auto &category : categories) {
|
||||
const auto categoryName = category.get("name")->string_value();
|
||||
if (categoryName == filter.getCategory()) {
|
||||
category["selected"] = true;
|
||||
}
|
||||
category["hf_name"] = humanFriendlyTitle(categoryName);
|
||||
}
|
||||
|
||||
for (auto &language : languages) {
|
||||
if (language.get("lang_code")->string_value() == filter.getLang()) {
|
||||
language["selected"] = true;
|
||||
}
|
||||
}
|
||||
|
||||
for ( const auto& bookId : filteredBooks ) {
|
||||
const auto bookObj = library->getBookById(bookId);
|
||||
const auto bookTitle = bookObj.getTitle();
|
||||
std::string contentId = "";
|
||||
try {
|
||||
contentId = urlEncode(nameMapper->getNameForId(bookId));
|
||||
} catch (...) {}
|
||||
const auto bookDescription = bookObj.getDescription();
|
||||
const auto langCode = bookObj.getCommaSeparatedLanguages();
|
||||
const auto bookIconUrl = rootLocation + "/catalog/v2/illustration/" + bookId + "/?size=48";
|
||||
const auto tags = bookObj.getTags();
|
||||
const auto downloadAvailable = (bookObj.getUrl() != "");
|
||||
std::string faviconAttr = "style=background-image:url(" + bookIconUrl + ")";
|
||||
|
||||
booksData.push_back(kainjow::mustache::object{
|
||||
{"id", contentId},
|
||||
{"title", bookTitle},
|
||||
{"description", bookDescription},
|
||||
{"langCode", langCode},
|
||||
{"faviconAttr", faviconAttr},
|
||||
{"tagList", getTagList(tags)},
|
||||
{"downloadAvailable", downloadAvailable}
|
||||
});
|
||||
}
|
||||
|
||||
auto getTranslation = i18n::GetTranslatedStringWithMsgId(m_userLang);
|
||||
|
||||
const auto translations = kainjow::mustache::object{
|
||||
getTranslation("search"),
|
||||
getTranslation("download"),
|
||||
getTranslation("count-of-matching-books", {{"COUNT", to_string(filteredBooks.size())}}),
|
||||
getTranslation("book-filtering-all-categories"),
|
||||
getTranslation("book-filtering-all-languages"),
|
||||
getTranslation("powered-by-kiwix-html"),
|
||||
getTranslation("welcome-to-kiwix-server"),
|
||||
getTranslation("preview-book"),
|
||||
getTranslation("welcome-page-overzealous-filter", {{"URL", "?lang="}})
|
||||
};
|
||||
|
||||
return render_template(
|
||||
RESOURCE::templates::no_js_library_page_html,
|
||||
kainjow::mustache::object{
|
||||
{"root", rootLocation},
|
||||
{"books", booksData },
|
||||
{"searchQuery", searchQuery},
|
||||
{"languages", languages},
|
||||
{"categories", categories},
|
||||
{"noResults", filteredBooks.size() == 0},
|
||||
{"translations", translations}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -9,7 +9,6 @@
|
||||
# include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "tools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
|
||||
477
src/library.cpp
477
src/library.cpp
@@ -19,20 +19,17 @@
|
||||
|
||||
#include "library.h"
|
||||
#include "book.h"
|
||||
#include "reader.h"
|
||||
#include "libxml_dumper.h"
|
||||
|
||||
#include "tools.h"
|
||||
#include "tools/base64.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/concurrent_cache.h"
|
||||
|
||||
#include <pugixml.hpp>
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
#include <cmath>
|
||||
#include <unicode/locid.h>
|
||||
#include <xapian.h>
|
||||
|
||||
@@ -51,152 +48,54 @@ std::string normalizeText(const std::string& text)
|
||||
return removeAccents(text);
|
||||
}
|
||||
|
||||
bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
|
||||
{
|
||||
return book1.isPathValid()
|
||||
&& book2.isPathValid()
|
||||
&& book1.getPath() == book2.getPath();
|
||||
}
|
||||
|
||||
template<typename Key, typename Value>
|
||||
class MultiKeyCache: public ConcurrentCache<std::set<Key>, Value>
|
||||
{
|
||||
public:
|
||||
explicit MultiKeyCache(size_t maxEntries)
|
||||
: ConcurrentCache<std::set<Key>, Value>(maxEntries)
|
||||
{}
|
||||
|
||||
bool drop(const Key& key)
|
||||
{
|
||||
std::unique_lock<std::mutex> l(this->lock_);
|
||||
bool removed = false;
|
||||
for(auto& cache_key: this->impl_.keys()) {
|
||||
if(cache_key.find(key)!=cache_key.end()) {
|
||||
removed |= this->impl_.drop(cache_key);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
};
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
struct Library::Impl
|
||||
class Library::BookDB : public Xapian::WritableDatabase
|
||||
{
|
||||
struct Entry : Book
|
||||
{
|
||||
Library::Revision lastUpdatedRevision = 0;
|
||||
};
|
||||
|
||||
Library::Revision m_revision;
|
||||
std::map<std::string, Entry> m_books;
|
||||
using ArchiveCache = ConcurrentCache<std::string, std::shared_ptr<zim::Archive>>;
|
||||
std::unique_ptr<ArchiveCache> mp_archiveCache;
|
||||
using SearcherCache = MultiKeyCache<std::string, std::shared_ptr<ZimSearcher>>;
|
||||
std::unique_ptr<SearcherCache> mp_searcherCache;
|
||||
std::vector<kiwix::Bookmark> m_bookmarks;
|
||||
Xapian::WritableDatabase m_bookDB;
|
||||
|
||||
unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
|
||||
|
||||
Impl();
|
||||
~Impl();
|
||||
|
||||
Impl(Impl&& );
|
||||
Impl& operator=(Impl&& );
|
||||
public:
|
||||
BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
|
||||
};
|
||||
|
||||
Library::Impl::Impl()
|
||||
: mp_archiveCache(new ArchiveCache(std::max(getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
|
||||
mp_searcherCache(new SearcherCache(std::max(getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
|
||||
m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
|
||||
{
|
||||
}
|
||||
|
||||
Library::Impl::~Impl()
|
||||
{
|
||||
}
|
||||
|
||||
Library::Impl::Impl(Library::Impl&& ) = default;
|
||||
Library::Impl& Library::Impl::operator=(Library::Impl&& ) = default;
|
||||
|
||||
unsigned int
|
||||
Library::Impl::getBookCount(const bool localBooks, const bool remoteBooks) const
|
||||
{
|
||||
unsigned int result = 0;
|
||||
for (auto& pair: m_books) {
|
||||
auto& book = pair.second;
|
||||
if ((!book.getPath().empty() && localBooks)
|
||||
|| (!book.getUrl().empty() && remoteBooks)) {
|
||||
result++;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Constructor */
|
||||
Library::Library()
|
||||
: mp_impl(new Library::Impl)
|
||||
: m_bookDB(new BookDB)
|
||||
{
|
||||
}
|
||||
|
||||
Library::Library(Library&& other)
|
||||
: mp_impl(std::move(other.mp_impl))
|
||||
{
|
||||
}
|
||||
Library::Library(Library&& ) = default;
|
||||
|
||||
Library& Library::operator=(Library&& other)
|
||||
{
|
||||
mp_impl = std::move(other.mp_impl);
|
||||
return *this;
|
||||
}
|
||||
Library& Library::operator=(Library&& ) = default;
|
||||
|
||||
/* Destructor */
|
||||
Library::~Library() = default;
|
||||
Library::~Library()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
bool Library::addBook(const Book& book)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
++mp_impl->m_revision;
|
||||
/* Try to find it */
|
||||
updateBookDB(book);
|
||||
try {
|
||||
auto& oldbook = mp_impl->m_books.at(book.getId());
|
||||
if ( ! booksReferToTheSameArchive(oldbook, book) ) {
|
||||
dropCache(book.getId());
|
||||
}
|
||||
oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
|
||||
// XXX: Then m_bookDB will become out-of-sync with
|
||||
// XXX: the real contents of the library.
|
||||
oldbook.lastUpdatedRevision = mp_impl->m_revision;
|
||||
auto& oldbook = m_books.at(book.getId());
|
||||
oldbook.update(book);
|
||||
return false;
|
||||
} catch (std::out_of_range&) {
|
||||
auto& newEntry = mp_impl->m_books[book.getId()];
|
||||
static_cast<Book&>(newEntry) = book;
|
||||
newEntry.lastUpdatedRevision = mp_impl->m_revision;
|
||||
size_t new_cache_size = static_cast<size_t>(std::ceil(mp_impl->getBookCount(true, true)*0.1));
|
||||
if (getEnvVar<int>("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
|
||||
mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
|
||||
}
|
||||
if (getEnvVar<int>("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
|
||||
mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
|
||||
}
|
||||
m_books[book.getId()] = book;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void Library::addBookmark(const Bookmark& bookmark)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
mp_impl->m_bookmarks.push_back(bookmark);
|
||||
m_bookmarks.push_back(bookmark);
|
||||
}
|
||||
|
||||
bool Library::removeBookmark(const std::string& zimId, const std::string& url)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
for(auto it=mp_impl->m_bookmarks.begin(); it!=mp_impl->m_bookmarks.end(); it++) {
|
||||
for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
|
||||
if (it->getBookId() == zimId && it->getUrl() == url) {
|
||||
mp_impl->m_bookmarks.erase(it);
|
||||
m_bookmarks.erase(it);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -204,74 +103,28 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
|
||||
}
|
||||
|
||||
|
||||
void Library::dropCache(const std::string& id)
|
||||
{
|
||||
mp_impl->mp_archiveCache->drop(id);
|
||||
mp_impl->mp_searcherCache->drop(id);
|
||||
}
|
||||
|
||||
bool Library::removeBookById(const std::string& id)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
mp_impl->m_bookDB.delete_document("Q" + id);
|
||||
dropCache(id);
|
||||
// We do not change the cache size here
|
||||
// Most of the time, the book is remove in case of library refresh, it is
|
||||
// often associated with addBook calls (which will properly set the cache size)
|
||||
// Having a too big cache is not a problem here (or it would have been before)
|
||||
// (And setMaxSize doesn't actually reduce the cache size, extra cached items
|
||||
// will be removed in put or getOrPut).
|
||||
const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
|
||||
if ( bookWasRemoved ) {
|
||||
++mp_impl->m_revision;
|
||||
}
|
||||
return bookWasRemoved;
|
||||
}
|
||||
|
||||
Library::Revision Library::getRevision() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
return mp_impl->m_revision;
|
||||
}
|
||||
|
||||
uint32_t Library::removeBooksNotUpdatedSince(Revision libraryRevision)
|
||||
{
|
||||
BookIdCollection booksToRemove;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
for ( const auto& entry : mp_impl->m_books) {
|
||||
if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
|
||||
booksToRemove.push_back(entry.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t countOfRemovedBooks = 0;
|
||||
for ( const auto& id : booksToRemove ) {
|
||||
if ( removeBookById(id) )
|
||||
++countOfRemovedBooks;
|
||||
}
|
||||
return countOfRemovedBooks;
|
||||
m_bookDB->delete_document("Q" + id);
|
||||
m_readers.erase(id);
|
||||
return m_books.erase(id) == 1;
|
||||
}
|
||||
|
||||
const Book& Library::getBookById(const std::string& id) const
|
||||
{
|
||||
// XXX: Doesn't make sense to lock this operation since it cannot
|
||||
// XXX: guarantee thread-safety because of its return type
|
||||
return mp_impl->m_books.at(id);
|
||||
return m_books.at(id);
|
||||
}
|
||||
|
||||
Book Library::getBookByIdThreadSafe(const std::string& id) const
|
||||
Book& Library::getBookById(const std::string& id)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
return getBookById(id);
|
||||
const Library& const_self = *this;
|
||||
return const_cast<Book&>(const_self.getBookById(id));
|
||||
}
|
||||
|
||||
const Book& Library::getBookByPath(const std::string& path) const
|
||||
{
|
||||
// XXX: Doesn't make sense to lock this operation since it cannot
|
||||
// XXX: guarantee thread-safety because of its return type
|
||||
for(auto& it: mp_impl->m_books) {
|
||||
for(auto& it: m_books) {
|
||||
auto& book = it.second;
|
||||
if (book.getPath() == path)
|
||||
return book;
|
||||
@@ -281,127 +134,78 @@ const Book& Library::getBookByPath(const std::string& path) const
|
||||
throw std::out_of_range(ss.str());
|
||||
}
|
||||
|
||||
std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
|
||||
Book& Library::getBookByPath(const std::string& path)
|
||||
{
|
||||
try {
|
||||
return mp_impl->mp_archiveCache->getOrPut(id,
|
||||
[&](){
|
||||
auto book = getBookById(id);
|
||||
if (!book.isPathValid()) {
|
||||
throw std::invalid_argument("");
|
||||
}
|
||||
return std::make_shared<zim::Archive>(book.getPath());
|
||||
});
|
||||
} catch (std::invalid_argument&) {
|
||||
return nullptr;
|
||||
}
|
||||
const Library& const_self = *this;
|
||||
return const_cast<Book&>(const_self.getBookByPath(path));
|
||||
}
|
||||
|
||||
std::shared_ptr<ZimSearcher> Library::getSearcherByIds(const BookIdSet& ids)
|
||||
std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
|
||||
{
|
||||
assert(!ids.empty());
|
||||
try {
|
||||
return mp_impl->mp_searcherCache->getOrPut(ids,
|
||||
[&](){
|
||||
std::vector<zim::Archive> archives;
|
||||
for(auto& id:ids) {
|
||||
auto archive = getArchiveById(id);
|
||||
if(!archive) {
|
||||
throw std::invalid_argument("");
|
||||
}
|
||||
archives.push_back(*archive);
|
||||
}
|
||||
return std::make_shared<ZimSearcher>(zim::Searcher(archives));
|
||||
});
|
||||
} catch (std::invalid_argument&) {
|
||||
return m_readers.at(id);
|
||||
} catch (std::out_of_range& e) {}
|
||||
|
||||
auto book = getBookById(id);
|
||||
if (!book.isPathValid())
|
||||
return nullptr;
|
||||
}
|
||||
auto sptr = make_shared<Reader>(book.getPath());
|
||||
m_readers[id] = sptr;
|
||||
return sptr;
|
||||
}
|
||||
|
||||
unsigned int Library::getBookCount(const bool localBooks,
|
||||
const bool remoteBooks) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
return mp_impl->getBookCount(localBooks, remoteBooks);
|
||||
unsigned int result = 0;
|
||||
for (auto& pair: m_books) {
|
||||
auto& book = pair.second;
|
||||
if ((!book.getPath().empty() && localBooks)
|
||||
|| (book.getPath().empty() && remoteBooks)) {
|
||||
result++;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool Library::writeToFile(const std::string& path) const
|
||||
{
|
||||
const auto allBookIds = getBooksIds();
|
||||
|
||||
auto baseDir = removeLastPathElement(path);
|
||||
LibXMLDumper dumper(this);
|
||||
dumper.setBaseDir(baseDir);
|
||||
std::string xml;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
xml = dumper.dumpLibXMLContent(allBookIds);
|
||||
};
|
||||
return writeTextFile(path, xml);
|
||||
return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
|
||||
}
|
||||
|
||||
bool Library::writeBookmarksToFile(const std::string& path) const
|
||||
{
|
||||
LibXMLDumper dumper(this);
|
||||
// NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
|
||||
const std::string xml = dumper.dumpLibXMLBookmark();
|
||||
return writeTextFile(path, xml);
|
||||
}
|
||||
|
||||
Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
AttributeCounts propValueCounts;
|
||||
|
||||
for (const auto& pair: mp_impl->m_books) {
|
||||
const auto& book = pair.second;
|
||||
if (book.getOrigId().empty()) {
|
||||
propValueCounts[(book.*p)()] += 1;
|
||||
}
|
||||
}
|
||||
return propValueCounts;
|
||||
}
|
||||
|
||||
std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
|
||||
{
|
||||
std::vector<std::string> result;
|
||||
for ( const auto& kv : getBookAttributeCounts(p) ) {
|
||||
result.push_back(kv.first);
|
||||
}
|
||||
return result;
|
||||
return writeTextFile(path, dumper.dumpLibXMLBookmark());
|
||||
}
|
||||
|
||||
std::vector<std::string> Library::getBooksLanguages() const
|
||||
{
|
||||
std::vector<std::string> langs;
|
||||
for ( const auto& langAndCount : getBooksLanguagesWithCounts() ) {
|
||||
langs.push_back(langAndCount.first);
|
||||
}
|
||||
return langs;
|
||||
}
|
||||
std::vector<std::string> booksLanguages;
|
||||
std::map<std::string, bool> booksLanguagesMap;
|
||||
|
||||
Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
AttributeCounts langsWithCounts;
|
||||
|
||||
for (const auto& pair: mp_impl->m_books) {
|
||||
const auto& book = pair.second;
|
||||
if (book.getOrigId().empty()) {
|
||||
for ( const auto& lang : book.getLanguages() ) {
|
||||
++langsWithCounts[lang];
|
||||
for (auto& pair: m_books) {
|
||||
auto& book = pair.second;
|
||||
auto& language = book.getLanguage();
|
||||
if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
|
||||
if (book.getOrigId().empty()) {
|
||||
booksLanguagesMap[language] = true;
|
||||
booksLanguages.push_back(language);
|
||||
}
|
||||
}
|
||||
}
|
||||
return langsWithCounts;
|
||||
|
||||
return booksLanguages;
|
||||
}
|
||||
|
||||
std::vector<std::string> Library::getBooksCategories() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
std::set<std::string> categories;
|
||||
|
||||
for (const auto& pair: mp_impl->m_books) {
|
||||
for (const auto& pair: m_books) {
|
||||
const auto& book = pair.second;
|
||||
const auto& c = book.getCategory();
|
||||
if ( !c.empty() ) {
|
||||
@@ -414,23 +218,50 @@ std::vector<std::string> Library::getBooksCategories() const
|
||||
|
||||
std::vector<std::string> Library::getBooksCreators() const
|
||||
{
|
||||
return getBookPropValueSet(&Book::getCreator);
|
||||
std::vector<std::string> booksCreators;
|
||||
std::map<std::string, bool> booksCreatorsMap;
|
||||
|
||||
for (auto& pair: m_books) {
|
||||
auto& book = pair.second;
|
||||
auto& creator = book.getCreator();
|
||||
if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
|
||||
if (book.getOrigId().empty()) {
|
||||
booksCreatorsMap[creator] = true;
|
||||
booksCreators.push_back(creator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return booksCreators;
|
||||
}
|
||||
|
||||
std::vector<std::string> Library::getBooksPublishers() const
|
||||
{
|
||||
return getBookPropValueSet(&Book::getPublisher);
|
||||
std::vector<std::string> booksPublishers;
|
||||
std::map<std::string, bool> booksPublishersMap;
|
||||
|
||||
for (auto& pair:m_books) {
|
||||
auto& book = pair.second;
|
||||
auto& publisher = book.getPublisher();
|
||||
if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
|
||||
if (book.getOrigId().empty()) {
|
||||
booksPublishersMap[publisher] = true;
|
||||
booksPublishers.push_back(publisher);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return booksPublishers;
|
||||
}
|
||||
|
||||
const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
|
||||
{
|
||||
if (!onlyValidBookmarks) {
|
||||
return mp_impl->m_bookmarks;
|
||||
return m_bookmarks;
|
||||
}
|
||||
std::vector<kiwix::Bookmark> validBookmarks;
|
||||
auto booksId = getBooksIds();
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
for(auto& bookmark:mp_impl->m_bookmarks) {
|
||||
for(auto& bookmark:m_bookmarks) {
|
||||
if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
|
||||
validBookmarks.push_back(bookmark);
|
||||
}
|
||||
@@ -440,29 +271,35 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
|
||||
|
||||
Library::BookIdCollection Library::getBooksIds() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
BookIdCollection bookIds;
|
||||
|
||||
for (auto& pair: mp_impl->m_books) {
|
||||
for (auto& pair: m_books) {
|
||||
bookIds.push_back(pair.first);
|
||||
}
|
||||
|
||||
return bookIds;
|
||||
}
|
||||
|
||||
Library::BookIdCollection Library::filter(const std::string& search) const
|
||||
{
|
||||
if (search.empty()) {
|
||||
return getBooksIds();
|
||||
}
|
||||
|
||||
return filter(Filter().query(search));
|
||||
}
|
||||
|
||||
|
||||
void Library::updateBookDB(const Book& book)
|
||||
{
|
||||
Xapian::Stem stemmer;
|
||||
Xapian::TermGenerator indexer;
|
||||
const auto langs = book.getLanguages();
|
||||
if ( langs.size() == 1 ) {
|
||||
try {
|
||||
stemmer = Xapian::Stem(iso639_3ToXapian(langs[0]));
|
||||
indexer.set_stemmer(stemmer);
|
||||
indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
|
||||
} catch (...) {}
|
||||
}
|
||||
const std::string lang = book.getLanguage();
|
||||
try {
|
||||
stemmer = Xapian::Stem(iso639_3ToXapian(lang));
|
||||
indexer.set_stemmer(stemmer);
|
||||
indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
|
||||
} catch (...) {}
|
||||
Xapian::Document doc;
|
||||
indexer.set_document(doc);
|
||||
|
||||
@@ -477,28 +314,21 @@ void Library::updateBookDB(const Book& book)
|
||||
// Index all fields for field-based search
|
||||
indexer.index_text(title, 1, "S");
|
||||
indexer.index_text(desc, 1, "XD");
|
||||
for ( const auto& lang : langs ) {
|
||||
indexer.index_text(lang, 1, "L");
|
||||
}
|
||||
indexer.index_text(lang, 1, "L");
|
||||
indexer.index_text(normalizeText(book.getCreator()), 1, "A");
|
||||
indexer.index_text(normalizeText(book.getPublisher()), 1, "XP");
|
||||
indexer.index_text(normalizeText(book.getName()), 1, "XN");
|
||||
indexer.index_text(normalizeText(book.getCategory()), 1, "XC");
|
||||
|
||||
for ( const auto& tag : split(normalizeText(book.getTags()), ";") ) {
|
||||
for ( const auto& tag : split(normalizeText(book.getTags()), ";") )
|
||||
doc.add_boolean_term("XT" + tag);
|
||||
if ( tag[0] != '_' ) {
|
||||
indexer.increase_termpos();
|
||||
indexer.index_text(tag);
|
||||
}
|
||||
}
|
||||
|
||||
const std::string idterm = "Q" + book.getId();
|
||||
doc.add_boolean_term(idterm);
|
||||
|
||||
doc.set_data(book.getId());
|
||||
|
||||
mp_impl->m_bookDB.replace_document(idterm, doc);
|
||||
m_bookDB->replace_document(idterm, doc);
|
||||
}
|
||||
|
||||
namespace
|
||||
@@ -554,20 +384,9 @@ Xapian::Query categoryQuery(const std::string& category)
|
||||
return Xapian::Query("XC" + normalizeText(category));
|
||||
}
|
||||
|
||||
Xapian::Query langQuery(const std::string& commaSeparatedLanguageList)
|
||||
Xapian::Query langQuery(const std::string& lang)
|
||||
{
|
||||
Xapian::Query q;
|
||||
bool firstIteration = true;
|
||||
for ( const auto& lang : kiwix::split(commaSeparatedLanguageList, ",") ) {
|
||||
const Xapian::Query singleLangQuery("L" + normalizeText(lang));
|
||||
if ( firstIteration ) {
|
||||
q = singleLangQuery;
|
||||
firstIteration = false;
|
||||
} else {
|
||||
q = Xapian::Query(Xapian::Query::OP_OR, q, singleLangQuery);
|
||||
}
|
||||
}
|
||||
return q;
|
||||
return Xapian::Query("L" + normalizeText(lang));
|
||||
}
|
||||
|
||||
Xapian::Query publisherQuery(const std::string& publisher)
|
||||
@@ -641,10 +460,9 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
|
||||
|
||||
BookIdCollection bookIds;
|
||||
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
Xapian::Enquire enquire(mp_impl->m_bookDB);
|
||||
Xapian::Enquire enquire(*m_bookDB);
|
||||
enquire.set_query(query);
|
||||
const auto results = enquire.get_mset(0, mp_impl->m_books.size());
|
||||
const auto results = enquire.get_mset(0, m_books.size());
|
||||
for ( auto it = results.begin(); it != results.end(); ++it ) {
|
||||
bookIds.push_back(it.get_document().get_data());
|
||||
}
|
||||
@@ -655,10 +473,8 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
|
||||
Library::BookIdCollection Library::filter(const Filter& filter) const
|
||||
{
|
||||
BookIdCollection result;
|
||||
const auto preliminaryResult = filterViaBookDB(filter);
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
for(auto id : preliminaryResult) {
|
||||
if(filter.accept(mp_impl->m_books.at(id))) {
|
||||
for(auto id : filterViaBookDB(filter)) {
|
||||
if(filter.accept(m_books.at(id))) {
|
||||
result.push_back(id);
|
||||
}
|
||||
}
|
||||
@@ -726,11 +542,6 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
|
||||
|
||||
void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
|
||||
{
|
||||
// NOTE: Can reimplement this method in a way that doesn't require locking
|
||||
// NOTE: for the entire duration of the sort. Will need to obtain (under a
|
||||
// NOTE: lock) the required atributes from the books once, and then the
|
||||
// NOTE: sorting will run on a copy of data without locking.
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
switch(sort) {
|
||||
case TITLE:
|
||||
std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
|
||||
@@ -753,6 +564,48 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
|
||||
}
|
||||
|
||||
|
||||
Library::BookIdCollection Library::listBooksIds(
|
||||
int mode,
|
||||
supportedListSortBy sortBy,
|
||||
const std::string& search,
|
||||
const std::string& language,
|
||||
const std::string& creator,
|
||||
const std::string& publisher,
|
||||
const std::vector<std::string>& tags,
|
||||
size_t maxSize) const {
|
||||
|
||||
Filter _filter;
|
||||
if (mode & LOCAL)
|
||||
_filter.local(true);
|
||||
if (mode & NOLOCAL)
|
||||
_filter.local(false);
|
||||
if (mode & VALID)
|
||||
_filter.valid(true);
|
||||
if (mode & NOVALID)
|
||||
_filter.valid(false);
|
||||
if (mode & REMOTE)
|
||||
_filter.remote(true);
|
||||
if (mode & NOREMOTE)
|
||||
_filter.remote(false);
|
||||
if (!tags.empty())
|
||||
_filter.acceptTags(tags);
|
||||
if (maxSize != 0)
|
||||
_filter.maxSize(maxSize);
|
||||
if (!language.empty())
|
||||
_filter.lang(language);
|
||||
if (!publisher.empty())
|
||||
_filter.publisher(publisher);
|
||||
if (!creator.empty())
|
||||
_filter.creator(creator);
|
||||
if (!search.empty())
|
||||
_filter.query(search);
|
||||
|
||||
auto bookIds = filter(_filter);
|
||||
|
||||
sort(bookIds, sortBy, true);
|
||||
return bookIds;
|
||||
}
|
||||
|
||||
Filter::Filter()
|
||||
: activeFilters(0),
|
||||
_maxSize(0)
|
||||
@@ -878,18 +731,6 @@ Filter& Filter::name(std::string name)
|
||||
return *this;
|
||||
}
|
||||
|
||||
Filter& Filter::clearLang()
|
||||
{
|
||||
activeFilters &= ~LANG;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Filter& Filter::clearCategory()
|
||||
{
|
||||
activeFilters &= ~CATEGORY;
|
||||
return *this;
|
||||
}
|
||||
|
||||
#define ACTIVE(X) (activeFilters & (X))
|
||||
#define FILTER(TAG, TEST) if (ACTIVE(TAG) && !(TEST)) { return false; }
|
||||
bool Filter::hasQuery() const
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
#include "library_dumper.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools.h"
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
/* Constructor */
|
||||
LibraryDumper::LibraryDumper(const Library* library, const NameMapper* nameMapper)
|
||||
: library(library),
|
||||
nameMapper(nameMapper)
|
||||
{
|
||||
}
|
||||
/* Destructor */
|
||||
LibraryDumper::~LibraryDumper()
|
||||
{
|
||||
}
|
||||
|
||||
void LibraryDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
|
||||
{
|
||||
m_totalResults = totalResults;
|
||||
m_startIndex = startIndex,
|
||||
m_count = count;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
std::map<std::string, std::string> iso639_3 = {
|
||||
{"atj", "atikamekw"},
|
||||
{"azb", "آذربایجان دیلی"},
|
||||
{"bcl", "central bikol"},
|
||||
{"bgs", "tagabawa"},
|
||||
{"bxr", "буряад хэлэн"},
|
||||
{"cbk", "chavacano"},
|
||||
{"cdo", "閩東語"},
|
||||
{"dag", "Dagbani"},
|
||||
{"diq", "dimli"},
|
||||
{"dty", "डोटेली"},
|
||||
{"eml", "emiliân-rumagnōl"},
|
||||
{"fbs", "српскохрватски"},
|
||||
{"hbs", "srpskohrvatski"},
|
||||
{"ido", "ido"},
|
||||
{"kbp", "kabɩyɛ"},
|
||||
{"kld", "Gamilaraay"},
|
||||
{"lbe", "лакку маз"},
|
||||
{"lbj", "ལ་དྭགས་སྐད་"},
|
||||
{"map", "Austronesian"},
|
||||
{"mhr", "марий йылме"},
|
||||
{"mnw", "ဘာသာမန်"},
|
||||
{"myn", "mayan"},
|
||||
{"nah", "nahuatl"},
|
||||
{"nai", "north American Indian"},
|
||||
{"nds", "plattdütsch"},
|
||||
{"nrm", "bhasa narom"},
|
||||
{"olo", "livvi"},
|
||||
{"pih", "Pitcairn-Norfolk"},
|
||||
{"pnb", "Western Panjabi"},
|
||||
{"rmr", "Caló"},
|
||||
{"rmy", "romani shib"},
|
||||
{"roa", "romance languages"},
|
||||
{"twi", "twi"},
|
||||
};
|
||||
|
||||
std::once_flag fillLanguagesFlag;
|
||||
|
||||
void fillLanguagesMap()
|
||||
{
|
||||
for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
|
||||
const ICULanguageInfo lang(*icuLangPtr);
|
||||
iso639_3.insert({lang.iso3Code(), lang.selfName()});
|
||||
}
|
||||
}
|
||||
|
||||
std::string getLanguageSelfName(const std::string& lang) {
|
||||
const auto itr = iso639_3.find(lang);
|
||||
if (itr != iso639_3.end()) {
|
||||
return itr->second;
|
||||
}
|
||||
return lang;
|
||||
};
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
kainjow::mustache::list LibraryDumper::getCategoryData() const
|
||||
{
|
||||
const auto now = gen_date_str();
|
||||
kainjow::mustache::list categoryData;
|
||||
for ( const auto& category : library->getBooksCategories() ) {
|
||||
const auto urlencodedCategoryName = urlEncode(category);
|
||||
categoryData.push_back(kainjow::mustache::object{
|
||||
{"name", category},
|
||||
{"urlencoded_name", urlencodedCategoryName},
|
||||
{"updated", now},
|
||||
{"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
|
||||
});
|
||||
}
|
||||
return categoryData;
|
||||
}
|
||||
|
||||
kainjow::mustache::list LibraryDumper::getLanguageData() const
|
||||
{
|
||||
const auto now = gen_date_str();
|
||||
kainjow::mustache::list languageData;
|
||||
std::call_once(fillLanguagesFlag, fillLanguagesMap);
|
||||
for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
|
||||
const std::string languageCode = langAndBookCount.first;
|
||||
const int bookCount = langAndBookCount.second;
|
||||
const auto languageSelfName = getLanguageSelfName(languageCode);
|
||||
languageData.push_back(kainjow::mustache::object{
|
||||
{"lang_code", languageCode},
|
||||
{"lang_self_name", languageSelfName},
|
||||
{"book_count", to_string(bookCount)},
|
||||
{"updated", now},
|
||||
{"id", gen_uuid(libraryId + "/languages/" + languageCode)}
|
||||
});
|
||||
}
|
||||
return languageData;
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -20,10 +20,10 @@
|
||||
#include "libxml_dumper.h"
|
||||
#include "book.h"
|
||||
|
||||
#include "tools.h"
|
||||
#include "tools/base64.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/pathTools.h"
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
@@ -54,19 +54,16 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
|
||||
if (book.getOrigId().empty()) {
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getCommaSeparatedLanguages());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
|
||||
try {
|
||||
auto defaultIllustration = book.getIllustration(48);
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
|
||||
if (!defaultIllustration->getData().empty())
|
||||
ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
|
||||
} catch(...) {}
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
|
||||
ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
|
||||
if (!book.getFavicon().empty())
|
||||
ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
|
||||
} else {
|
||||
ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
|
||||
}
|
||||
@@ -94,10 +91,10 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
|
||||
auto book_node = entry_node.append_child("book");
|
||||
|
||||
try {
|
||||
auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
|
||||
auto book = library->getBookById(bookmark.getBookId());
|
||||
ADD_TEXT_ENTRY(book_node, "id", book.getId());
|
||||
ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
|
||||
ADD_TEXT_ENTRY(book_node, "language", book.getCommaSeparatedLanguages());
|
||||
ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
|
||||
ADD_TEXT_ENTRY(book_node, "date", book.getDate());
|
||||
} catch (...) {
|
||||
ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
|
||||
|
||||
@@ -19,88 +19,34 @@
|
||||
|
||||
#include "manager.h"
|
||||
|
||||
#include "tools.h"
|
||||
#include "tools/pathTools.h"
|
||||
|
||||
#include <pugixml.hpp>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct NoDelete
|
||||
{
|
||||
template<class T> void operator()(T*) {}
|
||||
};
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// LibraryManipulator
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
LibraryManipulator::LibraryManipulator(Library* library)
|
||||
: library(*library)
|
||||
{}
|
||||
|
||||
LibraryManipulator::~LibraryManipulator()
|
||||
{}
|
||||
|
||||
bool LibraryManipulator::addBookToLibrary(const Book& book)
|
||||
{
|
||||
const auto ret = library.addBook(book);
|
||||
if ( ret ) {
|
||||
bookWasAddedToLibrary(book);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
|
||||
{
|
||||
library.addBookmark(bookmark);
|
||||
bookmarkWasAddedToLibrary(bookmark);
|
||||
}
|
||||
|
||||
uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
|
||||
{
|
||||
const auto n = library.removeBooksNotUpdatedSince(rev);
|
||||
if ( n != 0 ) {
|
||||
booksWereRemovedFromLibrary();
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
|
||||
{
|
||||
}
|
||||
|
||||
void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
|
||||
{
|
||||
}
|
||||
|
||||
void LibraryManipulator::booksWereRemovedFromLibrary()
|
||||
{
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Manager
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/* Constructor */
|
||||
Manager::Manager(LibraryManipulator* manipulator):
|
||||
writableLibraryPath(""),
|
||||
manipulator(manipulator, NoDelete())
|
||||
manipulator(manipulator),
|
||||
mustDeleteManipulator(false)
|
||||
{
|
||||
}
|
||||
|
||||
Manager::Manager(Library* library) :
|
||||
writableLibraryPath(""),
|
||||
manipulator(new LibraryManipulator(library))
|
||||
manipulator(new DefaultLibraryManipulator(library)),
|
||||
mustDeleteManipulator(true)
|
||||
{
|
||||
}
|
||||
|
||||
/* Destructor */
|
||||
Manager::~Manager()
|
||||
{
|
||||
if (mustDeleteManipulator) {
|
||||
delete manipulator;
|
||||
}
|
||||
}
|
||||
bool Manager::parseXmlDom(const pugi::xml_document& doc,
|
||||
bool readOnly,
|
||||
const std::string& libraryPath,
|
||||
@@ -238,7 +184,7 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
|
||||
}
|
||||
|
||||
if (!checkMetaData
|
||||
|| (!book.getTitle().empty() && !book.getLanguages().empty()
|
||||
|| (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
|
||||
&& !book.getDate().empty())) {
|
||||
book.setUrl(url);
|
||||
manipulator->addBookToLibrary(book);
|
||||
@@ -268,8 +214,8 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
|
||||
tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
|
||||
}
|
||||
try {
|
||||
zim::Archive archive(tmp_path);
|
||||
book->update(archive);
|
||||
kiwix::Reader reader(tmp_path);
|
||||
book->update(reader);
|
||||
book->setPathValid(true);
|
||||
} catch (const std::exception& e) {
|
||||
book->setPathValid(false);
|
||||
@@ -302,21 +248,4 @@ bool Manager::readBookmarkFile(const std::string& path)
|
||||
return true;
|
||||
}
|
||||
|
||||
void Manager::reload(const Paths& paths)
|
||||
{
|
||||
const auto libRevision = manipulator->getLibrary().getRevision();
|
||||
for (std::string path : paths) {
|
||||
if (!path.empty()) {
|
||||
if ( kiwix::isRelativePath(path) )
|
||||
path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
|
||||
|
||||
if (!readFile(path, false, true)) {
|
||||
throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
manipulator->removeBooksNotUpdatedSince(libRevision);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,10 +5,11 @@ kiwix_sources = [
|
||||
'manager.cpp',
|
||||
'libxml_dumper.cpp',
|
||||
'opds_dumper.cpp',
|
||||
'html_dumper.cpp',
|
||||
'library_dumper.cpp',
|
||||
'downloader.cpp',
|
||||
'reader.cpp',
|
||||
'entry.cpp',
|
||||
'server.cpp',
|
||||
'searcher.cpp',
|
||||
'search_renderer.cpp',
|
||||
'subprocess.cpp',
|
||||
'aria2.cpp',
|
||||
@@ -18,7 +19,6 @@ kiwix_sources = [
|
||||
'tools/stringTools.cpp',
|
||||
'tools/networkTools.cpp',
|
||||
'tools/otherTools.cpp',
|
||||
'tools/archiveTools.cpp',
|
||||
'kiwixserve.cpp',
|
||||
'name_mapper.cpp',
|
||||
'server/byte_range.cpp',
|
||||
@@ -26,13 +26,9 @@ kiwix_sources = [
|
||||
'server/request_context.cpp',
|
||||
'server/response.cpp',
|
||||
'server/internalServer.cpp',
|
||||
'server/internalServer_catalog_v2.cpp',
|
||||
'server/i18n.cpp',
|
||||
'opds_catalog.cpp',
|
||||
'version.cpp'
|
||||
'server/internalServer_catalog_v2.cpp'
|
||||
]
|
||||
kiwix_sources += lib_resources
|
||||
kiwix_sources += i18n_resources
|
||||
|
||||
if host_machine.system() == 'windows'
|
||||
kiwix_sources += 'subprocess_windows.cpp'
|
||||
@@ -40,14 +36,22 @@ else
|
||||
kiwix_sources += 'subprocess_unix.cpp'
|
||||
endif
|
||||
|
||||
install_dir = get_option('libdir')
|
||||
if wrapper.contains('android')
|
||||
install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
|
||||
else
|
||||
install_dir = get_option('libdir')
|
||||
endif
|
||||
|
||||
if wrapper.contains('android') or wrapper.contains('java')
|
||||
subdir('wrapper/java')
|
||||
endif
|
||||
|
||||
config_h = configure_file(output : 'kiwix_config.h',
|
||||
configuration : conf,
|
||||
input : 'config.h.in')
|
||||
install_headers(config_h, subdir:'kiwix')
|
||||
|
||||
libkiwix = library('kiwix',
|
||||
kiwixlib = library('kiwix',
|
||||
kiwix_sources,
|
||||
include_directories : inc,
|
||||
dependencies : all_deps,
|
||||
|
||||
@@ -51,54 +51,12 @@ HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool w
|
||||
}
|
||||
}
|
||||
|
||||
std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
|
||||
std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
|
||||
return m_idToName.at(id);
|
||||
}
|
||||
|
||||
std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
|
||||
std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
|
||||
return m_nameToId.at(name);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// UpdatableNameMapper
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
|
||||
: library(lib)
|
||||
, withAlias(withAlias)
|
||||
{
|
||||
update();
|
||||
}
|
||||
|
||||
void UpdatableNameMapper::update()
|
||||
{
|
||||
const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
nameMapper.reset(newNameMapper);
|
||||
}
|
||||
|
||||
UpdatableNameMapper::NameMapperHandle
|
||||
UpdatableNameMapper::currentNameMapper() const
|
||||
{
|
||||
// Return a copy of the handle to the current NameMapper object. It will
|
||||
// ensure that the object survives any call to UpdatableNameMapper::update()
|
||||
// made before the completion of any pending operation on that object.
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
return nameMapper;
|
||||
}
|
||||
|
||||
std::string UpdatableNameMapper::getNameForId(const std::string& id) const
|
||||
{
|
||||
// Ensure that the current nameMapper object survives a concurrent call
|
||||
// to UpdatableNameMapper::update()
|
||||
return currentNameMapper()->getNameForId(id);
|
||||
}
|
||||
|
||||
std::string UpdatableNameMapper::getIdForName(const std::string& name) const
|
||||
{
|
||||
// Ensure that the current nameMapper object survives a concurrent call
|
||||
// to UpdatableNameMapper::update()
|
||||
return currentNameMapper()->getIdForName(name);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "opds_catalog.h"
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
const char opdsSearchEndpoint[] = "/catalog/v2/entries";
|
||||
|
||||
enum Separator { AMP };
|
||||
|
||||
std::ostringstream& operator<<(std::ostringstream& oss, Separator sep)
|
||||
{
|
||||
if ( oss.tellp() > 0 )
|
||||
oss << "&";
|
||||
return oss;
|
||||
}
|
||||
|
||||
std::string buildSearchString(const Filter& f)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
if ( f.hasQuery() )
|
||||
oss << AMP << "q=" << urlEncode(f.getQuery());
|
||||
|
||||
if ( f.hasCategory() )
|
||||
oss << AMP << "category=" << urlEncode(f.getCategory());
|
||||
|
||||
if ( f.hasLang() )
|
||||
oss << AMP << "lang=" << urlEncode(f.getLang());
|
||||
|
||||
if ( f.hasName() )
|
||||
oss << AMP << "name=" << urlEncode(f.getName());
|
||||
|
||||
if ( !f.getAcceptTags().empty() )
|
||||
oss << AMP << "tag=" << urlEncode(join(f.getAcceptTags(), ";"));
|
||||
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::string getSearchUrl(const Filter& f)
|
||||
{
|
||||
const std::string searchString = buildSearchString(f);
|
||||
|
||||
if ( searchString.empty() )
|
||||
return opdsSearchEndpoint;
|
||||
else
|
||||
return opdsSearchEndpoint + ("?" + searchString);
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -20,18 +20,17 @@
|
||||
#include "opds_dumper.h"
|
||||
#include "book.h"
|
||||
|
||||
#include "libkiwix-resources.h"
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
|
||||
#include "kiwixlib-resources.h"
|
||||
#include <mustache.hpp>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/* Constructor */
|
||||
OPDSDumper::OPDSDumper(const Library* library, const NameMapper* nameMapper)
|
||||
: LibraryDumper(library, nameMapper)
|
||||
OPDSDumper::OPDSDumper(Library* library)
|
||||
: library(library)
|
||||
{
|
||||
}
|
||||
/* Destructor */
|
||||
@@ -39,44 +38,35 @@ OPDSDumper::~OPDSDumper()
|
||||
{
|
||||
}
|
||||
|
||||
void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count)
|
||||
{
|
||||
m_totalResults = totalResults;
|
||||
m_startIndex = startIndex,
|
||||
m_count = count;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
const std::string XML_HEADER(R"(<?xml version="1.0" encoding="UTF-8"?>)");
|
||||
|
||||
typedef kainjow::mustache::data MustacheData;
|
||||
typedef kainjow::mustache::list BooksData;
|
||||
typedef kainjow::mustache::list IllustrationInfo;
|
||||
typedef kainjow::mustache::list BookData;
|
||||
|
||||
IllustrationInfo getBookIllustrationInfo(const Book& book)
|
||||
BookData getBookData(const Library* library, const std::vector<std::string>& bookIds)
|
||||
{
|
||||
kainjow::mustache::list illustrations;
|
||||
if ( book.isPathValid() ) {
|
||||
for ( const auto& illustration : book.getIllustrations() ) {
|
||||
// For now, we are handling only sizexsize@1 illustration.
|
||||
// So we can simply pass one size to mustache.
|
||||
illustrations.push_back(kainjow::mustache::object{
|
||||
{"icon_size", to_string(illustration->width)},
|
||||
{"icon_mimetype", illustration->mimeType}
|
||||
});
|
||||
}
|
||||
}
|
||||
return illustrations;
|
||||
}
|
||||
|
||||
std::string fullEntryXML(const Book& book, const std::string& rootLocation, const std::string& contentId)
|
||||
{
|
||||
const auto bookDate = book.getDate() + "T00:00:00Z";
|
||||
const kainjow::mustache::object data{
|
||||
{"root", rootLocation},
|
||||
{"id", book.getId()},
|
||||
BookData bookData;
|
||||
for ( const auto& bookId : bookIds ) {
|
||||
const Book& book = library->getBookById(bookId);
|
||||
const MustacheData bookUrl = book.getUrl().empty()
|
||||
? MustacheData(false)
|
||||
: MustacheData(book.getUrl());
|
||||
bookData.push_back(kainjow::mustache::object{
|
||||
{"id", "urn:uuid:"+book.getId()},
|
||||
{"name", book.getName()},
|
||||
{"title", book.getTitle()},
|
||||
{"description", book.getDescription()},
|
||||
{"language", book.getCommaSeparatedLanguages()},
|
||||
{"content_id", urlEncode(contentId)},
|
||||
{"updated", bookDate}, // XXX: this should be the entry update datetime
|
||||
{"book_date", bookDate},
|
||||
{"language", book.getLanguage()},
|
||||
{"content_id", book.getHumanReadableIdFromPath()},
|
||||
{"updated", book.getDate() + "T00:00:00Z"},
|
||||
{"category", book.getCategory()},
|
||||
{"flavour", book.getFlavour()},
|
||||
{"tags", book.getTags()},
|
||||
@@ -84,101 +74,66 @@ std::string fullEntryXML(const Book& book, const std::string& rootLocation, cons
|
||||
{"media_count", to_string(book.getMediaCount())},
|
||||
{"author_name", book.getCreator()},
|
||||
{"publisher_name", book.getPublisher()},
|
||||
{"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
|
||||
{"url", bookUrl},
|
||||
{"size", to_string(book.getSize())},
|
||||
{"icons", getBookIllustrationInfo(book)},
|
||||
};
|
||||
return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
|
||||
}
|
||||
|
||||
std::string partialEntryXML(const Book& book, const std::string& rootLocation)
|
||||
{
|
||||
const auto bookDate = book.getDate() + "T00:00:00Z";
|
||||
const kainjow::mustache::object data{
|
||||
{"root", rootLocation},
|
||||
{"endpoint_root", rootLocation + "/catalog/v2"},
|
||||
{"id", book.getId()},
|
||||
{"title", book.getTitle()},
|
||||
{"updated", bookDate}, // XXX: this should be the entry update datetime
|
||||
};
|
||||
const auto xmlTemplate = RESOURCE::templates::catalog_v2_partial_entry_xml;
|
||||
return render_template(xmlTemplate, data);
|
||||
}
|
||||
|
||||
BooksData getBooksData(const Library* library, const NameMapper* nameMapper, const std::vector<std::string>& bookIds, const std::string& rootLocation, bool partial)
|
||||
{
|
||||
BooksData booksData;
|
||||
for ( const auto& bookId : bookIds ) {
|
||||
try {
|
||||
const Book book = library->getBookByIdThreadSafe(bookId);
|
||||
const std::string contentId = nameMapper->getNameForId(bookId);
|
||||
const auto entryXML = partial
|
||||
? partialEntryXML(book, rootLocation)
|
||||
: fullEntryXML(book, rootLocation, contentId);
|
||||
booksData.push_back(kainjow::mustache::object{ {"entry", entryXML} });
|
||||
} catch ( const std::out_of_range& ) {
|
||||
// the book was removed from the library since its id was obtained
|
||||
// ignore it
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return booksData;
|
||||
return bookData;
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
|
||||
{
|
||||
const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, false);
|
||||
const auto bookData = getBookData(library, bookIds);
|
||||
const kainjow::mustache::object template_data{
|
||||
{"date", gen_date_str()},
|
||||
{"root", rootLocation},
|
||||
{"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
|
||||
{"filter", onlyAsNonEmptyMustacheValue(query)},
|
||||
{"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
|
||||
{"totalResults", to_string(m_totalResults)},
|
||||
{"startIndex", to_string(m_startIndex)},
|
||||
{"itemsPerPage", to_string(m_count)},
|
||||
{"books", booksData }
|
||||
{"books", bookData }
|
||||
};
|
||||
|
||||
return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
|
||||
}
|
||||
|
||||
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
|
||||
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const
|
||||
{
|
||||
const auto endpointRoot = rootLocation + "/catalog/v2";
|
||||
const auto booksData = getBooksData(library, nameMapper, bookIds, rootLocation, partial);
|
||||
const auto bookData = getBookData(library, bookIds);
|
||||
|
||||
const char* const endpoint = partial ? "/partial_entries" : "/entries";
|
||||
const kainjow::mustache::object template_data{
|
||||
{"date", gen_date_str()},
|
||||
{"endpoint_root", endpointRoot},
|
||||
{"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
|
||||
{"filter", onlyAsNonEmptyMustacheValue(query)},
|
||||
{"query", query.empty() ? "" : "?" + query},
|
||||
{"endpoint_root", rootLocation + "/catalog/v2"},
|
||||
{"feed_id", gen_uuid(libraryId + "/entries?"+query)},
|
||||
{"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
|
||||
{"query", query.empty() ? "" : "?" + urlEncode(query)},
|
||||
{"totalResults", to_string(m_totalResults)},
|
||||
{"startIndex", to_string(m_startIndex)},
|
||||
{"itemsPerPage", to_string(m_count)},
|
||||
{"books", booksData },
|
||||
{"dump_partial_entries", MustacheData(partial)}
|
||||
{"books", bookData }
|
||||
};
|
||||
|
||||
return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
|
||||
}
|
||||
|
||||
std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
|
||||
{
|
||||
const auto book = library->getBookById(bookId);
|
||||
const std::string contentId = nameMapper->getNameForId(bookId);
|
||||
return XML_HEADER
|
||||
+ "\n"
|
||||
+ fullEntryXML(book, rootLocation, contentId);
|
||||
}
|
||||
|
||||
std::string OPDSDumper::categoriesOPDSFeed() const
|
||||
std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categories) const
|
||||
{
|
||||
const auto now = gen_date_str();
|
||||
kainjow::mustache::list categoryData = getCategoryData();
|
||||
kainjow::mustache::list categoryData;
|
||||
for ( const auto& category : categories ) {
|
||||
const auto urlencodedCategoryName = urlEncode(category);
|
||||
categoryData.push_back(kainjow::mustache::object{
|
||||
{"name", category},
|
||||
{"urlencoded_name", urlencodedCategoryName},
|
||||
{"updated", now},
|
||||
{"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)}
|
||||
});
|
||||
}
|
||||
|
||||
return render_template(
|
||||
RESOURCE::templates::catalog_v2_categories_xml,
|
||||
kainjow::mustache::object{
|
||||
@@ -190,19 +145,4 @@ std::string OPDSDumper::categoriesOPDSFeed() const
|
||||
);
|
||||
}
|
||||
|
||||
std::string OPDSDumper::languagesOPDSFeed() const
|
||||
{
|
||||
const auto now = gen_date_str();
|
||||
kainjow::mustache::list languageData = getLanguageData();
|
||||
return render_template(
|
||||
RESOURCE::templates::catalog_v2_languages_xml,
|
||||
kainjow::mustache::object{
|
||||
{"date", now},
|
||||
{"endpoint_root", rootLocation + "/catalog/v2"},
|
||||
{"feed_id", gen_uuid(libraryId + "/languages")},
|
||||
{"languages", languageData }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
581
src/reader.cpp
Normal file
581
src/reader.cpp
Normal file
@@ -0,0 +1,581 @@
|
||||
/*
|
||||
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "reader.h"
|
||||
#include <time.h>
|
||||
|
||||
#include <zim/search.h>
|
||||
#include <zim/item.h>
|
||||
#include <zim/error.h>
|
||||
|
||||
#include "tools/otherTools.h"
|
||||
|
||||
inline char hi(char v)
|
||||
{
|
||||
char hex[] = "0123456789abcdef";
|
||||
return hex[(v >> 4) & 0xf];
|
||||
}
|
||||
|
||||
inline char lo(char v)
|
||||
{
|
||||
char hex[] = "0123456789abcdef";
|
||||
return hex[v & 0xf];
|
||||
}
|
||||
|
||||
std::string hexUUID(std::string in)
|
||||
{
|
||||
std::ostringstream out;
|
||||
for (unsigned n = 0; n < 4; ++n) {
|
||||
out << hi(in[n]) << lo(in[n]);
|
||||
}
|
||||
out << '-';
|
||||
for (unsigned n = 4; n < 6; ++n) {
|
||||
out << hi(in[n]) << lo(in[n]);
|
||||
}
|
||||
out << '-';
|
||||
for (unsigned n = 6; n < 8; ++n) {
|
||||
out << hi(in[n]) << lo(in[n]);
|
||||
}
|
||||
out << '-';
|
||||
for (unsigned n = 8; n < 10; ++n) {
|
||||
out << hi(in[n]) << lo(in[n]);
|
||||
}
|
||||
out << '-';
|
||||
for (unsigned n = 10; n < 16; ++n) {
|
||||
out << hi(in[n]) << lo(in[n]);
|
||||
}
|
||||
std::string op = out.str();
|
||||
return op;
|
||||
}
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
/* Constructor */
|
||||
Reader::Reader(const string zimFilePath)
|
||||
: zimArchive(nullptr),
|
||||
zimFilePath(zimFilePath)
|
||||
{
|
||||
string tmpZimFilePath = zimFilePath;
|
||||
|
||||
/* Remove potential trailing zimaa */
|
||||
size_t found = tmpZimFilePath.rfind("zimaa");
|
||||
if (found != string::npos && tmpZimFilePath.size() > 5
|
||||
&& found == tmpZimFilePath.size() - 5) {
|
||||
tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
|
||||
}
|
||||
|
||||
zimArchive.reset(new zim::Archive(tmpZimFilePath));
|
||||
|
||||
/* initialize random seed: */
|
||||
srand(time(nullptr));
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
Reader::Reader(int fd)
|
||||
: zimArchive(new zim::Archive(fd)),
|
||||
zimFilePath("")
|
||||
{
|
||||
/* initialize random seed: */
|
||||
srand(time(nullptr));
|
||||
}
|
||||
|
||||
Reader::Reader(int fd, zim::offset_type offset, zim::size_type size)
|
||||
: zimArchive(new zim::Archive(fd, offset, size)),
|
||||
zimFilePath("")
|
||||
{
|
||||
/* initialize random seed: */
|
||||
srand(time(nullptr));
|
||||
}
|
||||
#endif // #ifndef _WIN32
|
||||
|
||||
zim::Archive* Reader::getZimArchive() const
|
||||
{
|
||||
return zimArchive.get();
|
||||
}
|
||||
|
||||
MimeCounterType Reader::parseCounterMetadata() const
|
||||
{
|
||||
try {
|
||||
auto counterContent = zimArchive->getMetadata("Counter");
|
||||
return parseMimetypeCounter(counterContent);
|
||||
} catch (zim::EntryNotFound& e) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/* Get the count of articles which can be indexed/displayed */
|
||||
unsigned int Reader::getArticleCount() const
|
||||
{
|
||||
std::map<const std::string, unsigned int> counterMap
|
||||
= this->parseCounterMetadata();
|
||||
unsigned int counter = 0;
|
||||
|
||||
for(auto &pair:counterMap) {
|
||||
if (startsWith(pair.first, "text/html")) {
|
||||
counter += pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
return counter;
|
||||
}
|
||||
|
||||
/* Get the count of medias content in the ZIM file */
|
||||
unsigned int Reader::getMediaCount() const
|
||||
{
|
||||
std::map<const std::string, unsigned int> counterMap
|
||||
= this->parseCounterMetadata();
|
||||
unsigned int counter = 0;
|
||||
|
||||
for (auto &pair:counterMap) {
|
||||
if (startsWith(pair.first, "image/") ||
|
||||
startsWith(pair.first, "video/") ||
|
||||
startsWith(pair.first, "audio/")) {
|
||||
counter += pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
return counter;
|
||||
}
|
||||
|
||||
/* Get the total of all items of a ZIM file, redirects included */
|
||||
unsigned int Reader::getGlobalCount() const
|
||||
{
|
||||
return zimArchive->getEntryCount();
|
||||
}
|
||||
|
||||
/* Return the UID of the ZIM file */
|
||||
string Reader::getId() const
|
||||
{
|
||||
std::ostringstream s;
|
||||
s << zimArchive->getUuid();
|
||||
return s.str();
|
||||
}
|
||||
|
||||
Entry Reader::getRandomPage() const
|
||||
{
|
||||
try {
|
||||
return zimArchive->getRandomEntry();
|
||||
} catch(...) {
|
||||
throw NoEntry();
|
||||
}
|
||||
}
|
||||
|
||||
Entry Reader::getMainPage() const
|
||||
{
|
||||
return zimArchive->getMainEntry();
|
||||
}
|
||||
|
||||
bool Reader::getFavicon(string& content, string& mimeType) const
|
||||
{
|
||||
try {
|
||||
auto item = zimArchive->getIllustrationItem();
|
||||
content = item.getData();
|
||||
mimeType = item.getMimetype();
|
||||
return true;
|
||||
} catch(zim::EntryNotFound& e) {};
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
string Reader::getZimFilePath() const
|
||||
{
|
||||
return zimFilePath;
|
||||
}
|
||||
/* Return a metatag value */
|
||||
bool Reader::getMetadata(const string& name, string& value) const
|
||||
{
|
||||
try {
|
||||
value = zimArchive->getMetadata(name);
|
||||
return true;
|
||||
} catch(zim::EntryNotFound& e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#define METADATA(NAME) std::string v; getMetadata(NAME, v); return v;
|
||||
|
||||
string Reader::getName() const
|
||||
{
|
||||
METADATA("Name")
|
||||
}
|
||||
|
||||
string Reader::getTitle() const
|
||||
{
|
||||
string value = zimArchive->getMetadata("Title");
|
||||
if (value.empty()) {
|
||||
value = getLastPathElement(zimFilePath);
|
||||
std::replace(value.begin(), value.end(), '_', ' ');
|
||||
size_t pos = value.find(".zim");
|
||||
value = value.substr(0, pos);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
string Reader::getCreator() const
|
||||
{
|
||||
METADATA("Creator")
|
||||
}
|
||||
|
||||
string Reader::getPublisher() const
|
||||
{
|
||||
METADATA("Publisher")
|
||||
}
|
||||
|
||||
string Reader::getDate() const
|
||||
{
|
||||
METADATA("Date")
|
||||
}
|
||||
|
||||
string Reader::getDescription() const
|
||||
{
|
||||
string value;
|
||||
this->getMetadata("Description", value);
|
||||
|
||||
/* Mediawiki Collection tends to use the "Subtitle" name */
|
||||
if (value.empty()) {
|
||||
this->getMetadata("Subtitle", value);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
string Reader::getLongDescription() const
|
||||
{
|
||||
METADATA("LongDescription")
|
||||
}
|
||||
|
||||
string Reader::getLanguage() const
|
||||
{
|
||||
METADATA("Language")
|
||||
}
|
||||
|
||||
string Reader::getLicense() const
|
||||
{
|
||||
METADATA("License")
|
||||
}
|
||||
|
||||
string Reader::getTags(bool original) const
|
||||
{
|
||||
string tags_str;
|
||||
getMetadata("Tags", tags_str);
|
||||
if (original) {
|
||||
return tags_str;
|
||||
}
|
||||
auto tags = convertTags(tags_str);
|
||||
return join(tags, ";");
|
||||
}
|
||||
|
||||
|
||||
string Reader::getTagStr(const std::string& tagName) const
|
||||
{
|
||||
string tags_str;
|
||||
getMetadata("Tags", tags_str);
|
||||
return getTagValueFromTagList(convertTags(tags_str), tagName);
|
||||
}
|
||||
|
||||
bool Reader::getTagBool(const std::string& tagName) const
|
||||
{
|
||||
return convertStrToBool(getTagStr(tagName));
|
||||
}
|
||||
|
||||
string Reader::getRelation() const
|
||||
{
|
||||
METADATA("Relation")
|
||||
}
|
||||
|
||||
string Reader::getFlavour() const
|
||||
{
|
||||
METADATA("Flavour")
|
||||
}
|
||||
|
||||
string Reader::getSource() const
|
||||
{
|
||||
METADATA("Source")
|
||||
}
|
||||
|
||||
string Reader::getScraper() const
|
||||
{
|
||||
METADATA("Scraper")
|
||||
}
|
||||
#undef METADATA
|
||||
|
||||
string Reader::getOrigId() const
|
||||
{
|
||||
string value;
|
||||
this->getMetadata("startfileuid", value);
|
||||
if (value.empty()) {
|
||||
return "";
|
||||
}
|
||||
std::string id = value;
|
||||
std::string origID;
|
||||
std::string temp = "";
|
||||
unsigned int k = 0;
|
||||
char tempArray[16] = "";
|
||||
for (unsigned int i = 0; i < id.size(); i++) {
|
||||
if (id[i] == '\n') {
|
||||
tempArray[k] = atoi(temp.c_str());
|
||||
temp = "";
|
||||
k++;
|
||||
} else {
|
||||
temp += id[i];
|
||||
}
|
||||
}
|
||||
origID = hexUUID(tempArray);
|
||||
return origID;
|
||||
}
|
||||
|
||||
Entry Reader::getEntryFromPath(const std::string& path) const
|
||||
{
|
||||
if (path.empty() || path == "/") {
|
||||
return getMainPage();
|
||||
}
|
||||
|
||||
try {
|
||||
return zimArchive->getEntryByPath(path);
|
||||
} catch (zim::EntryNotFound& e) {
|
||||
throw NoEntry();
|
||||
}
|
||||
}
|
||||
|
||||
Entry Reader::getEntryFromEncodedPath(const std::string& path) const
|
||||
{
|
||||
return getEntryFromPath(urlDecode(path, true));
|
||||
}
|
||||
|
||||
Entry Reader::getEntryFromTitle(const std::string& title) const
|
||||
{
|
||||
try {
|
||||
return zimArchive->getEntryByTitle(title);
|
||||
} catch(zim::EntryNotFound& e) {
|
||||
throw NoEntry();
|
||||
}
|
||||
}
|
||||
|
||||
bool Reader::pathExists(const string& path) const
|
||||
{
|
||||
return zimArchive->hasEntryByPath(path);
|
||||
}
|
||||
|
||||
/* Does the ZIM file has a fulltext index */
|
||||
bool Reader::hasFulltextIndex() const
|
||||
{
|
||||
return zimArchive->hasFulltextIndex();
|
||||
}
|
||||
|
||||
/* Search titles by prefix */
|
||||
|
||||
bool Reader::searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
const bool reset)
|
||||
{
|
||||
/* Reset the suggestions otherwise check if the suggestions number is less
|
||||
* than the suggestionsCount */
|
||||
if (reset) {
|
||||
this->suggestions.clear();
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
} else {
|
||||
if (this->suggestions.size() > suggestionsCount) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = searchSuggestions(prefix, suggestionsCount, this->suggestions);
|
||||
|
||||
/* Set the cursor to the begining */
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
bool Reader::searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results)
|
||||
{
|
||||
bool retVal = false;
|
||||
|
||||
/* Return if no prefix */
|
||||
if (prefix.size() == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto& entry: zimArchive->findByTitle(prefix)) {
|
||||
if (results.size() >= suggestionsCount) {
|
||||
break;
|
||||
}
|
||||
/* Extract the interesting part of article title & url */
|
||||
std::string normalizedArticleTitle
|
||||
= kiwix::normalize(entry.getTitle());
|
||||
|
||||
// Get the final path.
|
||||
auto item = entry.getItem(true);
|
||||
std::string articleFinalUrl = item.getPath();
|
||||
|
||||
/* Go through all already found suggestions and skip if this
|
||||
article is already in the suggestions list (with an other
|
||||
title) */
|
||||
bool insert = true;
|
||||
std::vector<SuggestionItem>::iterator suggestionItr;
|
||||
for (suggestionItr = results.begin();
|
||||
suggestionItr != results.end();
|
||||
suggestionItr++) {
|
||||
int result = normalizedArticleTitle.compare((*suggestionItr).getNormalizedTitle());
|
||||
if (result == 0 && articleFinalUrl.compare((*suggestionItr).getPath()) == 0) {
|
||||
insert = false;
|
||||
break;
|
||||
} else if (result < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Insert if possible */
|
||||
if (insert) {
|
||||
SuggestionItem suggestion(entry.getTitle(), normalizedArticleTitle, articleFinalUrl);
|
||||
results.insert(suggestionItr, suggestion);
|
||||
}
|
||||
|
||||
/* Suggestions where found */
|
||||
retVal = true;
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
std::vector<std::string> Reader::getTitleVariants(
|
||||
const std::string& title) const
|
||||
{
|
||||
std::vector<std::string> variants;
|
||||
variants.push_back(title);
|
||||
variants.push_back(kiwix::ucFirst(title));
|
||||
variants.push_back(kiwix::lcFirst(title));
|
||||
variants.push_back(kiwix::toTitle(title));
|
||||
return variants;
|
||||
}
|
||||
|
||||
|
||||
bool Reader::searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount)
|
||||
{
|
||||
this->suggestions.clear();
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
|
||||
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Try also a few variations of the prefix to have better results */
|
||||
bool Reader::searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results)
|
||||
{
|
||||
std::vector<std::string> variants = this->getTitleVariants(prefix);
|
||||
bool retVal = false;
|
||||
|
||||
/* Try to search in the title using fulltext search database */
|
||||
|
||||
auto suggestionSearcher = zim::Searcher(*zimArchive);
|
||||
zim::Query suggestionQuery;
|
||||
suggestionQuery.setQuery(prefix, true);
|
||||
auto suggestionSearch = suggestionSearcher.search(suggestionQuery);
|
||||
|
||||
if (suggestionSearch.getEstimatedMatches()) {
|
||||
const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
|
||||
for (auto current = suggestions.begin();
|
||||
current != suggestions.end();
|
||||
current++) {
|
||||
SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
|
||||
current.getPath(), current.getSnippet());
|
||||
results.push_back(suggestion);
|
||||
}
|
||||
retVal = true;
|
||||
} else {
|
||||
for (std::vector<std::string>::iterator variantsItr = variants.begin();
|
||||
variantsItr != variants.end();
|
||||
variantsItr++) {
|
||||
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
|
||||
|| retVal;
|
||||
}
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/* Get next suggestion */
|
||||
bool Reader::getNextSuggestion(string& title)
|
||||
{
|
||||
if (this->suggestionsOffset != this->suggestions.end()) {
|
||||
/* title */
|
||||
title = (*(this->suggestionsOffset)).getTitle();
|
||||
|
||||
/* increment the cursor for the next call */
|
||||
this->suggestionsOffset++;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Reader::getNextSuggestion(string& title, string& url)
|
||||
{
|
||||
if (this->suggestionsOffset != this->suggestions.end()) {
|
||||
/* title */
|
||||
title = (*(this->suggestionsOffset)).getTitle();
|
||||
url = (*(this->suggestionsOffset)).getPath();
|
||||
|
||||
/* increment the cursor for the next call */
|
||||
this->suggestionsOffset++;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Check if the file has as checksum */
|
||||
bool Reader::canCheckIntegrity() const
|
||||
{
|
||||
return zimArchive->hasChecksum();
|
||||
}
|
||||
|
||||
/* Return true if corrupted, false otherwise */
|
||||
bool Reader::isCorrupted() const
|
||||
{
|
||||
try {
|
||||
if (zimArchive->check() == true) {
|
||||
return false;
|
||||
}
|
||||
} catch (exception& e) {
|
||||
cerr << e.what() << endl;
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Return the file size, works also for splitted files */
|
||||
unsigned int Reader::getFileSize() const
|
||||
{
|
||||
return zimArchive->getFilesize() / 1024;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -21,35 +21,26 @@
|
||||
#include <cmath>
|
||||
|
||||
#include "search_renderer.h"
|
||||
#include "searcher.h"
|
||||
#include "reader.h"
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
|
||||
#include "tools/archiveTools.h"
|
||||
|
||||
#include <zim/search.h>
|
||||
|
||||
#include <mustache.hpp>
|
||||
#include "libkiwix-resources.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/* Constructor */
|
||||
SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
|
||||
unsigned int start, unsigned int estimatedResultCount)
|
||||
: SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
|
||||
{}
|
||||
|
||||
SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
|
||||
unsigned int start, unsigned int estimatedResultCount)
|
||||
: m_srs(srs),
|
||||
SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
|
||||
: mp_searcher(searcher),
|
||||
mp_nameMapper(mapper),
|
||||
mp_library(library),
|
||||
protocolPrefix("zim://"),
|
||||
searchProtocolPrefix("search://"),
|
||||
estimatedResultCount(estimatedResultCount),
|
||||
resultStart(start)
|
||||
searchProtocolPrefix("search://?")
|
||||
{}
|
||||
|
||||
/* Destructor */
|
||||
@@ -57,12 +48,12 @@ SearchRenderer::~SearchRenderer() = default;
|
||||
|
||||
void SearchRenderer::setSearchPattern(const std::string& pattern)
|
||||
{
|
||||
searchPattern = pattern;
|
||||
this->searchPattern = pattern;
|
||||
}
|
||||
|
||||
void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
|
||||
void SearchRenderer::setSearchContent(const std::string& name)
|
||||
{
|
||||
searchBookQuery = bookQuery;
|
||||
this->searchContent = name;
|
||||
}
|
||||
|
||||
void SearchRenderer::setProtocolPrefix(const std::string& prefix)
|
||||
@@ -75,162 +66,84 @@ void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
|
||||
this->searchProtocolPrefix = prefix;
|
||||
}
|
||||
|
||||
std::string extractValueFromQuery(const std::string& query, const std::string& key) {
|
||||
const std::string p = key + "=";
|
||||
const size_t i = query.find(p);
|
||||
if (i == std::string::npos) {
|
||||
return "";
|
||||
}
|
||||
std::string r = query.substr(i + p.size());
|
||||
return r.substr(0, r.find("&"));
|
||||
}
|
||||
|
||||
kainjow::mustache::data buildQueryData
|
||||
(
|
||||
const std::string& searchProtocolPrefix,
|
||||
const std::string& pattern,
|
||||
const std::string& bookQuery
|
||||
) {
|
||||
kainjow::mustache::data query;
|
||||
query.set("pattern", kiwix::encodeDiples(pattern));
|
||||
std::ostringstream ss;
|
||||
ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern);
|
||||
ss << "&" << bookQuery;
|
||||
query.set("unpaginatedQuery", ss.str());
|
||||
auto lang = extractValueFromQuery(bookQuery, "books.filter.lang");
|
||||
if(!lang.empty()) {
|
||||
query.set("lang", lang);
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
kainjow::mustache::data buildPagination(
|
||||
unsigned int pageLength,
|
||||
unsigned int resultsCount,
|
||||
unsigned int resultsStart
|
||||
)
|
||||
std::string SearchRenderer::getHtml()
|
||||
{
|
||||
assert(pageLength!=0);
|
||||
kainjow::mustache::data pagination;
|
||||
kainjow::mustache::data results{kainjow::mustache::data::type::list};
|
||||
|
||||
mp_searcher->restart_search();
|
||||
Result* p_result = NULL;
|
||||
while ((p_result = mp_searcher->getNextResult())) {
|
||||
kainjow::mustache::data result;
|
||||
result.set("title", p_result->get_title());
|
||||
result.set("url", p_result->get_url());
|
||||
result.set("snippet", p_result->get_snippet());
|
||||
result.set("resultContentId", mp_nameMapper->getNameForId(p_result->get_zimId()));
|
||||
|
||||
if (p_result->get_wordCount() >= 0) {
|
||||
result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
|
||||
}
|
||||
|
||||
results.push_back(result);
|
||||
delete p_result;
|
||||
}
|
||||
|
||||
// pages
|
||||
kainjow::mustache::data pages{kainjow::mustache::data::type::list};
|
||||
|
||||
if (resultsCount == 0) {
|
||||
// Easy case
|
||||
pagination.set("itemsPerPage", to_string(pageLength));
|
||||
pagination.set("hasPages", false);
|
||||
pagination.set("pages", pages);
|
||||
return pagination;
|
||||
}
|
||||
|
||||
// First we want to display pages starting at a multiple of `pageLength`
|
||||
// so, let's calculate the start index of the current page.
|
||||
auto currentPage = resultsStart/pageLength;
|
||||
auto lastPage = ((resultsCount-1)/pageLength);
|
||||
auto lastPageStart = lastPage*pageLength;
|
||||
auto nbPages = lastPage + 1;
|
||||
|
||||
auto firstPageGenerated = currentPage > 4 ? currentPage-4 : 0;
|
||||
auto lastPageGenerated = std::min(currentPage+4, lastPage);
|
||||
|
||||
if (nbPages != 1) {
|
||||
if (firstPageGenerated!=0) {
|
||||
kainjow::mustache::data page;
|
||||
page.set("label", "◀");
|
||||
page.set("start", to_string(0));
|
||||
page.set("current", false);
|
||||
pages.push_back(page);
|
||||
auto resultStart = mp_searcher->getResultStart();
|
||||
auto resultEnd = 0U;
|
||||
auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
|
||||
auto currentPage = 0U;
|
||||
auto pageStart = 0U;
|
||||
auto pageEnd = 0U;
|
||||
auto lastPageStart = 0U;
|
||||
if (pageLength) {
|
||||
currentPage = resultStart/pageLength;
|
||||
pageStart = currentPage > 4 ? currentPage-4 : 0;
|
||||
pageEnd = currentPage + 5;
|
||||
if (pageEnd > estimatedResultCount / pageLength) {
|
||||
pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
|
||||
}
|
||||
|
||||
for (auto i=firstPageGenerated; i<=lastPageGenerated; i++) {
|
||||
kainjow::mustache::data page;
|
||||
page.set("label", to_string(i+1));
|
||||
page.set("start", to_string(i*pageLength));
|
||||
page.set("current", bool(i == currentPage));
|
||||
pages.push_back(page);
|
||||
}
|
||||
|
||||
if (lastPageGenerated!=lastPage) {
|
||||
kainjow::mustache::data page;
|
||||
page.set("label", "▶");
|
||||
page.set("start", to_string(lastPageStart));
|
||||
page.set("current", false);
|
||||
pages.push_back(page);
|
||||
if (estimatedResultCount > pageLength) {
|
||||
lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
|
||||
}
|
||||
}
|
||||
|
||||
pagination.set("itemsPerPage", to_string(pageLength));
|
||||
pagination.set("hasPages", firstPageGenerated < lastPageGenerated);
|
||||
pagination.set("pages", pages);
|
||||
return pagination;
|
||||
}
|
||||
resultEnd = resultStart+pageLength; //setting result end
|
||||
|
||||
std::string SearchRenderer::renderTemplate(const std::string& tmpl_str)
|
||||
{
|
||||
const std::string absPathPrefix = protocolPrefix;
|
||||
// Build the results list
|
||||
kainjow::mustache::data items{kainjow::mustache::data::type::list};
|
||||
for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
|
||||
kainjow::mustache::data result;
|
||||
const std::string zim_id(it.getZimId());
|
||||
const auto path = mp_nameMapper->getNameForId(zim_id) + "/" + it.getPath();
|
||||
result.set("title", it.getTitle());
|
||||
result.set("absolutePath", absPathPrefix + urlEncode(path));
|
||||
result.set("snippet", it.getSnippet());
|
||||
if (mp_library) {
|
||||
result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
|
||||
}
|
||||
if (it.getWordCount() >= 0) {
|
||||
result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
|
||||
}
|
||||
for (unsigned int i = pageStart; i < pageEnd; i++) {
|
||||
kainjow::mustache::data page;
|
||||
page.set("label", to_string(i + 1));
|
||||
page.set("start", to_string(i * pageLength));
|
||||
|
||||
items.push_back(result);
|
||||
if (i == currentPage) {
|
||||
page.set("selected", true);
|
||||
}
|
||||
pages.push_back(page);
|
||||
}
|
||||
kainjow::mustache::data results;
|
||||
results.set("items", items);
|
||||
results.set("count", kiwix::beautifyInteger(estimatedResultCount));
|
||||
results.set("hasResults", estimatedResultCount != 0);
|
||||
results.set("start", kiwix::beautifyInteger(resultStart));
|
||||
results.set("end", kiwix::beautifyInteger(std::min(resultStart+pageLength-1, estimatedResultCount)));
|
||||
|
||||
// pagination
|
||||
auto pagination = buildPagination(
|
||||
pageLength,
|
||||
estimatedResultCount,
|
||||
resultStart
|
||||
);
|
||||
|
||||
kainjow::mustache::data query = buildQueryData(
|
||||
searchProtocolPrefix,
|
||||
searchPattern,
|
||||
searchBookQuery
|
||||
);
|
||||
|
||||
std::string template_str = RESOURCE::templates::search_result_html;
|
||||
kainjow::mustache::mustache tmpl(template_str);
|
||||
|
||||
kainjow::mustache::data allData;
|
||||
allData.set("searchProtocolPrefix", searchProtocolPrefix);
|
||||
allData.set("results", results);
|
||||
allData.set("pagination", pagination);
|
||||
allData.set("query", query);
|
||||
|
||||
kainjow::mustache::mustache tmpl(tmpl_str);
|
||||
allData.set("pages", pages);
|
||||
allData.set("hasResults", estimatedResultCount != 0);
|
||||
allData.set("hasPages", pageStart != pageEnd);
|
||||
allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
|
||||
allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
|
||||
allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
|
||||
allData.set("resultStart", to_string(resultStart + 1));
|
||||
allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
|
||||
allData.set("pageLength", to_string(pageLength));
|
||||
allData.set("resultLastPageStart", to_string(lastPageStart));
|
||||
allData.set("protocolPrefix", this->protocolPrefix);
|
||||
allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
|
||||
allData.set("contentId", this->searchContent);
|
||||
|
||||
std::stringstream ss;
|
||||
tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
|
||||
if (!tmpl.is_valid()) {
|
||||
throw std::runtime_error("Error while rendering search results: " + tmpl.error_message());
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::string SearchRenderer::getHtml()
|
||||
{
|
||||
return renderTemplate(RESOURCE::templates::search_result_html);
|
||||
}
|
||||
|
||||
std::string SearchRenderer::getXml()
|
||||
{
|
||||
return renderTemplate(RESOURCE::templates::search_result_xml);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
272
src/searcher.cpp
Normal file
272
src/searcher.cpp
Normal file
@@ -0,0 +1,272 @@
|
||||
/*
|
||||
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include "searcher.h"
|
||||
#include "reader.h"
|
||||
|
||||
#include <zim/search.h>
|
||||
|
||||
#include <mustache.hpp>
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#define MAX_SEARCH_LEN 140
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
class _Result : public Result
|
||||
{
|
||||
public:
|
||||
_Result(zim::SearchResultSet::iterator iterator);
|
||||
virtual ~_Result(){};
|
||||
|
||||
virtual std::string get_url();
|
||||
virtual std::string get_title();
|
||||
virtual int get_score();
|
||||
virtual std::string get_snippet();
|
||||
virtual std::string get_content();
|
||||
virtual int get_wordCount();
|
||||
virtual int get_size();
|
||||
virtual std::string get_zimId();
|
||||
|
||||
private:
|
||||
zim::SearchResultSet::iterator iterator;
|
||||
};
|
||||
|
||||
struct SearcherInternal : zim::SearchResultSet {
|
||||
explicit SearcherInternal(const zim::SearchResultSet& srs)
|
||||
: zim::SearchResultSet(srs)
|
||||
, current_iterator(srs.begin())
|
||||
{
|
||||
}
|
||||
|
||||
zim::SearchResultSet::iterator current_iterator;
|
||||
};
|
||||
|
||||
/* Constructor */
|
||||
Searcher::Searcher()
|
||||
: searchPattern(""),
|
||||
estimatedResultCount(0),
|
||||
resultStart(0),
|
||||
resultEnd(0)
|
||||
{
|
||||
loadICUExternalTables();
|
||||
}
|
||||
|
||||
/* Destructor */
|
||||
Searcher::~Searcher()
|
||||
{
|
||||
}
|
||||
|
||||
bool Searcher::add_reader(Reader* reader)
|
||||
{
|
||||
if (!reader->hasFulltextIndex()) {
|
||||
return false;
|
||||
}
|
||||
this->readers.push_back(reader);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
Reader* Searcher::get_reader(int readerIndex)
|
||||
{
|
||||
return readers.at(readerIndex);
|
||||
}
|
||||
|
||||
/* Search strings in the database */
|
||||
void Searcher::search(const std::string& search,
|
||||
unsigned int resultStart,
|
||||
unsigned int resultEnd,
|
||||
const bool verbose)
|
||||
{
|
||||
this->reset();
|
||||
|
||||
if (verbose == true) {
|
||||
cout << "Performing query `" << search << "'" << endl;
|
||||
}
|
||||
|
||||
this->searchPattern = search;
|
||||
this->resultStart = resultStart;
|
||||
this->resultEnd = resultEnd;
|
||||
/* Try to find results */
|
||||
if (resultStart != resultEnd) {
|
||||
/* Perform the search */
|
||||
string unaccentedSearch = removeAccents(search);
|
||||
std::vector<zim::Archive> archives;
|
||||
for (auto current = this->readers.begin(); current != this->readers.end();
|
||||
current++) {
|
||||
if ( (*current)->hasFulltextIndex() ) {
|
||||
archives.push_back(*(*current)->getZimArchive());
|
||||
}
|
||||
}
|
||||
zim::Searcher searcher(archives);
|
||||
zim::Query query;
|
||||
query.setQuery(unaccentedSearch, false);
|
||||
query.setVerbose(verbose);
|
||||
zim::Search search = searcher.search(query);
|
||||
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
|
||||
this->estimatedResultCount = search.getEstimatedMatches();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
void Searcher::geo_search(float latitude, float longitude, float distance,
|
||||
unsigned int resultStart,
|
||||
unsigned int resultEnd,
|
||||
const bool verbose)
|
||||
{
|
||||
this->reset();
|
||||
|
||||
if (verbose == true) {
|
||||
cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
|
||||
}
|
||||
|
||||
/* Perform the search */
|
||||
std::ostringstream oss;
|
||||
oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
|
||||
this->searchPattern = oss.str();
|
||||
this->resultStart = resultStart;
|
||||
this->resultEnd = resultEnd;
|
||||
|
||||
/* Try to find results */
|
||||
if (resultStart == resultEnd) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<zim::Archive> archives;
|
||||
for (auto current = this->readers.begin(); current != this->readers.end();
|
||||
current++) {
|
||||
archives.push_back(*(*current)->getZimArchive());
|
||||
}
|
||||
zim::Searcher searcher(archives);
|
||||
zim::Query query;
|
||||
query.setVerbose(verbose);
|
||||
query.setQuery("", false);
|
||||
query.setGeorange(latitude, longitude, distance);
|
||||
zim::Search search = searcher.search(query);
|
||||
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
|
||||
this->estimatedResultCount = search.getEstimatedMatches();
|
||||
}
|
||||
|
||||
|
||||
void Searcher::restart_search()
|
||||
{
|
||||
if (internal.get()) {
|
||||
internal->current_iterator = internal->begin();
|
||||
}
|
||||
}
|
||||
|
||||
Result* Searcher::getNextResult()
|
||||
{
|
||||
if (internal.get() &&
|
||||
internal->current_iterator != internal->end()) {
|
||||
Result* result = new _Result(internal->current_iterator);
|
||||
internal->current_iterator++;
|
||||
return result;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Reset the results */
|
||||
void Searcher::reset()
|
||||
{
|
||||
this->estimatedResultCount = 0;
|
||||
this->searchPattern = "";
|
||||
return;
|
||||
}
|
||||
|
||||
void Searcher::suggestions(std::string& searchPattern, const bool verbose)
|
||||
{
|
||||
this->reset();
|
||||
|
||||
if (verbose == true) {
|
||||
cout << "Performing suggestion query `" << searchPattern << "`" << endl;
|
||||
}
|
||||
|
||||
this->searchPattern = searchPattern;
|
||||
this->resultStart = 0;
|
||||
this->resultEnd = 10;
|
||||
string unaccentedSearch = removeAccents(searchPattern);
|
||||
|
||||
std::vector<zim::Archive> archives;
|
||||
for (auto current = this->readers.begin(); current != this->readers.end();
|
||||
current++) {
|
||||
archives.push_back(*(*current)->getZimArchive());
|
||||
}
|
||||
zim::Searcher searcher(archives);
|
||||
zim::Query query;
|
||||
query.setVerbose(verbose);
|
||||
query.setQuery(unaccentedSearch, true);
|
||||
zim::Search search = searcher.search(query);
|
||||
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
|
||||
this->estimatedResultCount = search.getEstimatedMatches();
|
||||
}
|
||||
|
||||
/* Return the result count estimation */
|
||||
unsigned int Searcher::getEstimatedResultCount()
|
||||
{
|
||||
return this->estimatedResultCount;
|
||||
}
|
||||
|
||||
_Result::_Result(zim::SearchResultSet::iterator iterator)
|
||||
: iterator(iterator)
|
||||
{
|
||||
}
|
||||
|
||||
std::string _Result::get_url()
|
||||
{
|
||||
return iterator.getPath();
|
||||
}
|
||||
std::string _Result::get_title()
|
||||
{
|
||||
return iterator.getTitle();
|
||||
}
|
||||
int _Result::get_score()
|
||||
{
|
||||
return iterator.getScore();
|
||||
}
|
||||
std::string _Result::get_snippet()
|
||||
{
|
||||
return iterator.getSnippet();
|
||||
}
|
||||
std::string _Result::get_content()
|
||||
{
|
||||
return iterator->getItem(true).getData();
|
||||
}
|
||||
int _Result::get_size()
|
||||
{
|
||||
return iterator.getSize();
|
||||
}
|
||||
int _Result::get_wordCount()
|
||||
{
|
||||
return iterator.getWordCount();
|
||||
}
|
||||
std::string _Result::get_zimId()
|
||||
{
|
||||
std::ostringstream s;
|
||||
s << iterator.getZimId();
|
||||
return s.str();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -24,7 +24,6 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <zim/item.h>
|
||||
#include "server/internalServer.h"
|
||||
|
||||
namespace kiwix {
|
||||
@@ -46,13 +45,10 @@ bool Server::start() {
|
||||
m_port,
|
||||
m_root,
|
||||
m_nbThreads,
|
||||
m_multizimSearchLimit,
|
||||
m_verbose,
|
||||
m_withTaskbar,
|
||||
m_withLibraryButton,
|
||||
m_blockExternalLinks,
|
||||
m_indexTemplateString,
|
||||
m_ipConnectionLimit));
|
||||
m_blockExternalLinks));
|
||||
return mp_server->start();
|
||||
}
|
||||
|
||||
@@ -74,14 +70,4 @@ void Server::setRoot(const std::string& root)
|
||||
}
|
||||
}
|
||||
|
||||
int Server::getPort()
|
||||
{
|
||||
return mp_server->getPort();
|
||||
}
|
||||
|
||||
std::string Server::getAddress()
|
||||
{
|
||||
return mp_server->getAddress();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -37,11 +37,11 @@ namespace {
|
||||
// into the ETag for ETag::Option opt.
|
||||
// IMPORTANT: The characters in all_options must come in sorted order (so that
|
||||
// IMPORTANT: isValidOptionsString() works correctly).
|
||||
const char all_options[] = "Zz";
|
||||
const char all_options[] = "cz";
|
||||
|
||||
static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
|
||||
|
||||
bool isValidETagBody(const std::string& s)
|
||||
bool isValidServerId(const std::string& s)
|
||||
{
|
||||
return !s.empty() && s.find_first_of("\"/") == std::string::npos;
|
||||
}
|
||||
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
|
||||
|
||||
std::string ETag::get_etag() const
|
||||
{
|
||||
if ( m_body.empty() )
|
||||
if ( m_serverId.empty() )
|
||||
return std::string();
|
||||
|
||||
return "\"" + m_body + "/" + m_options + "\"";
|
||||
return "\"" + m_serverId + "/" + m_options + "\"";
|
||||
}
|
||||
|
||||
ETag::ETag(const std::string& body, const std::string& options)
|
||||
ETag::ETag(const std::string& serverId, const std::string& options)
|
||||
{
|
||||
if ( isValidETagBody(body) && isValidOptionsString(options) )
|
||||
if ( isValidServerId(serverId) && isValidOptionsString(options) )
|
||||
{
|
||||
m_body = body;
|
||||
m_serverId = serverId;
|
||||
m_options = options;
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
|
||||
return ETag(s.substr(0, i), s.substr(i+1));
|
||||
}
|
||||
|
||||
ETag ETag::match(const std::string& etags, const std::string& body)
|
||||
ETag ETag::match(const std::string& etags, const std::string& server_id)
|
||||
{
|
||||
std::istringstream ss(etags);
|
||||
std::string etag_str;
|
||||
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& body)
|
||||
etag_str.pop_back();
|
||||
|
||||
const ETag etag = parse(etag_str);
|
||||
if ( etag && etag.m_body == body )
|
||||
if ( etag && etag.m_serverId == server_id )
|
||||
return etag;
|
||||
}
|
||||
|
||||
|
||||
@@ -28,11 +28,10 @@ namespace kiwix {
|
||||
// The ETag string used by Kiwix server (more precisely, its value inside the
|
||||
// double quotes) consists of two parts:
|
||||
//
|
||||
// 1. Body - A string uniquely identifying the object or state from which
|
||||
// the resource has been obtained.
|
||||
// 1. ServerId - The string obtained on server start up
|
||||
//
|
||||
// 2. Options - Zero or more characters encoding the type of the ETag and/or
|
||||
// the values of some of the headers of the response
|
||||
// 2. Options - Zero or more characters encoding the values of some of the
|
||||
// headers of the response
|
||||
//
|
||||
// The two parts are separated with a slash (/) symbol (which is always present,
|
||||
// even when the the options part is empty). Neither portion of a Kiwix ETag
|
||||
@@ -41,7 +40,7 @@ namespace kiwix {
|
||||
//
|
||||
// "abcdefghijklmn/"
|
||||
// "1234567890/z"
|
||||
// "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
|
||||
// "1234567890/cz"
|
||||
//
|
||||
// The options part of the Kiwix ETag allows to correctly set the required
|
||||
// headers when responding to a conditional If-None-Match request with a 304
|
||||
@@ -52,7 +51,7 @@ class ETag
|
||||
{
|
||||
public: // types
|
||||
enum Option {
|
||||
ZIM_CONTENT,
|
||||
CACHEABLE_ENTITY,
|
||||
COMPRESSED_CONTENT,
|
||||
OPTION_COUNT
|
||||
};
|
||||
@@ -60,10 +59,10 @@ class ETag
|
||||
public: // functions
|
||||
ETag() {}
|
||||
|
||||
void set_body(const std::string& s) { m_body = s; }
|
||||
void set_server_id(const std::string& id) { m_serverId = id; }
|
||||
void set_option(Option opt);
|
||||
|
||||
explicit operator bool() const { return !m_body.empty(); }
|
||||
explicit operator bool() const { return !m_serverId.empty(); }
|
||||
|
||||
bool get_option(Option opt) const;
|
||||
std::string get_etag() const;
|
||||
@@ -77,7 +76,7 @@ class ETag
|
||||
static ETag parse(std::string s);
|
||||
|
||||
private: // data
|
||||
std::string m_body;
|
||||
std::string m_serverId;
|
||||
std::string m_options;
|
||||
};
|
||||
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "i18n.h"
|
||||
|
||||
#include "tools/otherTools.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
const char* I18nStringTable::get(const std::string& key) const
|
||||
{
|
||||
const I18nString* const begin = entries;
|
||||
const I18nString* const end = begin + entryCount;
|
||||
const I18nString* found = std::lower_bound(begin, end, key,
|
||||
[](const I18nString& a, const std::string& k) {
|
||||
return a.key < k;
|
||||
});
|
||||
return (found == end || found->key != key) ? nullptr : found->value;
|
||||
}
|
||||
|
||||
namespace i18n
|
||||
{
|
||||
// this data is generated by the i18n resource compiler
|
||||
extern const I18nStringTable stringTables[];
|
||||
extern const size_t langCount;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class I18nStringDB
|
||||
{
|
||||
public: // functions
|
||||
I18nStringDB() {
|
||||
for ( size_t i = 0; i < kiwix::i18n::langCount; ++i ) {
|
||||
const auto& t = kiwix::i18n::stringTables[i];
|
||||
lang2TableMap[t.lang] = &t;
|
||||
}
|
||||
enStrings = lang2TableMap.at("en");
|
||||
};
|
||||
|
||||
std::string get(const std::string& lang, const std::string& key) const {
|
||||
const char* s = getStringsFor(lang)->get(key);
|
||||
if ( s == nullptr ) {
|
||||
s = enStrings->get(key);
|
||||
if ( s == nullptr ) {
|
||||
throw std::runtime_error("Invalid message id");
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
size_t getStringCount(const std::string& lang) const {
|
||||
try {
|
||||
return lang2TableMap.at(lang)->entryCount;
|
||||
} catch(const std::out_of_range&) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private: // functions
|
||||
const I18nStringTable* getStringsFor(const std::string& lang) const {
|
||||
try {
|
||||
return lang2TableMap.at(lang);
|
||||
} catch(const std::out_of_range&) {
|
||||
return enStrings;
|
||||
}
|
||||
}
|
||||
|
||||
private: // data
|
||||
std::map<std::string, const I18nStringTable*> lang2TableMap;
|
||||
const I18nStringTable* enStrings;
|
||||
};
|
||||
|
||||
const I18nStringDB& getStringDb()
|
||||
{
|
||||
static const I18nStringDB stringDb;
|
||||
return stringDb;
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::string getTranslatedString(const std::string& lang, const std::string& key)
|
||||
{
|
||||
return getStringDb().get(lang, key);
|
||||
}
|
||||
|
||||
namespace i18n
|
||||
{
|
||||
|
||||
std::string expandParameterizedString(const std::string& lang,
|
||||
const std::string& key,
|
||||
const Parameters& params)
|
||||
{
|
||||
const std::string tmpl = getTranslatedString(lang, key);
|
||||
return render_template(tmpl, params);
|
||||
}
|
||||
|
||||
} // namespace i18n
|
||||
|
||||
std::string ParameterizedMessage::getText(const std::string& lang) const
|
||||
{
|
||||
return i18n::expandParameterizedString(lang, msgId, params);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
LangPreference parseSingleLanguagePreference(const std::string& s)
|
||||
{
|
||||
const size_t langStart = s.find_first_not_of(" \t\n");
|
||||
if ( langStart == std::string::npos ) {
|
||||
return {"", 0};
|
||||
}
|
||||
|
||||
const size_t langEnd = s.find(';', langStart);
|
||||
if ( langEnd == std::string::npos ) {
|
||||
return {s.substr(langStart), 1};
|
||||
}
|
||||
|
||||
const std::string lang = s.substr(langStart, langEnd - langStart);
|
||||
// We don't care about langEnd == langStart which will result in an empty
|
||||
// language name - it will be dismissed by parseUserLanguagePreferences()
|
||||
|
||||
float q = 1.0;
|
||||
int nCharsScanned;
|
||||
if ( 1 == sscanf(s.c_str() + langEnd + 1, "q=%f%n", &q, &nCharsScanned)
|
||||
&& langEnd + 1 + nCharsScanned == s.size() ) {
|
||||
return {lang, q};
|
||||
}
|
||||
|
||||
return {"", 0};
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
UserLangPreferences parseUserLanguagePreferences(const std::string& s)
|
||||
{
|
||||
UserLangPreferences result;
|
||||
std::istringstream iss(s);
|
||||
std::string singleLangPrefStr;
|
||||
while ( std::getline(iss, singleLangPrefStr, ',') )
|
||||
{
|
||||
const auto langPref = parseSingleLanguagePreference(singleLangPrefStr);
|
||||
if ( !langPref.lang.empty() && langPref.preference > 0 ) {
|
||||
result.push_back(langPref);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string selectMostSuitableLanguage(const UserLangPreferences& prefs)
|
||||
{
|
||||
if ( prefs.empty() ) {
|
||||
return "en";
|
||||
}
|
||||
|
||||
std::string bestLangSoFar("en");
|
||||
float bestScoreSoFar = 0;
|
||||
const auto& stringDb = getStringDb();
|
||||
for ( const auto& entry : prefs ) {
|
||||
const float score = entry.preference * stringDb.getStringCount(entry.lang);
|
||||
if ( score > bestScoreSoFar ) {
|
||||
bestScoreSoFar = score;
|
||||
bestLangSoFar = entry.lang;
|
||||
}
|
||||
}
|
||||
return bestLangSoFar;
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -1,128 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_SERVER_I18N
|
||||
#define KIWIX_SERVER_I18N
|
||||
|
||||
#include <string>
|
||||
#include <mustache.hpp>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
struct I18nString {
|
||||
const char* const key;
|
||||
const char* const value;
|
||||
};
|
||||
|
||||
struct I18nStringTable {
|
||||
const char* const lang;
|
||||
const size_t entryCount;
|
||||
const I18nString* const entries;
|
||||
|
||||
const char* get(const std::string& key) const;
|
||||
};
|
||||
|
||||
std::string getTranslatedString(const std::string& lang, const std::string& key);
|
||||
|
||||
namespace i18n
|
||||
{
|
||||
|
||||
typedef kainjow::mustache::object Parameters;
|
||||
|
||||
std::string expandParameterizedString(const std::string& lang,
|
||||
const std::string& key,
|
||||
const Parameters& params);
|
||||
|
||||
class GetTranslatedString
|
||||
{
|
||||
public:
|
||||
explicit GetTranslatedString(const std::string& lang) : m_lang(lang) {}
|
||||
|
||||
std::string operator()(const std::string& key) const
|
||||
{
|
||||
return getTranslatedString(m_lang, key);
|
||||
}
|
||||
|
||||
std::string operator()(const std::string& key, const Parameters& params) const
|
||||
{
|
||||
return expandParameterizedString(m_lang, key, params);
|
||||
}
|
||||
|
||||
private:
|
||||
const std::string m_lang;
|
||||
};
|
||||
|
||||
class GetTranslatedStringWithMsgId
|
||||
{
|
||||
typedef kainjow::mustache::basic_data<std::string> MustacheString;
|
||||
typedef std::pair<std::string, MustacheString> MsgIdAndTranslation;
|
||||
|
||||
public:
|
||||
explicit GetTranslatedStringWithMsgId(const std::string& lang) : m_lang(lang) {}
|
||||
|
||||
MsgIdAndTranslation operator()(const std::string& key) const
|
||||
{
|
||||
return {key, getTranslatedString(m_lang, key)};
|
||||
}
|
||||
|
||||
MsgIdAndTranslation operator()(const std::string& key, const Parameters& params) const
|
||||
{
|
||||
return {key, expandParameterizedString(m_lang, key, params)};
|
||||
}
|
||||
|
||||
private:
|
||||
const std::string m_lang;
|
||||
};
|
||||
|
||||
} // namespace i18n
|
||||
|
||||
struct ParameterizedMessage
|
||||
{
|
||||
public: // types
|
||||
typedef kainjow::mustache::object Parameters;
|
||||
|
||||
public: // functions
|
||||
ParameterizedMessage(const std::string& msgId, const Parameters& params)
|
||||
: msgId(msgId)
|
||||
, params(params)
|
||||
{}
|
||||
|
||||
std::string getText(const std::string& lang) const;
|
||||
|
||||
private: // data
|
||||
const std::string msgId;
|
||||
const Parameters params;
|
||||
};
|
||||
|
||||
struct LangPreference
|
||||
{
|
||||
const std::string lang;
|
||||
const float preference;
|
||||
};
|
||||
|
||||
typedef std::vector<LangPreference> UserLangPreferences;
|
||||
|
||||
UserLangPreferences parseUserLanguagePreferences(const std::string& s);
|
||||
|
||||
std::string selectMostSuitableLanguage(const UserLangPreferences& prefs);
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
#endif // KIWIX_SERVER_I18N
|
||||
File diff suppressed because it is too large
Load Diff
@@ -28,9 +28,6 @@ extern "C" {
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
|
||||
#include <zim/search.h>
|
||||
#include <zim/suggestion.h>
|
||||
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include <atomic>
|
||||
@@ -39,55 +36,11 @@ extern "C" {
|
||||
#include "server/request_context.h"
|
||||
#include "server/response.h"
|
||||
|
||||
#include "tools/concurrent_cache.h"
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
struct GeoQuery {
|
||||
GeoQuery()
|
||||
: GeoQuery(0, 0, -1)
|
||||
{}
|
||||
|
||||
GeoQuery(float latitude, float longitude, float distance)
|
||||
: latitude(latitude), longitude(longitude), distance(distance)
|
||||
{}
|
||||
float latitude;
|
||||
float longitude;
|
||||
float distance;
|
||||
|
||||
explicit operator bool() const {
|
||||
return distance >= 0;
|
||||
}
|
||||
|
||||
friend bool operator<(const GeoQuery& l, const GeoQuery& r)
|
||||
{
|
||||
return std::tie(l.latitude, l.longitude, l.distance)
|
||||
< std::tie(r.latitude, r.longitude, r.distance); // keep the same order
|
||||
}
|
||||
};
|
||||
|
||||
class SearchInfo {
|
||||
public:
|
||||
SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
|
||||
|
||||
zim::Query getZimQuery(bool verbose) const;
|
||||
const Library::BookIdSet& getBookIds() const { return bookIds; }
|
||||
|
||||
friend bool operator<(const SearchInfo& l, const SearchInfo& r)
|
||||
{
|
||||
return std::tie(l.bookIds, l.pattern, l.geoQuery)
|
||||
< std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
|
||||
}
|
||||
|
||||
public: //data
|
||||
std::string pattern;
|
||||
GeoQuery geoQuery;
|
||||
Library::BookIdSet bookIds;
|
||||
std::string bookFilterQuery;
|
||||
};
|
||||
|
||||
|
||||
typedef kainjow::mustache::data MustacheData;
|
||||
|
||||
class Entry;
|
||||
class OPDSDumper;
|
||||
|
||||
class InternalServer {
|
||||
@@ -98,14 +51,11 @@ class InternalServer {
|
||||
int port,
|
||||
std::string root,
|
||||
int nbThreads,
|
||||
unsigned int multizimSearchLimit,
|
||||
bool verbose,
|
||||
bool withTaskbar,
|
||||
bool withLibraryButton,
|
||||
bool blockExternalLinks,
|
||||
std::string indexTemplateString,
|
||||
int ipConnectionLimit);
|
||||
virtual ~InternalServer();
|
||||
bool blockExternalLinks);
|
||||
virtual ~InternalServer() = default;
|
||||
|
||||
MHD_Result handlerCallback(struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
@@ -116,82 +66,55 @@ class InternalServer {
|
||||
void** cont_cls);
|
||||
bool start();
|
||||
void stop();
|
||||
std::string getAddress() { return m_addr; }
|
||||
int getPort() { return m_port; }
|
||||
|
||||
private: // functions
|
||||
std::unique_ptr<Response> handle_request(const RequestContext& request);
|
||||
std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
|
||||
std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
|
||||
std::unique_ptr<Response> build_homepage(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_viewer_settings(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_skin(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
|
||||
std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
|
||||
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_no_js(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_meta(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_search(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_search_request(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_suggest(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_random(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catch(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_content(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_raw(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_locally_customized_resource(const RequestContext& request);
|
||||
|
||||
std::vector<std::string> search_catalog(const RequestContext& request,
|
||||
kiwix::OPDSDumper& opdsDumper);
|
||||
|
||||
MustacheData get_default_data() const;
|
||||
|
||||
std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
|
||||
SearchInfo getSearchInfo(const RequestContext& r) const;
|
||||
|
||||
bool isLocallyCustomizedResource(const std::string& url) const;
|
||||
|
||||
std::string getLibraryId() const;
|
||||
|
||||
std::string getNoJSDownloadPageHTML(const std::string& bookId, const std::string& userLang) const;
|
||||
|
||||
private: // types
|
||||
class LockableSuggestionSearcher;
|
||||
typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
|
||||
typedef ConcurrentCache<std::string, std::shared_ptr<LockableSuggestionSearcher>> SuggestionSearcherCache;
|
||||
std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
|
||||
bool etag_not_needed(const RequestContext& r) const;
|
||||
ETag get_matching_if_none_match_etag(const RequestContext& request) const;
|
||||
|
||||
private: // data
|
||||
std::string m_addr;
|
||||
int m_port;
|
||||
std::string m_root; // URI-encoded
|
||||
std::string m_rootPrefixOfDecodedURL; // URI-decoded
|
||||
std::string m_root;
|
||||
int m_nbThreads;
|
||||
unsigned int m_multizimSearchLimit;
|
||||
std::atomic_bool m_verbose;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
bool m_blockExternalLinks;
|
||||
std::string m_indexTemplateString;
|
||||
int m_ipConnectionLimit;
|
||||
struct MHD_Daemon* mp_daemon;
|
||||
|
||||
Library* mp_library;
|
||||
NameMapper* mp_nameMapper;
|
||||
|
||||
SearchCache searchCache;
|
||||
SuggestionSearcherCache suggestionSearcherCache;
|
||||
|
||||
std::string m_server_id;
|
||||
|
||||
class CustomizedResources;
|
||||
std::unique_ptr<CustomizedResources> m_customizedResources;
|
||||
std::string m_library_id;
|
||||
|
||||
friend std::unique_ptr<Response> Response::build(const InternalServer& server);
|
||||
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
|
||||
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
|
||||
friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
|
||||
friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
#include "request_context.h"
|
||||
#include "response.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "libkiwix-resources.h"
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#include <mustache.hpp>
|
||||
|
||||
@@ -43,8 +43,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
|
||||
try {
|
||||
url = request.get_url_part(2);
|
||||
} catch (const std::out_of_range&) {
|
||||
return HTTP404Response(*this, request)
|
||||
+ urlNotFoundMsg;
|
||||
return Response::build_404(*this, request, "", "");
|
||||
}
|
||||
|
||||
if (url == "root.xml") {
|
||||
@@ -56,51 +55,38 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
|
||||
kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
|
||||
"application/opensearchdescription+xml"
|
||||
);
|
||||
} else if (url == "entry") {
|
||||
const std::string entryId = request.get_url_part(3);
|
||||
return handle_catalog_v2_complete_entry(request, entryId);
|
||||
} else if (url == "entries") {
|
||||
return handle_catalog_v2_entries(request, /*partial=*/false);
|
||||
} else if (url == "partial_entries") {
|
||||
return handle_catalog_v2_entries(request, /*partial=*/true);
|
||||
return handle_catalog_v2_entries(request);
|
||||
} else if (url == "categories") {
|
||||
return handle_catalog_v2_categories(request);
|
||||
} else if (url == "languages") {
|
||||
return handle_catalog_v2_languages(request);
|
||||
} else if (url == "illustration") {
|
||||
return handle_catalog_v2_illustration(request);
|
||||
} else {
|
||||
return HTTP404Response(*this, request)
|
||||
+ urlNotFoundMsg;
|
||||
return Response::build_404(*this, request, "", "");
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
|
||||
{
|
||||
const std::string libraryId = getLibraryId();
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
RESOURCE::templates::catalog_v2_root_xml,
|
||||
kainjow::mustache::object{
|
||||
{"date", gen_date_str()},
|
||||
{"endpoint_root", m_root + "/catalog/v2"},
|
||||
{"feed_id", gen_uuid(libraryId)},
|
||||
{"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
|
||||
{"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
|
||||
{"category_list_feed_id", gen_uuid(libraryId + "/categories")},
|
||||
{"language_list_feed_id", gen_uuid(libraryId + "/languages")}
|
||||
{"feed_id", gen_uuid(m_library_id)},
|
||||
{"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
|
||||
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")}
|
||||
},
|
||||
"application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||
);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request)
|
||||
{
|
||||
OPDSDumper opdsDumper(mp_library, mp_nameMapper);
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
const auto bookIds = search_catalog(request, opdsDumper);
|
||||
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
|
||||
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query());
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsFeed,
|
||||
@@ -108,66 +94,16 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
|
||||
);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId)
|
||||
{
|
||||
try {
|
||||
mp_library->getBookById(entryId);
|
||||
} catch (const std::out_of_range&) {
|
||||
return HTTP404Response(*this, request)
|
||||
+ urlNotFoundMsg;
|
||||
}
|
||||
|
||||
OPDSDumper opdsDumper(mp_library, mp_nameMapper);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsFeed,
|
||||
"application/atom+xml;type=entry;profile=opds-catalog"
|
||||
);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
|
||||
{
|
||||
OPDSDumper opdsDumper(mp_library, mp_nameMapper);
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsDumper.categoriesOPDSFeed(),
|
||||
opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()),
|
||||
"application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||
);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
|
||||
{
|
||||
OPDSDumper opdsDumper(mp_library, mp_nameMapper);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsDumper.languagesOPDSFeed(),
|
||||
"application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||
);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
|
||||
{
|
||||
try {
|
||||
const auto bookId = request.get_url_part(3);
|
||||
auto book = mp_library->getBookByIdThreadSafe(bookId);
|
||||
auto size = request.get_argument<unsigned int>("size");
|
||||
auto illustration = book.getIllustration(size);
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
illustration->getData(),
|
||||
illustration->mimeType
|
||||
);
|
||||
} catch(...) {
|
||||
return HTTP404Response(*this, request)
|
||||
+ urlNotFoundMsg;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
@@ -25,10 +25,8 @@
|
||||
#include <sstream>
|
||||
#include <cstdio>
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
#include "i18n.h"
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
@@ -49,35 +47,48 @@ RequestMethod str2RequestMethod(const std::string& method) {
|
||||
else return RequestMethod::OTHER;
|
||||
}
|
||||
|
||||
std::string
|
||||
fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
|
||||
{
|
||||
if (rootLocation.empty()) {
|
||||
// nothing special to handle.
|
||||
return full_url;
|
||||
} else if (full_url == rootLocation) {
|
||||
return "/";
|
||||
} else if (full_url.size() > rootLocation.size() &&
|
||||
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
|
||||
return full_url.substr(rootLocation.size());
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
RequestContext::RequestContext(struct MHD_Connection* connection,
|
||||
const std::string& _rootLocation, // URI-encoded
|
||||
const std::string& unrootedUrl, // URI-decoded
|
||||
std::string rootLocation,
|
||||
const std::string& _url,
|
||||
const std::string& _method,
|
||||
const std::string& version) :
|
||||
rootLocation(_rootLocation),
|
||||
url(unrootedUrl),
|
||||
full_url(_url),
|
||||
url(fullURL2LocalURL(_url, rootLocation)),
|
||||
method(str2RequestMethod(_method)),
|
||||
version(version),
|
||||
requestIndex(s_requestIndex++),
|
||||
acceptEncodingGzip(false),
|
||||
acceptEncodingDeflate(false),
|
||||
byteRange_()
|
||||
{
|
||||
MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
|
||||
MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
|
||||
MHD_get_connection_values(connection, MHD_COOKIE_KIND, &RequestContext::fill_cookie, this);
|
||||
|
||||
try {
|
||||
acceptEncodingGzip =
|
||||
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("gzip") != std::string::npos);
|
||||
acceptEncodingDeflate =
|
||||
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
try {
|
||||
byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
userlang = determine_user_language();
|
||||
}
|
||||
|
||||
RequestContext::~RequestContext()
|
||||
@@ -95,23 +106,7 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char* value)
|
||||
{
|
||||
RequestContext *_this = static_cast<RequestContext*>(__this);
|
||||
_this->arguments[key].push_back(value == nullptr ? "" : value);
|
||||
if ( ! _this->queryString.empty() ) {
|
||||
_this->queryString += "&";
|
||||
}
|
||||
_this->queryString += urlEncode(key);
|
||||
if ( value ) {
|
||||
_this->queryString += "=";
|
||||
_this->queryString += urlEncode(value);
|
||||
}
|
||||
return MHD_YES;
|
||||
}
|
||||
|
||||
MHD_Result RequestContext::fill_cookie(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char* value)
|
||||
{
|
||||
RequestContext *_this = static_cast<RequestContext*>(__this);
|
||||
_this->cookies[key] = value == nullptr ? "" : value;
|
||||
_this->arguments[key] = value == nullptr ? "" : value;
|
||||
return MHD_YES;
|
||||
}
|
||||
|
||||
@@ -126,18 +121,13 @@ void RequestContext::print_debug_info() const {
|
||||
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
|
||||
}
|
||||
printf("arguments :\n");
|
||||
for (auto& pair:arguments) {
|
||||
printf(" - %s :", pair.first.c_str());
|
||||
bool first = true;
|
||||
for (auto& v: pair.second) {
|
||||
printf("%s %s", first?"":",", v.c_str());
|
||||
first = false;
|
||||
}
|
||||
printf("\n");
|
||||
for (auto it=arguments.begin(); it!=arguments.end(); it++) {
|
||||
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
|
||||
}
|
||||
printf("Parsed : \n");
|
||||
printf("full_url: %s\n", full_url.c_str());
|
||||
printf("url : %s\n", url.c_str());
|
||||
printf("acceptEncodingGzip : %d\n", acceptEncodingGzip);
|
||||
printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
|
||||
printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
|
||||
printf("is_valid_url : %d\n", is_valid_url());
|
||||
printf(".............\n");
|
||||
@@ -173,15 +163,11 @@ std::string RequestContext::get_url_part(int number) const {
|
||||
}
|
||||
|
||||
std::string RequestContext::get_full_url() const {
|
||||
return rootLocation + urlEncode(url);
|
||||
}
|
||||
|
||||
std::string RequestContext::get_root_path() const {
|
||||
return rootLocation.empty() ? "/" : rootLocation;
|
||||
return full_url;
|
||||
}
|
||||
|
||||
bool RequestContext::is_valid_url() const {
|
||||
return url.empty() || url[0] == '/';
|
||||
return !url.empty();
|
||||
}
|
||||
|
||||
ByteRange RequestContext::get_range() const {
|
||||
@@ -190,46 +176,21 @@ ByteRange RequestContext::get_range() const {
|
||||
|
||||
template<>
|
||||
std::string RequestContext::get_argument(const std::string& name) const {
|
||||
return arguments.at(name)[0];
|
||||
return arguments.at(name);
|
||||
}
|
||||
|
||||
std::string RequestContext::get_header(const std::string& name) const {
|
||||
return headers.at(lcAll(name));
|
||||
}
|
||||
|
||||
std::string RequestContext::get_user_language() const
|
||||
{
|
||||
return userlang.lang;
|
||||
}
|
||||
|
||||
bool RequestContext::user_language_comes_from_cookie() const
|
||||
{
|
||||
return userlang.selectedBy == UserLanguage::SelectorKind::COOKIE;
|
||||
}
|
||||
|
||||
RequestContext::UserLanguage RequestContext::determine_user_language() const
|
||||
{
|
||||
try {
|
||||
return {UserLanguage::SelectorKind::QUERY_PARAM, get_argument("userlang")};
|
||||
} catch(const std::out_of_range&) {}
|
||||
|
||||
try {
|
||||
return {UserLanguage::SelectorKind::COOKIE, cookies.at("userlang")};
|
||||
} catch(const std::out_of_range&) {}
|
||||
|
||||
try {
|
||||
const std::string acceptLanguage = get_header("Accept-Language");
|
||||
const auto userLangPrefs = parseUserLanguagePreferences(acceptLanguage);
|
||||
const auto lang = selectMostSuitableLanguage(userLangPrefs);
|
||||
return {UserLanguage::SelectorKind::ACCEPT_LANGUAGE_HEADER, lang};
|
||||
} catch(const std::out_of_range&) {}
|
||||
|
||||
return {UserLanguage::SelectorKind::DEFAULT, "en"};
|
||||
}
|
||||
|
||||
std::string RequestContext::get_requested_format() const
|
||||
{
|
||||
return get_optional_param<std::string>("format", "html");
|
||||
std::string RequestContext::get_query() const {
|
||||
std::string q;
|
||||
const char* sep = "";
|
||||
for ( const auto& a : arguments ) {
|
||||
q += sep + a.first + '=' + a.second;
|
||||
sep = "&";
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,11 +25,9 @@
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "byte_range.h"
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
@@ -57,8 +55,8 @@ class IndexError: public std::runtime_error {};
|
||||
class RequestContext {
|
||||
public: // functions
|
||||
RequestContext(struct MHD_Connection* connection,
|
||||
const std::string& rootLocation, // URI-encoded
|
||||
const std::string& unrootedUrl, // URI-decoded
|
||||
std::string rootLocation,
|
||||
const std::string& url,
|
||||
const std::string& method,
|
||||
const std::string& version);
|
||||
~RequestContext();
|
||||
@@ -70,11 +68,10 @@ class RequestContext {
|
||||
std::string get_header(const std::string& name) const;
|
||||
template<typename T=std::string>
|
||||
T get_argument(const std::string& name) const {
|
||||
return extractFromString<T>(get_argument(name));
|
||||
}
|
||||
|
||||
std::vector<std::string> get_arguments(const std::string& name) const {
|
||||
return arguments.at(name);
|
||||
std::istringstream stream(arguments.at(name));
|
||||
T v;
|
||||
stream >> v;
|
||||
return v;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
@@ -91,72 +88,27 @@ class RequestContext {
|
||||
std::string get_url() const;
|
||||
std::string get_url_part(int part) const;
|
||||
std::string get_full_url() const;
|
||||
std::string get_root_path() const;
|
||||
|
||||
std::string get_query() const { return queryString; }
|
||||
|
||||
template<class F>
|
||||
std::string get_query(F filter, bool mustEncode) const {
|
||||
std::string q;
|
||||
const char* sep = "";
|
||||
auto encode = [=](const std::string& value) { return mustEncode?urlEncode(value):value; };
|
||||
for ( const auto& a : arguments ) {
|
||||
if (!filter(a.first)) {
|
||||
continue;
|
||||
}
|
||||
for (const auto& v: a.second) {
|
||||
q += sep + encode(a.first) + '=' + encode(v);
|
||||
sep = "&";
|
||||
}
|
||||
}
|
||||
return q;
|
||||
}
|
||||
std::string get_query() const;
|
||||
|
||||
ByteRange get_range() const;
|
||||
|
||||
bool can_compress() const { return acceptEncodingGzip; }
|
||||
|
||||
std::string get_user_language() const;
|
||||
std::string get_requested_format() const;
|
||||
|
||||
bool user_language_comes_from_cookie() const;
|
||||
|
||||
private: // types
|
||||
struct UserLanguage
|
||||
{
|
||||
enum SelectorKind
|
||||
{
|
||||
QUERY_PARAM,
|
||||
COOKIE,
|
||||
ACCEPT_LANGUAGE_HEADER,
|
||||
DEFAULT
|
||||
};
|
||||
|
||||
SelectorKind selectedBy;
|
||||
std::string lang;
|
||||
};
|
||||
bool can_compress() const { return acceptEncodingDeflate; }
|
||||
|
||||
private: // data
|
||||
std::string rootLocation;
|
||||
std::string full_url;
|
||||
std::string url;
|
||||
RequestMethod method;
|
||||
std::string version;
|
||||
unsigned long long requestIndex;
|
||||
|
||||
bool acceptEncodingGzip;
|
||||
bool acceptEncodingDeflate;
|
||||
|
||||
ByteRange byteRange_;
|
||||
std::map<std::string, std::string> headers;
|
||||
std::map<std::string, std::vector<std::string>> arguments;
|
||||
std::map<std::string, std::string> cookies;
|
||||
std::string queryString;
|
||||
UserLanguage userlang;
|
||||
std::map<std::string, std::string> arguments;
|
||||
|
||||
private: // functions
|
||||
UserLanguage determine_user_language() const;
|
||||
|
||||
static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
static MHD_Result fill_cookie(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
};
|
||||
|
||||
|
||||
@@ -20,28 +20,18 @@
|
||||
#include "response.h"
|
||||
#include "request_context.h"
|
||||
#include "internalServer.h"
|
||||
#include "libkiwix-resources.h"
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#include "tools/regexTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/archiveTools.h"
|
||||
|
||||
#include "string.h"
|
||||
#include <mustache.hpp>
|
||||
#include <zlib.h>
|
||||
|
||||
#include <array>
|
||||
|
||||
// This is somehow a magic value.
|
||||
// If this value is too small, we will compress (and lost cpu time) too much
|
||||
// content.
|
||||
// If this value is too big, we will not compress enough content and send too
|
||||
// much data.
|
||||
// If we assume that MTU is 1500 Bytes it is useless to compress
|
||||
// content smaller as the content will be sent in one packet anyway.
|
||||
// 1400 Bytes seems to be a common accepted limit.
|
||||
#define KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS 1400
|
||||
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
@@ -53,69 +43,20 @@ std::string get_mime_type(const zim::Item& item)
|
||||
{
|
||||
try {
|
||||
return item.getMimetype();
|
||||
} catch (std::exception& e) {
|
||||
} catch (exception& e) {
|
||||
return "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
bool is_compressible_mime_type(const std::string& mimeType)
|
||||
{
|
||||
return mimeType.find("text/") != std::string::npos
|
||||
|| mimeType.find("application/javascript") != std::string::npos
|
||||
|| mimeType.find("application/atom") != std::string::npos
|
||||
|| mimeType.find("application/opensearchdescription") != std::string::npos
|
||||
|| mimeType.find("application/json") != std::string::npos
|
||||
|
||||
// Web fonts
|
||||
|| mimeType.find("application/font-") != std::string::npos
|
||||
|| mimeType.find("application/x-font-") != std::string::npos
|
||||
|| mimeType.find("application/vnd.ms-fontobject") != std::string::npos
|
||||
|| mimeType.find("font/") != std::string::npos;
|
||||
return mimeType.find("text/") != string::npos
|
||||
|| mimeType.find("application/javascript") != string::npos
|
||||
|| mimeType.find("application/atom") != string::npos
|
||||
|| mimeType.find("application/opensearchdescription") != string::npos
|
||||
|| mimeType.find("application/json") != string::npos;
|
||||
}
|
||||
|
||||
bool compress(std::string &content) {
|
||||
z_stream strm;
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
|
||||
auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
|
||||
Z_DEFAULT_STRATEGY);
|
||||
if (ret != Z_OK) { return false; }
|
||||
|
||||
strm.avail_in = static_cast<decltype(strm.avail_in)>(content.size());
|
||||
strm.next_in =
|
||||
const_cast<Bytef *>(reinterpret_cast<const Bytef *>(content.data()));
|
||||
|
||||
std::string compressed;
|
||||
|
||||
std::array<char, 16384> buff{};
|
||||
do {
|
||||
strm.avail_out = buff.size();
|
||||
strm.next_out = reinterpret_cast<Bytef *>(buff.data());
|
||||
ret = deflate(&strm, Z_FINISH);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
compressed.append(buff.data(), buff.size() - strm.avail_out);
|
||||
} while (strm.avail_out == 0);
|
||||
|
||||
assert(ret == Z_STREAM_END);
|
||||
assert(strm.avail_in == 0);
|
||||
|
||||
content.swap(compressed);
|
||||
|
||||
deflateEnd(&strm);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
const char* getCacheControlHeader(Response::Kind k)
|
||||
{
|
||||
switch(k) {
|
||||
case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
|
||||
case Response::ZIM_CONTENT: return "max-age=3600, must-revalidate";
|
||||
default: return "max-age=0, must-revalidate";
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
@@ -126,13 +67,6 @@ Response::Response(bool verbose)
|
||||
add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
|
||||
}
|
||||
|
||||
void Response::set_kind(Kind k)
|
||||
{
|
||||
m_kind = k;
|
||||
if ( k == ZIM_CONTENT )
|
||||
m_etag.set_option(ETag::ZIM_CONTENT);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build(const InternalServer& server)
|
||||
{
|
||||
return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
|
||||
@@ -143,124 +77,23 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
|
||||
auto response = Response::build(server);
|
||||
response->set_code(MHD_HTTP_NOT_MODIFIED);
|
||||
response->m_etag = etag;
|
||||
if ( etag.get_option(ETag::ZIM_CONTENT) ) {
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
}
|
||||
if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
|
||||
response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
const UrlNotFoundMsg urlNotFoundMsg;
|
||||
const InvalidUrlMsg invalidUrlMsg;
|
||||
|
||||
std::string ContentResponseBlueprint::getMessage(const std::string& msgId) const
|
||||
std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
|
||||
{
|
||||
return getTranslatedString(m_request.get_user_language(), msgId);
|
||||
}
|
||||
MustacheData results;
|
||||
results.set("url", request.get_full_url());
|
||||
results.set("details", details);
|
||||
|
||||
std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
|
||||
{
|
||||
auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
|
||||
r->set_code(m_httpStatusCode);
|
||||
return r;
|
||||
}
|
||||
auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
|
||||
response->set_code(MHD_HTTP_NOT_FOUND);
|
||||
response->set_taskbar(bookName, bookTitle);
|
||||
|
||||
HTTPErrorResponse::HTTPErrorResponse(const InternalServer& server,
|
||||
const RequestContext& request,
|
||||
int httpStatusCode,
|
||||
const std::string& pageTitleMsgId,
|
||||
const std::string& headingMsgId,
|
||||
const std::string& cssUrl)
|
||||
: ContentResponseBlueprint(&server,
|
||||
&request,
|
||||
httpStatusCode,
|
||||
request.get_requested_format() == "html" ? "text/html; charset=utf-8" : "application/xml; charset=utf-8",
|
||||
request.get_requested_format() == "html" ? RESOURCE::templates::error_html : RESOURCE::templates::error_xml)
|
||||
{
|
||||
kainjow::mustache::list emptyList;
|
||||
this->m_data = kainjow::mustache::object{
|
||||
{"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
|
||||
{"PAGE_TITLE", getMessage(pageTitleMsgId)},
|
||||
{"PAGE_HEADING", getMessage(headingMsgId)},
|
||||
{"details", emptyList}
|
||||
};
|
||||
}
|
||||
|
||||
HTTP404Response::HTTP404Response(const InternalServer& server,
|
||||
const RequestContext& request)
|
||||
: HTTPErrorResponse(server,
|
||||
request,
|
||||
MHD_HTTP_NOT_FOUND,
|
||||
"404-page-title",
|
||||
"404-page-heading")
|
||||
{
|
||||
}
|
||||
|
||||
HTTPErrorResponse& HTTP404Response::operator+(UrlNotFoundMsg /*unused*/)
|
||||
{
|
||||
const std::string requestUrl = urlDecode(m_request.get_full_url(), false);
|
||||
return *this + ParameterizedMessage("url-not-found", {{"url", requestUrl}});
|
||||
}
|
||||
|
||||
HTTPErrorResponse& HTTPErrorResponse::operator+(const std::string& msg)
|
||||
{
|
||||
m_data["details"].push_back({"p", msg});
|
||||
return *this;
|
||||
}
|
||||
|
||||
HTTPErrorResponse& HTTPErrorResponse::operator+(const ParameterizedMessage& details)
|
||||
{
|
||||
return *this + details.getText(m_request.get_user_language());
|
||||
}
|
||||
|
||||
HTTPErrorResponse& HTTPErrorResponse::operator+=(const ParameterizedMessage& details)
|
||||
{
|
||||
// operator+() is already a state-modifying operator (akin to operator+=)
|
||||
return *this + details;
|
||||
}
|
||||
|
||||
|
||||
HTTP400Response::HTTP400Response(const InternalServer& server,
|
||||
const RequestContext& request)
|
||||
: HTTPErrorResponse(server,
|
||||
request,
|
||||
MHD_HTTP_BAD_REQUEST,
|
||||
"400-page-title",
|
||||
"400-page-heading")
|
||||
{
|
||||
}
|
||||
|
||||
HTTPErrorResponse& HTTP400Response::operator+(InvalidUrlMsg /*unused*/)
|
||||
{
|
||||
std::string requestUrl = urlDecode(m_request.get_full_url(), false);
|
||||
const auto query = m_request.get_query();
|
||||
if (!query.empty()) {
|
||||
requestUrl += "?" + encodeDiples(query);
|
||||
}
|
||||
kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
|
||||
return *this + msgTmpl.render({"url", requestUrl});
|
||||
}
|
||||
|
||||
HTTP500Response::HTTP500Response(const InternalServer& server,
|
||||
const RequestContext& request)
|
||||
: HTTPErrorResponse(server,
|
||||
request,
|
||||
MHD_HTTP_INTERNAL_SERVER_ERROR,
|
||||
"500-page-title",
|
||||
"500-page-heading")
|
||||
{
|
||||
// operator+() is a state-modifying operator (akin to operator+=)
|
||||
*this + "An internal server error occured. We are sorry about that :/";
|
||||
}
|
||||
|
||||
std::unique_ptr<ContentResponse> HTTP500Response::generateResponseObject() const
|
||||
{
|
||||
const std::string mimeType = "text/html;charset=utf-8";
|
||||
auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
|
||||
r->set_code(m_httpStatusCode);
|
||||
return r;
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
|
||||
@@ -276,6 +109,17 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
|
||||
return response;
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
|
||||
{
|
||||
MustacheData data;
|
||||
data.set("error", msg);
|
||||
auto content = render_template(RESOURCE::templates::_500_html, data);
|
||||
std::unique_ptr<Response> response (
|
||||
new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
|
||||
response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
|
||||
{
|
||||
@@ -311,7 +155,7 @@ static ssize_t callback_reader_from_item(void* cls,
|
||||
{
|
||||
RunningResponse* response = static_cast<RunningResponse*>(cls);
|
||||
|
||||
size_t max_size_to_set = std::min<size_t>(
|
||||
size_t max_size_to_set = min<size_t>(
|
||||
max,
|
||||
response->item.getSize() - pos - response->range_start);
|
||||
|
||||
@@ -341,12 +185,59 @@ void print_response_info(int retCode, MHD_Response* response)
|
||||
}
|
||||
|
||||
|
||||
void ContentResponse::introduce_taskbar()
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("root", m_root);
|
||||
data.set("content", m_bookName);
|
||||
data.set("hascontent", (!m_bookName.empty() && !m_bookTitle.empty()));
|
||||
data.set("title", m_bookTitle);
|
||||
data.set("withlibrarybutton", m_withLibraryButton);
|
||||
auto head_content = render_template(RESOURCE::templates::head_taskbar_html, data);
|
||||
m_content = prependToFirstOccurence(
|
||||
m_content,
|
||||
"</head[ \\t]*>",
|
||||
head_content);
|
||||
|
||||
auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
|
||||
m_content = appendToFirstOccurence(
|
||||
m_content,
|
||||
"<body[^>]*>",
|
||||
taskbar_part);
|
||||
}
|
||||
|
||||
|
||||
void ContentResponse::inject_externallinks_blocker()
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("root", m_root);
|
||||
auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
|
||||
m_content = prependToFirstOccurence(
|
||||
m_content,
|
||||
"</head[ \\t]*>",
|
||||
script_tag);
|
||||
}
|
||||
|
||||
void ContentResponse::inject_root_link(){
|
||||
m_content = prependToFirstOccurence(
|
||||
m_content,
|
||||
"</head[ \\t]*>",
|
||||
"<link type=\"root\" href=\"" + m_root + "\">");
|
||||
}
|
||||
|
||||
bool
|
||||
ContentResponse::can_compress(const RequestContext& request) const
|
||||
{
|
||||
return request.can_compress()
|
||||
&& is_compressible_mime_type(m_mimeType)
|
||||
&& (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_COMPRESS);
|
||||
&& (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
|
||||
}
|
||||
|
||||
bool
|
||||
ContentResponse::contentDecorationAllowed() const
|
||||
{
|
||||
return (startsWith(m_mimeType, "text/html")
|
||||
&& m_mimeType.find(";raw=true") == std::string::npos);
|
||||
}
|
||||
|
||||
MHD_Response*
|
||||
@@ -359,17 +250,46 @@ Response::create_mhd_response(const RequestContext& request)
|
||||
MHD_Response*
|
||||
ContentResponse::create_mhd_response(const RequestContext& request)
|
||||
{
|
||||
const bool isCompressed = can_compress(request) && compress(m_content);
|
||||
if (contentDecorationAllowed()) {
|
||||
inject_root_link();
|
||||
|
||||
if (m_withTaskbar) {
|
||||
introduce_taskbar();
|
||||
}
|
||||
if (m_blockExternalLinks) {
|
||||
inject_externallinks_blocker();
|
||||
}
|
||||
}
|
||||
|
||||
bool shouldCompress = can_compress(request);
|
||||
if (shouldCompress) {
|
||||
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
|
||||
uLongf comprLen = compr_buffer.capacity();
|
||||
int err = compress(&compr_buffer[0],
|
||||
&comprLen,
|
||||
(const Bytef*)(m_content.data()),
|
||||
m_content.size());
|
||||
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
|
||||
/* /!\ Internet Explorer has a bug with deflate compression.
|
||||
It can not handle the first two bytes (compression headers)
|
||||
We need to chunk them off (move the content 2bytes)
|
||||
It has no incidence on other browsers
|
||||
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
|
||||
m_content = string((char*)&compr_buffer[2], comprLen - 2);
|
||||
m_etag.set_option(ETag::COMPRESSED_CONTENT);
|
||||
} else {
|
||||
shouldCompress = false;
|
||||
}
|
||||
}
|
||||
|
||||
MHD_Response* response = MHD_create_response_from_buffer(
|
||||
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
|
||||
|
||||
if (isCompressed) {
|
||||
m_etag.set_option(ETag::COMPRESSED_CONTENT);
|
||||
if (shouldCompress) {
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "gzip");
|
||||
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
@@ -379,7 +299,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
|
||||
MHD_Response* response = create_mhd_response(request);
|
||||
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
|
||||
getCacheControlHeader(m_kind));
|
||||
m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
|
||||
const std::string etag = m_etag.get_etag();
|
||||
if ( ! etag.empty() )
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
|
||||
@@ -387,13 +307,6 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
|
||||
MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
|
||||
}
|
||||
|
||||
if ( ! request.user_language_comes_from_cookie() ) {
|
||||
const std::string cookie = "userlang=" + request.get_user_language()
|
||||
+ ";Path=" + request.get_root_path()
|
||||
+ ";Max-Age=31536000";
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_SET_COOKIE, cookie.c_str());
|
||||
}
|
||||
|
||||
if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
|
||||
m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
|
||||
|
||||
@@ -405,35 +318,42 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
|
||||
return ret;
|
||||
}
|
||||
|
||||
ContentResponse::ContentResponse(const std::string& root, bool verbose, const std::string& content, const std::string& mimetype) :
|
||||
void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
|
||||
{
|
||||
m_bookName = bookName;
|
||||
m_bookTitle = bookTitle;
|
||||
}
|
||||
|
||||
|
||||
ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
|
||||
Response(verbose),
|
||||
m_root(root),
|
||||
m_content(content),
|
||||
m_mimeType(mimetype)
|
||||
m_mimeType(mimetype),
|
||||
m_withTaskbar(withTaskbar),
|
||||
m_withLibraryButton(withLibraryButton),
|
||||
m_blockExternalLinks(blockExternalLinks),
|
||||
m_bookName(""),
|
||||
m_bookTitle("")
|
||||
{
|
||||
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
|
||||
}
|
||||
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(
|
||||
const InternalServer& server,
|
||||
const std::string& content,
|
||||
const std::string& mimetype)
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
|
||||
{
|
||||
return std::unique_ptr<ContentResponse>(new ContentResponse(
|
||||
server.m_root,
|
||||
server.m_verbose.load(),
|
||||
server.m_withTaskbar && !isHomePage,
|
||||
server.m_withLibraryButton,
|
||||
server.m_blockExternalLinks,
|
||||
content,
|
||||
mimetype));
|
||||
}
|
||||
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(
|
||||
const InternalServer& server,
|
||||
const std::string& template_str,
|
||||
kainjow::mustache::data data,
|
||||
const std::string& mimetype)
|
||||
{
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
|
||||
auto content = render_template(template_str, data);
|
||||
return ContentResponse::build(server, content, mimetype);
|
||||
return ContentResponse::build(server, content, mimetype, isHomePage);
|
||||
}
|
||||
|
||||
ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange) :
|
||||
@@ -442,7 +362,7 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
|
||||
m_mimeType(mimetype)
|
||||
{
|
||||
m_byteRange = byterange;
|
||||
set_kind(Response::ZIM_CONTENT);
|
||||
set_cacheable();
|
||||
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
|
||||
}
|
||||
|
||||
@@ -454,14 +374,14 @@ std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, cons
|
||||
if (noRange && is_compressible_mime_type(mimetype)) {
|
||||
// Return a contentResponse
|
||||
auto response = ContentResponse::build(server, item.getData(), mimetype);
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
response->set_cacheable();
|
||||
response->m_byteRange = byteRange;
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
|
||||
auto response = Response::build_416(server, item.getSize());
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
response->set_cacheable();
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,48 +26,37 @@
|
||||
|
||||
#include <mustache.hpp>
|
||||
#include "byte_range.h"
|
||||
#include "entry.h"
|
||||
#include "etag.h"
|
||||
#include "i18n.h"
|
||||
|
||||
#include <zim/item.h>
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
}
|
||||
|
||||
namespace zim {
|
||||
class Archive;
|
||||
} // namespace zim
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
class InternalServer;
|
||||
class RequestContext;
|
||||
|
||||
class Response {
|
||||
public:
|
||||
enum Kind
|
||||
{
|
||||
STATIC_RESOURCE,
|
||||
ZIM_CONTENT,
|
||||
DYNAMIC_CONTENT
|
||||
};
|
||||
class EntryResponse;
|
||||
|
||||
class Response {
|
||||
public:
|
||||
Response(bool verbose);
|
||||
virtual ~Response() = default;
|
||||
|
||||
static std::unique_ptr<Response> build(const InternalServer& server);
|
||||
static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
|
||||
static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
|
||||
static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
|
||||
static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
|
||||
static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
|
||||
|
||||
MHD_Result send(const RequestContext& request, MHD_Connection* connection);
|
||||
|
||||
void set_code(int code) { m_returnCode = code; }
|
||||
void set_kind(Kind k);
|
||||
Kind get_kind() const { return m_kind; }
|
||||
void set_etag_body(const std::string& id) { m_etag.set_body(id); }
|
||||
void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
|
||||
void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
|
||||
void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
|
||||
|
||||
int getReturnCode() const { return m_returnCode; }
|
||||
@@ -77,7 +66,6 @@ class Response {
|
||||
MHD_Response* create_error_response(const RequestContext& request) const;
|
||||
|
||||
protected: // data
|
||||
Kind m_kind = DYNAMIC_CONTENT;
|
||||
bool m_verbose;
|
||||
int m_returnCode;
|
||||
ByteRange m_byteRange;
|
||||
@@ -90,127 +78,33 @@ class Response {
|
||||
|
||||
class ContentResponse : public Response {
|
||||
public:
|
||||
ContentResponse(
|
||||
const std::string& root,
|
||||
bool verbose,
|
||||
const std::string& content,
|
||||
const std::string& mimetype);
|
||||
ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
|
||||
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
|
||||
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
|
||||
|
||||
static std::unique_ptr<ContentResponse> build(
|
||||
const InternalServer& server,
|
||||
const std::string& content,
|
||||
const std::string& mimetype);
|
||||
|
||||
static std::unique_ptr<ContentResponse> build(
|
||||
const InternalServer& server,
|
||||
const std::string& template_str,
|
||||
kainjow::mustache::data data,
|
||||
const std::string& mimetype);
|
||||
void set_taskbar(const std::string& bookName, const std::string& bookTitle);
|
||||
|
||||
private:
|
||||
MHD_Response* create_mhd_response(const RequestContext& request);
|
||||
|
||||
void introduce_taskbar();
|
||||
void inject_externallinks_blocker();
|
||||
void inject_root_link();
|
||||
bool can_compress(const RequestContext& request) const;
|
||||
bool contentDecorationAllowed() const;
|
||||
|
||||
|
||||
private:
|
||||
std::string m_root;
|
||||
std::string m_content;
|
||||
std::string m_mimeType;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
bool m_blockExternalLinks;
|
||||
std::string m_bookName;
|
||||
std::string m_bookTitle;
|
||||
};
|
||||
|
||||
class ContentResponseBlueprint
|
||||
{
|
||||
public: // functions
|
||||
ContentResponseBlueprint(const InternalServer* server,
|
||||
const RequestContext* request,
|
||||
int httpStatusCode,
|
||||
const std::string& mimeType,
|
||||
const std::string& templateStr)
|
||||
: m_server(*server)
|
||||
, m_request(*request)
|
||||
, m_httpStatusCode(httpStatusCode)
|
||||
, m_mimeType(mimeType)
|
||||
, m_template(templateStr)
|
||||
{}
|
||||
|
||||
virtual ~ContentResponseBlueprint() = default;
|
||||
|
||||
operator std::unique_ptr<ContentResponse>() const
|
||||
{
|
||||
return generateResponseObject();
|
||||
}
|
||||
|
||||
operator std::unique_ptr<Response>() const
|
||||
{
|
||||
return operator std::unique_ptr<ContentResponse>();
|
||||
}
|
||||
|
||||
|
||||
protected: // functions
|
||||
std::string getMessage(const std::string& msgId) const;
|
||||
virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
|
||||
|
||||
public: //data
|
||||
const InternalServer& m_server;
|
||||
const RequestContext& m_request;
|
||||
const int m_httpStatusCode;
|
||||
const std::string m_mimeType;
|
||||
const std::string m_template;
|
||||
kainjow::mustache::data m_data;
|
||||
};
|
||||
|
||||
struct HTTPErrorResponse : ContentResponseBlueprint
|
||||
{
|
||||
HTTPErrorResponse(const InternalServer& server,
|
||||
const RequestContext& request,
|
||||
int httpStatusCode,
|
||||
const std::string& pageTitleMsgId,
|
||||
const std::string& headingMsgId,
|
||||
const std::string& cssUrl = "");
|
||||
|
||||
HTTPErrorResponse& operator+(const std::string& msg);
|
||||
HTTPErrorResponse& operator+(const ParameterizedMessage& errorDetails);
|
||||
HTTPErrorResponse& operator+=(const ParameterizedMessage& errorDetails);
|
||||
};
|
||||
|
||||
class UrlNotFoundMsg {};
|
||||
|
||||
extern const UrlNotFoundMsg urlNotFoundMsg;
|
||||
|
||||
struct HTTP404Response : HTTPErrorResponse
|
||||
{
|
||||
HTTP404Response(const InternalServer& server,
|
||||
const RequestContext& request);
|
||||
|
||||
using HTTPErrorResponse::operator+;
|
||||
HTTPErrorResponse& operator+(UrlNotFoundMsg /*unused*/);
|
||||
};
|
||||
|
||||
class InvalidUrlMsg {};
|
||||
|
||||
extern const InvalidUrlMsg invalidUrlMsg;
|
||||
|
||||
struct HTTP400Response : HTTPErrorResponse
|
||||
{
|
||||
HTTP400Response(const InternalServer& server,
|
||||
const RequestContext& request);
|
||||
|
||||
using HTTPErrorResponse::operator+;
|
||||
HTTPErrorResponse& operator+(InvalidUrlMsg /*unused*/);
|
||||
};
|
||||
|
||||
struct HTTP500Response : HTTPErrorResponse
|
||||
{
|
||||
HTTP500Response(const InternalServer& server,
|
||||
const RequestContext& request);
|
||||
|
||||
private: // overrides
|
||||
// generateResponseObject() is overriden in order to produce a minimal
|
||||
// response without any need for additional resources from the server
|
||||
std::unique_ptr<ContentResponse> generateResponseObject() const override;
|
||||
};
|
||||
|
||||
class ItemResponse : public Response {
|
||||
public:
|
||||
ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "archiveTools.h"
|
||||
|
||||
#include "tools.h"
|
||||
#include "pathTools.h"
|
||||
#include "otherTools.h"
|
||||
#include "stringTools.h"
|
||||
|
||||
#include <zim/error.h>
|
||||
#include <zim/item.h>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
std::string getMetadata(const zim::Archive& archive, const std::string& name) {
|
||||
try {
|
||||
return archive.getMetadata(name);
|
||||
} catch (zim::EntryNotFound& e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
std::string getArchiveTitle(const zim::Archive& archive) {
|
||||
std::string value = getMetadata(archive, "Title");
|
||||
if (value.empty()) {
|
||||
value = getLastPathElement(archive.getFilename());
|
||||
std::replace(value.begin(), value.end(), '_', ' ');
|
||||
size_t pos = value.find(".zim");
|
||||
value = value.substr(0, pos);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
std::string getMetaDescription(const zim::Archive& archive) {
|
||||
std::string value;
|
||||
value = getMetadata(archive, "Description");
|
||||
|
||||
/* Mediawiki Collection tends to use the "Subtitle" name */
|
||||
if (value.empty()) {
|
||||
value = getMetadata(archive, "Subtitle");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
std::string getMetaTags(const zim::Archive& archive, bool original) {
|
||||
std::string tags_str = getMetadata(archive, "Tags");
|
||||
if (original) {
|
||||
return tags_str;
|
||||
}
|
||||
auto tags = convertTags(tags_str);
|
||||
return join(tags, ";");
|
||||
}
|
||||
|
||||
std::string getMetaLanguage(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Language");
|
||||
}
|
||||
|
||||
std::string getMetaName(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Name");
|
||||
}
|
||||
|
||||
std::string getMetaDate(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Date");
|
||||
}
|
||||
|
||||
std::string getMetaCreator(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Creator");
|
||||
}
|
||||
|
||||
std::string getMetaPublisher(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Publisher");
|
||||
}
|
||||
|
||||
std::string getMetaFlavour(const zim::Archive& archive) {
|
||||
return getMetadata(archive, "Flavour");
|
||||
}
|
||||
|
||||
bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
|
||||
std::string& content, std::string& mimeType){
|
||||
try {
|
||||
auto item = archive.getIllustrationItem(size);
|
||||
content = item.getData();
|
||||
mimeType = item.getMimetype();
|
||||
return true;
|
||||
} catch(zim::EntryNotFound& e) {};
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned int getArchiveFileSize(const zim::Archive& archive) {
|
||||
return archive.getFilesize() / 1024;
|
||||
}
|
||||
|
||||
zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry)
|
||||
{
|
||||
return entry.getItem(true);
|
||||
}
|
||||
|
||||
zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path)
|
||||
{
|
||||
try {
|
||||
return archive.getEntryByPath(path);
|
||||
} catch (zim::EntryNotFound& e) {
|
||||
if (path.empty() || path == "/") {
|
||||
return archive.getMainEntry();
|
||||
}
|
||||
}
|
||||
throw zim::EntryNotFound("Cannot find entry for non empty path");
|
||||
}
|
||||
} // kiwix
|
||||
@@ -1,56 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIX_ARCHIVETOOLS_H
|
||||
#define KIWIX_ARCHIVETOOLS_H
|
||||
|
||||
#include <zim/archive.h>
|
||||
#include <tools/otherTools.h>
|
||||
|
||||
/**
|
||||
* This file contains all the functions that would make handling data related to
|
||||
* an archive easier.
|
||||
**/
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
std::string getMetadata(const zim::Archive& archive, const std::string& name);
|
||||
std::string getArchiveTitle(const zim::Archive& archive);
|
||||
std::string getMetaDescription(const zim::Archive& archive);
|
||||
std::string getMetaTags(const zim::Archive& archive, bool original = false);
|
||||
std::string getMetaLanguage(const zim::Archive& archive);
|
||||
std::string getMetaName(const zim::Archive& archive);
|
||||
std::string getMetaDate(const zim::Archive& archive);
|
||||
std::string getMetaCreator(const zim::Archive& archive);
|
||||
std::string getMetaPublisher(const zim::Archive& archive);
|
||||
std::string getMetaFlavour(const zim::Archive& archive);
|
||||
|
||||
bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
|
||||
std::string& content, std::string& mimeType);
|
||||
|
||||
unsigned int getArchiveMediaCount(const zim::Archive& archive);
|
||||
unsigned int getArchiveArticleCount(const zim::Archive& archive);
|
||||
unsigned int getArchiveFileSize(const zim::Archive& archive);
|
||||
|
||||
zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
|
||||
|
||||
zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,208 +0,0 @@
|
||||
|
||||
/*
|
||||
* Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
* Copyright (C) 2020 Veloman Yunkan
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License as
|
||||
* published by the Free Software Foundation; either version 2 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but
|
||||
* is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
|
||||
* warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
|
||||
* NON-INFRINGEMENT. See the GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef ZIM_CONCURRENT_CACHE_H
|
||||
#define ZIM_CONCURRENT_CACHE_H
|
||||
|
||||
#include "lrucache.h"
|
||||
|
||||
#include <future>
|
||||
#include <mutex>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
|
||||
/**
|
||||
ConcurrentCache implements a concurrent thread-safe cache
|
||||
|
||||
Compared to kiwix::lru_cache, each access operation is slightly more expensive.
|
||||
However, different slots of the cache can be safely accessed concurrently
|
||||
with minimal blocking. Concurrent access to the same element is also
|
||||
safe, and, in case of a cache miss, will block until that element becomes
|
||||
available.
|
||||
*/
|
||||
template <typename Key, typename Value>
|
||||
class ConcurrentCache
|
||||
{
|
||||
private: // types
|
||||
typedef std::shared_future<Value> ValuePlaceholder;
|
||||
typedef lru_cache<Key, ValuePlaceholder> Impl;
|
||||
|
||||
public: // types
|
||||
explicit ConcurrentCache(size_t maxEntries)
|
||||
: impl_(maxEntries)
|
||||
{}
|
||||
|
||||
// Gets the entry corresponding to the given key. If the entry is not in the
|
||||
// cache, it is obtained by calling f() (without any arguments) and the
|
||||
// result is put into the cache.
|
||||
//
|
||||
// The cache as a whole is locked only for the duration of accessing
|
||||
// the respective slot. If, in the case of the a cache miss, the generation
|
||||
// of the missing element takes a long time, only attempts to access that
|
||||
// element will block - the rest of the cache remains open to concurrent
|
||||
// access.
|
||||
template<class F>
|
||||
Value getOrPut(const Key& key, F f)
|
||||
{
|
||||
std::promise<Value> valuePromise;
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
|
||||
l.unlock();
|
||||
if ( x.miss() ) {
|
||||
try {
|
||||
valuePromise.set_value(f());
|
||||
} catch (std::exception& e) {
|
||||
drop(key);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
return x.value().get();
|
||||
}
|
||||
|
||||
bool drop(const Key& key)
|
||||
{
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
return impl_.drop(key);
|
||||
}
|
||||
|
||||
size_t setMaxSize(size_t new_size) {
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
return impl_.setMaxSize(new_size);
|
||||
}
|
||||
|
||||
protected: // data
|
||||
Impl impl_;
|
||||
std::mutex lock_;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
WeakStore represent a thread safe store (map) of weak ptr.
|
||||
It allows to store weak_ptr from shared_ptr and retrieve shared_ptr from
|
||||
potential non expired weak_ptr.
|
||||
It is not limited in size.
|
||||
*/
|
||||
template<typename Key, typename Value>
|
||||
class WeakStore {
|
||||
private: // types
|
||||
typedef std::weak_ptr<Value> WeakValue;
|
||||
|
||||
public:
|
||||
explicit WeakStore() = default;
|
||||
|
||||
std::shared_ptr<Value> get(const Key& key)
|
||||
{
|
||||
std::lock_guard<std::mutex> l(m_lock);
|
||||
auto it = m_weakMap.find(key);
|
||||
if (it != m_weakMap.end()) {
|
||||
auto shared = it->second.lock();
|
||||
if (shared) {
|
||||
return shared;
|
||||
} else {
|
||||
m_weakMap.erase(it);
|
||||
}
|
||||
}
|
||||
throw std::runtime_error("No weak ptr");
|
||||
}
|
||||
|
||||
void add(const Key& key, std::shared_ptr<Value> shared)
|
||||
{
|
||||
std::lock_guard<std::mutex> l(m_lock);
|
||||
m_weakMap[key] = WeakValue(shared);
|
||||
}
|
||||
|
||||
|
||||
private: //data
|
||||
std::map<Key, WeakValue> m_weakMap;
|
||||
std::mutex m_lock;
|
||||
};
|
||||
|
||||
template <typename Key, typename RawValue>
|
||||
class ConcurrentCache<Key, std::shared_ptr<RawValue>>
|
||||
{
|
||||
private: // types
|
||||
typedef std::shared_ptr<RawValue> Value;
|
||||
typedef std::shared_future<Value> ValuePlaceholder;
|
||||
typedef lru_cache<Key, ValuePlaceholder> Impl;
|
||||
|
||||
public: // types
|
||||
explicit ConcurrentCache(size_t maxEntries)
|
||||
: impl_(maxEntries)
|
||||
{}
|
||||
|
||||
// Gets the entry corresponding to the given key. If the entry is not in the
|
||||
// cache, it is obtained by calling f() (without any arguments) and the
|
||||
// result is put into the cache.
|
||||
//
|
||||
// The cache as a whole is locked only for the duration of accessing
|
||||
// the respective slot. If, in the case of the a cache miss, the generation
|
||||
// of the missing element takes a long time, only attempts to access that
|
||||
// element will block - the rest of the cache remains open to concurrent
|
||||
// access.
|
||||
template<class F>
|
||||
Value getOrPut(const Key& key, F f)
|
||||
{
|
||||
std::promise<Value> valuePromise;
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
|
||||
l.unlock();
|
||||
if ( x.miss() ) {
|
||||
// Try to get back the shared_ptr from the weak_ptr first.
|
||||
try {
|
||||
valuePromise.set_value(m_weakStore.get(key));
|
||||
} catch(const std::runtime_error& e) {
|
||||
try {
|
||||
const auto value = f();
|
||||
valuePromise.set_value(value);
|
||||
m_weakStore.add(key, value);
|
||||
} catch (std::exception& e) {
|
||||
drop(key);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return x.value().get();
|
||||
}
|
||||
|
||||
bool drop(const Key& key)
|
||||
{
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
return impl_.drop(key);
|
||||
}
|
||||
|
||||
size_t setMaxSize(size_t new_size) {
|
||||
std::unique_lock<std::mutex> l(lock_);
|
||||
return impl_.setMaxSize(new_size);
|
||||
}
|
||||
|
||||
protected: // data
|
||||
std::mutex lock_;
|
||||
Impl impl_;
|
||||
WeakStore<Key, RawValue> m_weakStore;
|
||||
};
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
#endif // ZIM_CONCURRENT_CACHE_H
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
/*
|
||||
* Copyrigth (c) 2021, Matthieu Gautier <mgautier@kymeria.fr>
|
||||
* Copyright (c) 2020, Veloman Yunkan
|
||||
* Copyright (c) 2014, lamerman
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* * Neither the name of lamerman nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* File: lrucache.hpp
|
||||
* Author: Alexander Ponomarev
|
||||
*
|
||||
* Created on June 20, 2013, 5:09 PM
|
||||
*/
|
||||
|
||||
#ifndef _LRUCACHE_HPP_INCLUDED_
|
||||
#define _LRUCACHE_HPP_INCLUDED_
|
||||
|
||||
#include <map>
|
||||
#include <list>
|
||||
#include <set>
|
||||
#include <cstddef>
|
||||
#include <stdexcept>
|
||||
#include <cassert>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
template<typename key_t, typename value_t>
|
||||
class lru_cache {
|
||||
public: // types
|
||||
typedef typename std::pair<key_t, value_t> key_value_pair_t;
|
||||
typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
|
||||
|
||||
enum AccessStatus {
|
||||
HIT, // key was found in the cache
|
||||
PUT, // key was not in the cache but was created by the getOrPut() access
|
||||
MISS // key was not in the cache; get() access failed
|
||||
};
|
||||
|
||||
class AccessResult
|
||||
{
|
||||
const AccessStatus status_;
|
||||
const value_t val_;
|
||||
public:
|
||||
AccessResult(const value_t& val, AccessStatus status)
|
||||
: status_(status), val_(val)
|
||||
{}
|
||||
AccessResult() : status_(MISS), val_() {}
|
||||
|
||||
bool hit() const { return status_ == HIT; }
|
||||
bool miss() const { return !hit(); }
|
||||
const value_t& value() const
|
||||
{
|
||||
if ( status_ == MISS )
|
||||
throw std::range_error("There is no such key in cache");
|
||||
return val_;
|
||||
}
|
||||
|
||||
operator const value_t& () const { return value(); }
|
||||
};
|
||||
|
||||
public: // functions
|
||||
explicit lru_cache(size_t max_size) :
|
||||
_max_size(max_size) {
|
||||
}
|
||||
|
||||
// If 'key' is present in the cache, returns the associated value,
|
||||
// otherwise puts the given value into the cache (and returns it with
|
||||
// a status of a cache miss).
|
||||
AccessResult getOrPut(const key_t& key, const value_t& value) {
|
||||
auto it = _cache_items_map.find(key);
|
||||
if (it != _cache_items_map.end()) {
|
||||
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
|
||||
return AccessResult(it->second->second, HIT);
|
||||
} else {
|
||||
putMissing(key, value);
|
||||
return AccessResult(value, PUT);
|
||||
}
|
||||
}
|
||||
|
||||
void put(const key_t& key, const value_t& value) {
|
||||
auto it = _cache_items_map.find(key);
|
||||
if (it != _cache_items_map.end()) {
|
||||
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
|
||||
it->second->second = value;
|
||||
} else {
|
||||
putMissing(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
AccessResult get(const key_t& key) {
|
||||
auto it = _cache_items_map.find(key);
|
||||
if (it == _cache_items_map.end()) {
|
||||
return AccessResult();
|
||||
} else {
|
||||
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
|
||||
return AccessResult(it->second->second, HIT);
|
||||
}
|
||||
}
|
||||
|
||||
bool drop(const key_t& key) {
|
||||
try {
|
||||
auto list_it = _cache_items_map.at(key);
|
||||
_cache_items_list.erase(list_it);
|
||||
_cache_items_map.erase(key);
|
||||
return true;
|
||||
} catch (std::out_of_range& e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool exists(const key_t& key) const {
|
||||
return _cache_items_map.find(key) != _cache_items_map.end();
|
||||
}
|
||||
|
||||
size_t size() const {
|
||||
return _cache_items_map.size();
|
||||
}
|
||||
|
||||
size_t setMaxSize(size_t new_size) {
|
||||
size_t previous = _max_size;
|
||||
_max_size = new_size;
|
||||
return previous;
|
||||
}
|
||||
|
||||
std::set<key_t> keys() const {
|
||||
std::set<key_t> keys;
|
||||
for(auto& item:_cache_items_map) {
|
||||
keys.insert(item.first);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private: // functions
|
||||
void putMissing(const key_t& key, const value_t& value) {
|
||||
assert(_cache_items_map.find(key) == _cache_items_map.end());
|
||||
_cache_items_list.push_front(key_value_pair_t(key, value));
|
||||
_cache_items_map[key] = _cache_items_list.begin();
|
||||
while (_cache_items_map.size() > _max_size) {
|
||||
_cache_items_map.erase(_cache_items_list.back().first);
|
||||
_cache_items_list.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
private: // data
|
||||
std::list<key_value_pair_t> _cache_items_list;
|
||||
std::map<key_t, list_iterator_t> _cache_items_map;
|
||||
size_t _max_size;
|
||||
};
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
#endif /* _LRUCACHE_HPP_INCLUDED_ */
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
* Copyright 2021 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
@@ -18,7 +17,6 @@
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "tools.h"
|
||||
#include <tools/networkTools.h>
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -31,17 +29,6 @@
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <winsock2.h>
|
||||
#include <ws2tcpip.h>
|
||||
#include <iostream>
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/socket.h>
|
||||
#include <net/if.h>
|
||||
#include <netdb.h>
|
||||
#endif
|
||||
|
||||
size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
|
||||
{
|
||||
@@ -70,104 +57,3 @@ std::string kiwix::download(const std::string& url) {
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
|
||||
std::map<std::string, std::string> interfaces;
|
||||
|
||||
#ifdef _WIN32
|
||||
SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
|
||||
if (sd == INVALID_SOCKET) {
|
||||
std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
|
||||
return interfaces;
|
||||
}
|
||||
|
||||
INTERFACE_INFO InterfaceList[20];
|
||||
unsigned long nBytesReturned;
|
||||
if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
|
||||
sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
|
||||
std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
|
||||
return interfaces;
|
||||
}
|
||||
|
||||
int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
|
||||
for (int i = 0; i < nNumInterfaces; ++i) {
|
||||
sockaddr_in *pAddress;
|
||||
pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
|
||||
if(pAddress->sin_family == AF_INET) {
|
||||
/* Add to the map */
|
||||
std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
|
||||
interfaces[interfaceName] = interfaceName;
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* Get Network interfaces information */
|
||||
char buf[16384];
|
||||
struct ifconf ifconf;
|
||||
int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
|
||||
ifconf.ifc_len = sizeof(buf);
|
||||
ifconf.ifc_buf=buf;
|
||||
if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
|
||||
perror("ioctl(SIOCGIFCONF)");
|
||||
}
|
||||
|
||||
/* Go through each interface */
|
||||
struct ifreq *ifreq;
|
||||
ifreq = ifconf.ifc_req;
|
||||
for (int i = 0; i < ifconf.ifc_len; ) {
|
||||
if (ifreq->ifr_addr.sa_family == AF_INET) {
|
||||
/* Get the network interface ip */
|
||||
char host[128] = { 0 };
|
||||
const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
|
||||
host, sizeof(host),
|
||||
0, 0, NI_NUMERICHOST);
|
||||
if (!error) {
|
||||
std::string interfaceName = std::string(ifreq->ifr_name);
|
||||
std::string interfaceIp = std::string(host);
|
||||
/* Add to the map */
|
||||
interfaces[interfaceName] = interfaceIp;
|
||||
} else {
|
||||
perror("getnameinfo()");
|
||||
}
|
||||
}
|
||||
|
||||
/* some systems have ifr_addr.sa_len and adjust the length that
|
||||
* way, but not mine. weird */
|
||||
size_t len;
|
||||
#ifndef __linux__
|
||||
len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
|
||||
#else
|
||||
len = sizeof(*ifreq);
|
||||
#endif
|
||||
ifreq = (struct ifreq*)((char*)ifreq+len);
|
||||
i += len;
|
||||
}
|
||||
#endif
|
||||
return interfaces;
|
||||
}
|
||||
|
||||
std::string kiwix::getBestPublicIp() {
|
||||
auto interfaces = getNetworkInterfaces();
|
||||
|
||||
#ifndef _WIN32
|
||||
const char* const prioritizedNames[] =
|
||||
{ "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
|
||||
for(auto name: prioritizedNames) {
|
||||
auto it = interfaces.find(name);
|
||||
if(it != interfaces.end()) {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
|
||||
for(auto prefix : prefixes){
|
||||
for(auto& itr : interfaces) {
|
||||
auto interfaceIp = itr.second;
|
||||
if (interfaceIp.find(prefix) == 0) {
|
||||
return interfaceIp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "127.0.0.1";
|
||||
}
|
||||
|
||||
@@ -17,11 +17,7 @@
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
// Implement function declared in tools.h and tools/otherTools.h
|
||||
#include "tools.h"
|
||||
#include "tools/otherTools.h"
|
||||
|
||||
|
||||
#include <algorithm>
|
||||
#include <iomanip>
|
||||
|
||||
@@ -32,15 +28,12 @@
|
||||
#endif
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
#include "server/i18n.h"
|
||||
#include "libkiwix-resources.h"
|
||||
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <pugixml.hpp>
|
||||
|
||||
#include <zim/uuid.h>
|
||||
#include <zim/suggestion_iterator.h>
|
||||
|
||||
|
||||
static std::map<std::string, std::string> codeisomapping {
|
||||
@@ -291,6 +284,67 @@ bool kiwix::convertStrToBool(const std::string& value)
|
||||
throw std::domain_error(ss.str());
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// The counter metadata format is a list of item separated by a `;` :
|
||||
// item0;item1;item2
|
||||
// Each item is a "tuple" mimetype=number.
|
||||
// However, the mimetype may contains parameters:
|
||||
// text/html;raw=true;foo=bar
|
||||
// So the final format may be complex to parse:
|
||||
// key0=value0;key1;foo=bar=value1;key2=value2
|
||||
|
||||
typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
|
||||
|
||||
std::string readFullMimetypeAndCounterString(std::istream& in)
|
||||
{
|
||||
std::string mtcStr, params;
|
||||
getline(in, mtcStr, ';');
|
||||
if ( mtcStr.find('=') == std::string::npos )
|
||||
{
|
||||
do
|
||||
{
|
||||
if ( !getline(in, params, ';' ) )
|
||||
return std::string();
|
||||
mtcStr += ";" + params;
|
||||
}
|
||||
while ( std::count(params.begin(), params.end(), '=') != 2 );
|
||||
}
|
||||
return mtcStr;
|
||||
}
|
||||
|
||||
MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
|
||||
{
|
||||
const std::string::size_type k = s.find_last_of("=");
|
||||
if ( k != std::string::npos )
|
||||
{
|
||||
const std::string mimeType = s.substr(0, k);
|
||||
std::istringstream counterSS(s.substr(k+1));
|
||||
unsigned int counter;
|
||||
if (counterSS >> counter && counterSS.eof())
|
||||
return MimetypeAndCounter{mimeType, counter};
|
||||
}
|
||||
return MimetypeAndCounter{"", 0};
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
|
||||
{
|
||||
kiwix::MimeCounterType counters;
|
||||
std::istringstream ss(counterData);
|
||||
|
||||
while (ss)
|
||||
{
|
||||
const std::string mtcStr = readFullMimetypeAndCounterString(ss);
|
||||
const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
|
||||
if ( !mtc.first.empty() )
|
||||
counters.insert(mtc);
|
||||
}
|
||||
|
||||
return counters;
|
||||
}
|
||||
|
||||
std::string kiwix::gen_date_str()
|
||||
{
|
||||
auto now = std::time(0);
|
||||
@@ -312,86 +366,13 @@ std::string kiwix::gen_uuid(const std::string& s)
|
||||
return kiwix::to_string(zim::Uuid::generate(s));
|
||||
}
|
||||
|
||||
kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
|
||||
{
|
||||
return s.empty()
|
||||
? kainjow::mustache::data(false)
|
||||
: kainjow::mustache::data(s);
|
||||
}
|
||||
|
||||
std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
|
||||
{
|
||||
kainjow::mustache::mustache tmpl(template_str);
|
||||
kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
|
||||
[](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
|
||||
data.set("urlencoded", urlencode);
|
||||
std::stringstream ss;
|
||||
tmpl.render(data, [&ss](const std::string& str) { ss << str; });
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
std::string escapeBackslashes(const std::string& s)
|
||||
{
|
||||
std::string es;
|
||||
es.reserve(s.size());
|
||||
for (char c : s) {
|
||||
if ( c == '\\' ) {
|
||||
es.push_back('\\');
|
||||
}
|
||||
es.push_back(c);
|
||||
}
|
||||
return es;
|
||||
}
|
||||
|
||||
std::string makeFulltextSearchSuggestion(const std::string& lang,
|
||||
const std::string& queryString)
|
||||
{
|
||||
return kiwix::i18n::expandParameterizedString(lang, "suggest-full-text-search",
|
||||
{
|
||||
{"SEARCH_TERMS", queryString}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
kiwix::Suggestions::Suggestions()
|
||||
: m_data(kainjow::mustache::data::type::list)
|
||||
{
|
||||
}
|
||||
|
||||
void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
|
||||
{
|
||||
kainjow::mustache::data result;
|
||||
|
||||
const std::string label = suggestion.hasSnippet()
|
||||
? suggestion.getSnippet()
|
||||
: suggestion.getTitle();
|
||||
|
||||
result.set("label", escapeBackslashes(label));
|
||||
result.set("value", escapeBackslashes(suggestion.getTitle()));
|
||||
result.set("kind", "path");
|
||||
result.set("path", escapeBackslashes(suggestion.getPath()));
|
||||
result.set("first", m_data.is_empty_list());
|
||||
m_data.push_back(result);
|
||||
}
|
||||
|
||||
void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
|
||||
const std::string& queryString)
|
||||
{
|
||||
kainjow::mustache::data result;
|
||||
const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
|
||||
result.set("label", escapeBackslashes(label));
|
||||
result.set("value", escapeBackslashes(queryString + " "));
|
||||
result.set("kind", "pattern");
|
||||
result.set("first", m_data.is_empty_list());
|
||||
m_data.push_back(result);
|
||||
}
|
||||
|
||||
std::string kiwix::Suggestions::getJSON() const
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("suggestions", m_data);
|
||||
|
||||
return render_template(RESOURCE::templates::suggestion_json, data);
|
||||
}
|
||||
|
||||
@@ -17,10 +17,7 @@
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
// Implement method defined in <kiwix/tools.h> and "tools/pathTools.h"
|
||||
#include "tools.h"
|
||||
#include "tools/pathTools.h"
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#ifdef __APPLE__
|
||||
@@ -62,6 +59,7 @@
|
||||
#define PATH_MAX 1024
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
std::string WideToUtf8(const std::wstring& wstr)
|
||||
{
|
||||
@@ -80,7 +78,7 @@ std::wstring Utf8ToWide(const std::string& str)
|
||||
}
|
||||
#endif
|
||||
|
||||
bool kiwix::isRelativePath(const std::string& path)
|
||||
bool isRelativePath(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
if (path.size() < 3 ) {
|
||||
@@ -175,7 +173,7 @@ std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool ab
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string kiwix::computeRelativePath(const std::string& path, const std::string& absolutePath)
|
||||
std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
|
||||
{
|
||||
auto parts = kiwix::split(path, SEPARATOR, false);
|
||||
auto pathParts = normalizeParts(parts, false);
|
||||
@@ -200,11 +198,11 @@ std::string kiwix::computeRelativePath(const std::string& path, const std::strin
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string kiwix::computeAbsolutePath(const std::string& path, const std::string& relativePath)
|
||||
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
|
||||
{
|
||||
std::string absolutePath = path;
|
||||
if (path.empty()) {
|
||||
absolutePath = kiwix::getCurrentDirectory();
|
||||
absolutePath = getCurrentDirectory();
|
||||
}
|
||||
|
||||
auto parts = kiwix::split(absolutePath, SEPARATOR, false);
|
||||
@@ -217,7 +215,7 @@ std::string kiwix::computeAbsolutePath(const std::string& path, const std::strin
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string kiwix::removeLastPathElement(const std::string& path)
|
||||
std::string removeLastPathElement(const std::string& path)
|
||||
{
|
||||
auto parts_ = kiwix::split(path, SEPARATOR, false);
|
||||
auto parts = normalizeParts(parts_, false);
|
||||
@@ -228,7 +226,7 @@ std::string kiwix::removeLastPathElement(const std::string& path)
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string kiwix::appendToDirectory(const std::string& directoryPath, const std::string& filename)
|
||||
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
|
||||
{
|
||||
std::string newPath = directoryPath;
|
||||
if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
|
||||
@@ -238,7 +236,7 @@ std::string kiwix::appendToDirectory(const std::string& directoryPath, const std
|
||||
return newPath;
|
||||
}
|
||||
|
||||
std::string kiwix::getLastPathElement(const std::string& path)
|
||||
std::string getLastPathElement(const std::string& path)
|
||||
{
|
||||
auto parts_ = kiwix::split(path, SEPARATOR);
|
||||
auto parts = normalizeParts(parts_, false);
|
||||
@@ -269,7 +267,7 @@ std::string getFileSizeAsString(const std::string& path)
|
||||
return convert.str();
|
||||
}
|
||||
|
||||
std::string kiwix::getFileContent(const std::string& path)
|
||||
std::string getFileContent(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
auto wpath = Utf8ToWide(path);
|
||||
@@ -302,21 +300,19 @@ std::string kiwix::getFileContent(const std::string& path)
|
||||
return content;
|
||||
}
|
||||
|
||||
bool kiwix::fileExists(const std::string& path)
|
||||
bool fileExists(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return (_waccess_s(Utf8ToWide(path).c_str(), 0) == 0);
|
||||
return PathFileExistsW(Utf8ToWide(path).c_str());
|
||||
#else
|
||||
return (access(path.c_str(), F_OK) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool kiwix::fileReadable(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return (_waccess_s(Utf8ToWide(path).c_str(), 4) == 0);
|
||||
#else
|
||||
return (access(path.c_str(), R_OK) == 0);
|
||||
bool flag = false;
|
||||
std::fstream fin;
|
||||
fin.open(path.c_str(), std::ios::in);
|
||||
if (fin.is_open()) {
|
||||
flag = true;
|
||||
}
|
||||
fin.close();
|
||||
return flag;
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -370,7 +366,7 @@ bool copyFile(const std::string& sourcePath, const std::string& destPath)
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string kiwix::getExecutablePath(bool realPathOnly)
|
||||
std::string getExecutablePath(bool realPathOnly)
|
||||
{
|
||||
if (!realPathOnly) {
|
||||
char* cAppImage = ::getenv("APPIMAGE");
|
||||
@@ -424,7 +420,7 @@ bool writeTextFile(const std::string& path, const std::string& content)
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string kiwix::getCurrentDirectory()
|
||||
std::string getCurrentDirectory()
|
||||
{
|
||||
#ifdef _WIN32
|
||||
wchar_t* a_cwd = _wgetcwd(NULL, 0);
|
||||
@@ -438,7 +434,7 @@ std::string kiwix::getCurrentDirectory()
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string kiwix::getDataDirectory()
|
||||
std::string getDataDirectory()
|
||||
{
|
||||
// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
|
||||
#ifdef _WIN32
|
||||
@@ -493,14 +489,12 @@ static std::map<std::string, std::string> extMimeTypes = {
|
||||
{ "jpeg", "image/jpeg"},
|
||||
{ "jpg", "image/jpeg"},
|
||||
{ "gif", "image/gif"},
|
||||
{ "ico", "image/x-icon"},
|
||||
{ "svg", "image/svg+xml"},
|
||||
{ "txt", "text/plain"},
|
||||
{ "xml", "text/xml"},
|
||||
{ "pdf", "application/pdf"},
|
||||
{ "ogg", "application/ogg"},
|
||||
{ "js", "application/javascript"},
|
||||
{ "json", "application/json"},
|
||||
{ "css", "text/css"},
|
||||
{ "otf", "application/vnd.ms-opentype"},
|
||||
{ "ttf", "application/font-ttf"},
|
||||
@@ -509,7 +503,7 @@ static std::map<std::string, std::string> extMimeTypes = {
|
||||
};
|
||||
|
||||
/* Try to get the mimeType from the file extension */
|
||||
std::string kiwix::getMimeTypeForFile(const std::string& filename)
|
||||
std::string getMimeTypeForFile(const std::string& filename)
|
||||
{
|
||||
std::string mimeType = "text/plain";
|
||||
auto pos = filename.find_last_of(".");
|
||||
@@ -530,3 +524,4 @@ std::string kiwix::getMimeTypeForFile(const std::string& filename)
|
||||
|
||||
return mimeType;
|
||||
}
|
||||
|
||||
|
||||
@@ -75,3 +75,41 @@ std::string replaceRegex(const std::string& content,
|
||||
uresult.toUTF8String(tmp);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
std::string appendToFirstOccurence(const std::string& content,
|
||||
const std::string& regex,
|
||||
const std::string& replacement)
|
||||
{
|
||||
ucnv_setDefaultName("UTF-8");
|
||||
icu::UnicodeString ucontent(content.c_str());
|
||||
icu::UnicodeString ureplacement(replacement.c_str());
|
||||
auto matcher = buildMatcher(regex, ucontent);
|
||||
if (matcher->find()) {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
ucontent.insert(matcher->end(status), ureplacement);
|
||||
std::string tmp;
|
||||
ucontent.toUTF8String(tmp);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
std::string prependToFirstOccurence(const std::string& content,
|
||||
const std::string& regex,
|
||||
const std::string& replacement)
|
||||
{
|
||||
ucnv_setDefaultName("UTF-8");
|
||||
icu::UnicodeString ucontent(content.c_str());
|
||||
icu::UnicodeString ureplacement(replacement.c_str());
|
||||
auto matcher = buildMatcher(regex, ucontent);
|
||||
if (matcher->find()) {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
ucontent.insert(matcher->start(status), ureplacement);
|
||||
std::string tmp;
|
||||
ucontent.toUTF8String(tmp);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
@@ -17,11 +17,9 @@
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
// Implement function declared in tools.h and tools/stringTools.h
|
||||
#include "tools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include <tools/stringTools.h>
|
||||
|
||||
#include "tools/pathTools.h"
|
||||
#include <tools/pathTools.h>
|
||||
#include <unicode/normlzr.h>
|
||||
#include <unicode/rep.h>
|
||||
#include <unicode/translit.h>
|
||||
@@ -49,24 +47,6 @@ void kiwix::loadICUExternalTables()
|
||||
#endif
|
||||
}
|
||||
|
||||
kiwix::ICULanguageInfo::ICULanguageInfo(const std::string& langCode)
|
||||
: locale(langCode.c_str())
|
||||
{}
|
||||
|
||||
std::string kiwix::ICULanguageInfo::iso3Code() const
|
||||
{
|
||||
return locale.getISO3Language();
|
||||
}
|
||||
|
||||
std::string kiwix::ICULanguageInfo::selfName() const
|
||||
{
|
||||
icu::UnicodeString langSelfNameICUString;
|
||||
locale.getDisplayLanguage(locale, langSelfNameICUString);
|
||||
std::string langSelfName;
|
||||
langSelfNameICUString.toUTF8String(langSelfName);
|
||||
return langSelfName;
|
||||
}
|
||||
|
||||
std::string kiwix::removeAccents(const std::string& text)
|
||||
{
|
||||
loadICUExternalTables();
|
||||
@@ -161,14 +141,15 @@ std::string kiwix::encodeDiples(const std::string& str)
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
/* urlEncode() based on javascript encodeURI() &
|
||||
encodeURIComponent(). Mostly code from rstudio/httpuv (GPLv3) */
|
||||
|
||||
bool isReservedUrlChar(char c)
|
||||
{
|
||||
switch (c) {
|
||||
case ';':
|
||||
case ',':
|
||||
case '/':
|
||||
case '?':
|
||||
case ':':
|
||||
case '@':
|
||||
@@ -176,22 +157,22 @@ bool isReservedUrlChar(char c)
|
||||
case '=':
|
||||
case '+':
|
||||
case '$':
|
||||
case '#':
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool isHarmlessUriChar(char c)
|
||||
bool needsEscape(char c, bool encodeReserved)
|
||||
{
|
||||
if (c >= 'a' && c <= 'z')
|
||||
return true;
|
||||
return false;
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
return true;
|
||||
return false;
|
||||
if (c >= '0' && c <= '9')
|
||||
return true;
|
||||
|
||||
return false;
|
||||
if (isReservedUrlChar(c))
|
||||
return encodeReserved;
|
||||
switch (c) {
|
||||
case '-':
|
||||
case '_':
|
||||
@@ -202,10 +183,9 @@ bool isHarmlessUriChar(char c)
|
||||
case '\'':
|
||||
case '(':
|
||||
case ')':
|
||||
case '/':
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
int hexToInt(char c) {
|
||||
@@ -230,18 +210,18 @@ int hexToInt(char c) {
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::string kiwix::urlEncode(const std::string& value)
|
||||
std::string kiwix::urlEncode(const std::string& value, bool encodeReserved)
|
||||
{
|
||||
std::ostringstream os;
|
||||
os << std::hex << std::uppercase;
|
||||
for (const char c : value) {
|
||||
if (isHarmlessUriChar(c)) {
|
||||
os << c;
|
||||
for (std::string::const_iterator it = value.begin();
|
||||
it != value.end();
|
||||
it++) {
|
||||
|
||||
if (!needsEscape(*it, encodeReserved)) {
|
||||
os << *it;
|
||||
} else {
|
||||
const unsigned int charVal = static_cast<unsigned char>(c);
|
||||
os << '%' << std::setw(2) << std::setfill('0') << charVal;
|
||||
os << '%' << std::setw(2) << static_cast<unsigned int>(static_cast<unsigned char>(*it));
|
||||
}
|
||||
}
|
||||
return os.str();
|
||||
@@ -267,15 +247,15 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
|
||||
int iHi = hexToInt(hi);
|
||||
int iLo = hexToInt(lo);
|
||||
if (iHi < 0 || iLo < 0) {
|
||||
// Invalid escape sequence
|
||||
os << '%' << hi << lo;
|
||||
continue;
|
||||
// Invalid escape sequence
|
||||
os << '%' << hi << lo;
|
||||
continue;
|
||||
}
|
||||
char c = (char)(iHi << 4 | iLo);
|
||||
if (!component && isReservedUrlChar(c)) {
|
||||
os << '%' << hi << lo;
|
||||
os << '%' << hi << lo;
|
||||
} else {
|
||||
os << c;
|
||||
os << c;
|
||||
}
|
||||
} else {
|
||||
os << *it;
|
||||
@@ -288,7 +268,7 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
|
||||
/* Split string in a token array */
|
||||
std::vector<std::string> kiwix::split(const std::string& str,
|
||||
const std::string& delims,
|
||||
bool dropEmpty,
|
||||
bool trimEmpty,
|
||||
bool keepDelim)
|
||||
{
|
||||
std::string::size_type lastPos = 0;
|
||||
@@ -297,7 +277,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
|
||||
while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
|
||||
{
|
||||
auto token = str.substr(lastPos, pos - lastPos);
|
||||
if (!dropEmpty || !token.empty()) {
|
||||
if (!trimEmpty || !token.empty()) {
|
||||
tokens.push_back(token);
|
||||
}
|
||||
if (keepDelim) {
|
||||
@@ -307,7 +287,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
|
||||
}
|
||||
|
||||
auto token = str.substr(lastPos);
|
||||
if (!dropEmpty || !token.empty()) {
|
||||
if (!trimEmpty || !token.empty()) {
|
||||
tokens.push_back(token);
|
||||
}
|
||||
return tokens;
|
||||
@@ -415,27 +395,3 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
|
||||
&& std::equal(start.begin(), start.end(), base.begin());
|
||||
}
|
||||
|
||||
std::string kiwix::stripSuffix(const std::string& str, const std::string& suffix)
|
||||
{
|
||||
if (str.size() > suffix.size()) {
|
||||
const auto subStr = str.substr(str.size() - suffix.size(), str.size());
|
||||
if (subStr == suffix) {
|
||||
return str.substr(0, str.size() - suffix.size());
|
||||
}
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
|
||||
std::vector<std::string> variants;
|
||||
variants.push_back(title);
|
||||
variants.push_back(kiwix::ucFirst(title));
|
||||
variants.push_back(kiwix::lcFirst(title));
|
||||
variants.push_back(kiwix::toTitle(title));
|
||||
return variants;
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string kiwix::extractFromString(const std::string& str) {
|
||||
return str;
|
||||
}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <version.h>
|
||||
#include <zim/zim.h>
|
||||
#include <kiwix_config.h>
|
||||
#include <unicode/uversion.h>
|
||||
#include <pugixml.hpp>
|
||||
#include <curl/curl.h>
|
||||
#include <microhttpd.h>
|
||||
#include <xapian.h>
|
||||
#include <mustache.hpp>
|
||||
#include <zlib.h>
|
||||
|
||||
namespace kiwix
|
||||
{
|
||||
LibVersions getVersions() {
|
||||
LibVersions versions = {
|
||||
{ "libkiwix", LIBKIWIX_VERSION },
|
||||
{ "libzim", LIBZIM_VERSION },
|
||||
{ "libxapian", XAPIAN_VERSION },
|
||||
{ "libcurl", LIBCURL_VERSION },
|
||||
{ "libmicrohttpd", MHD_get_version() },
|
||||
{ "libz", ZLIB_VERSION }
|
||||
};
|
||||
|
||||
// U_ICU_VERSION does not include the patch level if 0
|
||||
std::ostringstream libicu_version;
|
||||
libicu_version << U_ICU_VERSION_MAJOR_NUM << "." << U_ICU_VERSION_MINOR_NUM << "." << U_ICU_VERSION_PATCHLEVEL_NUM;
|
||||
versions.push_back({ "libicu", libicu_version.str() });
|
||||
|
||||
// No human readable version string for pugixml
|
||||
const unsigned pugixml_major = (PUGIXML_VERSION - PUGIXML_VERSION % 1000) / 1000;
|
||||
const unsigned pugixml_minor = (PUGIXML_VERSION - pugixml_major * 1000 - PUGIXML_VERSION % 10) / 10;
|
||||
const unsigned pugixml_patch = PUGIXML_VERSION - pugixml_major * 1000 - pugixml_minor * 10;
|
||||
std::ostringstream libpugixml_version;
|
||||
libpugixml_version << pugixml_major << "." << pugixml_minor << "." << pugixml_patch;
|
||||
versions.push_back({ "libpugixml", libpugixml_version.str() });
|
||||
|
||||
// Needs version 5.0 of Mustache
|
||||
#if defined(KAINJOW_MUSTACHE_VERSION_MAJOR)
|
||||
std::ostringstream libmustache_version;
|
||||
libmustache_version << KAINJOW_MUSTACHE_VERSION_MAJOR << "." <<
|
||||
KAINJOW_MUSTACHE_VERSION_MINOR << "." << KAINJOW_MUSTACHE_VERSION_PATCH;
|
||||
versions.push_back({ "libmustache", libmustache_version.str() });
|
||||
#endif
|
||||
|
||||
return versions;
|
||||
}
|
||||
|
||||
void printVersions(std::ostream& out) {
|
||||
LibVersions versions = getVersions();
|
||||
for (const auto& iter : versions) {
|
||||
out << (iter != versions.front() ? "+ " : "")
|
||||
<< iter.first << " " << iter.second << std::endl;
|
||||
}
|
||||
}
|
||||
} //namespace kiwix
|
||||
13
src/wrapper/java/AndroidManifest.xml
Normal file
13
src/wrapper/java/AndroidManifest.xml
Normal file
@@ -0,0 +1,13 @@
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
|
||||
package="kiwix.org.kiwixlib"
|
||||
>
|
||||
|
||||
<application android:allowBackup="true"
|
||||
android:label="@string/app_name"
|
||||
android:supportsRtl="true"
|
||||
>
|
||||
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
88
src/wrapper/java/book.cpp
Normal file
88
src/wrapper/java/book.cpp
Normal file
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#include <jni.h>
|
||||
#include "org_kiwix_kiwixlib_Book.h"
|
||||
|
||||
#include "utils.h"
|
||||
#include "book.h"
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_org_kiwix_kiwixlib_Book_allocate(
|
||||
JNIEnv* env, jobject thisObj)
|
||||
{
|
||||
allocate<kiwix::Book>(env, thisObj);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_org_kiwix_kiwixlib_Book_dispose(JNIEnv* env, jobject thisObj)
|
||||
{
|
||||
dispose<kiwix::Book>(env, thisObj);
|
||||
}
|
||||
|
||||
#define BOOK (getPtr<kiwix::Book>(env, thisObj))
|
||||
|
||||
METHOD(void, Book, update__Lorg_kiwix_kiwixlib_Book_2, jobject otherBook)
|
||||
{
|
||||
BOOK->update(*getPtr<kiwix::Book>(env, otherBook));
|
||||
}
|
||||
|
||||
METHOD(void, Book, update__Lorg_kiwix_kiwixlib_JNIKiwixReader_2, jobject reader)
|
||||
{
|
||||
BOOK->update(**Handle<kiwix::Reader>::getHandle(env, reader));
|
||||
}
|
||||
|
||||
#define GETTER(retType, name) JNIEXPORT retType JNICALL \
|
||||
Java_org_kiwix_kiwixlib_Book_##name (JNIEnv* env, jobject thisObj) \
|
||||
{ \
|
||||
auto cRet = BOOK->name(); \
|
||||
retType ret = c2jni(cRet, env); \
|
||||
return ret; \
|
||||
}
|
||||
|
||||
GETTER(jstring, getId)
|
||||
GETTER(jstring, getPath)
|
||||
GETTER(jboolean, isPathValid)
|
||||
GETTER(jstring, getTitle)
|
||||
GETTER(jstring, getDescription)
|
||||
GETTER(jstring, getLanguage)
|
||||
GETTER(jstring, getCreator)
|
||||
GETTER(jstring, getPublisher)
|
||||
GETTER(jstring, getDate)
|
||||
GETTER(jstring, getUrl)
|
||||
GETTER(jstring, getName)
|
||||
GETTER(jstring, getFlavour)
|
||||
GETTER(jstring, getCategory)
|
||||
GETTER(jstring, getTags)
|
||||
GETTER(jlong, getArticleCount)
|
||||
GETTER(jlong, getMediaCount)
|
||||
GETTER(jlong, getSize)
|
||||
GETTER(jstring, getFavicon)
|
||||
GETTER(jstring, getFaviconUrl)
|
||||
GETTER(jstring, getFaviconMimeType)
|
||||
|
||||
METHOD(jstring, Book, getTagStr, jstring tagName) try {
|
||||
auto cRet = BOOK->getTagStr(jni2c(tagName, env));
|
||||
return c2jni(cRet, env);
|
||||
} catch(...) {
|
||||
return c2jni<std::string>("", env);
|
||||
}
|
||||
|
||||
#undef GETTER
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user