Compare commits

..

1 Commits

Author SHA1 Message Date
Matthieu Gautier
00c8f55cc4 Update name of the project in meson.build.
The right name is `kiwix-lib`, not `kiwixlib`.
Also update the version to be able to do a small fix release.
2018-02-01 16:10:54 +01:00
275 changed files with 7034 additions and 32743 deletions

View File

@@ -1,12 +0,0 @@
codecov:
notify:
require_ci_to_pass: yes
coverage:
status:
project:
default:
threshold: 1%
ignore:
- "test"

12
.github/FUNDING.yml vendored
View File

@@ -1,12 +0,0 @@
# These are supported funding model platforms
github: kiwix
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # https://kiwix.org/support-us/

15
.github/stale.yml vendored
View File

@@ -1,15 +0,0 @@
daysUntilClose: false
staleLabel: stale
issues:
daysUntilStale: 60
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be now be reviewed manually. Thank you
for your contributions.
pulls:
daysUntilStale: 7
markComment: >
This pull request has been automatically marked as stale because it has not had
recent activity. It will be now be reviewed manually. Thank you
for your contributions.

View File

@@ -1,149 +0,0 @@
name: CI
on:
push:
branches:
- main
pull_request:
jobs:
Macos:
runs-on: macos-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup python 3.9
uses: actions/setup-python@v1
with:
python-version: '3.9'
- name: Install packages
run: |
brew update
brew install gcovr pkg-config ninja || brew link --overwrite python
- name: Install python modules
run: pip3 install meson==0.49.2 pytest
- name: Install deps
shell: bash
run: |
ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
- name: Compile
shell: bash
run: |
export PKG_CONFIG_PATH=$HOME/BUILD_native_dyn/INSTALL/lib/pkgconfig
export CPPFLAGS="-I$HOME/BUILD_native_dyn/INSTALL/include"
meson . build --default-library=shared -Db_coverage=true
cd build
ninja
- name: Test
shell: bash
run: |
export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
cd build
meson test --verbose
env:
SKIP_BIG_MEMORY_TEST: 1
Linux:
strategy:
fail-fast: false
matrix:
name:
- native_static
- native_dyn
- android_arm
- android_arm64
- win32_static
- win32_dyn
include:
- name: native_static
target: native_static
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- name: native_dyn
target: native_dyn
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- name: android_arm
target: android_arm
image_variant: bionic
lib_postfix: '/arm-linux-androideabi'
- name: android_arm64
target: android_arm64
image_variant: bionic
lib_postfix: '/aarch64-linux-android'
- name: win32_static
target: win32_static
image_variant: f35
lib_postfix: '64'
- name: win32_dyn
target: win32_dyn
image_variant: f35
lib_postfix: '64'
env:
HOME: /home/runner
runs-on: ubuntu-latest
container:
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
steps:
- name: Checkout code
shell: python
run: |
from subprocess import check_call
from os import environ
command = [
'git', 'clone',
'https://github.com/${{github.repository}}',
'--depth=1',
'--branch', '${{ github.head_ref || github.ref_name }}'
]
check_call(command, cwd=environ['HOME'])
- name: Install deps
shell: bash
run: |
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_libkiwix.tar.xz
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
- name: Compile
shell: bash
run: |
meson --version
if [[ "${{matrix.target}}" =~ .*_dyn ]]; then
MESON_OPTION="--default-library=shared"
else
MESON_OPTION="--default-library=static"
fi
if [[ "${{matrix.target}}" =~ native_.* ]]; then
MESON_OPTION="$MESON_OPTION -Db_coverage=true"
else
MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
fi
if [[ "${{matrix.target}}" =~ android_.* ]]; then
MESON_OPTION="$MESON_OPTION -Dstatic-linkage=true"
fi
cd $HOME/libkiwix
meson . build ${MESON_OPTION}
cd build
ninja
env:
PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
- name: Test
if: startsWith(matrix.target, 'native_')
shell: bash
run: |
cd $HOME/libkiwix/build
meson test --verbose
ninja coverage
env:
LD_LIBRARY_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}"
SKIP_BIG_MEMORY_TEST: 1
- name: Publish coverage
shell: bash
run: |
cd $HOME/libkiwix
curl https://codecov.io/bash -o codecov.sh
bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
rm codecov.sh
if: startsWith(matrix.target, 'native_')
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

View File

@@ -1,91 +0,0 @@
name: Packages
on: [push, pull_request]
jobs:
build-deb:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
distro:
- ubuntu-kinetic
- ubuntu-jammy
- ubuntu-focal
- ubuntu-bionic
steps:
- uses: actions/checkout@v2
# Determine which PPA we should upload to
- name: PPA
id: ppa
run: |
if [[ $REF == refs/tags* ]]
then
echo "::set-output name=ppa::kiwixteam/release"
else
echo "::set-output name=ppa::kiwixteam/dev"
fi
env:
REF: ${{ github.ref }}
- uses: legoktm/gh-action-auto-dch@master
with:
fullname: Kiwix builder
email: release+launchpad@kiwix.org
distro: ${{ matrix.distro }}
- uses: legoktm/gh-action-build-deb@ubuntu-kinetic
if: matrix.distro == 'ubuntu-kinetic'
name: Build package for ubuntu-kinetic
id: build-ubuntu-kinetic
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-jammy
if: matrix.distro == 'ubuntu-jammy'
name: Build package for ubuntu-jammy
id: build-ubuntu-jammy
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-focal
if: matrix.distro == 'ubuntu-focal'
name: Build package for ubuntu-focal
id: build-ubuntu-focal
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-bionic
if: matrix.distro == 'ubuntu-bionic'
name: Build package for ubuntu-bionic
id: build-ubuntu-bionic
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: actions/upload-artifact@v2
with:
name: Packages for ${{ matrix.distro }}
path: output
- uses: legoktm/gh-action-dput@master
name: Upload dev package
# Only upload on pushes to git default branch
if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && startswith(matrix.distro, 'ubuntu-')
with:
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
repository: ppa:kiwixteam/dev
packages: output/*_source.changes
- uses: legoktm/gh-action-dput@master
name: Upload release package
# Only upload on pushes to master or tag
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
with:
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
repository: ppa:kiwixteam/release
packages: output/*_source.changes

9
.gitignore vendored
View File

@@ -1,9 +0,0 @@
.idea/
*.swp
subprojects/googletest-release*
*.class
build/
.vscode/
builddir/
.cache/
.clangd/

13
.travis.yml Normal file
View File

@@ -0,0 +1,13 @@
language: cpp
dist: trusty
sudo: required
cache: ccache
install: travis/install_deps.sh
script: travis/compile.sh
env:
- PLATFORM="native_static"
- PLATFORM="native_dyn"
- PLATFORM="win32_static"
- PLATFORM="win32_dyn"
- PLATFORM="android_arm"
- PLATFORM="android_arm64"

View File

@@ -77,7 +77,7 @@ modification follow.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties

558
ChangeLog
View File

@@ -1,561 +1,3 @@
libkiwix 12.0.0
===============
* [API Break] Remove wrapper around libzim (@mgautierfr #789)
* Allow kiwix-serve to use custom resource files (@veloman-yunkan #779)
* Properly handle searchProtocolPrefix when rendering search result (@veloman-yunkan #823)
* Prevent search on multi language content (@veloman-yunkan #838)
* Use new `zim::Archive::getMediaCount` from libzim (@mgautierfr #836)
* Catalog:
- Include tags in free text catalog search (@veloman-yunkan #802)
- Illustration's url is based on book's uuid (@veloman-yunkan #804)
- Cleanup of the opds-dumper (@veloman-yunkan #829)
- Allow filtering of catalog content using multiple languages (@veloman-yunkan #841)
- Make opds-dumper respect the namemapper (@mgautierfr #837)
* Server:
- Correctly handle `\` in suggestion json generation (@veloman-yunkan #843)
- Better http caching (@veloman-yunkan #833)
- Make `/suggest` endpoint thread-safe (@veloman-yunkan #834)
- Better redirection of main page (@veloman-yunkan #827)
- Remove jquery (@mgautierfr @juuz0 #796)
- Better Viewer of zim content :
. Introduce `/content` endpoints (@veloman-yunkan #806)
. Switch to iframe based content viewer (@veloman-yunkan #716)
- Optimised design of the welcome page:
. Alignement (@juuz0 @kelson42 #786)
. Exit download modal on pressing escape key (@juzz0 #800)
. Add favicon for different devices (@juzz0 #805)
. Fix auto hidding of the toolbar (@veloman-yunkan #821)
. Allow user to filter books by tags in the front page (@juuz0 #711)
* CI :
- Trigger CI on pull_request (@kelson42 #791)
- Drop Ubuntu Impish packaging (@legoktm #825)
- Add Ubuntu Kinetic packaging (@legoktm #801)
* Testing:
- Test ICULanguageInfo (@veloman-yunkan #795)
- Introduce fake `test` language to test i18n (@veloman-yunkan #848)
* Fix documentation (@kelson42 #816)
* Udpate translation (#787 #839 #847)
libkiwix 11.0.0
===============
* [server] Add support for internationalization (@veloman-yunkan #679)
* [server] Use gzip compression instead of deflat (mgautierfr #757)
* [server] Version the static resources. This allow better invalidating
browser cache when resources are changed (@veloman-yunkan #712)
* [server|front] Use integer to query the host for page length (@juuz0 #772)
* [server] Improve multizim search API:
- Improvement of the cache system
- Better API to select on which books to search in.
- SysAdmin is now able to limit the number of book we search in for a multizim search
* [server] Introduce a opensearch API for multizim fulltext search
* [wrapper] Remove java wrapper
* Testing:
- Testing of search result pages content (@veloman-yunkan #765)
- Better testing structure of xml search result (@veloman-yunkan #780)
libkiwix 10.1.1
===============
* Correctly detect the number of article for older zims (<=6) (@mgautier #743)
* [server] Fix fulltext search (@mgautierfr #724)
* [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
* Fix CI (@mgautierfr #736)
libkiwix 10.1.0
===============
This release is an important one as it fixes a Xss vulnerability introduced
in libkiwix 10.0.0
* [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
* [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
* [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
* [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
* Add `dc:issued` to opds output stream (@veloman-yunkan #715)
* Add handling of several languages not provided by ICU (@juuz0 #701)
* [server] Add a caching system for search and suggestion (@maneeshpm #620)
* Fix cross-compilation (@kelson42 #703)
* Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
* Better testing system of html response (@veloman-yunkan #725)
libkiwix 10.0.1
===============
* [server] The catalog search interpret `count=0` as no limit.
This was the case for a long time. This was changed unintentionally
(@veloman-yunkan #686)
* [server] Correctly generere a human friendly title in the server frontend.
(@juuz0 #687, @kelson42 #689)
* [server] Fix download button if there is no url do download from.
(@juuz0 #691)
* Add non-minified isotope.pkdg.js
Needed for debian packaging as we need the source and minified version is
not the source (@legoktm #693)
* [server] Add a tooltip with the full language for the lang tag.
* CI fixes (@kelson42 @legoktm)
libkiwix 10.0.0
===============
This release is huge release.
The project has been renamed to libkiwix, it is more coherent with the library name.
* Server front page :
- Use js in the front page to display the available book,
using the OPDS stream as source. The front page is now populated only with
the visible books and user can search for books. (@MananJethwany #530, #541, #534)
(@kelson42 #628)
- Revamp css (@MananJethwany #559)
- Correctly Convert 3iso language code to 2iso (@juuz0 #672)
* Server suggestions search :
- Add pagination for suggestion search (@maneeshpm #591)
- Fix suggestion system (@MananJethwany #498)
- Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
- The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
- Properly handle html encoding of suggestions (@veloman-yunkan #458)
* Server improvements :
- Remove meta endpoints (@mgautier #669)
- Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
- Add details on 404 error pages (@soumyankar #490)
- Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
- Better headbar insertion (after charset definition) (@kelson42 #442)
* New OPDS Stream v2 :
- Add a list of categories (@veloman-yunkan)
- Support for partial entries (@veloman-yunkan #602)
- Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
- Add language endpoint to catalog (@veloman-yunkan #553)
- Add illustration API to get the illustration of a book (@mgautierfr #645)
- OPDS search can now filter books by category (@veloman-yunkan #459)
* Library improvements :
- Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
- Properly handle removing of book from the library (@veloman-yunkan #485)
- Use xapian to search for books in the library (@veloman-yunkan #460, #488)
* Added methods/functions :
- Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
- Add `getVersions` and `printVersions` functions (@kelson42 #665)
- Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
- Add `get_zimid()` method to the search result (@maneeshpm #510)
* Various improvements :
- Better secret value for aria2c rpc (@juuz0 #666)
- Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
- Add basic documentation (@mgautierfr #640)
- Do not use Reader internally (@maneeshpm #536 #576)
- Remove dependency headers from our public headers (@mgautierfr #574)
- Downloader now don't write metalink on the filesystem (@kelson42 #502)
- Support opening a zim file using a fd (@veloman-yukan #429)
- Use C++11 std::thread instead of pthread (@mgautierfr #445)
- [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
- Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
- Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
- Better parsing of the counterMap (@kelson42 #437)
- Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
- Remove deprecated methods (@mgautierfr)
- CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
- Fix compilation for FreeBSD (@swills g#432)
- Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
kiwix-lib 9.4.1
===============
* Fix `M/Counter` parsing.
* [SERVER] Adjust body padding-top for taskbar
* Fix potential crash when stoping a server not started.
* Various fix in build system and the CI.
kiwix-lib 9.4.0
===============
* [SERVER] Make the headers handling case insensitive.
* [SERVER] Make server answer 204 http status code for empty search
* [PACKAGING] Made CI build deb packages.
* [SERVER] Add a way to prevent taskbar and external link bloquer at article
level.
* Fix meson file to be compatible with meson 0.45
* [SERVER] Update search requests to use pageStart/pageLength instead of
pageStart/pageEnd arguments.
* [SERVER] Set a fixed favicon size in the main page.
* [SERVER] Refactor the response system code to better handling future new
libzim api.
* Fix segmentation fault around exchange with aria2 process making
kiwix-desktop crash at exit.
kiwix-lib 9.3.1
===============
* Fix handling of samba path on windows.
* Do not include `kiwix_config.h` in public header.
* Fix compilation with libmicrohttpd v0.97.1
* Increase default test timeout to 160seconds/test.
* Add automatic debian packaging.
* Use non-minified version of jquery-ui.js
* Pass `-latomic` compile option for sh4 architecture.
* Make mesion install `kiwix-compile-resources` man page.
kiwix-lib 9.3.0
===============
* Add a thread safe method to search suggestions.
Old methods are now deprecated.
kiwix-lib 9.2.3
===============
* Add test on byte-range
* Fix compilation on bionic and windows.
* Allow building using debian packaged kainjow-mustache
* Pass `-latomic` compile option for architectures that need it
kiwix-lib 9.2.2
===============
* Fix handling on empty content in byte range management (wrong assert)
kiwix-lib 9.2.1
===============
* Fix support of byte range request.
kiwix-lib 9.2
=============
* Add tests
* Refactoring server code.
* [SERVER] Add HEAD, Etag and If-None-Match support.
* [SERVER] Compress opds catalog answers.
kiwix-lib 9.1.2
===============
* Do not use the pathToSave if it is empty.
kiwix-lib 9.1.1
===============
* Fix the detection of the dataDirectory on windows.
kiwix-lib 9.1.0
===============
* [JAVA] Add a method to get the size of an article.
* Add a new method to the libray to get the book by path.
* Add a option to make the server blocks external link.
Links are intercepted by js an redirected to a "portail" page.
* [ODPS] Correctly handle book's articleCount and mediaCount.
kiwix-lib 9.0.1
===============
* [JAVA] Use a long to store the offset of an article in the zim file instead
of an int.
kiwix-lib 9.0.0
===============
* [OPDS] Correctly set the id of the OPDS stream.
* [OPDS] Do not try to filter the catalog if no filter field is given in the
request.
* [WINDOWS] Correctly convert path to wide chars when opening the library.xml
* [LIBRARY] Remove the function the read file using a native path.
All path must be utf8, no need to pass a native path along the utf8 path.
* [TEST] Fix tests using the main function of gtest instead of custom one.
* [CI] Move to github CI instead of Travis.
* The `Book::update` method always update the book's fields. Even if they are
not empty.
* [JAVA] Add wrapping around the library manager (opds parsing)
* [ARIA2] Add api option to start download with option (destination folder)
* [OPDS] Fixes about opds parsing, generation (missing attributes)
and requesting (server)
* Add methods on `Book` to get specific tag values (was on `Reader` only)
* Add flavour attribute to `Book`
* Fix opensearch description.
* Trust the given library.xml (by default) instead of reading the value from
the zim files.
* [OPDS] Be able to filter the content by name or size.
* [WINDOWS] Fix launching subcommand when there is spaces in the path.
kiwix-lib 8.2.2
===============
* Improve a few compilation scripts
kiwix-lib 8.2.1
===============
* Reintroduce kiwix-serve taskbar
kiwix-lib 8.2.0
===============
* More debug information if aria2c command fails
* Allow to set kiwix-serve port
* Better (dead) bookmarks mgmt
kiwix-lib 8.1.0
===============
* Fix pathTools manipulation.
* Add missing implementation of getArticleCount and getMediaCount on android.
* Correctly convert windows path to utf8.
* Add code coverage in the CI
kiwix-lib 8.0.1
===============
* Fix join function
kiwix-lib 8.0.0
===============
* Add new methods to get all (and new) metadata from the zim file.
* Add methods to get the value of a specific tag.
* [API Change] Convert tags value to the new convention.
* [API Change] Rename `getMatatag` method to `getMetadata`
* [ABI Change] Correctly detect executable path in appimage.
kiwix-lib 7.0.0
===============
* [API break] Add a argument to kiwix-serve to specify the library to use.
kiwix-lib 6.0.4
===============
* Fix HTML rendering of the search result if there is no result.
* Do not crash at html rendering if request ask for 0 results (start == end)
* Correctly find the executable path if we are using AppImage
kiwix-lib 6.0.3
===============
* force one column suggestion in kiwix-serve suggestions
* fix fulltext search link in suggestions
* UI fixes in kiwix-serve rendering
kiwix-lib 6.0.2
===============
* Correctly set the groupId in the pom file.
kiwix-lib 6.0.1
===============
* Generate the pom file for android/maven
kiwix-lib 6.0.0
===============
* Move the server code in kiwix-lib (from kiwix-serve).
* Add unit test on regex functions.
* Fix computerAbsolutePath (thread safe, memory leak).
* Correctly set the book's path as valid if we construct the book from a
reader.
* [JNI] Add a method to know if a article is a redirection.
* Do not embed the gtest dependency.
* [JNI] Add a constructor to JNIKiwixString.
* Change order of search of the favicon urls.
* Clean a lot of unecessary includes in headers. (potential "API break")
kiwix-lib 5.2.0
===============
* kiwix-serve integration (as a seperated process).
* Fix crash in the suggestion search.
* Better API to filter the library books.
* New kiwix-lib application for android. (.aar)
* Use ReLinker to link with libkiwix.so in android.
* Correctly set the verbosity of zim search.
kiwix-lib 5.1.0
===============
* Add function to pause, resume and stop downloads.
* Add zim's tags in the opds stream.
* Addapt to new libzim 5.0.0 API.
kiwix-lib 5.0.0
===============
* Remove error message when trying to open an wrong zim file.
* Rewrite `makeTmpDirectory` to not use uuid functions on windows.
* [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
* Remove rpath
* Detect infinite (and too long) redirect loops.
kiwix-lib 4.1.0
===============
* Allow the library to be filtered by tags.
* Fix language mapping.
* Update README about mustache dependency.
kiwix-lib 4.0.1
===============
* Fix "maybe uninitialize variable" issue.
* Ensure path are stored correctly (absolute path) in the library.
* [CI] Use the new deps archive xz
kiwix-lib 4.0.0
===============
* [API break] Remove support for external index.
* Move to the mustache templating system instead of ctpp2.
* Make meson.build works for meson>=0.43.0
* [API break] Move the basic tools from the `common` directory to `tools`.
kiwix-lib 3.1.1
===============
* The OPDS feed book's date must be the date of the book, not the date of the
feed generation.
* Convert the standard opds date to our format (YYYY-MM-DD)
* Remove duplicate language attribute in the libxml dumper.
* Create the datadirectory to not fail to write a file in a non-existent
directory
kiwix-lib 3.1.0
===============
* Add a method to get the favicon url of book (if available).
* Move dump code of library.xml in a specific class.
* Add a first support to bookmarks
kiwix-lib 3.0.3
===============
* Add the 'en' language to the mapping alpha2-code ('en') to alpha3-code
('eng').
* Correctly write the 'ArticleCount' and 'MediaCount' in the library.xml.
* Correctly fill the book size for the zim file size.
* Fix launch of aria2c.
kiwix-lib 3.0.2
===============
* Use the correct path separator when computing relativePath on Windows.
kiwix-lib 3.0.1
===============
* Small fix about parsing the opdsStream.
kiwix-lib 3.0.0
===============
* Change the downloader to use aria2 using a separated process (with rpc)
instead of using the libaria2. This simplify a lot the link process to
libaria2 on Windows.
- kiwix-lib doesn't depend on libaria2 anymore.
- kiwix-lib now depends on libcurl.
* [API break] Library class API has been updated :
- Books are referenced by id, not index. A lot of methods have been
updated this way.
- Books "list" is now private.
- There is no more "current" book.
- listBooksIds's filters have been updated.
* [API break] Book class API has been updated :
- Move the definition of Book in `book.h`.
- Use getter/setter methods instead public members.
- Size (getSize/setSize) is now returned in bytes, not kB.
- Dependending of how the book has been initialized (opdsfeed), the
faviconUrl may be stored in the book, the favicon being downloaded when
using `getFavicon`.
- The path (and indexPath) are always absolute path.
- Book has now a downloadId, corresponding to the aria2 download id (if
exists)
* [API break] Manager class API has been updated :
- The manager is mainly use to fill a Libray from a "library.xml" file or
opds feed. Other operations (has removeBookById, setBookPath, filter, ...)
have been removed.
- The manager use a intermediate class (LibraryManipulator) to add book to
the library. This dependency injection allow caller code to hook the add
of a book to the library.
- The manager work on a existing Library. It doesn't how a internal
Library.
* [API break] OpdsDumper class API has been updated :
- dumpOPDSFeed method now take the list of bookIds to dump instead of
dumping all books in the library.
- OpdsDumper can now dump openSearch result information (total result
count, start index, ...).
* [API break] Common tools API has been updated :
- `base64_encode` and `base64_decode` take std::string as arguments.
- New `download` function in networkTools.h using libcurl.
- New `getDataDirectory` function in pathTools.
- Better `beautifyInteger` and `beautifyFileSize` functions.
- New `nodeToString` function serializing a pugi::xml_node to a string.
- New `converta2toa3` function to convert alpha2 language code to aplha3
language code.
kiwix-lib 2.0.2
===============
* [Android] Forward c++ errors message de Java world.
* Follow redirection of favicon.
* Make aria2 dependency optional.
* Inculde unistd.h only on unix platform.
kiwix-lib 2.0.1
===============
* Fix parsing of url.
* Remove unused static resources.
* Correctly decode reserved characters in URLs.
* Explicitly use icu namespace to allow use of packaged icu lib.
kiwix-lib 2.0.0
===============
* Introduce a new API to retrive content from a reader.
* Introduce the `Entry` class.
* Reader's methods return an `Entry`.
* Content and other information can be retrieved from the `Entry`.
* Older Reader's methods are depreciated.
* Add an `OPDSDumper` class to dump a whole `Library` as an OPDS feed.
* Add a tool function to get the content of a file.
* Add a tool function to create a tempory directory.
* Add a `Downloader` class to download a file.
* Allow the manager to populate a `Library` from an OPDS feed.
* Try to locate libctpp2 in default system libdir and then fallback in 'lib'
directory.
* Build kiwix-lib setting RPATH.
* Build kiwix-lib without warning (werror=true)
* Build kiwix-lib on macos.
kiwix-lib 1.1.1
===============
* Correct the name of kiwix-lib (from `kiwixlib`) in meson.build to generate
dist archive with the correct name.
* Libzim version need to be at least 3.2.0
kiwix-lib 1.1.0
===============
* Allow for more than 70 search result per page in html results rendering
(kiwix/kiwix-tools#92)
* Add a small api to do geo queries.
* Add multi-search support in the JNI (#67)
* Add an API to get only one part of an article.
* Add an API to get direct location of an article content in the zim file.
* Improve urlencoding
* Fix pagination in html results rendering.
* Compile using gcc-5 on Travis.
* Allow JNI to access search snippets.
* JNI throw an exception instead of returning an invalid object if something
goes wrong.
* Add doctext documentation. (#116)
* Various bug fixes.
kiwix-lib 1.0.0
===============

263
README.md
View File

@@ -1,55 +1,42 @@
Libkiwix
========
Kiwix library
=============
The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
core. It contains the code shared by all Kiwix ports (Windows,
GNU/Linux, macOS, Android, iOS, ...).
[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
[![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amain)
[![Doc](https://readthedocs.org/projects/libkiwix/badge/?style=flat)](https://libkiwix.readthedocs.org/en/latest/?badge=latest)
[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
[![Codecov](https://codecov.io/gh/kiwix/libkiwix/branch/main/graph/badge.svg)](https://codecov.io/gh/kiwix/libkiwix)
[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
The Kiwix library provides the Kiwix software core. It contains the
code shared by all Kiwix ports (Windows, Linux, OSX, Android, ...).
Disclaimer
----------
This document assumes you have a little knowledge about software
compilation. If you experience difficulties with the dependencies or
with the Libkiwix compilation itself, we recommend to have a look to
[kiwix-build](https://github.com/kiwix/kiwix-build).
with the Kiwix libary compilation itself, we recommend to have a look
to [kiwix-build](https://github.com/kiwix/kiwix-build).
Preamble
--------
Although the Libkiwix can be (cross-)compiled on/for many sytems, the
following documentation explains how to do it on POSIX ones. It is
primarly thought for GNU/Linux systems and has been tested on recent
releases of Ubuntu and Fedora.
Although the Kiwix library can be compiled/cross-compiled on/for many
sytems, the following documentation explains how to do it on POSIX
ones. It is primarly though for GNU/Linux systems and has been tested
on recent releases of Ubuntu and Fedora.
Dependencies
------------
The Libkiwix relies on many third party software libraries. They are
prerequisites to the Libkiwix compilation. Following libraries need to
be available:
* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
header `mustache.hpp` somewhere it can be found by the compiler and/or
set CPPFLAGS with correct `-I` option). Use Mustache version 4.1 or above.
* [Libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
* [Microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
* [Zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
The Kiwix library relies on many third parts software libraries. They
are prerequisites to the Kiwix library compilation. Following
libraries need to be available:
To test the code:
* [Google Test](https://github.com/google/googletest) (package `googletest` on Ubuntu)
The following dependency needs to be available at runtime:
* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
* ICU ................................... http://site.icu-project.org/
(package libicu-dev on Ubuntu)
* ZIM ........................................ http://www.openzim.org/
(package libzim-dev on Ubuntu)
* Pugixml ........................................ http://pugixml.org/
(package libpugixml-dev on Ubuntu)
* ctpp2 ........................................ http://ctpp.havoc.ru/
(package libctpp2-dev on Ubuntu)
* Xapian ......................................... https://xapian.org/
(package libxapian-dev on Ubuntu)
These dependencies may or may not be packaged by your operating
system. They may also be packaged but only in an older version. The
@@ -58,182 +45,76 @@ In the worse case, you will have to download and compile bleeding edge
version by hand.
If you want to install these dependencies locally, then use the
`libkiwix` directory as install prefix.
kiwix-lib directory as install prefix.
Environment
If you compile ctpp2 from source and want to compile the Kiwix library
statically then you will probably need to rename ctpp2 static library
from ctpp2-st.a to ctpp2.a.
Environnement
-------------
The Libkiwix builds using [Meson](https://mesonbuild.com/) version
0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
The Kiwix library builds using [Meson](http://mesonbuild.com/) version
0.34 or higher. Meson relies itself on Ninja, pkg-config and few other
compilation tools.
Install first the few common compilation tools:
* [Meson](https://mesonbuild.com/)
* [Ninja](https://ninja-build.org/)
* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
* Automake
* Libtool
* Virtualenv
* Pkg-config
These tools should be packaged if you use a cutting edge operating
system. If not, have a look to the [Troubleshooting](#Troubleshooting)
section.
Compilation
-----------
Once all dependencies are installed, you can compile the Libkiwix
with:
```bash
meson . build
ninja -C build
Then install Meson itself:
```
By default, it will compile dynamic linked libraries. All binary files
will be created in the `build` directory created automatically by
Meson. If you want statically linked libraries, you can add
`--default-library=static` option to the Meson command.
Depending of you system, `ninja` may be called `ninja-build`.
The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
```bash
meson . build -Dwrapper=android -Dwerror=false
ninja -C build
```
Static files compilation
------------------------
Libkiwix has a few static files 'compiled' within the binary
code. This is mostly Javascript/HTML/pictures necessary for the HTTP
daemon.
These static files are available in the `static` directory and are
compiled by custom Python code available in this repository `scripts`
directory. This happens automatically at compilation time without any
additional command to run.
To avoid HTTP caching issues, the URLs (to the static content) are
appended with a `cacheid` parameter (this is called "cache
busting"). This `cacheid` value derived from the
[sha1sum](https://en.wikipedia.org/wiki/Sha1sum) of each targeted
static file. As a consequence, each time you change a static file, the
corresponding `cacheid` value will change.
To properly test this feature, this `cacheid` needs to be added
manually to the automated tests and has to be commited. After
modifying the needed static file, [run the automated
tests](#Testing). They will fail, but the inspection of the testing
log will give you the new `cacheid` value(s). Finally update
`test/server.cpp` with the appropriate `cacheid` value(s) which have
changed.
Testing
-------
To run the automated tests:
```bash
cd build
meson test
```
Installation
------------
If you want to install the Libkiwix and the headers you just have
compiled on your system, here we go:
```bash
ninja -C build install
```
You might need to run the command as root (or using `sudo`), depending
where you want to install the libraries. After the installation
succeeded, you may need to run `ldconfig` (as `root`).
Uninstallation
--------------
If you want to uninstall the Kiwix library:
```bash
ninja -C build uninstall
```
Like for the installation, you might need to run the command as `root`
(or using `sudo`).
Custom Index Page
-----------------
to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
(note - while using custom html file please mention all external links as absolute path.)
to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
To use JS provided by kiwix-serve you can use the following template to start with ->
```
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title><-- Custom Tittle --></title>
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
<script src="{{root}}/skin/iso6391To3.js"></script>
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
</head>
<body>
</body>
</html>
```
- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
- To add language select box add - `<select id="languageFilter"></select>` under body tag.
- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
- To add search box for books use following form -
```
<form id='kiwixSearchForm'>
<input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
<input type="submit" class="kiwixButton" value="Search"/>
</form>
```
If you compile manually Libmicrohttpd, you might need to compile it
without GNU TLS, a bug here will empeach further compilation
otherwise.
If the compilation still fails, you might need to get a more recent
version of a dependency than the one packaged by your Linux
distribution. Try then with a source tarball distributed by the
problematic upstream project or even directly from the source code
repository.
Troubleshooting
---------------
If you need to install Meson "manually":
```bash
virtualenv -p python3 ./ # Create virtualenv
source bin/activate # Activate the virtualenv
pip3 install meson # Install Meson
pip install meson # Install Meson
hash -r # Refresh bash paths
```
If you need to install Ninja "manually":
```bash
Finally download and build Ninja locally:
```
git clone git://github.com/ninja-build/ninja.git
cd ninja
git checkout release
./configure.py --bootstrap
mkdir ../bin
mkdir ../bin
cp ninja ../bin
cd ..
```
Compilation
-----------
Once all dependencies are installed, you can compile kiwix-lib with:
```
mkdir build
meson . build
cd build
ninja
```
By default, it will compile dynamic linked libraries. If you want
statically linked libraries, you can add `--default-library=static`
option to the Meson command.
Depending of you system, `ninja` may be called `ninja-build`.
Installation
------------
If you want to install the libraries you just have compiled on your
system, here we go:
```
ninja install
cd ..
```
You might need to run the command as root, depending where you want to
install the libraries.
License
-------
[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
[COPYING](COPYING) for more details.
GPLv3 or later, see COPYING for more details.

View File

@@ -1,13 +0,0 @@
*.iml
.gradle
/local.properties
/.idea/caches
/.idea/libraries
/.idea/modules.xml
/.idea/workspace.xml
/.idea/navEditor.xml
/.idea/assetWizardSettings.xml
.DS_Store
/build
/captures
.externalNativeBuild

View File

@@ -1,25 +0,0 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
google()
jcenter()
}
dependencies {
classpath 'com.android.tools.build:gradle:3.4.1'
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files
}
}
allprojects {
repositories {
google()
jcenter()
}
}
task clean(type: Delete) {
delete rootProject.buildDir
}

View File

@@ -1,15 +0,0 @@
# Project-wide Gradle settings.
# IDE (e.g. Android Studio) users:
# Gradle settings configured through the IDE *will override*
# any settings specified in this file.
# For more details on how to configure your build environment visit
# http://www.gradle.org/docs/current/userguide/build_environment.html
# Specifies the JVM arguments used for the daemon process.
# The setting is particularly useful for tweaking memory settings.
org.gradle.jvmargs=-Xmx1536m
# When configured, Gradle will run in incubating parallel mode.
# This option should only be used with decoupled projects. More details, visit
# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
# org.gradle.parallel=true
# Kotlin code style for this project: "official" or "obsolete":
kotlin.code.style=official

View File

Binary file not shown.

View File

@@ -1,6 +0,0 @@
#Wed Jun 19 15:28:39 BST 2019
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

View File

@@ -1,172 +0,0 @@
#!/usr/bin/env sh
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn () {
echo "$*"
}
die () {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=$(save "$@")
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi
exec "$JAVACMD" "$@"

View File

@@ -1,84 +0,0 @@
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windows variants
if not "%OS%" == "Windows_NT" goto win9xME_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

View File

@@ -1 +0,0 @@
/build

View File

@@ -1,64 +0,0 @@
apply plugin: 'com.android.library'
apply plugin: 'maven'
android {
compileSdkVersion 28
defaultConfig {
minSdkVersion 15
targetSdkVersion 28
versionCode 1
versionName "1.0"
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
}
dependencies {
implementation 'com.getkeepsafe.relinker:relinker:1.3.1'
}
task writePom {
pom {
project {
groupId 'org.kiwix.kiwixlib'
artifactId 'kiwixlib'
version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
packaging 'aar'
name 'kiwixlib'
url 'https://github.com/kiwix/libkiwix'
licenses {
license {
name 'GPLv3'
url 'https://www.gnu.org/licenses/gpl-3.0.en.html'
}
}
developers {
developer {
id 'kiwix'
name 'kiwix'
email 'contact@kiwix.org'
}
}
scm {
connection 'https://github.com/kiwix/libkiwix.git'
developerConnection 'https://github.com/kiwix/libkiwix.git'
url 'https://github.com/kiwix/libkiwix'
}
}
}.withXml {
def dependenciesNode = asNode().appendNode('dependencies')
//Iterate over the implementation dependencies, adding a <dependency> node for each
configurations.implementation.allDependencies.each {
def dependencyNode = dependenciesNode.appendNode('dependency')
dependencyNode.appendNode('groupId', it.group)
dependencyNode.appendNode('artifactId', it.name)
dependencyNode.appendNode('version', it.version)
}
}.writeTo("$buildDir/pom.xml")
}

View File

@@ -1,21 +0,0 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile

View File

@@ -1,10 +0,0 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.kiwix.kiwixlib">
<application
android:allowBackup="true"
android:supportsRtl="true">
</application>
</manifest>

View File

@@ -1 +0,0 @@
include ':kiwixLibAndroid'

5
debian/changelog vendored
View File

@@ -1,5 +0,0 @@
libkiwix (0.0.0) unstable; urgency=medium
* Initial release
-- Kunal Mehta <legoktm@debian.org> Wed, 08 Jul 2020 18:12:32 -0700

46
debian/control vendored
View File

@@ -1,46 +0,0 @@
Source: libkiwix
Priority: optional
Maintainer: Kiwix team <kiwix@kiwix.org>
Build-Depends: debhelper-compat (= 13),
meson,
pkg-config,
libzim-dev (>= 7.2.0~),
libcurl4-gnutls-dev,
libicu-dev,
libgtest-dev,
libkainjow-mustache-dev,
liblzma-dev,
libmicrohttpd-dev,
libpugixml-dev,
zlib1g-dev
Standards-Version: 4.5.0
Section: libs
Homepage: https://github.com/kiwix/libkiwix
Rules-Requires-Root: no
Package: libkiwix-dev
Section: libdevel
Architecture: any
Multi-Arch: same
Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
libzim-dev (>= 7.2.0~),
libicu-dev,
libpugixml-dev,
libcurl4-gnutls-dev,
libmicrohttpd-dev
Description: library of common code for Kiwix (development)
Kiwix is an offline Wikipedia reader. libkiwix provides the
software core for Kiwix, and contains the code shared by all
Kiwix ports (Windows, Linux, OSX, Android, etc.).
.
This package contains development files.
Package: libkiwix10
Architecture: any
Multi-Arch: same
Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
Conflicts: libkiwix0, libkiwix3, libkiwix9
Description: library of common code for Kiwix
Kiwix is an offline Wikipedia reader. libkiwix provides the
software core for Kiwix, and contains the code shared by all
Kiwix ports (Windows, Linux, OSX, Android, etc.).

1
debian/copyright vendored
View File

@@ -1 +0,0 @@
See COPYING in the repository root.

View File

@@ -1,4 +0,0 @@
usr/include
usr/lib/*/libkiwix.so
usr/lib/*/pkgconfig
usr/bin

View File

@@ -1,2 +0,0 @@
usr/share/man/man1/kiwix-compile-resources.1*
usr/share/man/man1/kiwix-compile-i18n.1*

View File

@@ -1 +0,0 @@
usr/lib/*/libkiwix.so.*

8
debian/rules vendored
View File

@@ -1,8 +0,0 @@
#!/usr/bin/make -f
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
%:
dh $@ --buildsystem=meson
override_dh_auto_test:
dh_auto_test -- -t 3

View File

@@ -1 +0,0 @@
3.0 (native)

2
docs/.gitignore vendored
View File

@@ -1,2 +0,0 @@
api
xml

View File

@@ -1,72 +0,0 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'libkiwix'
copyright = '2022, libkiwix-team'
author = 'libkiwix-team'
# -- General configuration ---------------------------------------------------
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'breathe',
'exhale'
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
if not on_rtd:
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
breathe_projects = {
"libkiwix": "./xml"
}
breathe_default_project = 'libkiwix'
exhale_args = {
"containmentFolder": "./api",
"rootFileName": "ref_api.rst",
"rootFileTitle": "Reference API",
"doxygenStripFromPath":"..",
"treeViewIsBootstrap": True,
"createTreeView" : True,
"exhaleExecutesDoxygen": True,
"exhaleDoxygenStdin": "INPUT = ../include"
}
primary_domain = 'cpp'
highlight_language = 'cpp'

View File

@@ -1,14 +0,0 @@
.. libkiwix documentation master file, created by
sphinx-quickstart on Fri Jul 24 15:40:50 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to libkiwix's documentation!
==================================
.. toctree::
:maxdepth: 2
:caption: Contents:
usage
api/ref_api

View File

@@ -1,7 +0,0 @@
sphinx = find_program('sphinx-build', native:true)
sphinx_target = run_target('doc',
command: [sphinx, '-bhtml',
meson.current_source_dir(),
meson.current_build_dir()])

View File

@@ -1,2 +0,0 @@
breathe
exhale

View File

@@ -1,15 +0,0 @@
Libkiwix programming
====================
Introduction
------------
libkiwix is written in C++. To use the library, you need the include files of libkiwix have
to link against libzim.
Errors are handled with exceptions. When something goes wrong, libkiwix throws an error,
which is always derived from std::exception.
All classes are defined in the namespace kiwix.
libkiwix is a set of tools to manage zim files and provide some common functionnality.

36
format_code.sh Executable file
View File

@@ -0,0 +1,36 @@
#!/usr/bin/bash
files=(
"include/library.h"
"include/common/stringTools.h"
"include/common/pathTools.h"
"include/common/otherTools.h"
"include/common/regexTools.h"
"include/common/networkTools.h"
"include/manager.h"
"include/reader.h"
"include/kiwix.h"
"include/xapianSearcher.h"
"include/searcher.h"
"src/library.cpp"
"src/android/kiwix.cpp"
"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
"src/searcher.cpp"
"src/common/pathTools.cpp"
"src/common/regexTools.cpp"
"src/common/otherTools.cpp"
"src/common/networkTools.cpp"
"src/common/stringTools.cpp"
"src/xapianSearcher.cpp"
"src/manager.cpp"
"src/reader.cpp"
)
for i in "${files[@]}"
do
echo $i
clang-format -i -style=file $i
done

View File

@@ -1,160 +0,0 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_BOOK_H
#define KIWIX_BOOK_H
#include <string>
#include <vector>
#include <memory>
#include <mutex>
#include "common.h"
namespace pugi {
class xml_node;
}
namespace zim {
class Archive;
}
namespace kiwix
{
class OPDSDumper;
/**
* A class to store information about a book (a zim file)
*/
class Book
{
public: // types
class Illustration
{
friend class Book;
public:
uint16_t width = 48;
uint16_t height = 48;
std::string mimeType;
std::string url;
const std::string& getData() const;
private:
mutable std::string data;
mutable std::mutex mutex;
};
typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
public: // functions
Book();
~Book();
bool update(const Book& other);
void update(const zim::Archive& archive);
void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
std::string getHumanReadableIdFromPath() const;
bool readOnly() const { return m_readOnly; }
const std::string& getId() const { return m_id; }
const std::string& getPath() const { return m_path; }
bool isPathValid() const { return m_pathValid; }
const std::string& getTitle() const { return m_title; }
const std::string& getDescription() const { return m_description; }
const std::string& getLanguage() const { return m_language; }
const std::string& getCreator() const { return m_creator; }
const std::string& getPublisher() const { return m_publisher; }
const std::string& getDate() const { return m_date; }
const std::string& getUrl() const { return m_url; }
const std::string& getName() const { return m_name; }
std::string getCategory() const;
const std::string& getTags() const { return m_tags; }
std::string getTagStr(const std::string& tagName) const;
bool getTagBool(const std::string& tagName) const;
const std::string& getFlavour() const { return m_flavour; }
const std::string& getOrigId() const { return m_origId; }
const uint64_t& getArticleCount() const { return m_articleCount; }
const uint64_t& getMediaCount() const { return m_mediaCount; }
const uint64_t& getSize() const { return m_size; }
DEPRECATED const std::string& getFavicon() const;
DEPRECATED const std::string& getFaviconUrl() const;
DEPRECATED const std::string& getFaviconMimeType() const;
Illustrations getIllustrations() const;
std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
const std::string& getDownloadId() const { return m_downloadId; }
void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
void setId(const std::string& id) { m_id = id; }
void setPath(const std::string& path);
void setPathValid(bool valid) { m_pathValid = valid; }
void setTitle(const std::string& title) { m_title = title; }
void setDescription(const std::string& description) { m_description = description; }
void setLanguage(const std::string& language) { m_language = language; }
void setCreator(const std::string& creator) { m_creator = creator; }
void setPublisher(const std::string& publisher) { m_publisher = publisher; }
void setDate(const std::string& date) { m_date = date; }
void setUrl(const std::string& url) { m_url = url; }
void setName(const std::string& name) { m_name = name; }
void setFlavour(const std::string& flavour) { m_flavour = flavour; }
void setTags(const std::string& tags) { m_tags = tags; }
void setOrigId(const std::string& origId) { m_origId = origId; }
void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
void setSize(uint64_t size) { m_size = size; }
void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
private: // functions
std::string getCategoryFromTags() const;
const Illustration& getDefaultIllustration() const;
protected: // data
std::string m_id;
std::string m_downloadId;
std::string m_path;
bool m_pathValid = false;
std::string m_title;
std::string m_description;
std::string m_category;
std::string m_language;
std::string m_creator;
std::string m_publisher;
std::string m_date;
std::string m_url;
std::string m_name;
std::string m_flavour;
std::string m_tags;
std::string m_origId;
uint64_t m_articleCount = 0;
uint64_t m_mediaCount = 0;
bool m_readOnly = false;
uint64_t m_size = 0;
Illustrations m_illustrations;
// Used as the return value of getDefaultIllustration() when no default
// illustration is found in the book
static const Illustration missingDefaultIllustration;
};
}
#endif

View File

@@ -1,68 +0,0 @@
/*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_BOOKMARK_H
#define KIWIX_BOOKMARK_H
#include <string>
namespace pugi {
class xml_node;
}
namespace kiwix
{
/**
* A class to store information about a bookmark (an article in a book)
*/
class Bookmark
{
public:
Bookmark();
~Bookmark();
void updateFromXml(const pugi::xml_node& node);
const std::string& getBookId() const { return m_bookId; }
const std::string& getBookTitle() const { return m_bookTitle; }
const std::string& getUrl() const { return m_url; }
const std::string& getTitle() const { return m_title; }
const std::string& getLanguage() const { return m_language; }
const std::string& getDate() const { return m_date; }
void setBookId(const std::string& bookId) { m_bookId = bookId; }
void setBookTitle(const std::string& bookTitle) { m_bookTitle = bookTitle; }
void setUrl(const std::string& url) { m_url = url; }
void setTitle(const std::string& title) { m_title = title; }
void setLanguage(const std::string& language) { m_language = language; }
void setDate(const std::string& date) { m_date = date; }
protected:
std::string m_bookId;
std::string m_bookTitle;
std::string m_url;
std::string m_title;
std::string m_language;
std::string m_date;
};
}
#endif

View File

@@ -1,24 +0,0 @@
#ifndef _KIWIX_COMMON_H_
#define _KIWIX_COMMON_H_
#include <zim/zim.h>
#ifdef __GNUC__
#define DEPRECATED __attribute__((deprecated))
#elif defined(_MSC_VER)
#define DEPRECATED __declspec(deprecated)
#else
#praga message("WARNING: You need to implement DEPRECATED for this compiler")
#define DEPRECATED
#endif
namespace kiwix {
typedef zim::size_type size_type;
typedef zim::offset_type offset_type;
}
#endif //_KIWIX_COMMON_H_

4
include/common/base64.h Normal file
View File

@@ -0,0 +1,4 @@
#include <string>
std::string base64_encode(unsigned char const* , unsigned int len);
std::string base64_decode(std::string const& s);

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -17,22 +17,33 @@
* MA 02110-1301, USA.
*/
#ifndef KIWIX_PATHTOOLS_H
#define KIWIX_PATHTOOLS_H
#include <string>
#ifndef KIWIX_NETWORKTOOLS_H
#define KIWIX_NETWORKTOOLS_H
#ifdef _WIN32
std::string WideToUtf8(const std::wstring& wstr);
std::wstring Utf8ToWide(const std::string& str);
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <net/if.h>
#include <netdb.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#endif
unsigned int getFileSize(const std::string& path);
std::string getFileSizeAsString(const std::string& path);
bool makeDirectory(const std::string& path);
std::string makeTmpDirectory();
bool copyFile(const std::string& sourcePath, const std::string& destPath);
bool writeTextFile(const std::string& path, const std::string& content);
#include <iostream>
#include <map>
#include <string>
#include <vector>
namespace kiwix
{
std::map<std::string, std::string> getNetworkInterfaces();
std::string getBestPublicIp();
}
#endif

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
* Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -17,17 +17,18 @@
* MA 02110-1301, USA.
*/
#ifndef KIWIX_OPDS_CATALOG_H
#define KIWIX_OPDS_CATALOG_H
#ifndef KIWIX_OTHERTOOLS_H
#define KIWIX_OTHERTOOLS_H
#include "library.h"
#ifdef _WIN32
#include <windows.h>
#else
#include <unistd.h>
#endif
namespace kiwix
{
void sleep(unsigned int milliseconds);
}
std::string getSearchUrl(const Filter& f);
} // namespace kiwix
#endif // KIWIX_OPDS_CATALOG_H
#endif

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_PATHTOOLS_H
#define KIWIX_PATHTOOLS_H
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fstream>
#include <ios>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#ifdef _WIN32
#include <direct.h>
#endif
#include "stringTools.h"
using namespace std;
bool isRelativePath(const string& path);
string computeAbsolutePath(const string path, const string relativePath);
string computeRelativePath(const string path, const string absolutePath);
string removeLastPathElement(const string path,
const bool removePreSeparator = false,
const bool removePostSeparator = false);
string appendToDirectory(const string& directoryPath, const string& filename);
unsigned int getFileSize(const string& path);
string getFileSizeAsString(const string& path);
bool fileExists(const string& path);
bool makeDirectory(const string& path);
bool copyFile(const string& sourcePath, const string& destPath);
string getLastPathElement(const string& path);
string getExecutablePath();
string getCurrentDirectory();
bool writeTextFile(const string& path, const string& content);
#endif

View File

@@ -20,11 +20,17 @@
#ifndef KIWIX_REGEXTOOLS_H
#define KIWIX_REGEXTOOLS_H
#include <unicode/regex.h>
#include <unicode/ucnv.h>
#include <map>
#include <string>
bool matchRegex(const std::string& content, const std::string& regex);
std::string replaceRegex(const std::string& content,
const std::string& replacement,
const std::string& regex);
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& replacement);
#endif

View File

@@ -0,0 +1,67 @@
/*
* Copyright 2011-2012 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_STRINGTOOLS_H
#define KIWIX_STRINGTOOLS_H
#include <unicode/unistr.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "pathTools.h"
namespace kiwix
{
#ifndef __ANDROID__
std::string beautifyInteger(const unsigned int number);
std::string beautifyFileSize(const unsigned int number);
std::string urlEncode(const std::string& c);
void printStringInHexadecimal(const char* s);
void printStringInHexadecimal(UnicodeString s);
void stringReplacement(std::string& str,
const std::string& oldStr,
const std::string& newStr);
std::string encodeDiples(const std::string& str);
#endif
std::string removeAccents(const std::string& text);
void loadICUExternalTables();
std::string urlDecode(const std::string& c);
std::vector<std::string> split(const std::string&, const std::string&);
std::vector<std::string> split(const char*, const char*);
std::vector<std::string> split(const std::string&, const char*);
std::vector<std::string> split(const char*, const std::string&);
std::string ucAll(const std::string& word);
std::string lcAll(const std::string& word);
std::string ucFirst(const std::string& word);
std::string lcFirst(const std::string& word);
std::string toTitle(const std::string& word);
std::string normalize(const std::string& word);
}
#endif

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013 Renaud Gaudin <reg@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef _CTPP2_VM_STRING_LOADER_HPP__
#define _CTPP2_VM_STRING_LOADER_HPP__ 1
#include <ctpp2/CTPP2VMLoader.hpp>
#include <ctpp2/CTPP2Util.hpp>
#include <ctpp2/CTPP2Exception.hpp>
#include <ctpp2/CTPP2VMExecutable.hpp>
#include <ctpp2/CTPP2VMInstruction.hpp>
#include <ctpp2/CTPP2VMMemoryCore.hpp>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <string>
/**
@file VMStringLoader.hpp
@brief Load program core from file
*/
namespace CTPP // C++ Template Engine
{
// FWD
struct VMExecutable;
/**
@class VMStringLoader CTPP2VMStringLoader.hpp <CTPP2VMStringLoader.hpp>
@brief Load program core from file
*/
class CTPP2DECL VMStringLoader:
public VMLoader
{
public:
/**
*/
VMStringLoader(CCHAR_P rawContent, size_t rawContentSize);
/**
@brief Get ready-to-run program
*/
const VMMemoryCore * GetCore() const;
/**
@brief A destructor
*/
~VMStringLoader() throw();
private:
/** Program core */
VMExecutable * oCore;
/** Ready-to-run program */
VMMemoryCore * pVMMemoryCore;
};
} // namespace CTPP
#endif // _CTPP2_VM_STRING_LOADER_HPP__
// End.

View File

@@ -1,107 +0,0 @@
/*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_DOWNLOADER_H
#define KIWIX_DOWNLOADER_H
#include <string>
#include <vector>
#include <map>
#include <memory>
#include <stdexcept>
namespace kiwix
{
class Aria2;
struct DownloadedFile {
DownloadedFile()
: success(false) {}
bool success;
std::string path;
};
class AriaError : public std::runtime_error {
public:
AriaError(const std::string& message) : std::runtime_error(message) {}
};
class Download {
public:
typedef enum { K_ACTIVE, K_WAITING, K_PAUSED, K_ERROR, K_COMPLETE, K_REMOVED, K_UNKNOWN } StatusResult;
Download() :
m_status(K_UNKNOWN) {}
Download(std::shared_ptr<Aria2> p_aria, std::string did)
: mp_aria(p_aria),
m_status(K_UNKNOWN),
m_did(did) {};
void updateStatus(bool follow=false);
void pauseDownload();
void resumeDownload();
void cancelDownload();
StatusResult getStatus() { return m_status; }
std::string getDid() { return m_did; }
std::string getFollowedBy() { return m_followedBy; }
uint64_t getTotalLength() { return m_totalLength; }
uint64_t getCompletedLength() { return m_completedLength; }
uint64_t getDownloadSpeed() { return m_downloadSpeed; }
uint64_t getVerifiedLength() { return m_verifiedLength; }
std::string getPath() { return m_path; }
std::vector<std::string>& getUris() { return m_uris; }
protected:
std::shared_ptr<Aria2> mp_aria;
StatusResult m_status;
std::string m_did = "";
std::string m_followedBy = "";
uint64_t m_totalLength;
uint64_t m_completedLength;
uint64_t m_downloadSpeed;
uint64_t m_verifiedLength;
std::vector<std::string> m_uris;
std::string m_path;
};
/**
* A tool to download things.
*
*/
class Downloader
{
public:
Downloader();
virtual ~Downloader();
void close();
Download* startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
Download* getDownload(const std::string& did);
size_t getNbDownload() { return m_knownDownloads.size(); }
std::vector<std::string> getDownloadIds();
private:
std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;
std::shared_ptr<Aria2> mp_aria;
};
}
#endif

View File

@@ -22,4 +22,4 @@
#include "library.h"
#endif
#endif

View File

@@ -1,30 +0,0 @@
#ifndef KIWIXLIB_KIWIX_SERVE_H_
#define KIWIXLIB_KIWIX_SERVE_H_
#include <memory>
#include <string>
class Subprocess;
namespace kiwix {
class KiwixServe
{
public:
KiwixServe(const std::string& libraryPath, int port = 8181);
~KiwixServe();
void run();
void shutDown();
bool isRunning();
int getPort() { return m_port; }
int setPort(int port);
private:
std::unique_ptr<Subprocess> mp_kiwixServe;
int m_port;
std::string m_libraryPath;
};
}; //end namespace kiwix
#endif // KIWIXLIB_KIWIX_SERVE_H_

View File

@@ -20,359 +20,87 @@
#ifndef KIWIX_LIBRARY_H
#define KIWIX_LIBRARY_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stack>
#include <string>
#include <vector>
#include <map>
#include <memory>
#include <mutex>
#include <zim/archive.h>
#include <zim/search.h>
#include "book.h"
#include "bookmark.h"
#include "common.h"
#include "common/regexTools.h"
#include "common/stringTools.h"
#define KIWIX_LIBRARY_VERSION "20110515"
using namespace std;
namespace kiwix
{
enum supportedIndexType { UNKNOWN, XAPIAN };
class OPDSDumper;
class Library;
enum supportedListSortBy { UNSORTED, TITLE, SIZE, DATE, CREATOR, PUBLISHER };
enum supportedListMode {
ALL = 0,
LOCAL = 1,
REMOTE = 1 << 1,
NOLOCAL = 1 << 2,
NOREMOTE = 1 << 3,
VALID = 1 << 4,
NOVALID = 1 << 5
};
class Filter {
public: // types
using Tags = std::vector<std::string>;
private: // data
uint64_t activeFilters;
Tags _acceptTags;
Tags _rejectTags;
std::string _category;
std::string _lang;
std::string _publisher;
std::string _creator;
size_t _maxSize;
std::string _query;
bool _queryIsPartial;
std::string _name;
public: // functions
Filter();
~Filter() = default;
/**
* Set the filter to check local.
*
* A local book is a book with a path.
* If accept is true, only local book are accepted.
* If accept is false, only non local book are accepted.
*/
Filter& local(bool accept);
/**
* Set the filter to check remote.
*
* A remote book is a book with a url.
* If accept is true, only remote book are accepted.
* If accept is false, only non remote book are accepted.
*/
Filter& remote(bool accept);
/**
* Set the filter to check validity.
*
* A valid book is a book with a path pointing to a existing zim file.
* If accept is true, only valid book are accepted.
* If accept is false, only non valid book are accepted.
*/
Filter& valid(bool accept);
/**
* Set the filter to only accept book with corresponding tag.
*/
Filter& acceptTags(const Tags& tags);
Filter& rejectTags(const Tags& tags);
Filter& category(std::string category);
/**
* Set the filter to only accept books in the specified language.
*
* Multiple languages can be specified as a comma-separated list (in
* which case a book in any of those languages will match).
*/
Filter& lang(std::string lang);
Filter& publisher(std::string publisher);
Filter& creator(std::string creator);
Filter& maxSize(size_t size);
Filter& query(std::string query, bool partial=true);
Filter& name(std::string name);
bool hasQuery() const;
const std::string& getQuery() const { return _query; }
bool queryIsPartial() const { return _queryIsPartial; }
bool hasName() const;
const std::string& getName() const { return _name; }
bool hasCategory() const;
const std::string& getCategory() const { return _category; }
bool hasLang() const;
const std::string& getLang() const { return _lang; }
bool hasPublisher() const;
const std::string& getPublisher() const { return _publisher; }
bool hasCreator() const;
const std::string& getCreator() const { return _creator; }
const Tags& getAcceptTags() const { return _acceptTags; }
const Tags& getRejectTags() const { return _rejectTags; }
private: // functions
friend class Library;
bool accept(const Book& book) const;
};
class ZimSearcher : public zim::Searcher
class Book
{
public:
explicit ZimSearcher(zim::Searcher&& searcher)
: zim::Searcher(searcher)
{}
public:
Book();
~Book();
std::unique_lock<std::mutex> getLock() {
return std::unique_lock<std::mutex>(m_mutex);
}
virtual ~ZimSearcher() = default;
private:
std::mutex m_mutex;
static bool sortByLastOpen(const Book& a, const Book& b);
static bool sortByTitle(const Book& a, const Book& b);
static bool sortBySize(const Book& a, const Book& b);
static bool sortByDate(const Book& a, const Book& b);
static bool sortByCreator(const Book& a, const Book& b);
static bool sortByPublisher(const Book& a, const Book& b);
static bool sortByLanguage(const Book& a, const Book& b);
string getHumanReadableIdFromPath();
string id;
string path;
string pathAbsolute;
string last;
string indexPath;
string indexPathAbsolute;
supportedIndexType indexType;
string title;
string description;
string language;
string creator;
string publisher;
string date;
string url;
string name;
string tags;
string origId;
string articleCount;
string mediaCount;
bool readOnly;
string size;
string favicon;
string faviconMimeType;
};
/**
* A Library store several books.
*/
class Library
{
// all data fields must be added in LibraryBase
mutable std::mutex m_mutex;
public:
typedef uint64_t Revision;
typedef std::vector<std::string> BookIdCollection;
typedef std::map<std::string, int> AttributeCounts;
typedef std::set<std::string> BookIdSet;
public:
Library();
~Library();
/**
* Library is not a copiable object. However it can be moved.
*/
Library(const Library& ) = delete;
Library(Library&& );
void operator=(const Library& ) = delete;
Library& operator=(Library&& );
/**
* Add a book to the library.
*
* If a book already exist in the library with the same id, update
* the existing book instead of adding a new one.
*
* @param book The book to add.
* @return True if the book has been added.
* False if a book has been updated.
*/
string version;
bool addBook(const Book& book);
bool removeBookByIndex(const unsigned int bookIndex);
vector<kiwix::Book> books;
/**
* A self-explanatory alias for addBook()
/*
* 'current' is the variable storing the current content/book id
* in the library. This is used to be able to load per default a
* content. As Kiwix may work with many library XML files, you may
* have "current" defined many time with different values. The
* last XML file read has the priority, Although we do not have an
* library object for each file, we want to be able to fallback to
* an 'old' current book if the one which should be load
* failed. That is the reason why we need a stack here
*/
bool addOrUpdateBook(const Book& book) { return addBook(book); }
/**
* Add a bookmark to the library.
*
* @param bookmark the book to add.
*/
void addBookmark(const Bookmark& bookmark);
/**
* Remove a bookmarkk
*
* @param zimId The zimId of the bookmark.
* @param url The url of the bookmark.
* @return True if the bookmark has been removed.
*/
bool removeBookmark(const std::string& zimId, const std::string& url);
// XXX: This is a non-thread-safe operation
const Book& getBookById(const std::string& id) const;
// XXX: This is a non-thread-safe operation
const Book& getBookByPath(const std::string& path) const;
Book getBookByIdThreadSafe(const std::string& id) const;
std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
std::shared_ptr<ZimSearcher> getSearcherById(const std::string& id) {
return getSearcherByIds(BookIdSet{id});
}
std::shared_ptr<ZimSearcher> getSearcherByIds(const BookIdSet& ids);
/**
* Remove a book from the library.
*
* @param id the id of the book to remove.
* @return True if the book were in the lirbrary and has been removed.
*/
bool removeBookById(const std::string& id);
/**
* Write the library to a file.
*
* @param path the path of the file to write to.
* @return True if the library has been correctly saved.
*/
bool writeToFile(const std::string& path) const;
/**
* Write the library bookmarks to a file.
*
* @param path the path of the file to write to.
* @return True if the library has been correctly saved.
*/
bool writeBookmarksToFile(const std::string& path) const;
/**
* Get the number of book in the library.
*
* @param localBooks If we must count local books (books with a path).
* @param remoteBooks If we must count remote books (books with an url)
* @return The number of books.
*/
unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const;
/**
* Get all languagues of the books in the library.
*
* @return A list of languages.
*/
std::vector<std::string> getBooksLanguages() const;
/**
* Get all languagues of the books in the library with counts.
*
* @return A list of languages with the count of books in each language.
*/
AttributeCounts getBooksLanguagesWithCounts() const;
/**
* Get all categories of the books in the library.
*
* @return A list of categories.
*/
std::vector<std::string> getBooksCategories() const;
/**
* Get all book creators of the books in the library.
*
* @return A list of book creators.
*/
std::vector<std::string> getBooksCreators() const;
/**
* Get all book publishers of the books in the library.
*
* @return A list of book publishers.
*/
std::vector<std::string> getBooksPublishers() const;
/**
* Get all bookmarks.
*
* @return A list of bookmarks
*/
const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true) const;
/**
* Get all book ids of the books in the library.
*
* @return A list of book ids.
*/
BookIdCollection getBooksIds() const;
/**
* Filter the library and return the id of the keep elements.
*
* @param filter The filter to use.
* @return The list of bookIds corresponding to the filter.
*/
BookIdCollection filter(const Filter& filter) const;
/**
* Sort (in place) bookIds using the given comparator.
*
* @param bookIds the list of book Ids to sort
* @param comparator how to sort the books
* @return The sorted list of books
*/
void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
/**
* Return the current revision of the library.
*
* The revision of the library is updated (incremented by one) by
* the addBook() and removeBookById() operations.
*
* @return Current revision of the library.
*/
Revision getRevision() const;
/**
* Remove books that have not been updated since the specified revision.
*
* @param rev the library revision to use
* @return Count of books that were removed by this operation.
*/
uint32_t removeBooksNotUpdatedSince(Revision rev);
friend class OPDSDumper;
friend class libXMLDumper;
private: // types
typedef const std::string& (Book::*BookStrPropMemFn)() const;
struct Impl;
private: // functions
AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
BookIdCollection filterViaBookDB(const Filter& filter) const;
void updateBookDB(const Book& book);
void dropCache(const std::string& bookId);
private: //data
std::unique_ptr<Impl> mp_impl;
stack<string> current;
};
}
#endif

View File

@@ -1,83 +0,0 @@
/*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_LIBXML_DUMPER_H
#define KIWIX_LIBXML_DUMPER_H
#include <string>
#include <vector>
#include <pugixml.hpp>
#include "library.h"
namespace kiwix
{
/**
* A tool to dump a `Library` into a basic library.xml
*
*/
class LibXMLDumper
{
public:
LibXMLDumper() = default;
LibXMLDumper(const Library* library);
~LibXMLDumper();
/**
* Dump the library.xml
*
* @param id The id of the library.
* @return The library.xml content.
*/
std::string dumpLibXMLContent(const std::vector<std::string>& bookIds);
/**
* Dump the bookmark of the library.
*
* @return The bookmark.xml content.
*/
std::string dumpLibXMLBookmark();
/**
* Set the base directory used.
*
* @param baseDir the base directory to use.
*/
void setBaseDir(const std::string& baseDir) { this->baseDir = baseDir; }
/**
* Set the library to dump.
*
* @param library The library to dump.
*/
void setLibrary(const Library* library) { this->library = library; }
protected:
const kiwix::Library* library;
std::string baseDir;
private:
void handleBook(Book book, pugi::xml_node root_node);
void handleBookmark(Bookmark bookmark, pugi::xml_node root_node);
};
}
#endif // KIWIX_OPDS_DUMPER_H

View File

@@ -20,159 +20,88 @@
#ifndef KIWIX_MANAGER_H
#define KIWIX_MANAGER_H
#include "book.h"
#include "library.h"
#include <time.h>
#include <sstream>
#include <string>
#include <vector>
#include <memory>
namespace pugi {
class xml_document;
}
#include <pugixml.hpp>
#include "common/base64.h"
#include "common/pathTools.h"
#include "common/regexTools.h"
#include "library.h"
#include "reader.h"
using namespace std;
namespace kiwix
{
enum supportedListMode { LASTOPEN, REMOTE, LOCAL };
enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
class LibraryManipulator
{
public: // functions
explicit LibraryManipulator(Library* library);
virtual ~LibraryManipulator();
Library& getLibrary() const { return library; }
bool addBookToLibrary(const Book& book);
void addBookmarkToLibrary(const Bookmark& bookmark);
uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
protected: // overrides
virtual void bookWasAddedToLibrary(const Book& book);
virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
virtual void booksWereRemovedFromLibrary();
private: // data
kiwix::Library& library;
};
/**
* A tool to manage a `Library`.
*/
class Manager
{
public: // types
typedef std::vector<std::string> Paths;
public:
Manager();
~Manager();
public: // functions
explicit Manager(LibraryManipulator* manipulator);
explicit Manager(Library* library);
/**
* Read a `library.xml` and add book in the file to the library.
*
* @param path The (utf8) path to the `library.xml`.
* @param readOnly Set if the libray path could be overwritten latter with
* updated content.
* @param trustLibrary use book metadata coming from XML.
* @return True if file has been properly parsed.
*/
bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
/**
* Sync the contents of the library with one or more `library.xml` files.
*
* The metadata of the library files is trusted unconditionally.
* Any books not present in the input library.xml files are removed
* from the library.
*
* @param paths The (utf8) paths to the `library.xml` files.
*/
void reload(const Paths& paths);
/**
* Load a library content store in the string.
*
* @param xml The content corresponding of the library xml
* @param readOnly Set if the libray path could be overwritten latter with
* updated content.
* @param libraryPath The library path (used to resolve relative path)
* @return True if the content has been properly parsed.
*/
bool readXml(const std::string& xml,
bool readFile(const string path, const bool readOnly = true);
bool readFile(const string nativePath,
const string UTF8Path,
const bool readOnly = true);
bool readXml(const string xml,
const bool readOnly = true,
const std::string& libraryPath = "",
bool trustLibrary = true);
/**
* Load a library content stored in a OPDS stream.
*
* @param content The content of the OPDS stream.
* @param readOnly Set if the library path could be overwritten later with
* updated content.
* @param libraryPath The library path (used to resolve relative path)
* @return True if the content has been properly parsed.
*/
bool readOpds(const std::string& content, const std::string& urlHost);
/**
* Load a bookmark file.
*
* @param path The path of the file to read.
* @return True if the content has been properly parsed.
*/
bool readBookmarkFile(const std::string& path);
/**
* Add a book to the library.
*
* @param pathToOpen The path to the zim file to add.
* @param pathToSave The path to store in the library in place of pathToOpen.
* @param url The url of the book to store in the library.
* @param checMetaData Tell if we check metadata before adding book to the
* library.
* @return The id of the book if the book has been added to the library.
* Else, an empty string.
*/
std::string addBookFromPathAndGetId(const std::string& pathToOpen,
const std::string& pathToSave = "",
const std::string& url = "",
const string libraryPath = "");
bool writeFile(const string path);
bool removeBookByIndex(const unsigned int bookIndex);
bool removeBookById(const string id);
bool setCurrentBookId(const string id);
string getCurrentBookId();
bool setBookIndex(const string id,
const string path,
const supportedIndexType type);
bool setBookIndex(const string id, const string path);
bool setBookPath(const string id, const string path);
string addBookFromPathAndGetId(const string pathToOpen,
const string pathToSave = "",
const string url = "",
const bool checkMetaData = false);
/**
* Add a book to the library.
*
* @param pathToOpen The path to the zim file to add.
* @param pathToSave The path to store in the library in place of pathToOpen.
* @param url The url of the book to store in the library.
* @param checMetaData Tell if we check metadata before adding book to the
* library.
* @return True if the book has been added to the library.
*/
bool addBookFromPath(const std::string& pathToOpen,
const std::string& pathToSave = "",
const std::string& url = "",
bool addBookFromPath(const string pathToOpen,
const string pathToSave = "",
const string url = "",
const bool checkMetaData = false);
Library cloneLibrary();
bool getBookById(const string id, Book& book);
bool getCurrentBook(Book& book);
unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
bool updateBookLastOpenDateById(const string id);
void removeBookPaths();
bool listBooks(const supportedListMode mode,
const supportedListSortBy sortBy,
const unsigned int maxSize,
const string language,
const string creator,
const string publisher,
const string search);
vector<string> getBooksLanguages();
vector<string> getBooksCreators();
vector<string> getBooksPublishers();
vector<string> getBooksIds();
std::string writableLibraryPath;
string writableLibraryPath;
bool m_hasSearchResult = false;
uint64_t m_totalBooks = 0;
uint64_t m_startIndex = 0;
uint64_t m_itemsPerPage = 0;
vector<std::string> bookIdList;
protected:
std::shared_ptr<kiwix::LibraryManipulator> manipulator;
kiwix::Library library;
bool readBookFromPath(const std::string& path, Book* book);
bool readBookFromPath(const string path, Book* book = NULL);
bool parseXmlDom(const pugi::xml_document& doc,
bool readOnly,
const std::string& libraryPath,
bool trustLibrary);
bool parseOpdsDom(const pugi::xml_document& doc,
const std::string& urlHost);
const bool readOnly,
const string libraryPath);
private:
void checkAndCleanBookPaths(Book& book, const string& libraryPath);
};
}

View File

@@ -1,18 +1,30 @@
headers = [
'book.h',
'bookmark.h',
'common.h',
'library.h',
'manager.h',
'libxml_dumper.h',
'opds_dumper.h',
'downloader.h',
'search_renderer.h',
'server.h',
'kiwixserve.h',
'name_mapper.h',
'tools.h',
'version.h'
'reader.h',
'searcher.h'
]
if xapian_dep.found()
headers += ['xapianSearcher.h']
endif
install_headers(headers, subdir:'kiwix')
install_headers(
'common/base64.h',
'common/networkTools.h',
'common/otherTools.h',
'common/pathTools.h',
'common/regexTools.h',
'common/stringTools.h',
subdir:'kiwix/common'
)
if has_ctpp2_dep
install_headers(
'ctpp2/CTPP2VMStringLoader.hpp',
subdir:'kiwix/ctpp2'
)
endif

View File

@@ -1,81 +0,0 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_NAMEMAPPER_H
#define KIWIX_NAMEMAPPER_H
#include <string>
#include <map>
#include <memory>
#include <mutex>
namespace kiwix
{
class Library;
class NameMapper {
public:
virtual ~NameMapper() = default;
virtual std::string getNameForId(const std::string& id) const = 0;
virtual std::string getIdForName(const std::string& name) const = 0;
};
class IdNameMapper : public NameMapper {
public:
virtual std::string getNameForId(const std::string& id) const { return id; };
virtual std::string getIdForName(const std::string& name) const { return name; };
};
class HumanReadableNameMapper : public NameMapper {
private:
std::map<std::string, std::string> m_idToName;
std::map<std::string, std::string> m_nameToId;
public:
HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
virtual ~HumanReadableNameMapper() = default;
virtual std::string getNameForId(const std::string& id) const;
virtual std::string getIdForName(const std::string& name) const;
};
class UpdatableNameMapper : public NameMapper {
typedef std::shared_ptr<NameMapper> NameMapperHandle;
public:
UpdatableNameMapper(Library& library, bool withAlias);
virtual std::string getNameForId(const std::string& id) const;
virtual std::string getIdForName(const std::string& name) const;
void update();
private:
NameMapperHandle currentNameMapper() const;
private:
mutable std::mutex mutex;
Library& library;
NameMapperHandle nameMapper;
const bool withAlias;
};
}
#endif

View File

@@ -1,123 +0,0 @@
/*
* Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_OPDS_DUMPER_H
#define KIWIX_OPDS_DUMPER_H
#include <time.h>
#include <sstream>
#include <string>
#include <pugixml.hpp>
#include "library.h"
#include "name_mapper.h"
using namespace std;
namespace kiwix
{
/**
* A tool to dump a `Library` into a opds stream.
*
*/
class OPDSDumper
{
public:
OPDSDumper() = default;
OPDSDumper(Library* library, NameMapper* NameMapper);
~OPDSDumper();
/**
* Dump the OPDS feed.
*
* @param bookIds the ids of the books to include in the feed
* @param query the query used to obtain the list of book ids
* @return The OPDS feed.
*/
std::string dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const;
/**
* Dump the OPDS feed.
*
* @param bookIds the ids of the books to include in the feed
* @param query the query used to obtain the list of book ids
* @param partial whether the feed should include partial or complete entries
* @return The OPDS feed.
*/
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
/**
* Dump the OPDS complete entry document.
*
* @param bookId the id of the book
* @return The OPDS complete entry document.
*/
std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
/**
* Dump the categories OPDS feed.
*
* @return The OPDS feed.
*/
std::string categoriesOPDSFeed() const;
/**
* Dump the languages OPDS feed.
*
* @return The OPDS feed.
*/
std::string languagesOPDSFeed() const;
/**
* Set the id of the library.
*
* @param id the id to use.
*/
void setLibraryId(const std::string& id) { this->libraryId = id;}
/**
* Set the root location used when generating url.
*
* @param rootLocation the root location to use.
*/
void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; }
/**
* Set some informations about the search results.
*
* @param totalResult the total number of results of the search.
* @param startIndex the start index of the result.
* @param count the number of result of the current set (or page).
*/
void setOpenSearchInfo(int totalResult, int startIndex, int count);
protected:
kiwix::Library* library;
kiwix::NameMapper* nameMapper;
std::string libraryId;
std::string rootLocation;
int m_totalResults;
int m_startIndex;
int m_count;
};
}
#endif // KIWIX_OPDS_DUMPER_H

129
include/reader.h Normal file
View File

@@ -0,0 +1,129 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_READER_H
#define KIWIX_READER_H
#include <stdio.h>
#include <zim/article.h>
#include <zim/file.h>
#include <zim/fileiterator.h>
#include <zim/zim.h>
#include <exception>
#include <map>
#include <sstream>
#include <string>
#include "common/pathTools.h"
#include "common/stringTools.h"
using namespace std;
namespace kiwix
{
class Reader
{
public:
Reader(const string zimFilePath);
~Reader();
void reset();
unsigned int getArticleCount() const;
unsigned int getMediaCount() const;
unsigned int getGlobalCount() const;
string getZimFilePath() const;
string getId() const;
string getRandomPageUrl() const;
string getFirstPageUrl() const;
string getMainPageUrl() const;
bool getMetatag(const string& url, string& content) const;
string getTitle() const;
string getDescription() const;
string getLanguage() const;
string getName() const;
string getTags() const;
string getDate() const;
string getCreator() const;
string getPublisher() const;
string getOrigId() const;
bool getFavicon(string& content, string& mimeType) const;
bool getPageUrlFromTitle(const string& title, string& url) const;
bool getMimeTypeByUrl(const string& url, string& mimeType) const;
bool getContentByUrl(const string& url,
string& content,
string& title,
unsigned int& contentLength,
string& contentType) const;
bool getContentByEncodedUrl(const string& url,
string& content,
string& title,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const;
bool getContentByEncodedUrl(const string& url,
string& content,
string& title,
unsigned int& contentLength,
string& contentType) const;
bool getContentByDecodedUrl(const string& url,
string& content,
string& title,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const;
bool getContentByDecodedUrl(const string& url,
string& content,
string& title,
unsigned int& contentLength,
string& contentType) const;
bool searchSuggestions(const string& prefix,
unsigned int suggestionsCount,
const bool reset = true);
bool searchSuggestionsSmart(const string& prefix,
unsigned int suggestionsCount);
bool urlExists(const string& url) const;
bool hasFulltextIndex() const;
std::vector<std::string> getTitleVariants(const std::string& title) const;
bool getNextSuggestion(string& title);
bool getNextSuggestion(string& title, string& url);
bool canCheckIntegrity() const;
bool isCorrupted() const;
bool parseUrl(const string& url, char* ns, string& title) const;
unsigned int getFileSize() const;
zim::File* getZimFileHandler() const;
bool getArticleObjectByDecodedUrl(const string& url,
zim::Article& article) const;
protected:
zim::File* zimFileHandler;
zim::size_type firstArticleOffset;
zim::size_type lastArticleOffset;
zim::size_type currentArticleOffset;
zim::size_type nsACount;
zim::size_type nsICount;
std::string zimFilePath;
std::vector<std::vector<std::string>> suggestions;
std::vector<std::vector<std::string>>::iterator suggestionsOffset;
private:
std::map<const std::string, unsigned int> parseCounterMetadata() const;
};
}
#endif

View File

@@ -1,123 +0,0 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SEARCH_RENDERER_H
#define KIWIX_SEARCH_RENDERER_H
#include <string>
#include <zim/search.h>
#include "library.h"
namespace kiwix
{
class NameMapper;
/**
* The SearcherRenderer class is used to render a search result to a html page.
*/
class SearchRenderer
{
public:
/**
* Construct a SearchRenderer from a SearchResultSet.
*
* The constructed version of the SearchRenderer will not introduce
* the book name for each result. It is better to use the other constructor
* with a Library pointer to have a better html page.
*
* @param srs The `SearchResultSet` to render.
* @param mapper The `NameMapper` to use to do the rendering.
* @param start The start offset used for the srs.
* @param estimatedResultCount The estimatedResultCount of the whole search
*/
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
unsigned int start, unsigned int estimatedResultCount);
/**
* Construct a SearchRenderer from a SearchResultSet.
*
* @param srs The `SearchResultSet` to render.
* @param mapper The `NameMapper` to use to do the rendering.
* @param library The `Library` to use to look up book details for search results.
* @param start The start offset used for the srs.
* @param estimatedResultCount The estimatedResultCount of the whole search
*/
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
unsigned int start, unsigned int estimatedResultCount);
~SearchRenderer();
/**
* Set the search pattern used to do the search
*/
void setSearchPattern(const std::string& pattern);
/**
* Set the querystring used to select books
*/
void setSearchBookQuery(const std::string& bookQuery);
/**
* Set protocol prefix.
*/
void setProtocolPrefix(const std::string& prefix);
/**
* Set search protocol prefix.
*/
void setSearchProtocolPrefix(const std::string& prefix);
/**
* set result count per page
*/
void setPageLength(unsigned int pageLength){
this->pageLength = pageLength;
}
std::string renderTemplate(const std::string& tmpl_str);
/**
* Generate the html page with the resutls of the search.
*/
std::string getHtml();
/**
* Generate the xml page with the resutls of the search.
*/
std::string getXml();
protected:
std::string beautifyInteger(const unsigned int number);
zim::SearchResultSet m_srs;
NameMapper* mp_nameMapper;
Library* mp_library;
std::string searchBookQuery;
std::string searchPattern;
std::string protocolPrefix;
std::string searchProtocolPrefix;
unsigned int pageLength;
unsigned int estimatedResultCount;
unsigned int resultStart;
};
}
#endif

104
include/searcher.h Normal file
View File

@@ -0,0 +1,104 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SEARCHER_H
#define KIWIX_SEARCHER_H
#include <stdio.h>
#include <stdlib.h>
#include <unicode/putil.h>
#include <algorithm>
#include <cctype>
#include <locale>
#include <string>
#include <vector>
#include <vector>
#include "common/pathTools.h"
#include "common/stringTools.h"
#include "kiwix_config.h"
using namespace std;
namespace kiwix
{
class Reader;
class Result
{
public:
virtual ~Result(){};
virtual std::string get_url() = 0;
virtual std::string get_title() = 0;
virtual int get_score() = 0;
virtual std::string get_snippet() = 0;
virtual std::string get_content() = 0;
virtual int get_wordCount() = 0;
virtual int get_size() = 0;
virtual int get_readerIndex() = 0;
};
struct SearcherInternal;
class Searcher
{
public:
Searcher();
Searcher(const string& xapianDirectoryPath,
Reader* reader,
const string& humanReadableName);
~Searcher();
void add_reader(Reader* reader, const std::string& humanReaderName);
void search(std::string& search,
unsigned int resultStart,
unsigned int resultEnd,
const bool verbose = false);
void suggestions(std::string& search, const bool verbose = false);
Result* getNextResult();
void restart_search();
unsigned int getEstimatedResultCount();
bool setProtocolPrefix(const std::string prefix);
bool setSearchProtocolPrefix(const std::string prefix);
void reset();
#ifdef ENABLE_CTPP2
string getHtml();
#endif
protected:
std::string beautifyInteger(const unsigned int number);
void closeIndex();
void searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int resultEnd,
const bool verbose = false);
std::vector<Reader*> readers;
std::vector<std::string> humanReaderNames;
SearcherInternal* internal;
std::string searchPattern;
std::string protocolPrefix;
std::string searchProtocolPrefix;
unsigned int resultCountPerPage;
unsigned int estimatedResultCount;
unsigned int resultStart;
unsigned int resultEnd;
std::string contentHumanReadableId;
};
}
#endif

View File

@@ -1,86 +0,0 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SERVER_H
#define KIWIX_SERVER_H
#include <string>
#include <memory>
namespace kiwix
{
class Library;
class NameMapper;
class InternalServer;
class Server {
public:
/**
* The default constructor.
*
* @param library The library to serve.
*/
Server(Library* library, NameMapper* nameMapper=nullptr);
virtual ~Server();
/**
* Serve the content.
*/
bool start();
/**
* Stop the daemon.
*/
void stop();
void setRoot(const std::string& root);
void setAddress(const std::string& addr) { m_addr = addr; }
void setPort(int port) { m_port = port; }
void setNbThreads(int threads) { m_nbThreads = threads; }
void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
void setVerbose(bool verbose) { m_verbose = verbose; }
void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
void setTaskbar(bool withTaskbar, bool withLibraryButton)
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
void setBlockExternalLinks(bool blockExternalLinks)
{ m_blockExternalLinks = blockExternalLinks; }
int getPort();
std::string getAddress();
protected:
Library* mp_library;
NameMapper* mp_nameMapper;
std::string m_root = "";
std::string m_addr = "";
std::string m_indexTemplateString = "";
int m_port = 80;
int m_nbThreads = 1;
unsigned int m_multizimSearchLimit = 0;
bool m_verbose = false;
bool m_withTaskbar = true;
bool m_withLibraryButton = true;
bool m_blockExternalLinks = false;
int m_ipConnectionLimit = 0;
std::unique_ptr<InternalServer> mp_server;
};
}
#endif

View File

@@ -1,220 +0,0 @@
/*
* Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_TOOLS_H
#define KIWIX_TOOLS_H
#include <string>
#include <vector>
#include <map>
namespace kiwix {
/**
* Return the current directory.
*
* @return the current directory (utf8 encoded)
*/
std::string getCurrentDirectory();
/**
* Return the data directory.
*
* The data directory is a directory where to put data (zim files, ...)
* It depends of the platform and it may be changed by user using environment variable.
*
* The resolution order is :
* - `KIWIX_DATA_DIR` env variable (if set).
* - On Windows :
* . `$APPDATA/kiwix` if $APPDATA is set
* . `$USERPROFILE/kiwix` if $USERPROFILE is set
* - Else :
* . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
* . `$HOME/.local/share/kiwx` if $HOWE is set
* - current directory
*
* @return the path of the data directory (utf8 encoded)
*/
std::string getDataDirectory();
/** Return the path of the executable
*
* Some application may be packaged in auto extractible archive (Appimage) and the
* real executable is different of the path of the archive.
* If `realPathOnly` is true, return the path of the real executable instead of the
* archive launched by the user.
*
* @param realPathOnly If we must return the real path of the executable.
* @return the path of the executable (utf8 encoded)
*/
std::string getExecutablePath(bool realPathOnly = false);
/** Tell if the path is a relative path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path A utf8 encoded path.
* @return true if the path is relative.
*/
bool isRelativePath(const std::string& path);
/** Append a path to another one.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param basePath the base path.
* @param relativePath a path to add to the base path, must be a relative path.
* @return The concatenation of the paths, using the right separator.
*/
std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
/** Remove the last element of a path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path a path.
* @return The parent directory (or empty string if none).
*/
std::string removeLastPathElement(const std::string& path);
/** Get the last element of a path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path a path.
* @return The base name of the path or empty string if none (ending with a separator).
*/
std::string getLastPathElement(const std::string& path);
/** Compute the absolute path of a relative path based on another one
*
* Equivalent to appendToDirectory followed by a normalization of the path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path the base path (if empty, current directory is taken).
* @param relativePath the relative path.
* @return a absolute path.
*/
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
/** Compute the relative path of a path relative to another one
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path the base path.
* @param absolutePath the absolute path to find the relative path for.
* @return a relative path (pointing to absolutePath, relative to path).
*/
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
/** Sleep the current thread.
*
* This function is provided as a small helper. It is probably better to use native tools.
*
* @param milliseconds The number of milliseconds to wait for.
*/
void sleep(unsigned int milliseconds);
/** Split a string
*
* This function is provided as a small helper. It is probably better to use native tools.
*
* Assuming text = "foo:;bar;baz,oups;"
*
* split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
* split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
* split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
* split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
*
* @param str The string to split.
* @param delims A string of potential delimiters.
* Each charater in the string can be a individual delimiters.
* @param dropEmpty true if empty part must be dropped from the result.
* @param keepDelim true if delimiter must be included from the result.
* @return a list of part (potentially containing delimiters)
*/
std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
/** Convert language code from iso2 code to iso3
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate locales.
*
* @param a2code a iso2 code string.
* @return the corresponding iso3 code.
* @throw std::out_of_range if iso2 code is not known.
*/
std::string converta2toa3(const std::string& a2code);
/** Extracts content from given file.
*
* This function provides content of a file provided it's path.
*
* @param path The absolute path provided in string format.
* @return Content of corresponding file in string format.
*/
std::string getFileContent(const std::string& path);
/** Checks if file exists.
*
* This function returns boolean stating if file exists.
*
* @param path The absolute path provided in string format.
* @return Boolean representing if file exists or not.
*/
bool fileExists(const std::string& path);
/** Checks if file is readable.
*
* This function returns boolean stating if file is readable.
*
* @param path The absolute path provided in string format.
* @return Boolean representing if file is readale or not.
*/
bool fileReadable(const std::string& path);
/** Provides mimetype from filename.
*
* This function provides mimetype from file-name.
*
* @param filename string containing filename.
* @return mimetype from filename in string format.
*/
std::string getMimeTypeForFile(const std::string& filename);
/** Provides all available network interfaces
*
* This function provides the available IPv4 network interfaces
*/
std::map<std::string, std::string> getNetworkInterfaces();
/** Provides the best IP address
* This function provides the best IP address from the list given by getNetworkInterfaces
*/
std::string getBestPublicIp();
}
#endif // KIWIX_TOOLS_H

98
include/xapianSearcher.h Normal file
View File

@@ -0,0 +1,98 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_XAPIAN_SEARCHER_H
#define KIWIX_XAPIAN_SEARCHER_H
#include <xapian.h>
#include "reader.h"
#include "searcher.h"
#include <map>
#include <string>
using namespace std;
namespace kiwix
{
class XapianSearcher;
class XapianResult : public Result
{
public:
XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
virtual ~XapianResult(){};
virtual std::string get_url();
virtual std::string get_title();
virtual int get_score();
virtual std::string get_snippet();
virtual std::string get_content();
virtual int get_wordCount();
virtual int get_size();
virtual int get_readerIndex() { return 0; };
private:
XapianSearcher* searcher;
Xapian::MSetIterator iterator;
Xapian::Document document;
};
class NoXapianIndexInZim : public exception
{
virtual const char* what() const throw()
{
return "There is no fulltext index in the zim file";
}
};
class XapianSearcher
{
friend class XapianResult;
public:
XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
virtual ~XapianSearcher(){};
void searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int resultEnd,
const bool verbose = false);
virtual Result* getNextResult();
void restart_search();
Xapian::MSet results;
protected:
void closeIndex();
void openIndex(const string& xapianDirectoryPath);
void setup_queryParser();
Reader* reader;
Xapian::Database readableDatabase;
std::string language;
std::string stopwords;
Xapian::QueryParser queryParser;
Xapian::Stem stemmer;
Xapian::SimpleStopper stopper;
Xapian::MSetIterator current_result;
std::map<std::string, int> valuesmap;
};
}
#endif

View File

@@ -1,79 +1,100 @@
project('libkiwix', 'cpp',
version : '12.0.0',
license : 'GPLv3+',
default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
project('kiwix-lib', 'cpp',
version : '1.0.2',
license : 'GPL',
default_options : ['c_std=c11', 'cpp_std=c++11'])
compiler = meson.get_compiler('cpp')
find_library_in_compiler = meson.version().version_compare('>=0.31.0')
static_deps = get_option('static-linkage') or get_option('default_library') == 'static'
static_deps = get_option('android') or get_option('default_library') == 'static'
# See https://github.com/kiwix/libkiwix/issues/371
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
extra_libs = ['-latomic']
else
extra_libs = []
endif
if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
# C++ std::thread is implemented using pthread on linux by gcc
thread_dep = dependency('threads')
else
thread_dep = dependency('', required:false)
endif
thread_dep = dependency('threads')
libicu_dep = dependency('icu-i18n', static:static_deps)
libzim_dep = dependency('libzim', version : '>=3.0.0', static:static_deps)
pugixml_dep = dependency('pugixml', static:static_deps)
libcurl_dep = dependency('libcurl', static:static_deps)
microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
zlib_dep = dependency('zlib', static:static_deps)
xapian_dep = dependency('xapian-core', static:static_deps)
if compiler.has_header('mustache.hpp')
extra_include = []
elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
extra_include = ['/usr/include/kainjow']
ctpp2_include_path = ''
has_ctpp2_dep = false
ctpp2_prefix_install = get_option('ctpp2-install-prefix')
ctpp2_link_args = []
if ctpp2_prefix_install == ''
if compiler.has_header('ctpp2/CTPP2Logger.hpp')
if find_library_in_compiler
ctpp2_lib = compiler.find_library('ctpp2')
else
ctpp2_lib = find_library('ctpp2')
endif
ctpp2_link_args = ['-lctpp2']
if meson.is_cross_build() and host_machine.system() == 'windows'
if find_library_in_compiler
iconv_lib = compiler.find_library('iconv', required:false)
else
iconv_lib = find_library('iconv', required:false)
endif
if iconv_lib.found()
ctpp2_link_args += ['-liconv']
endif
endif
has_ctpp2_dep = true
ctpp2_dep = declare_dependency(link_args:ctpp2_link_args)
else
message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
endif
else
error('Cannot found header mustache.hpp')
if not find_library_in_compiler
error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
endif
ctpp2_include_path = ctpp2_prefix_install + '/include'
ctpp2_include_args = ['-I'+ctpp2_include_path]
if compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
ctpp2_include_dir = include_directories(ctpp2_include_path, is_system:true)
ctpp2_lib_path = ctpp2_prefix_install+'/lib'
ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
ctpp2_link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
if meson.is_cross_build() and host_machine.system() == 'windows'
iconv_lib = compiler.find_library('iconv', required:false)
if iconv_lib.found()
ctpp2_link_args += ['-liconv']
endif
endif
has_ctpp2_dep = true
ctpp2_dep = declare_dependency(include_directories:ctpp2_include_dir, link_args:ctpp2_link_args)
else
message('ctpp2/CTPP2Logger.hpp not found. Compiling without CTPP2 support')
endif
endif
libzim_dep = dependency('libzim', version : '>=8.1.0', static:static_deps)
if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
xapian_dep = dependency('xapian-core', required:false, static:static_deps)
all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep]
if has_ctpp2_dep
all_deps += [ctpp2_dep]
endif
extra_cflags = ''
if host_machine.system() == 'windows' and static_deps
add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
extra_cflags += '-DCURL_STATICLIB'
endif
if host_machine.system() == 'windows'
add_project_arguments('-DNOMINMAX', language: 'cpp')
endif
all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep, xapian_dep]
inc = include_directories('include', extra_include)
inc = include_directories('include')
conf = configuration_data()
conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
if build_machine.system() == 'windows'
extra_link_args = ['-lshlwapi', '-lwinmm']
else
extra_link_args = []
endif
conf.set('VERSION', '"@0@"'.format(meson.project_version()))
conf.set('ENABLE_CTPP2', has_ctpp2_dep)
subdir('include')
subdir('scripts')
subdir('static')
subdir('src')
subdir('test')
if get_option('doc')
subdir('docs')
pkg_requires = ['libzim', 'icu-i18n', 'pugixml']
if xapian_dep.found()
pkg_requires += ['xapian-core']
endif
pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']
extra_libs = []
extra_cflags = ''
if has_ctpp2_dep
extra_libs += ctpp2_link_args
if ctpp2_include_path != ''
extra_cflags = '-I'+ctpp2_include_path
endif
endif
pkg_conf = configuration_data()
pkg_conf.set('prefix', get_option('prefix'))

View File

@@ -1,4 +1,4 @@
option('static-linkage', type : 'boolean', value : false,
description : 'Link statically with the dependencies.')
option('doc', type : 'boolean', value : false,
description : 'Build the documentations.')
option('ctpp2-install-prefix', type : 'string', value : '',
description : 'Prefix where ctpp libs has been installed')
option('android', type : 'boolean', value : false,
description : 'Do we make a kiwix-lib for android')

8
scripts/ctpp2c.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/usr/bin/env bash
ctpp2c=$1
SOURCE=$(pwd)/$2
DEST=$3
$ctpp2c $SOURCE $DEST

View File

@@ -1,14 +0,0 @@
#!/usr/bin/bash
# Compute 'src' path
SCRIPT_DIR=$(dirname "$0")
REPO_DIR=$(readlink -f "$SCRIPT_DIR"/..)
DIRS="src include"
# Apply formating to all *.cpp and *.h files
cd "$REPO_DIR"
for FILE in $(find $DIRS -name '*.h' -o -name '*.cpp')
do
echo $FILE
clang-format -i -style=file "$FILE"
done

View File

@@ -1,148 +0,0 @@
#!/usr/bin/env python3
'''
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
'''
import argparse
import os.path
import re
import json
def to_identifier(name):
ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
if ident[0].isnumeric():
return "_"+ident
return ident
def lang_code(filename):
filename = os.path.basename(filename)
lang = to_identifier(os.path.splitext(filename)[0])
print(filename, '->', lang)
return lang
from string import Template
def expand_cxx_template(t, **kwargs):
return Template(t).substitute(**kwargs)
def cxx_string_literal(s):
# Taking advantage of the fact the JSON string escape rules match
# those of C++
return 'u8' + json.dumps(s)
string_table_cxx_template = '''
const I18nString $TABLE_NAME[] = {
$TABLE_ENTRIES
};
'''
lang_table_entry_cxx_template = '''
{
$LANG_STRING_LITERAL,
ARRAY_ELEMENT_COUNT($STRING_TABLE_NAME),
$STRING_TABLE_NAME
}'''
cxxfile_template = '''// This file is automatically generated. Do not modify it.
#include "server/i18n.h"
namespace kiwix {
namespace i18n {
namespace
{
$STRING_DATA
} // unnamed namespace
#define ARRAY_ELEMENT_COUNT(a) (sizeof(a)/sizeof(a[0]))
extern const I18nStringTable stringTables[] = {
$LANG_TABLE
};
extern const size_t langCount = $LANG_COUNT;
} // namespace i18n
} // namespace kiwix
'''
class Resource:
def __init__(self, filename):
filename = filename.strip()
self.filename = filename
self.lang_code = lang_code(filename)
with open(filename, 'r', encoding='utf-8') as f:
self.data = f.read()
def get_string_table_name(self):
return "string_table_for_" + self.lang_code
def get_string_table(self):
table_entries = ",\n ".join(self.get_string_table_entries())
return expand_cxx_template(string_table_cxx_template,
TABLE_NAME=self.get_string_table_name(),
TABLE_ENTRIES=table_entries)
def get_string_table_entries(self):
d = json.loads(self.data)
for k in sorted(d.keys()):
if k != "@metadata":
key_string = cxx_string_literal(k)
value_string = cxx_string_literal(d[k])
yield '{ ' + key_string + ', ' + value_string + ' }'
def get_lang_table_entry(self):
return expand_cxx_template(lang_table_entry_cxx_template,
LANG_STRING_LITERAL=cxx_string_literal(self.lang_code),
STRING_TABLE_NAME=self.get_string_table_name())
def gen_c_file(resources):
string_data = []
lang_table = []
for r in resources:
string_data.append(r.get_string_table())
lang_table.append(r.get_lang_table_entry())
return expand_cxx_template(cxxfile_template,
STRING_DATA="\n".join(string_data),
LANG_TABLE=",\n ".join(lang_table),
LANG_COUNT=len(resources)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--cxxfile',
required=True,
help='The Cpp file name to generate')
parser.add_argument('i18n_resource_files', nargs='+',
help='The list of resources to compile.')
args = parser.parse_args()
resources = [Resource(filename) for filename in args.i18n_resource_files]
with open(args.cxxfile, 'w') as f:
f.write(gen_c_file(resources))

View File

@@ -1,18 +0,0 @@
.TH KIWIX-COMPILE-I18N "1" "January 2022" "Kiwix" "User Commands"
.SH NAME
kiwix-compile-i18n \- helper to compile Kiwix i18n (internationalization) data
.SH SYNOPSIS
\fBkiwix\-compile\-i18n\fR [\-h] \-\-cxxfile CXXFILE i18n_resource_files ...\fR
.SH DESCRIPTION
.TP
i18n_resource_files ...
The list of i18n resources to compile.
.TP
\fB\-h\fR, \fB\-\-help\fR
show a help message and exit
.TP
\fB\-\-cxxfile\fR CXXFILE
The Cpp file name to generate
.TP
.SH AUTHOR
Veloman Yunkan <veloman.yunkan@gmail.com>

View File

@@ -52,21 +52,15 @@ resource_getter_template = """
return RESOURCE::{identifier};
"""
resource_cacheid_getter_template = """
if (name == "{common_name}")
return "{cacheid}";
"""
resource_decl_template = """{namespaces_open}
extern const std::string {identifier};
{namespaces_close}"""
class Resource:
def __init__(self, base_dirs, filename, cacheid=None):
filename = filename
def __init__(self, base_dirs, filename):
filename = filename.strip()
self.filename = filename
self.identifier = full_identifier(filename)
self.cacheid = cacheid
found = False
for base_dir in base_dirs:
try:
@@ -77,7 +71,7 @@ class Resource:
except FileNotFoundError:
continue
if not found:
raise Exception("Resource not found: {}".format(filename))
raise Exception("Impossible to found {}".format(filename))
def dump_impl(self):
nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -87,34 +81,28 @@ class Resource:
data_identifier="_".join([""]+self.identifier),
resource_content=",\n ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
resource_len=len(self.data),
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
identifier=self.identifier[-1],
env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
)
def dump_getter(self):
return resource_getter_template.format(
common_name=self.filename,
identifier="::".join(self.identifier)
)
def dump_cacheid_getter(self):
return resource_cacheid_getter_template.format(
common_name=self.filename,
cacheid=self.cacheid
)
def dump_decl(self):
return resource_decl_template.format(
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
identifier=self.identifier[-1]
)
master_c_template = """//This file is automatically generated. Do not modify it.
master_c_template = """//This file is automaically generated. Do not modify it.
#include <stdlib.h>
#include <fstream>
@@ -125,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
char * resPath = getenv(name);
if (NULL == resPath)
return std::string(reinterpret_cast<const char*>(content), len);
std::ifstream ifs(resPath);
if (!ifs.good())
return std::string(reinterpret_cast<const char*>(content), len);
@@ -135,12 +123,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
const std::string& getResource_{basename}(const std::string& name) {{
{RESOURCES_GETTER}
throw ResourceNotFound("Resource not found: " + name);
}}
const char* getResourceCacheId_{basename}(const std::string& name) {{
{RESOURCE_CACHEID_GETTER}
return nullptr;
throw ResourceNotFound("Resource not found.");
}}
{RESOURCES}
@@ -151,11 +134,10 @@ def gen_c_file(resources, basename):
return master_c_template.format(
RESOURCES="\n\n".join(r.dump_impl() for r in resources),
RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
include_file=basename,
basename=to_identifier(basename)
)
master_h_template = """//This file is automaically generated. Do not modify it.
@@ -177,10 +159,8 @@ class ResourceNotFound : public std::runtime_error {{
}};
const std::string& getResource_{basename}(const std::string& name);
const char* getResourceCacheId_{basename}(const std::string& name);
#define getResource(a) (getResource_{basename}(a))
#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
#endif // KIWIX_{BASENAME}
@@ -209,8 +189,8 @@ if __name__ == "__main__":
base_dir = os.path.dirname(os.path.realpath(args.resource_file))
source_dir = args.source_dir or []
with open(args.resource_file, 'r') as f:
resources = [Resource([base_dir]+source_dir, *line.strip().split())
for line in f.readlines()]
resources = [Resource([base_dir]+source_dir, filename)
for filename in f.readlines()]
h_identifier = to_identifier(os.path.basename(args.hfile))
with open(args.hfile, 'w') as f:

View File

@@ -1,20 +0,0 @@
.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
.SH NAME
kiwix-compile-resources \- helper to compile and generate some Kiwix resources
.SH SYNOPSIS
\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
.SH DESCRIPTION
.TP
resource_file
The list of resources to compile.
.TP
\fB\-h\fR, \fB\-\-help\fR
show a help message and exit
.TP
\fB\-\-cxxfile\fR CXXFILE
The Cpp file name to generate
.TP
\fB\-\-hfile\fR HFILE
The h file name to generate
.SH AUTHOR
Matthieu Gautier <mgautier@kymeria.fr>

View File

@@ -1,135 +0,0 @@
#!/usr/bin/env python3
'''
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
'''
import argparse
import hashlib
import os.path
import re
def read_resource_file(resource_file_path):
with open(resource_file_path, 'r') as f:
return [line.strip() for line in f]
def list_resources(resource_file_path):
for resource_path in read_resource_file(resource_file_path):
print(resource_path)
def compute_resource_revision(resource_path):
with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
return hashlib.sha1(f.read()).hexdigest()[:8]
resource_revisions = {}
def get_resource_revision(res):
if not res in resource_revisions:
preprocess_resource(res)
resource_revisions[res] = compute_resource_revision(res)
return resource_revisions[res]
RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
def set_cacheid(resource_matchobj):
pre = resource_matchobj.group('pre')
resource = resource_matchobj.group('resource')
post = resource_matchobj.group('post')
cacheid = 'cacheid=' + get_resource_revision(resource)
return pre + cacheid + post
def preprocess_text(s):
if 'KIWIXCACHEID' in s:
s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
assert not 'KIWIXCACHEID' in s
return s
def get_preprocessed_resource(srcpath):
"""Get the transformed content of a resource
If the resource at srcpath is modified by preprocessing then this function
returns the transformed content of the resource. Otherwise it returns None.
"""
try:
with open(srcpath, 'r') as resource_file:
content = resource_file.read()
preprocessed_content = preprocess_text(content)
return preprocessed_content if preprocessed_content != content else None
except UnicodeDecodeError:
# It was a binary resource
return None
def symlink_resource(src, resource_path):
if os.path.exists(resource_path):
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
return
os.remove(resource_path)
os.symlink(src, resource_path)
def preprocess_resource(resource_path):
print('Preprocessing', resource_path, '...')
resource_dir = os.path.dirname(resource_path)
if resource_dir != '':
os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
srcpath = os.path.join(BASE_DIR, resource_path)
outpath = os.path.join(OUT_DIR, resource_path)
if os.path.exists(outpath):
os.remove(outpath)
preprocessed_content = get_preprocessed_resource(srcpath)
if preprocessed_content is None:
symlink_resource(srcpath, outpath)
else:
with open(outpath, 'w') as target:
print(preprocessed_content, end='', file=target)
def copy_resource_list_file(src_path, dst_path):
with open(src_path, 'r') as src:
with open(dst_path, 'w') as dst:
for line in src:
res = line.strip()
if line.startswith("skin/") and res in resource_revisions:
dst.write(res + " " + resource_revisions[res] + "\n")
else:
dst.write(line)
def preprocess_resources(resource_file_path):
resource_filename = os.path.basename(resource_file_path)
for resource in read_resource_file(resource_file_path):
if resource.startswith('skin/'):
get_resource_revision(resource)
else:
preprocess_resource(resource)
copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
commands = parser.add_mutually_exclusive_group()
commands.add_argument('--list-all', action='store_true')
commands.add_argument('--preprocess', action='store_true')
parser.add_argument('--outdir')
parser.add_argument('resource_file')
args = parser.parse_args()
BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
OUT_DIR = args.outdir
if args.list_all:
list_resources(args.resource_file)
elif args.preprocess:
preprocess_resources(args.resource_file)

View File

@@ -1,13 +1,5 @@
res_manager = find_program('kiwix-resources')
res_compiler = find_program('kiwix-compile-resources')
intermediate_ctpp2c = find_program('ctpp2c.sh')
install_data(res_compiler.path(), install_dir:get_option('bindir'))
install_man('kiwix-compile-resources.1')
i18n_compiler = find_program('kiwix-compile-i18n')
install_data(i18n_compiler.path(), install_dir:get_option('bindir'))
install_man('kiwix-compile-i18n.1')

View File

@@ -0,0 +1,13 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="kiwix.org.kiwixlib"
>
<application android:allowBackup="true"
android:label="@string/app_name"
android:supportsRtl="true"
>
</application>
</manifest>

11
src/android/gen_kiwix.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -e
BUILD_PATH=$(pwd)
javac -d $BUILD_PATH/src/android $1 $2 $3 $4
cd $BUILD_PATH/src/android
javah -jni org.kiwix.kiwixlib.JNIKiwix
cd $BUILD_PATH

559
src/android/kiwix.cpp Normal file
View File

@@ -0,0 +1,559 @@
/*
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <jni.h>
#include "org_kiwix_kiwixlib_JNIKiwix.h"
#include <stdio.h>
#include <string.h>
#include <iostream>
#include <string>
#include "common/base64.h"
#include "reader.h"
#include "searcher.h"
#include "unicode/putil.h"
#include <android/log.h>
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, "kiwix", __VA_ARGS__)
#include <xapian.h>
#include <zim/article.h>
#include <zim/error.h>
#include <zim/file.h>
#include <zim/zim.h>
/* global variables */
kiwix::Reader* reader = NULL;
kiwix::Searcher* searcher = NULL;
static pthread_mutex_t readerLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t searcherLock = PTHREAD_MUTEX_INITIALIZER;
/* c2jni type conversion functions */
jboolean c2jni(const bool& val)
{
return val ? JNI_TRUE : JNI_FALSE;
}
jstring c2jni(const std::string& val, JNIEnv* env)
{
return env->NewStringUTF(val.c_str());
}
jint c2jni(const int val)
{
return (jint)val;
}
jint c2jni(const unsigned val)
{
return (unsigned)val;
}
/* jni2c type conversion functions */
bool jni2c(const jboolean& val)
{
return val == JNI_TRUE;
}
std::string jni2c(const jstring& val, JNIEnv* env)
{
return std::string(env->GetStringUTFChars(val, 0));
}
int jni2c(const jint val)
{
return (int)val;
}
/* Method to deal with variable passed by reference */
void setStringObjValue(const std::string& value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
env->SetObjectField(obj, objFid, c2jni(value, env));
}
void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "I");
env->SetIntField(obj, objFid, value);
}
void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
env->SetIntField(obj, objFid, c2jni(value));
}
/* Kiwix library functions */
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv* env, jobject obj)
{
jstring url;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cUrl = reader->getMainPageUrl();
url = c2jni(cUrl, env);
} catch (...) {
std::cerr << "Unable to get ZIM main page" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return url;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv* env,
jobject obj)
{
jstring id;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cId = reader->getId();
id = c2jni(cId, env);
} catch (...) {
std::cerr << "Unable to get ZIM id" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return id;
}
JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv* env,
jobject obj)
{
jint size;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
int cSize = reader->getFileSize();
size = c2jni(cSize);
} catch (...) {
std::cerr << "Unable to get ZIM file size" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return size;
}
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv* env, jobject obj)
{
jstring creator;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cCreator = reader->getCreator();
creator = c2jni(cCreator, env);
} catch (...) {
std::cerr << "Unable to get ZIM creator" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return creator;
}
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv* env, jobject obj)
{
jstring publisher;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cPublisher = reader->getPublisher();
publisher = c2jni(cPublisher, env);
} catch (...) {
std::cerr << "Unable to get ZIM creator" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return publisher;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv* env,
jobject obj)
{
jstring name;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cName = reader->getName();
name = c2jni(cName, env);
} catch (...) {
std::cerr << "Unable to get ZIM name" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return name;
}
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv* env, jobject obj)
{
jstring favicon;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cContent;
std::string cMime;
reader->getFavicon(cContent, cMime);
favicon
= c2jni(base64_encode(
reinterpret_cast<const unsigned char*>(cContent.c_str()),
cContent.length()),
env);
} catch (...) {
std::cerr << "Unable to get ZIM favicon" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return favicon;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv* env,
jobject obj)
{
jstring date;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cDate = reader->getDate();
date = c2jni(cDate, env);
} catch (...) {
std::cerr << "Unable to get ZIM date" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return date;
}
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv* env, jobject obj)
{
jstring language;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cLanguage = reader->getLanguage();
language = c2jni(cLanguage, env);
} catch (...) {
std::cerr << "Unable to get ZIM language" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return language;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(
JNIEnv* env, jobject obj, jstring url)
{
jstring mimeType;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
std::string cUrl = jni2c(url, env);
try {
std::string cMimeType;
reader->getMimeTypeByUrl(cUrl, cMimeType);
mimeType = c2jni(cMimeType, env);
} catch (...) {
std::cerr << "Unable to get mime-type for url " << cUrl << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return mimeType;
}
JNIEXPORT jboolean JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv* env, jobject obj, jstring path)
{
jboolean retVal = JNI_TRUE;
std::string cPath = jni2c(path, env);
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
delete reader;
}
reader = new kiwix::Reader(cPath);
} catch (...) {
std::cerr << "Unable to load ZIM " << cPath << std::endl;
reader = NULL;
retVal = JNI_FALSE;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(
JNIEnv* env, jobject obj, jstring url, jobject titleObj, jobject mimeTypeObj, jobject sizeObj)
{
/* Default values */
setStringObjValue("", titleObj, env);
setStringObjValue("", mimeTypeObj, env);
setIntObjValue(0, sizeObj, env);
jbyteArray data = env->NewByteArray(0);
/* Retrieve the content */
if (reader != NULL) {
std::string cUrl = jni2c(url, env);
std::string cData;
std::string cTitle;
std::string cMimeType;
unsigned int cSize = 0;
pthread_mutex_lock(&readerLock);
try {
if (reader->getContentByUrl(cUrl, cData, cTitle, cSize, cMimeType)) {
data = env->NewByteArray(cSize);
env->SetByteArrayRegion(
data, 0, cSize, reinterpret_cast<const jbyte*>(cData.c_str()));
setStringObjValue(cMimeType, mimeTypeObj, env);
setStringObjValue(cTitle, titleObj, env);
setIntObjValue(cSize, sizeObj, env);
}
} catch (...) {
std::cerr << "Unable to get content for url " << cUrl << std::endl;
}
pthread_mutex_unlock(&readerLock);
}
return data;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions(
JNIEnv* env, jobject obj, jstring prefix, jint count)
{
jboolean retVal = JNI_FALSE;
std::string cPrefix = jni2c(prefix, env);
unsigned int cCount = jni2c(count);
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
if (reader->searchSuggestionsSmart(cPrefix, cCount)) {
retVal = JNI_TRUE;
}
}
} catch (...) {
std::cerr << "Unable to search suggestions for pattern " << cPrefix
<< std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion(
JNIEnv* env, jobject obj, jobject titleObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle;
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
if (reader->getNextSuggestion(cTitle)) {
setStringObjValue(cTitle, titleObj, env);
retVal = JNI_TRUE;
}
}
} catch (...) {
std::cerr << "Unable to get next suggestion" << std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle(
JNIEnv* env, jobject obj, jstring title, jobject urlObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle = jni2c(title, env);
std::string cUrl;
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
if (reader->getPageUrlFromTitle(cTitle, cUrl)) {
setStringObjValue(cUrl, urlObj, env);
retVal = JNI_TRUE;
}
}
} catch (...) {
std::cerr << "Unable to get URL for title " << cTitle << std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle(
JNIEnv* env, jobject obj, jobject titleObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle;
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
std::string cTitle = reader->getTitle();
setStringObjValue(cTitle, titleObj, env);
retVal = JNI_TRUE;
}
} catch (...) {
std::cerr << "Unable to get ZIM title" << std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv* env, jobject obj)
{
jstring description;
pthread_mutex_lock(&readerLock);
if (reader != NULL) {
try {
std::string cDescription = reader->getDescription();
description = c2jni(cDescription, env);
} catch (...) {
std::cerr << "Unable to get ZIM description" << std::endl;
}
}
pthread_mutex_unlock(&readerLock);
return description;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage(
JNIEnv* env, jobject obj, jobject urlObj)
{
jboolean retVal = JNI_FALSE;
std::string cUrl;
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) {
std::string cUrl = reader->getRandomPageUrl();
setStringObjValue(cUrl, urlObj, env);
retVal = JNI_TRUE;
}
} catch (...) {
std::cerr << "Unable to get random page" << std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
JNIEnv* env, jobject obj, jstring dirStr)
{
std::string cPath = jni2c(dirStr, env);
pthread_mutex_lock(&readerLock);
try {
u_setDataDirectory(cPath.c_str());
} catch (...) {
std::cerr << "Unable to set data directory " << cPath << std::endl;
}
pthread_mutex_unlock(&readerLock);
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(
JNIEnv* env, jobject obj, jstring path)
{
jboolean retVal = JNI_TRUE;
std::string cPath = jni2c(path, env);
pthread_mutex_lock(&searcherLock);
try {
if (searcher != NULL) {
delete searcher;
}
if (!reader || !reader->hasFulltextIndex()) {
// Use old API (no embedded full text index).
searcher = new kiwix::Searcher(cPath, reader, "");
} else {
// Use the new API. We don't care about the human readable name as
// we don't use it (in android).
searcher = new kiwix::Searcher();
searcher->add_reader(reader, "");
}
} catch (...) {
searcher = NULL;
retVal = JNI_FALSE;
std::cerr << "Unable to load full text index " << cPath << std::endl;
}
pthread_mutex_unlock(&searcherLock);
return retVal;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery(
JNIEnv* env, jclass obj, jstring query, jint count)
{
std::string cQuery = jni2c(query, env);
unsigned int cCount = jni2c(count);
kiwix::Result* p_result;
std::string result;
pthread_mutex_lock(&searcherLock);
try {
if (searcher != NULL) {
searcher->search(cQuery, 0, count);
while ((p_result = searcher->getNextResult())
&& !(p_result->get_title().empty())
&& !(p_result->get_url().empty())) {
result += p_result->get_title() + "\n";
delete p_result;
}
}
} catch (...) {
std::cerr << "Unable to make indexed query " << cQuery << std::endl;
}
pthread_mutex_unlock(&searcherLock);
return env->NewStringUTF(result.c_str());
}

17
src/android/meson.build Normal file
View File

@@ -0,0 +1,17 @@
jni_generator = find_program('gen_kiwix.sh')
kiwix_jni = custom_target('jni',
input: ['org/kiwix/kiwixlib/JNIKiwix.java',
'org/kiwix/kiwixlib/JNIKiwixInt.java',
'org/kiwix/kiwixlib/JNIKiwixString.java',
'org/kiwix/kiwixlib/JNIKiwixBool.java'],
output: ['org_kiwix_kiwixlib_JNIKiwix.h'],
command:[jni_generator, '@INPUT@']
)
kiwix_sources += ['android/kiwix.cpp', kiwix_jni]
install_subdir('org', install_dir: 'kiwix-lib/java')
install_subdir('res', install_dir: 'kiwix-lib')
install_data('AndroidManifest.xml', install_dir: 'kiwix-lib')

View File

@@ -0,0 +1,74 @@
/*
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.kiwix.kiwixlib;
import org.kiwix.kiwixlib.JNIKiwixString;
import org.kiwix.kiwixlib.JNIKiwixBool;
import org.kiwix.kiwixlib.JNIKiwixInt;
public class JNIKiwix
{
static { System.loadLibrary("kiwix"); }
public native String getMainPage();
public native String getId();
public native String getLanguage();
public native String getMimeType(String url);
public native boolean loadZIM(String path);
public native boolean loadFulltextIndex(String path);
public native byte[] getContent(String url, JNIKiwixString title, JNIKiwixString mimeType, JNIKiwixInt size);
public native boolean searchSuggestions(String prefix, int count);
public native boolean getNextSuggestion(JNIKiwixString title);
public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
public native boolean getTitle(JNIKiwixString title);
public native String getDescription();
public native String getDate();
public native String getFavicon();
public native String getCreator();
public native String getPublisher();
public native String getName();
public native int getFileSize();
public native int getArticleCount();
public native int getMediaCount();
public native boolean getRandomPage(JNIKiwixString url);
public native void setDataDirectory(String icuDataDir);
public static native String indexedQuery(String db, int count);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -17,14 +17,9 @@
* MA 02110-1301, USA.
*/
#ifndef KIWIX_NETWORKTOOLS_H
#define KIWIX_NETWORKTOOLS_H
package org.kiwix.kiwixlib;
#include <string>
namespace kiwix
public class JNIKiwixBool
{
std::string download(const std::string& url);
public boolean value;
}
#endif

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -17,8 +17,9 @@
* MA 02110-1301, USA.
*/
#include <microhttpd.h>
package org.kiwix.kiwixlib;
#if MHD_VERSION < 0x00097002
typedef int MHD_Result;
#endif
public class JNIKiwixInt
{
public int value;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -17,18 +17,9 @@
* MA 02110-1301, USA.
*/
#ifndef KIWIX_VERSION_H
#define KIWIX_VERSION_H
package org.kiwix.kiwixlib;
#include <string>
#include <vector>
#include <iostream>
namespace kiwix
public class JNIKiwixString
{
typedef std::vector<std::pair<std::string, std::string>> LibVersions;
LibVersions getVersions();
void printVersions(std::ostream& out = std::cout);
public String value;
}
#endif // KIWIX_VERSION_H

View File

@@ -0,0 +1,3 @@
<resources>
<string name="app_name">Kiwix Lib</string>
</resources>

View File

@@ -1,281 +0,0 @@
#include "aria2.h"
#include "xmlrpc.h"
#include <iostream>
#include <algorithm>
#include <sstream>
#include <thread>
#include <chrono>
#include "tools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "downloader.h" // For AriaError
#ifdef _WIN32
# define ARIA2_CMD "aria2c.exe"
#else
# define ARIA2_CMD "aria2c"
# include <unistd.h>
#endif
#define LOG_ARIA_ERROR() \
{ \
std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
}
namespace kiwix {
Aria2::Aria2():
mp_aria(nullptr),
m_port(42042),
m_secret(getNewRpcSecret()),
m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
mp_curl(nullptr)
{
m_downloadDir = getDataDirectory();
makeDirectory(m_downloadDir);
std::vector<const char*> callCmd;
std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
std::string download_dir = "--dir=" + getDataDirectory();
std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
std::string session = "--save-session=" + session_file;
std::string inputFile = "--input-file=" + session_file;
// std::string log_dir = "--log=\"" + logDir + "\"";
#ifdef _WIN32
int pid = GetCurrentProcessId();
#else
pid_t pid = getpid();
#endif
std::string stop_with_pid = "--stop-with-process=" + to_string(pid);
std::string rpc_secret = "--rpc-secret=" + m_secret;
m_secret = "token:"+m_secret;
std::string aria2cmd = appendToDirectory(
removeLastPathElement(getExecutablePath(true)),
ARIA2_CMD);
if (fileExists(aria2cmd)) {
// A local aria2c exe exists (packaged with kiwix-desktop), use it.
callCmd.push_back(aria2cmd.c_str());
} else {
// Try to use a potential installed aria2c.
callCmd.push_back(ARIA2_CMD);
}
callCmd.push_back("--follow-metalink=mem");
callCmd.push_back("--enable-rpc");
callCmd.push_back(rpc_secret.c_str());
callCmd.push_back(rpc_port.c_str());
callCmd.push_back(download_dir.c_str());
if (fileReadable(session_file)) {
callCmd.push_back(inputFile.c_str());
}
callCmd.push_back(session.c_str());
// callCmd.push_back(log_dir.c_str());
callCmd.push_back("--auto-save-interval=10");
callCmd.push_back(stop_with_pid.c_str());
callCmd.push_back("--allow-overwrite=true");
callCmd.push_back("--dht-entry-point=router.bittorrent.com:6881");
callCmd.push_back("--dht-entry-point6=router.bittorrent.com:6881");
callCmd.push_back("--quiet=true");
callCmd.push_back("--bt-enable-lpd=true");
callCmd.push_back("--always-resume=true");
callCmd.push_back("--max-concurrent-downloads=42");
callCmd.push_back("--rpc-max-request-size=6M");
callCmd.push_back("--file-allocation=none");
std::string launchCmd;
for (auto &cmd : callCmd) {
launchCmd.append(cmd).append(" ");
}
mp_aria = Subprocess::run(callCmd);
mp_curl = curl_easy_init();
curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
int watchdog = 50;
while(--watchdog) {
sleep(10);
m_curlErrorBuffer[0] = 0;
auto res = curl_easy_perform(mp_curl);
if (res == CURLE_OK) {
break;
} else if (watchdog == 1) {
LOG_ARIA_ERROR();
}
}
if (!watchdog) {
curl_easy_cleanup(mp_curl);
throw std::runtime_error("Cannot connect to aria2c rpc. Aria2c launch cmd : " + launchCmd);
}
}
Aria2::~Aria2()
{
std::unique_lock<std::mutex> lock(m_lock);
curl_easy_cleanup(mp_curl);
}
void Aria2::close()
{
saveSession();
shutdown();
}
size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
{
auto outStream = static_cast<std::stringstream*>(userdata);
outStream->write(ptr, nmemb);
return nmemb;
}
std::string Aria2::doRequest(const MethodCall& methodCall)
{
auto requestContent = methodCall.toString();
std::stringstream outStream;
CURLcode res;
long response_code;
{
std::unique_lock<std::mutex> lock(m_lock);
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
m_curlErrorBuffer[0] = 0;
res = curl_easy_perform(mp_curl);
if (res != CURLE_OK) {
LOG_ARIA_ERROR();
throw std::runtime_error("Cannot perform request");
}
curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
}
auto responseContent = outStream.str();
if (response_code != 200) {
std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
std::cerr << responseContent << std::endl;
throw std::runtime_error("Invalid return code from aria");
}
MethodResponse response(responseContent);
if (response.isFault()) {
throw AriaError(response.getFault().getFaultString());
}
return responseContent;
}
std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
{
MethodCall methodCall("aria2.addUri", m_secret);
auto uriParams = methodCall.newParamValue().getArray();
for (auto& uri : uris) {
uriParams.addValue().set(uri);
}
for (auto& option : options) {
methodCall.newParamValue().getStruct().addMember(option.first).getValue().set(option.second);
}
auto ret = doRequest(methodCall);
MethodResponse response(ret);
return response.getParamValue(0).getAsS();
}
std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::string>& statusKey)
{
MethodCall methodCall("aria2.tellStatus", m_secret);
methodCall.newParamValue().set(gid);
if (!statusKey.empty()) {
auto statusArray = methodCall.newParamValue().getArray();
for (auto& key : statusKey) {
statusArray.addValue().set(key);
}
}
return doRequest(methodCall);
}
std::string Aria2::getNewRpcSecret()
{
std::string uuid = gen_uuid("");
uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
return uuid.substr(0, 9);
}
std::vector<std::string> Aria2::tellActive()
{
MethodCall methodCall("aria2.tellActive", m_secret);
auto statusArray = methodCall.newParamValue().getArray();
statusArray.addValue().set(std::string("gid"));
auto responseContent = doRequest(methodCall);
MethodResponse response(responseContent);
std::vector<std::string> activeGID;
int index = 0;
while(true) {
try {
auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
auto gidNode = structNode.getMember("gid");
activeGID.push_back(gidNode.getValue().getAsS());
} catch (InvalidRPCNode& e) { break; }
}
return activeGID;
}
std::vector<std::string> Aria2::tellWaiting()
{
MethodCall methodCall("aria2.tellWaiting", m_secret);
methodCall.newParamValue().set(0);
methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
auto statusArray = methodCall.newParamValue().getArray();
statusArray.addValue().set(std::string("gid"));
auto responseContent = doRequest(methodCall);
MethodResponse response(responseContent);
std::vector<std::string> waitingGID;
int index = 0;
while(true) {
try {
auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
auto gidNode = structNode.getMember("gid");
waitingGID.push_back(gidNode.getValue().getAsS());
} catch (InvalidRPCNode& e) { break; }
}
return waitingGID;
}
void Aria2::saveSession()
{
MethodCall methodCall("aria2.saveSession", m_secret);
doRequest(methodCall);
std::cout << "session saved" << std::endl;
}
void Aria2::shutdown()
{
MethodCall methodCall("aria2.shutdown", m_secret);
doRequest(methodCall);
}
void Aria2::pause(const std::string& gid)
{
MethodCall methodCall("aria2.pause", m_secret);
methodCall.newParamValue().set(gid);
doRequest(methodCall);
}
void Aria2::unpause(const std::string& gid)
{
MethodCall methodCall("aria2.unpause", m_secret);
methodCall.newParamValue().set(gid);
doRequest(methodCall);
}
void Aria2::remove(const std::string& gid)
{
MethodCall methodCall("aria2.remove", m_secret);
methodCall.newParamValue().set(gid);
doRequest(methodCall);
}
} // end namespace kiwix

View File

@@ -1,52 +0,0 @@
#ifndef KIWIXLIB_ARIA2_H_
#define KIWIXLIB_ARIA2_H_
#ifdef _WIN32
// winsock2.h need to be included before windows.h (included by curl.h)
# include <winsock2.h>
#endif
#include "subprocess.h"
#include "xmlrpc.h"
#include <memory>
#include <mutex>
#include <curl/curl.h>
namespace kiwix {
class Aria2
{
private:
std::unique_ptr<Subprocess> mp_aria;
int m_port;
std::string m_secret;
std::string m_downloadDir;
std::unique_ptr<char[]> m_curlErrorBuffer;
CURL* mp_curl;
std::mutex m_lock;
std::string doRequest(const MethodCall& methodCall);
public:
Aria2();
virtual ~Aria2();
void close();
std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
static std::string getNewRpcSecret();
std::vector<std::string> tellActive();
std::vector<std::string> tellWaiting();
void saveSession();
void shutdown();
void pause(const std::string& gid);
void unpause(const std::string& gid);
void remove(const std::string& gid);
};
}; //end namespace kiwix
#endif // KIWIXLIB_ARIA2_H_

View File

@@ -1,289 +0,0 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "book.h"
#include "tools.h"
#include "tools/base64.h"
#include "tools/regexTools.h"
#include "tools/networkTools.h"
#include "tools/otherTools.h"
#include "tools/stringTools.h"
#include "tools/pathTools.h"
#include "tools/archiveTools.h"
#include <zim/archive.h>
#include <zim/item.h>
#include <pugixml.hpp>
namespace kiwix
{
/* Constructor */
Book::Book() :
m_pathValid(false),
m_readOnly(false)
{
}
/* Destructor */
Book::~Book()
{
}
Book::Illustrations Book::getIllustrations() const
{
return m_illustrations;
}
bool Book::update(const kiwix::Book& other)
{
if (m_readOnly)
return false;
if (m_id != other.m_id)
return false;
*this = other;
return true;
}
void Book::update(const zim::Archive& archive) {
m_path = archive.getFilename();
m_pathValid = true;
m_id = std::string(archive.getUuid());
m_title = getArchiveTitle(archive);
m_description = getMetaDescription(archive);
m_language = getMetaLanguage(archive);
m_creator = getMetaCreator(archive);
m_publisher = getMetaPublisher(archive);
m_date = getMetaDate(archive);
m_name = getMetaName(archive);
m_flavour = getMetaFlavour(archive);
m_tags = getMetaTags(archive);
m_category = getCategoryFromTags();
m_articleCount = archive.getArticleCount();
m_mediaCount = archive.getMediaCount();
m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
m_illustrations.clear();
for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
const auto illustration = std::make_shared<Illustration>();
const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
illustration->width = illustration->height = illustrationSize;
illustration->mimeType = illustrationItem.getMimetype();
illustration->data = illustrationItem.getData();
// NOTE: illustration->url is left uninitialized
m_illustrations.push_back(illustration);
}
}
#define ATTR(name) node.attribute(name).value()
void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
{
m_id = ATTR("id");
std::string path = ATTR("path");
if (isRelativePath(path)) {
path = computeAbsolutePath(baseDir, path);
}
m_path = path;
m_pathValid = fileReadable(path);
m_title = ATTR("title");
m_description = ATTR("description");
m_language = ATTR("language");
m_creator = ATTR("creator");
m_publisher = ATTR("publisher");
m_date = ATTR("date");
m_url = ATTR("url");
m_name = ATTR("name");
m_flavour = ATTR("flavour");
m_tags = ATTR("tags");
m_origId = ATTR("origId");
m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
m_size = strtoull(ATTR("size"), 0, 0) << 10;
std::string favicon_mimetype = ATTR("faviconMimeType");
if (! favicon_mimetype.empty()) {
const auto favicon = std::make_shared<Illustration>();
favicon->data = base64_decode(ATTR("favicon"));
favicon->mimeType = favicon_mimetype;
favicon->url = ATTR("faviconUrl");
m_illustrations.assign(1, favicon);
}
try {
m_downloadId = ATTR("downloadId");
} catch(...) {}
const auto catattr = node.attribute("category");
m_category = catattr.empty() ? getCategoryFromTags() : catattr.value();
}
#undef ATTR
static std::string fromOpdsDate(const std::string& date)
{
//The opds date use the standard <YYYY>-<MM>-<DD>T<HH>:<mm>:<SS>Z
//and we want <YYYY>-<MM>-<DD>. That's easy, let's take the first 10 char
return date.substr(0, 10);
}
#define VALUE(name) node.child(name).child_value()
void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost)
{
m_id = VALUE("id");
if (!m_id.compare(0, 9, "urn:uuid:")) {
m_id.erase(0, 9);
}
// No path on opds.
m_title = VALUE("title");
m_description = VALUE("summary");
m_language = VALUE("language");
m_creator = node.child("author").child("name").child_value();
m_publisher = node.child("publisher").child("name").child_value();
const std::string dcIssuedDate = VALUE("dc:issued");
m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
m_date = fromOpdsDate(m_date);
m_name = VALUE("name");
m_flavour = VALUE("flavour");
m_tags = VALUE("tags");
const auto catnode = node.child("category");
m_category = catnode.empty() ? getCategoryFromTags() : catnode.child_value();
m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
for(auto linkNode = node.child("link"); linkNode;
linkNode = linkNode.next_sibling("link")) {
std::string rel = linkNode.attribute("rel").value();
if (rel == "http://opds-spec.org/acquisition/open-access") {
m_url = linkNode.attribute("href").value();
m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
}
if (rel == "http://opds-spec.org/image/thumbnail") {
const auto favicon = std::make_shared<Illustration>();
favicon->data.clear();
favicon->url = urlHost + linkNode.attribute("href").value();
favicon->mimeType = linkNode.attribute("type").value();
m_illustrations.assign(1, favicon);
}
}
}
#undef VALUE
std::string Book::getHumanReadableIdFromPath() const
{
std::string id = m_path;
if (!id.empty()) {
id = kiwix::removeAccents(id);
#ifdef _WIN32
id = replaceRegex(id, "", "^.*\\\\");
#else
id = replaceRegex(id, "", "^.*/");
#endif
id = replaceRegex(id, "", "\\.zim[a-z]*$");
id = replaceRegex(id, "_", " ");
id = replaceRegex(id, "plus", "\\+");
}
return id;
}
void Book::setPath(const std::string& path)
{
m_path = isRelativePath(path)
? computeAbsolutePath(getCurrentDirectory(), path)
: path;
}
const Book::Illustration Book::missingDefaultIllustration;
std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
{
for ( const auto& ilPtr : m_illustrations ) {
if (ilPtr->width == size && ilPtr->height == size) {
return ilPtr;
}
}
throw std::runtime_error("Cannot find illustration");
}
const Book::Illustration& Book::getDefaultIllustration() const
{
try {
return *getIllustration(48);
} catch (...) {
return missingDefaultIllustration;
}
}
const std::string& Book::Illustration::getData() const
{
if (data.empty() && !url.empty()) {
const std::lock_guard<std::mutex> l(mutex);
if ( data.empty() ) {
try {
data = download(url);
} catch(...) {
std::cerr << "Cannot download favicon from " << url;
}
}
}
return data;
}
const std::string& Book::getFavicon() const {
return getDefaultIllustration().getData();
}
const std::string& Book::getFaviconUrl() const
{
return getDefaultIllustration().url;
}
const std::string& Book::getFaviconMimeType() const
{
return getDefaultIllustration().mimeType;
}
std::string Book::getTagStr(const std::string& tagName) const {
return getTagValueFromTagList(convertTags(m_tags), tagName);
}
bool Book::getTagBool(const std::string& tagName) const {
return convertStrToBool(getTagStr(tagName));
}
std::string Book::getCategory() const
{
return m_category;
}
std::string Book::getCategoryFromTags() const
{
try
{
return getTagStr("category");
}
catch ( const std::out_of_range& )
{
return "";
}
}
}

View File

@@ -1,47 +0,0 @@
/*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "bookmark.h"
#include <pugixml.hpp>
namespace kiwix
{
/* Constructor */
Bookmark::Bookmark()
{
}
/* Destructor */
Bookmark::~Bookmark()
{
}
void Bookmark::updateFromXml(const pugi::xml_node& node)
{
auto bookNode = node.child("book");
m_bookId = bookNode.child("id").child_value();
m_bookTitle = bookNode.child("title").child_value();
m_language = bookNode.child("language").child_value();
m_date = bookNode.child("date").child_value();
m_title = node.child("title").child_value();
m_url = node.child("url").child_value();
}
}

View File

@@ -1,4 +1,4 @@
/*
/*
base64.cpp and base64.h
Copyright (C) 2004-2008 René Nyffenegger
@@ -24,10 +24,10 @@
René Nyffenegger rene.nyffenegger@adp-gmbh.ch
*/
#include <tools/base64.h>
#include <common/base64.h>
#include <iostream>
static const std::string base64_chars =
static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
@@ -37,10 +37,8 @@ static inline bool is_base64(unsigned char c) {
return (isalnum(c) || (c == '+') || (c == '/'));
}
std::string base64_encode(const std::string& inString) {
std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) {
std::string ret;
auto in_len = inString.size();
const unsigned char* bytes_to_encode = reinterpret_cast<const unsigned char*>(inString.data());
int i = 0;
int j = 0;
unsigned char char_array_3[3];

160
src/common/networkTools.cpp Normal file
View File

@@ -0,0 +1,160 @@
/*
* Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <common/networkTools.h>
std::map<std::string, std::string> kiwix::getNetworkInterfaces()
{
std::map<std::string, std::string> interfaces;
#ifdef _WIN32
SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
if (sd == SOCKET_ERROR) {
std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
<< std::endl;
return interfaces;
}
INTERFACE_INFO InterfaceList[20];
unsigned long nBytesReturned;
if (WSAIoctl(sd,
SIO_GET_INTERFACE_LIST,
0,
0,
&InterfaceList,
sizeof(InterfaceList),
&nBytesReturned,
0,
0)
== SOCKET_ERROR) {
std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
<< std::endl;
return interfaces;
}
int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
for (int i = 0; i < nNumInterfaces; ++i) {
sockaddr_in* pAddress;
pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
/* Add to the map */
std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
interfaces.insert(
std::pair<std::string, std::string>(interfaceName, interfaceIp));
}
#else
/* Get Network interfaces information */
char buf[16384];
struct ifconf ifconf;
int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
ifconf.ifc_len = sizeof buf;
ifconf.ifc_buf = buf;
if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
perror("ioctl(SIOCGIFCONF)");
exit(EXIT_FAILURE);
}
/* Go through each interface */
int i;
size_t len;
struct ifreq* ifreq;
ifreq = ifconf.ifc_req;
for (i = 0; i < ifconf.ifc_len;) {
if (ifreq->ifr_addr.sa_family == AF_INET) {
/* Get the network interface ip */
char host[128] = {0};
const int error = getnameinfo(&(ifreq->ifr_addr),
sizeof ifreq->ifr_addr,
host,
sizeof host,
0,
0,
NI_NUMERICHOST);
if (!error) {
std::string interfaceName = std::string(ifreq->ifr_name);
std::string interfaceIp = std::string(host);
/* Add to the map */
interfaces.insert(
std::pair<std::string, std::string>(interfaceName, interfaceIp));
} else {
perror("getnameinfo()");
}
}
/* some systems have ifr_addr.sa_len and adjust the length that
* way, but not mine. weird */
#ifndef __linux__
len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
#else
len = sizeof *ifreq;
#endif
ifreq = (struct ifreq*)((char*)ifreq + len);
i += len;
}
#endif
return interfaces;
}
std::string kiwix::getBestPublicIp()
{
std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
#ifndef _WIN32
const char* const prioritizedNames[]
= {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
for (int i = 0; i < count; ++i) {
std::map<std::string, std::string>::const_iterator it
= interfaces.find(prioritizedNames[i]);
if (it != interfaces.end()) {
return it->second;
}
}
#endif
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
return interfaceIp;
}
}
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
return interfaceIp;
}
}
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
return interfaceIp;
}
}
return "127.0.0.1";
}

29
src/common/otherTools.cpp Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright 2014 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <common/otherTools.h>
void kiwix::sleep(unsigned int milliseconds)
{
#ifdef _WIN32
Sleep(milliseconds);
#else
usleep(1000 * milliseconds);
#endif
}

274
src/common/pathTools.cpp Normal file
View File

@@ -0,0 +1,274 @@
/*
* Copyright 2011-2014 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <common/pathTools.h>
#ifdef __APPLE__
#include <limits.h>
#include <mach-o/dyld.h>
#elif _WIN32
#include <direct.h>
#include <windows.h>
#include "shlwapi.h"
#define getcwd _getcwd // stupid MSFT "deprecation" warning
#endif
#ifdef _WIN32
#else
#include <unistd.h>
#endif
#ifdef _WIN32
#define SEPARATOR "\\"
#else
#define SEPARATOR "/"
#include <unistd.h>
#endif
#include <stdlib.h>
#ifndef PATH_MAX
#define PATH_MAX 1024
#endif
bool isRelativePath(const string& path)
{
#ifdef _WIN32
return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
#else
return path.empty() || path.substr(0, 1) == "/" ? false : true;
#endif
}
string computeRelativePath(const string path, const string absolutePath)
{
std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
std::vector<std::string> absolutePathParts
= kiwix::split(absolutePath, SEPARATOR);
unsigned int commonCount = 0;
while (commonCount < pathParts.size()
&& commonCount < absolutePathParts.size()
&& pathParts[commonCount] == absolutePathParts[commonCount]) {
if (!pathParts[commonCount].empty()) {
commonCount++;
}
}
string relativePath;
#ifdef _WIN32
/* On Windows you have a token more because the root is represented
by a letter */
if (commonCount == 0) {
relativePath = "../";
}
#endif
for (unsigned int i = commonCount; i < pathParts.size(); i++) {
relativePath += "../";
}
for (unsigned int i = commonCount; i < absolutePathParts.size(); i++) {
relativePath += absolutePathParts[i];
relativePath += i + 1 < absolutePathParts.size() ? "/" : "";
}
return relativePath;
}
/* Warning: the relative path must be with slashes */
string computeAbsolutePath(const string path, const string relativePath)
{
string absolutePath;
if (path.empty()) {
char* path = NULL;
size_t size = 0;
#ifdef _WIN32
path = _getcwd(path, size);
#else
path = getcwd(path, size);
#endif
absolutePath = string(path) + SEPARATOR;
} else {
absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
? path
: path + SEPARATOR;
}
#if _WIN32
char* cRelativePath = _strdup(relativePath.c_str());
#else
char* cRelativePath = strdup(relativePath.c_str());
#endif
char* token = strtok(cRelativePath, "/");
while (token != NULL) {
if (string(token) == "..") {
absolutePath = removeLastPathElement(absolutePath, true, false);
token = strtok(NULL, "/");
} else if (strcmp(token, ".") && strcmp(token, "")) {
absolutePath += string(token);
token = strtok(NULL, "/");
if (token != NULL) {
absolutePath += SEPARATOR;
}
} else {
token = strtok(NULL, "/");
}
}
return absolutePath;
}
string removeLastPathElement(const string path,
const bool removePreSeparator,
const bool removePostSeparator)
{
string newPath = path;
size_t offset = newPath.find_last_of(SEPARATOR);
if (removePreSeparator &&
#ifndef _WIN32
offset != newPath.find_first_of(SEPARATOR) &&
#endif
offset == newPath.length() - 1) {
newPath = newPath.substr(0, offset);
offset = newPath.find_last_of(SEPARATOR);
}
newPath = removePostSeparator ? newPath.substr(0, offset)
: newPath.substr(0, offset + 1);
return newPath;
}
string appendToDirectory(const string& directoryPath, const string& filename)
{
string newPath = directoryPath + SEPARATOR + filename;
return newPath;
}
string getLastPathElement(const string& path)
{
return path.substr(path.find_last_of(SEPARATOR) + 1);
}
unsigned int getFileSize(const string& path)
{
#ifdef _WIN32
struct _stat filestatus;
_stat(path.c_str(), &filestatus);
#else
struct stat filestatus;
stat(path.c_str(), &filestatus);
#endif
return filestatus.st_size / 1024;
}
string getFileSizeAsString(const string& path)
{
ostringstream convert;
convert << getFileSize(path);
return convert.str();
}
bool fileExists(const string& path)
{
#ifdef _WIN32
return PathFileExists(path.c_str());
#else
bool flag = false;
fstream fin;
fin.open(path.c_str(), ios::in);
if (fin.is_open()) {
flag = true;
}
fin.close();
return flag;
#endif
}
bool makeDirectory(const string& path)
{
#ifdef _WIN32
int status = _mkdir(path.c_str());
#else
int status = mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
#endif
return status == 0;
}
/* Try to create a link and if does not work then make a copy */
bool copyFile(const string& sourcePath, const string& destPath)
{
try {
#ifndef _WIN32
if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
#endif
std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
outfile << infile.rdbuf();
#ifndef _WIN32
}
#endif
} catch (exception& e) {
cerr << e.what() << endl;
return false;
}
return true;
}
string getExecutablePath()
{
char binRootPath[PATH_MAX];
#ifdef _WIN32
GetModuleFileName(NULL, binRootPath, PATH_MAX);
return std::string(binRootPath);
#elif __APPLE__
uint32_t max = (uint32_t)PATH_MAX;
_NSGetExecutablePath(binRootPath, &max);
return std::string(binRootPath);
#else
ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
if (size != -1) {
return std::string(binRootPath, size);
}
#endif
return "";
}
bool writeTextFile(const string& path, const string& content)
{
std::ofstream file;
file.open(path.c_str());
file << content;
file.close();
return true;
}
string getCurrentDirectory()
{
char* a_cwd = getcwd(NULL, 0);
string s_cwd(a_cwd);
free(a_cwd);
return s_cwd;
}

93
src/common/regexTools.cpp Normal file
View File

@@ -0,0 +1,93 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <common/regexTools.h>
std::map<std::string, RegexMatcher*> regexCache;
RegexMatcher* buildRegex(const std::string& regex)
{
RegexMatcher* matcher;
std::map<std::string, RegexMatcher*>::iterator itr = regexCache.find(regex);
/* Regex is in cache */
if (itr != regexCache.end()) {
matcher = itr->second;
}
/* Regex needs to be parsed (and cached) */
else {
UErrorCode status = U_ZERO_ERROR;
UnicodeString uregex = UnicodeString(regex.c_str());
matcher = new RegexMatcher(uregex, UREGEX_CASE_INSENSITIVE, status);
regexCache[regex] = matcher;
}
return matcher;
}
/* todo */
void freeRegexCache()
{
}
bool matchRegex(const std::string& content, const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
RegexMatcher* matcher = buildRegex(regex);
matcher->reset(ucontent);
return matcher->find();
}
std::string replaceRegex(const std::string& content,
const std::string& replacement,
const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
UnicodeString ureplacement = UnicodeString(replacement.c_str());
RegexMatcher* matcher = buildRegex(regex);
matcher->reset(ucontent);
UErrorCode status = U_ZERO_ERROR;
UnicodeString uresult = matcher->replaceAll(ureplacement, status);
std::string tmp;
uresult.toUTF8String(tmp);
return tmp;
}
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& replacement)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
UnicodeString ureplacement = UnicodeString(replacement.c_str());
RegexMatcher* matcher = buildRegex(regex);
matcher->reset(ucontent);
if (matcher->find()) {
UErrorCode status = U_ZERO_ERROR;
ucontent.insert(matcher->end(status), ureplacement);
std::string tmp;
ucontent.toUTF8String(tmp);
return tmp;
}
return content;
}

316
src/common/stringTools.cpp Normal file
View File

@@ -0,0 +1,316 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <common/stringTools.h>
#include <unicode/normlzr.h>
#include <unicode/rep.h>
#include <unicode/translit.h>
#include <unicode/ucnv.h>
#include <unicode/uniset.h>
#include <unicode/ustring.h>
/* tell ICU where to find its dat file (tables) */
void kiwix::loadICUExternalTables()
{
#ifdef __APPLE__
std::string executablePath = getExecutablePath();
std::string executableDirectory = removeLastPathElement(executablePath);
std::string datPath
= computeAbsolutePath(executableDirectory, "icudt49l.dat");
try {
u_setDataDirectory(datPath.c_str());
} catch (exception& e) {
std::cerr << e.what() << std::endl;
}
#endif
}
std::string kiwix::removeAccents(const std::string& text)
{
loadICUExternalTables();
ucnv_setDefaultName("UTF-8");
UErrorCode status = U_ZERO_ERROR;
Transliterator* removeAccentsTrans = Transliterator::createInstance(
"Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
UnicodeString ustring = UnicodeString(text.c_str());
removeAccentsTrans->transliterate(ustring);
delete removeAccentsTrans;
std::string unaccentedText;
ustring.toUTF8String(unaccentedText);
return unaccentedText;
}
#ifndef __ANDROID__
/* Prepare integer for display */
std::string kiwix::beautifyInteger(const unsigned int number)
{
std::stringstream numberStream;
numberStream << number;
std::string numberString = numberStream.str();
signed int offset = numberString.size() - 3;
while (offset > 0) {
numberString.insert(offset, ",");
offset -= 3;
}
return numberString;
}
std::string kiwix::beautifyFileSize(const unsigned int number)
{
if (number > 1024 * 1024) {
return kiwix::beautifyInteger(number / (1024 * 1024)) + " GB";
} else {
return kiwix::beautifyInteger(number / 1024 != 0 ? number / 1024 : 1)
+ " MB";
}
}
void kiwix::printStringInHexadecimal(UnicodeString s)
{
std::cout << std::showbase << std::hex;
for (int i = 0; i < s.length(); i++) {
char c = (char)((s.getTerminatedBuffer())[i]);
if (c & 0x80) {
std::cout << (c & 0xffff) << " ";
} else {
std::cout << c << " ";
}
}
std::cout << std::endl;
}
void kiwix::printStringInHexadecimal(const char* s)
{
std::cout << std::showbase << std::hex;
for (char const* pc = s; *pc; ++pc) {
if (*pc & 0x80) {
std::cout << (*pc & 0xffff);
} else {
std::cout << *pc;
}
std::cout << ' ';
}
std::cout << std::endl;
}
void kiwix::stringReplacement(std::string& str,
const std::string& oldStr,
const std::string& newStr)
{
size_t pos = 0;
while ((pos = str.find(oldStr, pos)) != std::string::npos) {
str.replace(pos, oldStr.length(), newStr);
pos += newStr.length();
}
}
/* Encode string to avoid XSS attacks */
std::string kiwix::encodeDiples(const std::string& str)
{
std::string result = str;
kiwix::stringReplacement(result, "<", "&lt;");
kiwix::stringReplacement(result, ">", "&gt;");
return result;
}
// Urlencode
// based on javascript encodeURIComponent()
std::string char2hex(char dec)
{
char dig1 = (dec & 0xF0) >> 4;
char dig2 = (dec & 0x0F);
if (0 <= dig1 && dig1 <= 9) {
dig1 += 48; // 0,48inascii
}
if (10 <= dig1 && dig1 <= 15) {
dig1 += 97 - 10; // a,97inascii
}
if (0 <= dig2 && dig2 <= 9) {
dig2 += 48;
}
if (10 <= dig2 && dig2 <= 15) {
dig2 += 97 - 10;
}
std::string r;
r.append(&dig1, 1);
r.append(&dig2, 1);
return r;
}
std::string kiwix::urlEncode(const std::string& c)
{
std::string escaped = "";
int max = c.length();
for (int i = 0; i < max; i++) {
if ((48 <= c[i] && c[i] <= 57) || // 0-9
(65 <= c[i] && c[i] <= 90)
|| // abc...xyz
(97 <= c[i] && c[i] <= 122)
|| // ABC...XYZ
(c[i] == '~' || c[i] == '!' || c[i] == '*' || c[i] == '(' || c[i] == ')'
|| c[i] == '\'')) {
escaped.append(&c[i], 1);
} else {
escaped.append("%");
escaped.append(char2hex(c[i])); // converts char 255 to string "ff"
}
}
return escaped;
}
#endif
static char charFromHex(std::string a)
{
std::istringstream Blat(a);
int Z;
Blat >> std::hex >> Z;
return char(Z);
}
std::string kiwix::urlDecode(const std::string& originalUrl)
{
std::string url = originalUrl;
std::string::size_type pos = 0;
while ((pos = url.find('%', pos)) != std::string::npos
&& pos + 2 < url.length()) {
url.replace(pos, 3, 1, charFromHex(url.substr(pos + 1, 2)));
++pos;
}
return url;
}
/* Split string in a token array */
std::vector<std::string> kiwix::split(const std::string& str,
const std::string& delims = " *-")
{
std::string::size_type lastPos = str.find_first_not_of(delims, 0);
std::string::size_type pos = str.find_first_of(delims, lastPos);
std::vector<std::string> tokens;
while (std::string::npos != pos || std::string::npos != lastPos) {
tokens.push_back(str.substr(lastPos, pos - lastPos));
lastPos = str.find_first_not_of(delims, pos);
pos = str.find_first_of(delims, lastPos);
}
return tokens;
}
std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
{
const std::string m1(lhs), m2(rhs);
return split(m1, m2);
}
std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
{
return split(lhs, rhs.c_str());
}
std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
{
return split(lhs.c_str(), rhs);
}
std::string kiwix::ucFirst(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
UnicodeString unicodeWord(word.c_str());
UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toUpper();
unicodeWord.replace(0, 1, unicodeFirstLetter);
unicodeWord.toUTF8String(result);
return result;
}
std::string kiwix::ucAll(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
UnicodeString unicodeWord(word.c_str());
unicodeWord.toUpper().toUTF8String(result);
return result;
}
std::string kiwix::lcFirst(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
UnicodeString unicodeWord(word.c_str());
UnicodeString unicodeFirstLetter = UnicodeString(unicodeWord, 0, 1).toLower();
unicodeWord.replace(0, 1, unicodeFirstLetter);
unicodeWord.toUTF8String(result);
return result;
}
std::string kiwix::lcAll(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
UnicodeString unicodeWord(word.c_str());
unicodeWord.toLower().toUTF8String(result);
return result;
}
std::string kiwix::toTitle(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
UnicodeString unicodeWord(word.c_str());
unicodeWord = unicodeWord.toTitle(0);
unicodeWord.toUTF8String(result);
return result;
}
std::string kiwix::normalize(const std::string& word)
{
return kiwix::lcAll(word);
}

View File

@@ -1,3 +1,4 @@
#mesondefine LIBKIWIX_VERSION
#mesondefine VERSION
#mesondefine ENABLE_CTPP2

View File

@@ -0,0 +1,210 @@
/*
* Copyright 2013 Renaud Gaudin <reg@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <ctpp2/CTPP2VMStringLoader.hpp>
namespace CTPP // C++ Template Engine
{
//
// Convert byte order
//
static void ConvertExecutable(VMExecutable * oCore)
{
// Code entry point
oCore -> entry_point = Swap32(oCore -> entry_point);
// Offset of code segment
oCore -> code_offset = Swap32(oCore -> code_offset);
// Code segment size
oCore -> code_size = Swap32(oCore -> code_size);
// Offset of static text segment
oCore -> syscalls_offset = Swap32(oCore -> syscalls_offset);
// Static text segment size
oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size);
// Offset of static text index segment
oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset);
// Static text index segment size
oCore -> syscalls_index_size = Swap32(oCore -> syscalls_index_size);
// Offset of static data segment
oCore -> static_data_offset = Swap32(oCore -> static_data_offset);
// Static data segment size
oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size);
// Offset of static text segment
oCore -> static_text_offset = Swap32(oCore -> static_text_offset);
// Static text segment size
oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size);
// Offset of static text index segment
oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset);
// Static text index segment size
oCore -> static_text_index_size = Swap32(oCore -> static_text_index_size);
// Version 2.2+
// Offset of static data bit index
oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset);
/// Offset of static data bit index
oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size);
// Platform
oCore -> platform = Swap64(oCore -> platform);
// Ugly-jolly hack!
// ... dereferencing type-punned pointer will break strict-aliasing rules ...
UINT_64 iTMP;
memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64));
iTMP = Swap64(iTMP);
memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64));
// Cyclic Redundancy Check
oCore -> crc = 0;
// Convert data structures
// Convert code segment
VMInstruction * pInstructions = const_cast<VMInstruction *>(VMExecutable::GetCodeSeg(oCore));
UINT_32 iI = 0;
UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction);
for(iI = 0; iI < iSteps; ++iI)
{
pInstructions -> instruction = Swap32(pInstructions -> instruction);
pInstructions -> argument = Swap32(pInstructions -> argument);
pInstructions -> reserved = Swap64(pInstructions -> reserved);
++pInstructions;
}
// Convert syscalls index
TextDataIndex * pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetSyscallsIndexSeg(oCore));
iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex);
for(iI = 0; iI < iSteps; ++iI)
{
pTextIndex -> offset = Swap32(pTextIndex -> offset);
pTextIndex -> length = Swap32(pTextIndex -> length);
++pTextIndex;
}
// Convert static text index
pTextIndex = const_cast<TextDataIndex *>(VMExecutable::GetStaticTextIndexSeg(oCore));
iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex);
for(iI = 0; iI < iSteps; ++iI)
{
pTextIndex -> offset = Swap32(pTextIndex -> offset);
pTextIndex -> length = Swap32(pTextIndex -> length);
++pTextIndex;
}
// Convert static data
StaticDataVar * pStaticDataVar = const_cast<StaticDataVar *>(VMExecutable::GetStaticDataSeg(oCore));
iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar);
for(iI = 0; iI < iSteps; ++iI)
{
(*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data);
++pStaticDataVar;
}
}
//
// Constructor
//
VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize)
{
oCore = (VMExecutable *)malloc(rawContentSize + 1);
memcpy(oCore, rawContent, rawContentSize);
if (oCore -> magic[0] == 'C' &&
oCore -> magic[1] == 'T' &&
oCore -> magic[2] == 'P' &&
oCore -> magic[3] == 'P')
{
// Check version
if (oCore -> version[0] >= 1)
{
// Platform-dependent data (byte order)
if (oCore -> platform == 0x4142434445464748ull)
{
#ifdef _DEBUG
fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n");
#endif
// Nothing to do, only check crc
UINT_32 iCRC = oCore -> crc;
oCore -> crc = 0;
// Calculate CRC of file
// KELSON: next line used to refer to oStat.st_size
// changed it to rawContentSize
if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize))
{
free(oCore);
throw CTPPLogicError("CRC checksum invalid");
}
}
// Platform-dependent data (byte order)
else if (oCore -> platform == 0x4847464544434241ull)
{
// Need to reconvert data
#ifdef _DEBUG
fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n");
#endif
ConvertExecutable(oCore);
}
else
{
free(oCore);
throw CTPPLogicError("Conversion of middle-end architecture does not supported.");
}
// Check IEEE 754 format
if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0)
{
free(oCore);
throw CTPPLogicError("IEEE 754 format is broken, cannot convert file");
}
}
pVMMemoryCore = new VMMemoryCore(oCore);
}
else
{
free(oCore);
throw CTPPLogicError("Not an CTPP bytecode file.");
}
}
//
// Get ready-to-run program
//
const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; }
//
// A destructor
//
VMStringLoader::~VMStringLoader() throw()
{
delete pVMMemoryCore;
free(oCore);
}
} // namespace CTPP
// End.

View File

@@ -1,204 +0,0 @@
/*
* Copyright 2018 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "downloader.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
#include <algorithm>
#include <thread>
#include <chrono>
#include <iostream>
#include "aria2.h"
#include "xmlrpc.h"
#include "tools/otherTools.h"
#include <pugixml.hpp>
namespace kiwix
{
void Download::updateStatus(bool follow)
{
if (m_status == Download::K_REMOVED)
return;
static std::vector<std::string> statusKey = {"status", "files", "totalLength",
"completedLength", "followedBy",
"downloadSpeed", "verifiedLength"};
std::string strStatus;
if(follow && !m_followedBy.empty()) {
strStatus = mp_aria->tellStatus(m_followedBy, statusKey);
} else {
strStatus = mp_aria->tellStatus(m_did, statusKey);
}
// std::cout << strStatus << std::endl;
MethodResponse response(strStatus);
if (response.isFault()) {
m_status = Download::K_UNKNOWN;
return;
}
auto structNode = response.getParams().getParam(0).getValue().getStruct();
auto _status = structNode.getMember("status").getValue().getAsS();
auto status = _status == "active" ? Download::K_ACTIVE
: _status == "waiting" ? Download::K_WAITING
: _status == "paused" ? Download::K_PAUSED
: _status == "error" ? Download::K_ERROR
: _status == "complete" ? Download::K_COMPLETE
: _status == "removed" ? Download::K_REMOVED
: Download::K_UNKNOWN;
if (status == K_COMPLETE) {
try {
auto followedByMember = structNode.getMember("followedBy");
m_followedBy = followedByMember.getValue().getArray().getValue(0).getAsS();
if (follow) {
status = K_ACTIVE;
updateStatus(true);
return;
}
} catch (InvalidRPCNode& e) { }
}
m_status = status;
m_totalLength = extractFromString<uint64_t>(structNode.getMember("totalLength").getValue().getAsS());
m_completedLength = extractFromString<uint64_t>(structNode.getMember("completedLength").getValue().getAsS());
m_downloadSpeed = extractFromString<uint64_t>(structNode.getMember("downloadSpeed").getValue().getAsS());
try {
auto verifiedLengthValue = structNode.getMember("verifiedLength").getValue();
m_verifiedLength = extractFromString<uint64_t>(verifiedLengthValue.getAsS());
} catch (InvalidRPCNode& e) { m_verifiedLength = 0; }
auto filesMember = structNode.getMember("files");
auto fileStruct = filesMember.getValue().getArray().getValue(0).getStruct();
m_path = fileStruct.getMember("path").getValue().getAsS();
auto urisArray = fileStruct.getMember("uris").getValue().getArray();
int index = 0;
m_uris.clear();
while(true) {
try {
auto uriNode = urisArray.getValue(index++).getStruct().getMember("uri");
m_uris.push_back(uriNode.getValue().getAsS());
} catch(InvalidRPCNode& e) { break; }
}
}
void Download::resumeDownload()
{
if (!m_followedBy.empty())
mp_aria->unpause(m_followedBy);
else
mp_aria->unpause(m_did);
updateStatus(true);
}
void Download::pauseDownload()
{
if (!m_followedBy.empty())
mp_aria->pause(m_followedBy);
else
mp_aria->pause(m_did);
updateStatus(true);
}
void Download::cancelDownload()
{
if (!m_followedBy.empty())
mp_aria->remove(m_followedBy);
else
mp_aria->remove(m_did);
m_status = Download::K_REMOVED;
}
/* Constructor */
Downloader::Downloader() :
mp_aria(new Aria2())
{
try {
for (auto gid : mp_aria->tellActive()) {
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
m_knownDownloads[gid]->updateStatus();
}
} catch (std::exception& e) {
std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
}
try {
for (auto gid : mp_aria->tellWaiting()) {
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
m_knownDownloads[gid]->updateStatus();
}
} catch (std::exception& e) {
std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
}
}
/* Destructor */
Downloader::~Downloader()
{
}
void Downloader::close()
{
mp_aria->close();
}
std::vector<std::string> Downloader::getDownloadIds() {
std::vector<std::string> ret;
for(auto& p:m_knownDownloads) {
ret.push_back(p.first);
}
return ret;
}
Download* Downloader::startDownload(const std::string& uri, const std::vector<std::pair<std::string, std::string>>& options)
{
for (auto& p: m_knownDownloads) {
auto& d = p.second;
auto& uris = d->getUris();
if (std::find(uris.begin(), uris.end(), uri) != uris.end())
return d.get();
}
std::vector<std::string> uris = {uri};
auto gid = mp_aria->addUri(uris, options);
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
return m_knownDownloads[gid].get();
}
Download* Downloader::getDownload(const std::string& did)
{
try {
m_knownDownloads.at(did).get()->updateStatus(true);
return m_knownDownloads.at(did).get();
} catch(std::exception& e) {
for (auto gid : mp_aria->tellActive()) {
if (gid == did) {
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
m_knownDownloads.at(gid).get()->updateStatus(true);
return m_knownDownloads[gid].get();
}
}
for (auto gid : mp_aria->tellWaiting()) {
if (gid == did) {
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
m_knownDownloads.at(gid).get()->updateStatus(true);
return m_knownDownloads[gid].get();
}
}
throw e;
}
}
}

View File

@@ -1,82 +0,0 @@
#include "kiwixserve.h"
#include "subprocess.h"
#ifdef _WIN32
# define KIWIXSERVE_CMD "kiwix-serve.exe"
# include <windows.h>
#else
# define KIWIXSERVE_CMD "kiwix-serve"
# include <unistd.h>
#endif
#include "tools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
namespace kiwix {
KiwixServe::KiwixServe(const std::string& libraryPath, int port)
: m_port(port),
m_libraryPath(libraryPath)
{
}
KiwixServe::~KiwixServe()
{
shutDown();
}
void KiwixServe::run()
{
#ifdef _WIN32
int pid = GetCurrentProcessId();
#else
pid_t pid = getpid();
#endif
std::vector<const char*> callCmd;
std::string kiwixServeCmd = appendToDirectory(
removeLastPathElement(getExecutablePath(true)),
KIWIXSERVE_CMD);
if (fileExists(kiwixServeCmd)) {
// A local kiwix-serve exe exists (packaged with kiwix-desktop), use it.
callCmd.push_back(kiwixServeCmd.c_str());
} else {
// Try to use a potential installed kiwix-serve.
callCmd.push_back(KIWIXSERVE_CMD);
}
std::string attachProcessOpt = "-a" + to_string(pid);
std::string portOpt = "-p" + to_string(m_port);
callCmd.push_back(attachProcessOpt.c_str());
callCmd.push_back(portOpt.c_str());
callCmd.push_back("-l");
callCmd.push_back(m_libraryPath.c_str());
mp_kiwixServe = Subprocess::run(callCmd);
}
void KiwixServe::shutDown()
{
if (mp_kiwixServe)
mp_kiwixServe->kill();
}
bool KiwixServe::isRunning()
{
if (mp_kiwixServe) {
return (mp_kiwixServe->isRunning());
}
return false;
}
int KiwixServe::setPort(int port)
{
if (port >= 1 && port <= 65535) {
m_port = port;
} else {
return -1;
}
return m_port;
}
}

View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,145 +0,0 @@
/*
* Copyright 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "libxml_dumper.h"
#include "book.h"
#include "tools.h"
#include "tools/base64.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
namespace kiwix
{
/* Constructor */
LibXMLDumper::LibXMLDumper(const Library* library)
: library(library)
{
}
/* Destructor */
LibXMLDumper::~LibXMLDumper()
{
}
#define ADD_ATTRIBUTE(node, name, value) { (node).append_attribute((name)) = (value).c_str(); }
#define ADD_ATTR_NOT_EMPTY(node, name, value) { if (!(value).empty()) ADD_ATTRIBUTE(node, name, value); }
void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
if (book.readOnly())
return;
auto entry_node = root_node.append_child("book");
ADD_ATTRIBUTE(entry_node, "id", book.getId());
if (!book.getPath().empty()) {
ADD_ATTRIBUTE(entry_node, "path", computeRelativePath(baseDir, book.getPath()));
}
if (book.getOrigId().empty()) {
ADD_ATTR_NOT_EMPTY(entry_node, "title", book.getTitle());
ADD_ATTR_NOT_EMPTY(entry_node, "description", book.getDescription());
ADD_ATTR_NOT_EMPTY(entry_node, "language", book.getLanguage());
ADD_ATTR_NOT_EMPTY(entry_node, "creator", book.getCreator());
ADD_ATTR_NOT_EMPTY(entry_node, "publisher", book.getPublisher());
ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
try {
auto defaultIllustration = book.getIllustration(48);
ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
if (!defaultIllustration->getData().empty())
ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
} catch(...) {}
} else {
ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
}
ADD_ATTR_NOT_EMPTY(entry_node, "date", book.getDate());
ADD_ATTR_NOT_EMPTY(entry_node, "url", book.getUrl());
if (book.getArticleCount())
ADD_ATTRIBUTE(entry_node, "articleCount", to_string(book.getArticleCount()));
if (book.getMediaCount())
ADD_ATTRIBUTE(entry_node, "mediaCount", to_string(book.getMediaCount()));
if (book.getSize())
ADD_ATTRIBUTE(entry_node, "size", to_string(book.getSize()>>10));
ADD_ATTR_NOT_EMPTY(entry_node, "downloadId", book.getDownloadId());
}
#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str())
void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
auto entry_node = root_node.append_child("bookmark");
auto book_node = entry_node.append_child("book");
try {
auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
ADD_TEXT_ENTRY(book_node, "id", book.getId());
ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());
ADD_TEXT_ENTRY(book_node, "date", book.getDate());
} catch (...) {
ADD_TEXT_ENTRY(book_node, "id", bookmark.getBookId());
ADD_TEXT_ENTRY(book_node, "title", bookmark.getBookTitle());
ADD_TEXT_ENTRY(book_node, "language", bookmark.getLanguage());
ADD_TEXT_ENTRY(book_node, "date", bookmark.getDate());
}
ADD_TEXT_ENTRY(entry_node, "title", bookmark.getTitle());
ADD_TEXT_ENTRY(entry_node, "url", bookmark.getUrl());
}
std::string LibXMLDumper::dumpLibXMLContent(const std::vector<std::string>& bookIds)
{
pugi::xml_document doc;
/* Add the library node */
pugi::xml_node libraryNode = doc.append_child("library");
libraryNode.append_attribute("version") = KIWIX_LIBRARY_VERSION;
if (library) {
for (auto& bookId: bookIds) {
handleBook(library->getBookById(bookId), libraryNode);
}
}
return nodeToString(libraryNode);
}
std::string LibXMLDumper::dumpLibXMLBookmark()
{
pugi::xml_document doc;
/* Add the library node */
pugi::xml_node bookmarksNode = doc.append_child("bookmarks");
if (library) {
for (auto& bookmark: library->getBookmarks()) {
handleBookmark(bookmark, bookmarksNode);
}
}
return nodeToString(bookmarksNode);
}
}

View File

@@ -19,191 +19,105 @@
#include "manager.h"
#include "tools.h"
#include "tools/pathTools.h"
#include <pugixml.hpp>
namespace kiwix
{
namespace
{
struct NoDelete
{
template<class T> void operator()(T*) {}
};
} // unnamed namespace
////////////////////////////////////////////////////////////////////////////////
// LibraryManipulator
////////////////////////////////////////////////////////////////////////////////
LibraryManipulator::LibraryManipulator(Library* library)
: library(*library)
{}
LibraryManipulator::~LibraryManipulator()
{}
bool LibraryManipulator::addBookToLibrary(const Book& book)
{
const auto ret = library.addBook(book);
if ( ret ) {
bookWasAddedToLibrary(book);
}
return ret;
}
void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
{
library.addBookmark(bookmark);
bookmarkWasAddedToLibrary(bookmark);
}
uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
{
const auto n = library.removeBooksNotUpdatedSince(rev);
if ( n != 0 ) {
booksWereRemovedFromLibrary();
}
return n;
}
void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
{
}
void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
{
}
void LibraryManipulator::booksWereRemovedFromLibrary()
{
}
////////////////////////////////////////////////////////////////////////////////
// Manager
////////////////////////////////////////////////////////////////////////////////
/* Constructor */
Manager::Manager(LibraryManipulator* manipulator):
writableLibraryPath(""),
manipulator(manipulator, NoDelete())
Manager::Manager() : writableLibraryPath("")
{
}
Manager::Manager(Library* library) :
writableLibraryPath(""),
manipulator(new LibraryManipulator(library))
/* Destructor */
Manager::~Manager()
{
}
bool Manager::parseXmlDom(const pugi::xml_document& doc,
bool readOnly,
const std::string& libraryPath,
bool trustLibrary)
const bool readOnly,
const string libraryPath)
{
pugi::xml_node libraryNode = doc.child("library");
std::string libraryVersion = libraryNode.attribute("version").value();
if (strlen(libraryNode.attribute("current").value()))
this->setCurrentBookId(libraryNode.attribute("current").value());
string libraryVersion = libraryNode.attribute("version").value();
for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode;
bookNode = bookNode.next_sibling("book")) {
bool ok = true;
kiwix::Book book;
book.setReadOnly(readOnly);
book.updateFromXml(bookNode,
removeLastPathElement(libraryPath));
book.readOnly = readOnly;
book.id = bookNode.attribute("id").value();
book.path = bookNode.attribute("path").value();
book.last = (std::string(bookNode.attribute("last").value()) != "undefined"
? bookNode.attribute("last").value()
: "");
book.indexPath = bookNode.attribute("indexPath").value();
book.indexType = XAPIAN;
book.title = bookNode.attribute("title").value();
book.name = bookNode.attribute("name").value();
book.tags = bookNode.attribute("tags").value();
book.description = bookNode.attribute("description").value();
book.language = bookNode.attribute("language").value();
book.date = bookNode.attribute("date").value();
book.creator = bookNode.attribute("creator").value();
book.publisher = bookNode.attribute("publisher").value();
book.url = bookNode.attribute("url").value();
book.origId = bookNode.attribute("origId").value();
book.articleCount = bookNode.attribute("articleCount").value();
book.mediaCount = bookNode.attribute("mediaCount").value();
book.size = bookNode.attribute("size").value();
book.favicon = bookNode.attribute("favicon").value();
book.faviconMimeType = bookNode.attribute("faviconMimeType").value();
if (!trustLibrary && !book.getPath().empty()) {
this->readBookFromPath(book.getPath(), &book);
}
manipulator->addBookToLibrary(book);
}
return true;
}
bool Manager::readXml(const std::string& xml,
bool readOnly,
const std::string& libraryPath,
bool trustLibrary)
{
pugi::xml_document doc;
pugi::xml_parse_result result
= doc.load_buffer((void*)xml.data(), xml.size());
if (result) {
this->parseXmlDom(doc, readOnly, libraryPath, trustLibrary);
}
return true;
}
bool Manager::parseOpdsDom(const pugi::xml_document& doc, const std::string& urlHost)
{
pugi::xml_node libraryNode = doc.child("feed");
try {
m_totalBooks = strtoull(libraryNode.child("totalResults").child_value(), 0, 0);
m_startIndex = strtoull(libraryNode.child("startIndex").child_value(), 0, 0);
m_itemsPerPage = strtoull(libraryNode.child("itemsPerPage").child_value(), 0, 0);
m_hasSearchResult = true;
} catch(...) {
m_hasSearchResult = false;
}
for (pugi::xml_node entryNode = libraryNode.child("entry"); entryNode;
entryNode = entryNode.next_sibling("entry")) {
kiwix::Book book;
book.setReadOnly(false);
book.updateFromOpds(entryNode, urlHost);
/* Check absolute and relative paths */
this->checkAndCleanBookPaths(book, libraryPath);
/* Update the book properties with the new importer */
manipulator->addBookToLibrary(book);
if (libraryVersion.empty()
|| atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
if (!book.path.empty()) {
ok = this->readBookFromPath(book.pathAbsolute);
}
}
if (ok) {
library.addBook(book);
}
}
return true;
}
bool Manager::readOpds(const std::string& content, const std::string& urlHost)
bool Manager::readXml(const string xml,
const bool readOnly,
const string libraryPath)
{
pugi::xml_document doc;
pugi::xml_parse_result result
= doc.load_buffer((void*)content.data(), content.size());
= doc.load_buffer_inplace((void*)xml.data(), xml.size());
if (result) {
this->parseOpdsDom(doc, urlHost);
return true;
this->parseXmlDom(doc, readOnly, libraryPath);
}
return false;
return true;
}
bool Manager::readFile(
const std::string& path,
bool readOnly,
bool trustLibrary)
bool Manager::readFile(const string path, const bool readOnly)
{
return this->readFile(path, path, readOnly);
}
bool Manager::readFile(const string nativePath,
const string UTF8Path,
const bool readOnly)
{
bool retVal = true;
pugi::xml_document doc;
#ifdef _WIN32
pugi::xml_parse_result result = doc.load_file(Utf8ToWide(path).c_str());
#else
pugi::xml_parse_result result = doc.load_file(path.c_str());
#endif
pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
if (result) {
this->parseXmlDom(doc, readOnly, path, trustLibrary);
this->parseXmlDom(doc, readOnly, UTF8Path);
} else {
retVal = false;
}
@@ -212,37 +126,154 @@ bool Manager::readFile(
* able to know where to save the library if new content are
* available */
if (!readOnly) {
this->writableLibraryPath = path;
this->writableLibraryPath = UTF8Path;
}
return retVal;
}
bool Manager::writeFile(const string path)
{
pugi::xml_document doc;
/* Add the library node */
pugi::xml_node libraryNode = doc.append_child("library");
if (!getCurrentBookId().empty()) {
libraryNode.append_attribute("current") = getCurrentBookId().c_str();
}
if (!library.version.empty())
libraryNode.append_attribute("version") = library.version.c_str();
/* Add each book */
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (!itr->readOnly) {
this->checkAndCleanBookPaths(*itr, path);
pugi::xml_node bookNode = libraryNode.append_child("book");
bookNode.append_attribute("id") = itr->id.c_str();
if (!itr->path.empty()) {
bookNode.append_attribute("path") = itr->path.c_str();
}
if (!itr->last.empty() && itr->last != "undefined") {
bookNode.append_attribute("last") = itr->last.c_str();
}
if (!itr->indexPath.empty())
bookNode.append_attribute("indexPath") = itr->indexPath.c_str();
if (!itr->indexPath.empty() || !itr->indexPathAbsolute.empty()) {
if (itr->indexType == XAPIAN) {
bookNode.append_attribute("indexType") = "xapian";
}
}
if (itr->origId.empty()) {
if (!itr->title.empty())
bookNode.append_attribute("title") = itr->title.c_str();
if (!itr->name.empty())
bookNode.append_attribute("name") = itr->name.c_str();
if (!itr->tags.empty())
bookNode.append_attribute("tags") = itr->tags.c_str();
if (!itr->description.empty())
bookNode.append_attribute("description") = itr->description.c_str();
if (!itr->language.empty())
bookNode.append_attribute("language") = itr->language.c_str();
if (!itr->creator.empty())
bookNode.append_attribute("creator") = itr->creator.c_str();
if (!itr->publisher.empty())
bookNode.append_attribute("publisher") = itr->publisher.c_str();
if (!itr->favicon.empty())
bookNode.append_attribute("favicon") = itr->favicon.c_str();
if (!itr->faviconMimeType.empty())
bookNode.append_attribute("faviconMimeType")
= itr->faviconMimeType.c_str();
}
if (!itr->date.empty()) {
bookNode.append_attribute("date") = itr->date.c_str();
}
if (!itr->url.empty()) {
bookNode.append_attribute("url") = itr->url.c_str();
}
if (!itr->origId.empty())
bookNode.append_attribute("origId") = itr->origId.c_str();
if (!itr->articleCount.empty())
bookNode.append_attribute("articleCount") = itr->articleCount.c_str();
if (!itr->mediaCount.empty())
bookNode.append_attribute("mediaCount") = itr->mediaCount.c_str();
if (!itr->size.empty()) {
bookNode.append_attribute("size") = itr->size.c_str();
}
}
}
/* saving file */
doc.save_file(path.c_str());
return true;
}
bool Manager::setCurrentBookId(const string id)
{
if (library.current.empty() || library.current.top() != id) {
if (id.empty() && !library.current.empty()) {
library.current.pop();
} else {
library.current.push(id);
}
}
return true;
}
string Manager::getCurrentBookId()
{
return library.current.empty() ? "" : library.current.top();
}
/* Add a book to the library. Return empty string if failed, book id otherwise
*/
std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
const std::string& pathToSave,
const std::string& url,
const bool checkMetaData)
string Manager::addBookFromPathAndGetId(const string pathToOpen,
const string pathToSave,
const string url,
const bool checkMetaData)
{
kiwix::Book book;
if (this->readBookFromPath(pathToOpen, &book)) {
if (!pathToSave.empty() && pathToSave != pathToOpen) {
book.setPath(isRelativePath(pathToSave)
if (pathToSave != pathToOpen) {
book.path = pathToSave;
book.pathAbsolute
= isRelativePath(pathToSave)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath),
removeLastPathElement(writableLibraryPath, true, false),
pathToSave)
: pathToSave);
: pathToSave;
}
if (!checkMetaData
|| (checkMetaData && !book.getTitle().empty() && !book.getLanguage().empty()
&& !book.getDate().empty())) {
book.setUrl(url);
manipulator->addBookToLibrary(book);
return book.getId();
|| (checkMetaData && !book.title.empty() && !book.language.empty()
&& !book.date.empty())) {
book.url = url;
library.addBook(book);
return book.id;
}
}
@@ -251,9 +282,9 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
/* Wrapper over Manager::addBookFromPath which return a bool instead of a string
*/
bool Manager::addBookFromPath(const std::string& pathToOpen,
const std::string& pathToSave,
const std::string& url,
bool Manager::addBookFromPath(const string pathToOpen,
const string pathToSave,
const string url,
const bool checkMetaData)
{
return !(
@@ -261,62 +292,367 @@ bool Manager::addBookFromPath(const std::string& pathToOpen,
.empty());
}
bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
bool Manager::readBookFromPath(const string path, kiwix::Book* book)
{
std::string tmp_path = path;
if (isRelativePath(path)) {
tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
}
try {
zim::Archive archive(tmp_path);
book->update(archive);
book->setPathValid(true);
kiwix::Reader* reader = new kiwix::Reader(path);
if (book != NULL) {
book->path = path;
book->pathAbsolute = path;
book->id = reader->getId();
book->description = reader->getDescription();
book->language = reader->getLanguage();
book->date = reader->getDate();
book->creator = reader->getCreator();
book->publisher = reader->getPublisher();
book->title = reader->getTitle();
book->name = reader->getName();
book->tags = reader->getTags();
book->origId = reader->getOrigId();
std::ostringstream articleCountStream;
articleCountStream << reader->getArticleCount();
book->articleCount = articleCountStream.str();
std::ostringstream mediaCountStream;
mediaCountStream << reader->getMediaCount();
book->mediaCount = mediaCountStream.str();
ostringstream convert;
convert << reader->getFileSize();
book->size = convert.str();
string favicon;
string faviconMimeType;
if (reader->getFavicon(favicon, faviconMimeType)) {
book->favicon = base64_encode(
reinterpret_cast<const unsigned char*>(favicon.c_str()),
favicon.length());
book->faviconMimeType = faviconMimeType;
}
}
delete reader;
} catch (const std::exception& e) {
book->setPathValid(false);
std::cerr << e.what() << std::endl;
return false;
}
return true;
}
bool Manager::readBookmarkFile(const std::string& path)
bool Manager::removeBookByIndex(const unsigned int bookIndex)
{
pugi::xml_document doc;
pugi::xml_parse_result result = doc.load_file(path.c_str());
if (!result) {
return false;
}
pugi::xml_node libraryNode = doc.child("bookmarks");
for (pugi::xml_node node = libraryNode.child("bookmark"); node;
node = node.next_sibling("bookmark")) {
kiwix::Bookmark bookmark;
bookmark.updateFromXml(node);
manipulator->addBookmarkToLibrary(bookmark);
}
return true;
return this->library.removeBookByIndex(bookIndex);
}
void Manager::reload(const Paths& paths)
bool Manager::removeBookById(const string id)
{
const auto libRevision = manipulator->getLibrary().getRevision();
for (std::string path : paths) {
if (!path.empty()) {
if ( kiwix::isRelativePath(path) )
path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
unsigned int bookIndex = 0;
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
return this->library.removeBookByIndex(bookIndex);
}
bookIndex++;
}
return false;
}
if (!readFile(path, false, true)) {
throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
vector<string> Manager::getBooksLanguages()
{
std::vector<string> booksLanguages;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksLanguagesMap;
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksLanguagesMap.find(itr->language) == booksLanguagesMap.end()) {
if (itr->origId.empty()) {
booksLanguagesMap[itr->language] = true;
booksLanguages.push_back(itr->language);
}
}
}
manipulator->removeBooksNotUpdatedSince(libRevision);
return booksLanguages;
}
vector<string> Manager::getBooksCreators()
{
std::vector<string> booksCreators;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksCreatorsMap;
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksCreatorsMap.find(itr->creator) == booksCreatorsMap.end()) {
if (itr->origId.empty()) {
booksCreatorsMap[itr->creator] = true;
booksCreators.push_back(itr->creator);
}
}
}
return booksCreators;
}
vector<string> Manager::getBooksIds()
{
std::vector<string> booksIds;
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
booksIds.push_back(itr->id);
}
return booksIds;
}
vector<string> Manager::getBooksPublishers()
{
std::vector<string> booksPublishers;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksPublishersMap;
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksPublishersMap.find(itr->publisher) == booksPublishersMap.end()) {
if (itr->origId.empty()) {
booksPublishersMap[itr->publisher] = true;
booksPublishers.push_back(itr->publisher);
}
}
}
return booksPublishers;
}
kiwix::Library Manager::cloneLibrary()
{
return this->library;
}
bool Manager::getCurrentBook(Book& book)
{
string currentBookId = getCurrentBookId();
if (currentBookId.empty()) {
return false;
} else {
getBookById(currentBookId, book);
return true;
}
}
bool Manager::getBookById(const string id, Book& book)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
book = *itr;
return true;
}
}
return false;
}
bool Manager::updateBookLastOpenDateById(const string id)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
char unixdate[12];
sprintf(unixdate, "%d", (int)time(NULL));
itr->last = unixdate;
return true;
}
}
return false;
}
bool Manager::setBookIndex(const string id,
const string path,
const supportedIndexType type)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
itr->indexPath = path;
itr->indexPathAbsolute
= isRelativePath(path)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath, true, false),
path)
: path;
itr->indexType = type;
return true;
}
}
return false;
}
bool Manager::setBookIndex(const string id, const string path)
{
return this->setBookIndex(id, path, XAPIAN);
}
bool Manager::setBookPath(const string id, const string path)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
itr->path = path;
itr->pathAbsolute
= isRelativePath(path)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath, true, false),
path)
: path;
return true;
}
}
return false;
}
void Manager::removeBookPaths()
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
itr->path = "";
itr->pathAbsolute = "";
}
}
unsigned int Manager::getBookCount(const bool localBooks,
const bool remoteBooks)
{
unsigned int result = 0;
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if ((!itr->path.empty() && localBooks)
|| (itr->path.empty() && remoteBooks)) {
result++;
}
}
return result;
}
bool Manager::listBooks(const supportedListMode mode,
const supportedListSortBy sortBy,
const unsigned int maxSize,
const string language,
const string creator,
const string publisher,
const string search)
{
this->bookIdList.clear();
std::vector<kiwix::Book>::iterator itr;
/* Sort */
if (sortBy == TITLE) {
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
} else if (sortBy == SIZE) {
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
} else if (sortBy == DATE) {
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
} else if (sortBy == CREATOR) {
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
} else if (sortBy == PUBLISHER) {
std::sort(library.books.begin(),
library.books.end(),
kiwix::Book::sortByPublisher);
}
/* Special sort for LASTOPEN */
if (mode == LASTOPEN) {
std::sort(library.books.begin(),
library.books.end(),
kiwix::Book::sortByLastOpen);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (!itr->last.empty()) {
this->bookIdList.push_back(itr->id);
}
}
} else {
/* Generate the list of book id */
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
bool ok = true;
if (mode == LOCAL && itr->path.empty()) {
ok = false;
}
if (ok == true && mode == REMOTE
&& (!itr->path.empty() || itr->url.empty())) {
ok = false;
}
if (ok == true && maxSize != 0
&& (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024) {
ok = false;
}
if (ok == true && !language.empty()
&& !matchRegex(itr->language, language)) {
ok = false;
}
if (ok == true && !creator.empty() && itr->creator != creator) {
ok = false;
}
if (ok == true && !publisher.empty() && itr->publisher != publisher) {
ok = false;
}
if ((ok == true && !search.empty())
&& !(matchRegex(itr->title, "\\Q" + search + "\\E")
|| matchRegex(itr->description, "\\Q" + search + "\\E")
|| matchRegex(itr->language, "\\Q" + search + "\\E"))) {
ok = false;
}
if (ok == true) {
this->bookIdList.push_back(itr->id);
}
}
}
return true;
}
void Manager::checkAndCleanBookPaths(Book& book, const string& libraryPath)
{
if (!book.path.empty()) {
if (isRelativePath(book.path)) {
book.pathAbsolute = computeAbsolutePath(
removeLastPathElement(libraryPath, true, false), book.path);
} else {
book.pathAbsolute = book.path;
book.path = computeRelativePath(
removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
}
}
if (!book.indexPath.empty()) {
if (isRelativePath(book.indexPath)) {
book.indexPathAbsolute = computeAbsolutePath(
removeLastPathElement(libraryPath, true, false), book.indexPath);
} else {
book.indexPathAbsolute = book.indexPath;
book.indexPath
= computeRelativePath(removeLastPathElement(libraryPath, true, false),
book.indexPathAbsolute);
}
}
}
}

View File

@@ -1,55 +1,44 @@
kiwix_sources = [
'book.cpp',
'bookmark.cpp',
'library.cpp',
'manager.cpp',
'libxml_dumper.cpp',
'opds_dumper.cpp',
'downloader.cpp',
'server.cpp',
'search_renderer.cpp',
'subprocess.cpp',
'aria2.cpp',
'tools/base64.cpp',
'tools/pathTools.cpp',
'tools/regexTools.cpp',
'tools/stringTools.cpp',
'tools/networkTools.cpp',
'tools/otherTools.cpp',
'tools/archiveTools.cpp',
'kiwixserve.cpp',
'name_mapper.cpp',
'server/byte_range.cpp',
'server/etag.cpp',
'server/request_context.cpp',
'server/response.cpp',
'server/internalServer.cpp',
'server/internalServer_catalog_v2.cpp',
'server/i18n.cpp',
'opds_catalog.cpp',
'version.cpp'
'reader.cpp',
'searcher.cpp',
'common/base64.cpp',
'common/pathTools.cpp',
'common/regexTools.cpp',
'common/stringTools.cpp',
'common/networkTools.cpp',
'common/otherTools.cpp',
'xapian/htmlparse.cc',
'xapian/myhtmlparse.cc'
]
kiwix_sources += lib_resources
kiwix_sources += i18n_resources
if host_machine.system() == 'windows'
kiwix_sources += 'subprocess_windows.cpp'
else
kiwix_sources += 'subprocess_unix.cpp'
if xapian_dep.found()
kiwix_sources += ['xapianSearcher.cpp']
endif
install_dir = get_option('libdir')
if get_option('android')
subdir('android')
install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
else
install_dir = get_option('libdir')
endif
if has_ctpp2_dep
kiwix_sources += ['ctpp2/CTPP2VMStringLoader.cpp']
endif
config_h = configure_file(output : 'kiwix_config.h',
configuration : conf,
input : 'config.h.in')
install_headers(config_h, subdir:'kiwix')
libkiwix = library('kiwix',
kiwixlib = library('kiwix',
kiwix_sources,
include_directories : inc,
dependencies : all_deps,
link_args: extra_libs,
version: meson.project_version(),
install: true,
install_dir: install_dir)

View File

@@ -1,104 +0,0 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "name_mapper.h"
#include "library.h"
#include "tools/regexTools.h"
#include <iostream>
namespace kiwix {
HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
auto& currentBook = library.getBookById(bookId);
auto bookName = currentBook.getHumanReadableIdFromPath();
m_idToName[bookId] = bookName;
m_nameToId[bookName] = bookId;
if (!withAlias)
continue;
auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
if (aliasName == bookName) {
continue;
}
if (m_nameToId.find(aliasName) == m_nameToId.end()) {
m_nameToId[aliasName] = bookId;
} else {
auto alreadyPresentPath = library.getBookById(m_nameToId[aliasName]).getPath();
std::cerr << "Path collision: " << alreadyPresentPath
<< " and " << currentBook.getPath()
<< " can't share the same URL path '" << aliasName << "'."
<< " Therefore, only " << alreadyPresentPath
<< " will be served." << std::endl;
}
}
}
std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
return m_idToName.at(id);
}
std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
return m_nameToId.at(name);
}
////////////////////////////////////////////////////////////////////////////////
// UpdatableNameMapper
////////////////////////////////////////////////////////////////////////////////
UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
: library(lib)
, withAlias(withAlias)
{
update();
}
void UpdatableNameMapper::update()
{
const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
std::lock_guard<std::mutex> lock(mutex);
nameMapper.reset(newNameMapper);
}
UpdatableNameMapper::NameMapperHandle
UpdatableNameMapper::currentNameMapper() const
{
// Return a copy of the handle to the current NameMapper object. It will
// ensure that the object survives any call to UpdatableNameMapper::update()
// made before the completion of any pending operation on that object.
std::lock_guard<std::mutex> lock(mutex);
return nameMapper;
}
std::string UpdatableNameMapper::getNameForId(const std::string& id) const
{
// Ensure that the current nameMapper object survives a concurrent call
// to UpdatableNameMapper::update()
return currentNameMapper()->getNameForId(id);
}
std::string UpdatableNameMapper::getIdForName(const std::string& name) const
{
// Ensure that the current nameMapper object survives a concurrent call
// to UpdatableNameMapper::update()
return currentNameMapper()->getIdForName(name);
}
}

Some files were not shown because too many files have changed in this diff Show More