mirror of
https://github.com/kiwix/libkiwix.git
synced 2025-12-31 10:28:04 -05:00
Compare commits
1 Commits
9.4.1
...
profile-di
| Author | SHA1 | Date | |
|---|---|---|---|
|
6b04eb3214 |
79
.github/workflows/package.yml
vendored
79
.github/workflows/package.yml
vendored
@@ -1,79 +0,0 @@
|
||||
name: Packages
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build-deb:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
distro: [ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# Determine which PPA we should upload to
|
||||
- name: PPA
|
||||
id: ppa
|
||||
run: |
|
||||
if [[ $REF == refs/tags* ]]
|
||||
then
|
||||
echo "::set-output name=ppa::kiwixteam/release"
|
||||
else
|
||||
echo "::set-output name=ppa::kiwixteam/dev"
|
||||
fi
|
||||
env:
|
||||
REF: ${{ github.ref }}
|
||||
|
||||
- uses: legoktm/gh-action-auto-dch@master
|
||||
with:
|
||||
fullname: Kiwix builder
|
||||
email: release+launchpad@kiwix.org
|
||||
distro: ${{ matrix.distro }}
|
||||
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-groovy
|
||||
if: matrix.distro == 'ubuntu-groovy'
|
||||
name: Build package for ubuntu-groovy
|
||||
id: build-ubuntu-groovy
|
||||
with:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-focal
|
||||
if: matrix.distro == 'ubuntu-focal'
|
||||
name: Build package for ubuntu-focal
|
||||
id: build-ubuntu-focal
|
||||
with:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
|
||||
- uses: legoktm/gh-action-build-deb@ubuntu-bionic
|
||||
if: matrix.distro == 'ubuntu-bionic'
|
||||
name: Build package for ubuntu-bionic
|
||||
id: build-ubuntu-bionic
|
||||
with:
|
||||
args: --no-sign
|
||||
ppa: ${{ steps.ppa.outputs.ppa }}
|
||||
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: Packages for ${{ matrix.distro }}
|
||||
path: output
|
||||
|
||||
- uses: legoktm/gh-action-dput@master
|
||||
name: Upload dev package
|
||||
# Only upload on pushes to master
|
||||
if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-')
|
||||
with:
|
||||
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
|
||||
repository: ppa:kiwixteam/dev
|
||||
packages: output/*_source.changes
|
||||
|
||||
- uses: legoktm/gh-action-dput@master
|
||||
name: Upload release package
|
||||
# Only upload on pushes to master or tag
|
||||
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-')
|
||||
with:
|
||||
gpg_key: ${{ secrets.LAUNCHPAD_GPG }}
|
||||
repository: ppa:kiwixteam/release
|
||||
packages: output/*_source.changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: CI
|
||||
|
||||
on: [push]
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
Macos:
|
||||
@@ -13,8 +13,9 @@ jobs:
|
||||
with:
|
||||
python-version: '3.5'
|
||||
- name: Install packages
|
||||
run:
|
||||
brew install gcovr pkg-config ninja
|
||||
uses: mstksg/get-package@v1
|
||||
with:
|
||||
brew: gcovr pkg-config ninja
|
||||
- name: Install python modules
|
||||
run: pip3 install meson==0.49.2 pytest
|
||||
- name: Install deps
|
||||
@@ -52,41 +53,30 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
name:
|
||||
target:
|
||||
- native_static
|
||||
- native_dyn
|
||||
- native_dyn_bionic
|
||||
- android_arm
|
||||
- android_arm64
|
||||
- win32_static
|
||||
- win32_dyn
|
||||
include:
|
||||
- name: native_static
|
||||
target: native_static
|
||||
- target: native_static
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: native_dyn
|
||||
target: native_dyn
|
||||
- target: native_dyn
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: native_dyn_bionic
|
||||
target: native_dyn
|
||||
image_variant: bionic
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: android_arm
|
||||
target: android_arm
|
||||
- target: android_arm
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: android_arm64
|
||||
target: android_arm64
|
||||
- target: android_arm64
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- name: win32_static
|
||||
target: win32_static
|
||||
- target: win32_static
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
- name: win32_dyn
|
||||
target: win32_dyn
|
||||
- target: win32_dyn
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
env:
|
||||
@@ -95,20 +85,25 @@ jobs:
|
||||
container:
|
||||
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
|
||||
steps:
|
||||
- name: Extract branch name
|
||||
shell: bash
|
||||
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
|
||||
id: extract_branch
|
||||
- name: Checkout code
|
||||
shell: python
|
||||
run: |
|
||||
import json
|
||||
from subprocess import check_call
|
||||
from os import environ
|
||||
with open(environ['GITHUB_EVENT_PATH'], 'r') as f:
|
||||
content = f.read()
|
||||
event_data = json.loads(content)
|
||||
try:
|
||||
branch_ref = event_data['ref'].split('/')[-1]
|
||||
except KeyError:
|
||||
branch_ref = event_data['pull_request']['head']['ref']
|
||||
print("Cloning branch", branch_ref)
|
||||
command = [
|
||||
'git', 'clone',
|
||||
'https://github.com/${{github.repository}}',
|
||||
'--depth=1',
|
||||
'--branch', '${{steps.extract_branch.outputs.branch}}'
|
||||
'--branch', branch_ref
|
||||
]
|
||||
check_call(command, cwd=environ['HOME'])
|
||||
- name: Install deps
|
||||
@@ -157,6 +152,6 @@ jobs:
|
||||
curl https://codecov.io/bash -o codecov.sh
|
||||
bash codecov.sh -n "${OS_NAME}_${{matrix.target}}" -Z
|
||||
rm codecov.sh
|
||||
if: startsWith(matrix.target, 'native_') && matrix.image_variant == 'xenial'
|
||||
if: startsWith(matrix.target, 'native_')
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -2,5 +2,3 @@
|
||||
*.swp
|
||||
subprojects/googletest-release*
|
||||
*.class
|
||||
build/
|
||||
.vscode/
|
||||
|
||||
88
ChangeLog
88
ChangeLog
@@ -1,91 +1,3 @@
|
||||
kiwix-lib 9.4.1
|
||||
===============
|
||||
|
||||
* Fix `M/Counter` parsing.
|
||||
* [SERVER] Adjust body padding-top for taskbar
|
||||
* Fix potential crash when stoping a server not started.
|
||||
* Various fix in build system and the CI.
|
||||
|
||||
kiwix-lib 9.4.0
|
||||
===============
|
||||
|
||||
* [SERVER] Make the headers handling case insensitive.
|
||||
* [SERVER] Make server answer 204 http status code for empty search
|
||||
* [PACKAGING] Made CI build deb packages.
|
||||
* [SERVER] Add a way to prevent taskbar and external link bloquer at article
|
||||
level.
|
||||
* Fix meson file to be compatible with meson 0.45
|
||||
* [SERVER] Update search requests to use pageStart/pageLength instead of
|
||||
pageStart/pageEnd arguments.
|
||||
* [SERVER] Set a fixed favicon size in the main page.
|
||||
* [SERVER] Refactor the response system code to better handling future new
|
||||
libzim api.
|
||||
* Fix segmentation fault around exchange with aria2 process making
|
||||
kiwix-desktop crash at exit.
|
||||
|
||||
kiwix-lib 9.3.1
|
||||
===============
|
||||
|
||||
* Fix handling of samba path on windows.
|
||||
* Do not include `kiwix_config.h` in public header.
|
||||
* Fix compilation with libmicrohttpd v0.97.1
|
||||
* Increase default test timeout to 160seconds/test.
|
||||
* Add automatic debian packaging.
|
||||
* Use non-minified version of jquery-ui.js
|
||||
* Pass `-latomic` compile option for sh4 architecture.
|
||||
* Make mesion install `kiwix-compile-resources` man page.
|
||||
|
||||
kiwix-lib 9.3.0
|
||||
===============
|
||||
|
||||
* Add a thread safe method to search suggestions.
|
||||
Old methods are now deprecated.
|
||||
|
||||
kiwix-lib 9.2.3
|
||||
===============
|
||||
|
||||
* Add test on byte-range
|
||||
* Fix compilation on bionic and windows.
|
||||
* Allow building using debian packaged kainjow-mustache
|
||||
* Pass `-latomic` compile option for architectures that need it
|
||||
|
||||
kiwix-lib 9.2.2
|
||||
===============
|
||||
|
||||
* Fix handling on empty content in byte range management (wrong assert)
|
||||
|
||||
kiwix-lib 9.2.1
|
||||
===============
|
||||
|
||||
* Fix support of byte range request.
|
||||
|
||||
kiwix-lib 9.2
|
||||
=============
|
||||
|
||||
* Add tests
|
||||
* Refactoring server code.
|
||||
* [SERVER] Add HEAD, Etag and If-None-Match support.
|
||||
* [SERVER] Compress opds catalog answers.
|
||||
|
||||
kiwix-lib 9.1.2
|
||||
===============
|
||||
|
||||
* Do not use the pathToSave if it is empty.
|
||||
|
||||
kiwix-lib 9.1.1
|
||||
===============
|
||||
|
||||
* Fix the detection of the dataDirectory on windows.
|
||||
|
||||
kiwix-lib 9.1.0
|
||||
===============
|
||||
|
||||
* [JAVA] Add a method to get the size of an article.
|
||||
* Add a new method to the libray to get the book by path.
|
||||
* Add a option to make the server blocks external link.
|
||||
Links are intercepted by js an redirected to a "portail" page.
|
||||
* [ODPS] Correctly handle book's articleCount and mediaCount.
|
||||
|
||||
kiwix-lib 9.0.1
|
||||
===============
|
||||
|
||||
|
||||
25
README.md
25
README.md
@@ -6,13 +6,12 @@ suite core. It contains the code shared by all Kiwix ports (Windows,
|
||||
GNU/Linux, macOS, Android, iOS, ...).
|
||||
|
||||
[](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
|
||||
[](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
|
||||
[](https://aur.archlinux.org/packages/kiwix-lib/)
|
||||
[](https://travis-ci.com/kiwix/kiwix-lib)
|
||||
[](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
|
||||
[](https://codecov.io/gh/kiwix/kiwix-lib)
|
||||
[](https://www.gnu.org/licenses/gpl-3.0)
|
||||
|
||||
[](https://repology.org/project/kiwix-lib/versions)
|
||||
|
||||
Disclaimer
|
||||
----------
|
||||
|
||||
@@ -39,15 +38,10 @@ libraries need to be available:
|
||||
* [ICU](https://site.icu-project.org/) (package `libicu-dev` on Ubuntu)
|
||||
* [ZIM](https://openzim.org/) (package `libzim-dev` on Ubuntu)
|
||||
* [Pugixml](https://pugixml.org/) (package `libpugixml-dev` on Ubuntu)
|
||||
* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
|
||||
* [Mustache](https://github.com/kainjow/Mustache) (Just copy the
|
||||
header `mustache.hpp` somewhere it can be found by the compiler and/or
|
||||
set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
|
||||
* [libcurl](https://curl.se/libcurl) (`libcurl4-gnutls-dev`, `libcurl4-nss-dev` or `libcurl4-openssl-dev` on Ubuntu)
|
||||
* [microhttpd](https://www.gnu.org/software/libmicrohttpd) (package `libmicrohttpd-dev` on Ubuntu)
|
||||
* [zlib](https://zlib.net/) (package `zlib1g-dev` on Ubuntu)
|
||||
|
||||
The following dependency needs to be available at runtime:
|
||||
* [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
|
||||
set CPPFLAGS with correct `-I` option)
|
||||
|
||||
These dependencies may or may not be packaged by your operating
|
||||
system. They may also be packaged but only in an older version. The
|
||||
@@ -62,7 +56,7 @@ Environment
|
||||
-------------
|
||||
|
||||
The Kiwix library builds using [Meson](https://mesonbuild.com/) version
|
||||
0.45 or higher. Meson relies itself on Ninja, pkg-config and few other
|
||||
0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
|
||||
compilation tools.
|
||||
|
||||
Install first the few common compilation tools:
|
||||
@@ -91,15 +85,6 @@ Meson. If you want statically linked libraries, you can add
|
||||
|
||||
Depending of you system, `ninja` may be called `ninja-build`.
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
To run the automated tests:
|
||||
```bash
|
||||
cd build
|
||||
meson test
|
||||
```
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ task writePom {
|
||||
project {
|
||||
groupId 'org.kiwix.kiwixlib'
|
||||
artifactId 'kiwixlib'
|
||||
version '9.4.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
|
||||
version '9.0.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
|
||||
packaging 'aar'
|
||||
name 'kiwixlib'
|
||||
url 'https://github.com/kiwix/kiwix-lib'
|
||||
|
||||
5
debian/changelog
vendored
5
debian/changelog
vendored
@@ -1,5 +0,0 @@
|
||||
libkiwix (0.0.0) unstable; urgency=medium
|
||||
|
||||
* Initial release
|
||||
|
||||
-- Kunal Mehta <legoktm@debian.org> Wed, 08 Jul 2020 18:12:32 -0700
|
||||
46
debian/control
vendored
46
debian/control
vendored
@@ -1,46 +0,0 @@
|
||||
Source: libkiwix
|
||||
Priority: optional
|
||||
Maintainer: Kiwix team <kiwix@kiwix.org>
|
||||
Build-Depends: debhelper-compat (= 13),
|
||||
meson,
|
||||
pkg-config,
|
||||
libzim-dev (>= 6.1.8),
|
||||
libcurl4-gnutls-dev,
|
||||
libicu-dev,
|
||||
libgtest-dev,
|
||||
libkainjow-mustache-dev,
|
||||
liblzma-dev,
|
||||
libmicrohttpd-dev,
|
||||
libpugixml-dev,
|
||||
zlib1g-dev
|
||||
Standards-Version: 4.5.0
|
||||
Section: libs
|
||||
Homepage: https://github.com/kiwix/kiwix-lib
|
||||
Rules-Requires-Root: no
|
||||
|
||||
Package: libkiwix-dev
|
||||
Section: libdevel
|
||||
Architecture: any
|
||||
Multi-Arch: same
|
||||
Depends: libkiwix9 (= ${binary:Version}), ${misc:Depends}, python3,
|
||||
libzim-dev (>= 6.0.0),
|
||||
libicu-dev,
|
||||
libpugixml-dev,
|
||||
libcurl4-gnutls-dev,
|
||||
libmicrohttpd-dev
|
||||
Description: library of common code for Kiwix (development)
|
||||
Kiwix is an offline Wikipedia reader. libkiwix provides the
|
||||
software core for Kiwix, and contains the code shared by all
|
||||
Kiwix ports (Windows, Linux, OSX, Android, etc.).
|
||||
.
|
||||
This package contains development files.
|
||||
|
||||
Package: libkiwix9
|
||||
Architecture: any
|
||||
Multi-Arch: same
|
||||
Depends: ${shlibs:Depends}, ${misc:Depends}, aria2
|
||||
Conflicts: libkiwix0, libkiwix3
|
||||
Description: library of common code for Kiwix
|
||||
Kiwix is an offline Wikipedia reader. libkiwix provides the
|
||||
software core for Kiwix, and contains the code shared by all
|
||||
Kiwix ports (Windows, Linux, OSX, Android, etc.).
|
||||
1
debian/copyright
vendored
1
debian/copyright
vendored
@@ -1 +0,0 @@
|
||||
See COPYING in the repository root.
|
||||
4
debian/libkiwix-dev.install
vendored
4
debian/libkiwix-dev.install
vendored
@@ -1,4 +0,0 @@
|
||||
usr/include
|
||||
usr/lib/*/libkiwix.so
|
||||
usr/lib/*/pkgconfig
|
||||
usr/bin
|
||||
1
debian/libkiwix-dev.manpages
vendored
1
debian/libkiwix-dev.manpages
vendored
@@ -1 +0,0 @@
|
||||
usr/share/man/man1/kiwix-compile-resources.1*
|
||||
1
debian/libkiwix9.install
vendored
1
debian/libkiwix9.install
vendored
@@ -1 +0,0 @@
|
||||
usr/lib/*/libkiwix.so.*
|
||||
8
debian/rules
vendored
8
debian/rules
vendored
@@ -1,8 +0,0 @@
|
||||
#!/usr/bin/make -f
|
||||
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
|
||||
|
||||
%:
|
||||
dh $@ --buildsystem=meson
|
||||
|
||||
override_dh_auto_test:
|
||||
dh_auto_test -- -t 3
|
||||
1
debian/source/format
vendored
1
debian/source/format
vendored
@@ -1 +0,0 @@
|
||||
3.0 (native)
|
||||
@@ -98,7 +98,7 @@ class Book
|
||||
std::string m_id;
|
||||
std::string m_downloadId;
|
||||
std::string m_path;
|
||||
bool m_pathValid = false;
|
||||
bool m_pathValid;
|
||||
std::string m_title;
|
||||
std::string m_description;
|
||||
std::string m_language;
|
||||
@@ -110,10 +110,10 @@ class Book
|
||||
std::string m_flavour;
|
||||
std::string m_tags;
|
||||
std::string m_origId;
|
||||
uint64_t m_articleCount = 0;
|
||||
uint64_t m_mediaCount = 0;
|
||||
bool m_readOnly = false;
|
||||
uint64_t m_size = 0;
|
||||
uint64_t m_articleCount;
|
||||
uint64_t m_mediaCount;
|
||||
bool m_readOnly;
|
||||
uint64_t m_size;
|
||||
mutable std::string m_favicon;
|
||||
std::string m_faviconUrl;
|
||||
std::string m_faviconMimeType;
|
||||
|
||||
@@ -88,7 +88,7 @@ class Download {
|
||||
class Downloader
|
||||
{
|
||||
public:
|
||||
Downloader();
|
||||
Downloader(std::string sessionFileDir = "");
|
||||
virtual ~Downloader();
|
||||
|
||||
void close();
|
||||
|
||||
@@ -18,6 +18,7 @@ class KiwixServe
|
||||
bool isRunning();
|
||||
int getPort() { return m_port; }
|
||||
int setPort(int port);
|
||||
void setLibraryPath(std::string path) { m_libraryPath = path; }
|
||||
|
||||
private:
|
||||
std::unique_ptr<Subprocess> mp_kiwixServe;
|
||||
|
||||
@@ -43,8 +43,6 @@ namespace kiwix
|
||||
* The Reader class is the class who allow to get an entry content from a zim
|
||||
* file.
|
||||
*/
|
||||
|
||||
using SuggestionsList_t = std::vector<std::vector<std::string>>;
|
||||
class Reader
|
||||
{
|
||||
public:
|
||||
@@ -421,10 +419,6 @@ class Reader
|
||||
*
|
||||
* Suggestions are stored in an internal vector and can be retrieved using
|
||||
* `getNextSuggestion` method.
|
||||
* This method is not thread safe and is deprecated. Use :
|
||||
* bool searchSuggestions(const string& prefix,
|
||||
* unsigned int suggestionsCount,
|
||||
* SuggestionsList_t& results);
|
||||
*
|
||||
* @param prefix The prefix to search.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
@@ -432,27 +426,12 @@ class Reader
|
||||
* If false, add suggestions to the internal vector
|
||||
* (until internal vector size is suggestionCount (or no more
|
||||
* suggestion))
|
||||
* @return True if some suggestions have been added to the internal vector.
|
||||
* @return True if some suggestions where added to the internal vector.
|
||||
*/
|
||||
DEPRECATED bool searchSuggestions(const string& prefix,
|
||||
bool searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
const bool reset = true);
|
||||
|
||||
/**
|
||||
* Search for entries with title starting with prefix (case sensitive).
|
||||
*
|
||||
* Suggestions are added to the `result` vector.
|
||||
*
|
||||
* @param prefix The prefix to search.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
* @param result The vector where to store the suggestions.
|
||||
* @return True if some suggestions have been added to the vector.
|
||||
*/
|
||||
|
||||
bool searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& resuls);
|
||||
|
||||
/**
|
||||
* Search for entries for the given prefix.
|
||||
*
|
||||
@@ -464,39 +443,13 @@ class Reader
|
||||
* In any case, suggestions are stored in an internal vector and can be
|
||||
* retrieved using `getNextSuggestion` method.
|
||||
* The internal vector will be reset.
|
||||
* This method is not thread safe and is deprecated. Use :
|
||||
* bool searchSuggestionsSmart(const string& prefix,
|
||||
* unsigned int suggestionsCount,
|
||||
* SuggestionsList_t& results);
|
||||
*
|
||||
* @param prefix The prefix to search for.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
*/
|
||||
DEPRECATED bool searchSuggestionsSmart(const string& prefix,
|
||||
bool searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount);
|
||||
|
||||
/**
|
||||
* Search for entries for the given prefix.
|
||||
*
|
||||
* If the zim file has a internal fulltext index, the suggestions will be
|
||||
* searched using it.
|
||||
* Else the suggestions will be search using `searchSuggestions` while trying
|
||||
* to be smart about case sensitivity (using `getTitleVariants`).
|
||||
*
|
||||
* In any case, suggestions are stored in an internal vector and can be
|
||||
* retrieved using `getNextSuggestion` method.
|
||||
* The internal vector will be reset.
|
||||
*
|
||||
* @param prefix The prefix to search for.
|
||||
* @param suggestionsCount How many suggestions to search for.
|
||||
* @param results The vector where to store the suggestions
|
||||
* @return True if some suggestions have been added to the results.
|
||||
*/
|
||||
bool searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results);
|
||||
|
||||
|
||||
/**
|
||||
* Check if the url exists in the zim file.
|
||||
*
|
||||
@@ -537,7 +490,7 @@ class Reader
|
||||
* @param[out] title the title of the suggestion.
|
||||
* @return True if title has been set.
|
||||
*/
|
||||
DEPRECATED bool getNextSuggestion(string& title);
|
||||
bool getNextSuggestion(string& title);
|
||||
|
||||
/**
|
||||
* Get the next suggestion title and url.
|
||||
@@ -546,7 +499,7 @@ class Reader
|
||||
* @param[out] url the url of the suggestion.
|
||||
* @return True if title and url have been set.
|
||||
*/
|
||||
DEPRECATED bool getNextSuggestion(string& title, string& url);
|
||||
bool getNextSuggestion(string& title, string& url);
|
||||
|
||||
/**
|
||||
* Get if we can check zim file integrity (has a checksum).
|
||||
@@ -606,8 +559,8 @@ class Reader
|
||||
zim::size_type nsICount;
|
||||
std::string zimFilePath;
|
||||
|
||||
SuggestionsList_t suggestions;
|
||||
SuggestionsList_t::iterator suggestionsOffset;
|
||||
std::vector<std::vector<std::string>> suggestions;
|
||||
std::vector<std::vector<std::string>>::iterator suggestionsOffset;
|
||||
|
||||
private:
|
||||
std::map<const std::string, unsigned int> parseCounterMetadata() const;
|
||||
|
||||
@@ -60,13 +60,6 @@ class SearchRenderer
|
||||
*/
|
||||
void setSearchProtocolPrefix(const std::string& prefix);
|
||||
|
||||
/**
|
||||
* set result count per page
|
||||
*/
|
||||
void setPageLength(unsigned int pageLength){
|
||||
this->pageLength = pageLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the html page with the resutls of the search.
|
||||
*/
|
||||
@@ -80,9 +73,10 @@ class SearchRenderer
|
||||
std::string searchPattern;
|
||||
std::string protocolPrefix;
|
||||
std::string searchProtocolPrefix;
|
||||
unsigned int pageLength;
|
||||
unsigned int resultCountPerPage;
|
||||
unsigned int estimatedResultCount;
|
||||
unsigned int resultStart;
|
||||
unsigned int resultEnd;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
#include <vector>
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "kiwix_config.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
|
||||
@@ -56,9 +56,7 @@ namespace kiwix
|
||||
void setNbThreads(int threads) { m_nbThreads = threads; }
|
||||
void setVerbose(bool verbose) { m_verbose = verbose; }
|
||||
void setTaskbar(bool withTaskbar, bool withLibraryButton)
|
||||
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
|
||||
void setBlockExternalLinks(bool blockExternalLinks)
|
||||
{ m_blockExternalLinks = blockExternalLinks; }
|
||||
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
|
||||
|
||||
protected:
|
||||
Library* mp_library;
|
||||
@@ -70,7 +68,6 @@ namespace kiwix
|
||||
bool m_verbose = false;
|
||||
bool m_withTaskbar = true;
|
||||
bool m_withLibraryButton = true;
|
||||
bool m_blockExternalLinks = false;
|
||||
std::unique_ptr<InternalServer> mp_server;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -22,8 +22,6 @@
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <zim/zim.h>
|
||||
|
||||
namespace pugi {
|
||||
class xml_node;
|
||||
@@ -43,8 +41,6 @@ namespace kiwix
|
||||
const std::string& tagName);
|
||||
bool convertStrToBool(const std::string& value);
|
||||
|
||||
using MimeCounterType = std::map<const std::string, zim::article_index_type>;
|
||||
MimeCounterType parseMimetypeCounter(const std::string& counterData);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -43,7 +43,7 @@ void loadICUExternalTables();
|
||||
std::string urlEncode(const std::string& value, bool encodeReserved = false);
|
||||
std::string urlDecode(const std::string& value, bool component = false);
|
||||
|
||||
std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
|
||||
std::vector<std::string> split(const std::string&, const std::string&, bool trimEmpty = true);
|
||||
std::string join(const std::vector<std::string>& list, const std::string& sep);
|
||||
|
||||
std::string ucAll(const std::string& word);
|
||||
|
||||
30
meson.build
30
meson.build
@@ -1,41 +1,31 @@
|
||||
project('kiwix-lib', 'cpp',
|
||||
version : '9.4.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
|
||||
license : 'GPLv3+',
|
||||
version : '9.0.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
|
||||
license : 'GPL',
|
||||
default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
|
||||
|
||||
compiler = meson.get_compiler('cpp')
|
||||
|
||||
wrapper = get_option('wrapper')
|
||||
|
||||
static_deps = wrapper.contains('android') or wrapper.contains('java') or get_option('default_library') == 'static'
|
||||
if wrapper.contains('android')
|
||||
static_deps = 'android' in wrapper or 'java' in wrapper or get_option('default_library') == 'static'
|
||||
if 'android' in wrapper
|
||||
extra_libs = ['-llog']
|
||||
else
|
||||
extra_libs = []
|
||||
endif
|
||||
|
||||
if wrapper.contains('java')
|
||||
if 'java' in wrapper
|
||||
add_languages('java')
|
||||
endif
|
||||
|
||||
# See https://github.com/kiwix/kiwix-lib/issues/371
|
||||
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
|
||||
extra_libs += '-latomic'
|
||||
endif
|
||||
|
||||
thread_dep = dependency('threads')
|
||||
libicu_dep = dependency('icu-i18n', static:static_deps)
|
||||
libzim_dep = dependency('libzim', version : '>=6.3.0', static:static_deps)
|
||||
libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
|
||||
pugixml_dep = dependency('pugixml', static:static_deps)
|
||||
libcurl_dep = dependency('libcurl', static:static_deps)
|
||||
microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
|
||||
zlib_dep = dependency('zlib', static:static_deps)
|
||||
|
||||
if compiler.has_header('mustache.hpp')
|
||||
extra_include = []
|
||||
elif compiler.has_header('mustache.hpp', args: '-I/usr/include/kainjow')
|
||||
extra_include = ['/usr/include/kainjow']
|
||||
else
|
||||
if not compiler.has_header('mustache.hpp')
|
||||
error('Cannot found header mustache.hpp')
|
||||
endif
|
||||
|
||||
@@ -45,9 +35,9 @@ if target_machine.system() == 'windows' and static_deps
|
||||
extra_cflags += '-DCURL_STATICLIB'
|
||||
endif
|
||||
|
||||
all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep, zlib_dep]
|
||||
all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
|
||||
|
||||
inc = include_directories('include', extra_include)
|
||||
inc = include_directories('include')
|
||||
|
||||
conf = configuration_data()
|
||||
conf.set('VERSION', '"@0@"'.format(meson.project_version()))
|
||||
@@ -64,7 +54,7 @@ subdir('static')
|
||||
subdir('src')
|
||||
subdir('test')
|
||||
|
||||
pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd']
|
||||
pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl']
|
||||
|
||||
pkg_conf = configuration_data()
|
||||
pkg_conf.set('prefix', get_option('prefix'))
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
.TH KIWIX-COMPILE-RESOURCES "1" "August 2017" "Kiwix" "User Commands"
|
||||
.SH NAME
|
||||
kiwix-compile-resources \- helper to compile and generate some Kiwix resources
|
||||
.SH SYNOPSIS
|
||||
\fBkiwix\-compile\-resources\fR [\-h] [\-\-cxxfile CXXFILE] [\-\-hfile HFILE] resource_file\fR
|
||||
.SH DESCRIPTION
|
||||
.TP
|
||||
resource_file
|
||||
The list of resources to compile.
|
||||
.TP
|
||||
\fB\-h\fR, \fB\-\-help\fR
|
||||
show a help message and exit
|
||||
.TP
|
||||
\fB\-\-cxxfile\fR CXXFILE
|
||||
The Cpp file name to generate
|
||||
.TP
|
||||
\fB\-\-hfile\fR HFILE
|
||||
The h file name to generate
|
||||
.SH AUTHOR
|
||||
Matthieu Gautier <mgautier@kymeria.fr>
|
||||
@@ -2,5 +2,3 @@
|
||||
res_compiler = find_program('kiwix-compile-resources')
|
||||
|
||||
install_data(res_compiler.path(), install_dir:get_option('bindir'))
|
||||
|
||||
install_man('kiwix-compile-resources.1')
|
||||
|
||||
@@ -19,20 +19,15 @@
|
||||
#endif
|
||||
|
||||
|
||||
#define LOG_ARIA_ERROR() \
|
||||
{ \
|
||||
std::cerr << "ERROR: aria2 RPC request failed. (" << res << ")." << std::endl; \
|
||||
std::cerr << (m_curlErrorBuffer[0] ? m_curlErrorBuffer.get() : curl_easy_strerror(res)) << std::endl; \
|
||||
}
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
Aria2::Aria2():
|
||||
Aria2::Aria2(std::string sessionFileDir):
|
||||
mp_aria(nullptr),
|
||||
m_port(42042),
|
||||
m_secret("kiwixariarpc"),
|
||||
m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
|
||||
mp_curl(nullptr)
|
||||
mp_curl(nullptr),
|
||||
m_lock(PTHREAD_MUTEX_INITIALIZER)
|
||||
{
|
||||
m_downloadDir = getDataDirectory();
|
||||
makeDirectory(m_downloadDir);
|
||||
@@ -40,7 +35,12 @@ Aria2::Aria2():
|
||||
|
||||
std::string rpc_port = "--rpc-listen-port=" + to_string(m_port);
|
||||
std::string download_dir = "--dir=" + getDataDirectory();
|
||||
std::string session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
|
||||
std::string session_file;
|
||||
if (sessionFileDir.empty()) {
|
||||
session_file = appendToDirectory(getDataDirectory(), "kiwix.session");
|
||||
} else {
|
||||
session_file = appendToDirectory(sessionFileDir, "kiwix.session");
|
||||
}
|
||||
std::string session = "--save-session=" + session_file;
|
||||
std::string inputFile = "--input-file=" + session_file;
|
||||
// std::string log_dir = "--log=\"" + logDir + "\"";
|
||||
@@ -89,21 +89,27 @@ Aria2::Aria2():
|
||||
}
|
||||
mp_aria = Subprocess::run(callCmd);
|
||||
mp_curl = curl_easy_init();
|
||||
|
||||
char errbuf[CURL_ERROR_SIZE];
|
||||
curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
|
||||
curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, m_curlErrorBuffer.get());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
|
||||
|
||||
int watchdog = 50;
|
||||
while(--watchdog) {
|
||||
sleep(10);
|
||||
m_curlErrorBuffer[0] = 0;
|
||||
errbuf[0] = 0;
|
||||
auto res = curl_easy_perform(mp_curl);
|
||||
if (res == CURLE_OK) {
|
||||
break;
|
||||
} else if (watchdog == 1) {
|
||||
LOG_ARIA_ERROR();
|
||||
std::cerr <<" curl_easy_perform() failed." << std::endl;
|
||||
fprintf(stderr, "\nlibcurl: (%d) ", res);
|
||||
if (errbuf[0] != 0) {
|
||||
std::cerr << errbuf << std::endl;
|
||||
} else {
|
||||
std::cerr << curl_easy_strerror(res) << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!watchdog) {
|
||||
@@ -114,7 +120,6 @@ Aria2::Aria2():
|
||||
|
||||
Aria2::~Aria2()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
curl_easy_cleanup(mp_curl);
|
||||
}
|
||||
|
||||
@@ -126,44 +131,38 @@ void Aria2::close()
|
||||
|
||||
size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
|
||||
{
|
||||
auto outStream = static_cast<std::stringstream*>(userdata);
|
||||
outStream->write(ptr, nmemb);
|
||||
auto str = static_cast<std::stringstream*>(userdata);
|
||||
str->write(ptr, nmemb);
|
||||
return nmemb;
|
||||
}
|
||||
|
||||
std::string Aria2::doRequest(const MethodCall& methodCall)
|
||||
{
|
||||
pthread_mutex_lock(&m_lock);
|
||||
auto requestContent = methodCall.toString();
|
||||
std::stringstream outStream;
|
||||
std::stringstream stringstream;
|
||||
CURLcode res;
|
||||
long response_code;
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_lock);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &outStream);
|
||||
m_curlErrorBuffer[0] = 0;
|
||||
res = curl_easy_perform(mp_curl);
|
||||
if (res != CURLE_OK) {
|
||||
LOG_ARIA_ERROR();
|
||||
throw std::runtime_error("Cannot perform request");
|
||||
}
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str());
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss);
|
||||
curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream);
|
||||
res = curl_easy_perform(mp_curl);
|
||||
if (res == CURLE_OK) {
|
||||
long response_code;
|
||||
curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
pthread_mutex_unlock(&m_lock);
|
||||
if (response_code != 200) {
|
||||
throw std::runtime_error("Invalid return code from aria");
|
||||
}
|
||||
auto responseContent = stringstream.str();
|
||||
MethodResponse response(responseContent);
|
||||
if (response.isFault()) {
|
||||
throw AriaError(response.getFault().getFaultString());
|
||||
}
|
||||
return responseContent;
|
||||
}
|
||||
|
||||
auto responseContent = outStream.str();
|
||||
if (response_code != 200) {
|
||||
std::cerr << "ERROR: Invalid return code (" << response_code << ") from aria :" << std::endl;
|
||||
std::cerr << responseContent << std::endl;
|
||||
throw std::runtime_error("Invalid return code from aria");
|
||||
}
|
||||
|
||||
MethodResponse response(responseContent);
|
||||
if (response.isFault()) {
|
||||
throw AriaError(response.getFault().getFaultString());
|
||||
}
|
||||
return responseContent;
|
||||
pthread_mutex_unlock(&m_lock);
|
||||
throw std::runtime_error("Cannot perform request");
|
||||
}
|
||||
|
||||
std::string Aria2::addUri(const std::vector<std::string>& uris, const std::vector<std::pair<std::string, std::string>>& options)
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
#include "xmlrpc.h"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <curl/curl.h>
|
||||
#include <pthread.h>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
@@ -24,14 +24,13 @@ class Aria2
|
||||
int m_port;
|
||||
std::string m_secret;
|
||||
std::string m_downloadDir;
|
||||
std::unique_ptr<char[]> m_curlErrorBuffer;
|
||||
CURL* mp_curl;
|
||||
std::mutex m_lock;
|
||||
pthread_mutex_t m_lock;
|
||||
|
||||
std::string doRequest(const MethodCall& methodCall);
|
||||
|
||||
public:
|
||||
Aria2();
|
||||
Aria2(std::string sessionFileDir = "");
|
||||
virtual ~Aria2();
|
||||
void close();
|
||||
|
||||
|
||||
@@ -156,8 +156,6 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
|
||||
m_name = VALUE("name");
|
||||
m_flavour = VALUE("flavour");
|
||||
m_tags = VALUE("tags");
|
||||
m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
|
||||
m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
|
||||
for(auto linkNode = node.child("link"); linkNode;
|
||||
linkNode = linkNode.next_sibling("link")) {
|
||||
std::string rel = linkNode.attribute("rel").value();
|
||||
|
||||
@@ -124,8 +124,8 @@ void Download::cancelDownload()
|
||||
}
|
||||
|
||||
/* Constructor */
|
||||
Downloader::Downloader() :
|
||||
mp_aria(new Aria2())
|
||||
Downloader::Downloader(std::string sessionFileDir) :
|
||||
mp_aria(new Aria2(sessionFileDir))
|
||||
{
|
||||
try {
|
||||
for (auto gid : mp_aria->tellActive()) {
|
||||
@@ -133,7 +133,7 @@ Downloader::Downloader() :
|
||||
m_knownDownloads[gid]->updateStatus();
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "aria2 tellActive failed : " << e.what() << std::endl;
|
||||
std::cerr << "aria2 tellActive failed : " << e.what();
|
||||
}
|
||||
try {
|
||||
for (auto gid : mp_aria->tellWaiting()) {
|
||||
@@ -141,7 +141,7 @@ Downloader::Downloader() :
|
||||
m_knownDownloads[gid]->updateStatus();
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "aria2 tellWaiting failed : " << e.what() << std::endl;
|
||||
std::cerr << "aria2 tellWaiting failed : " << e.what();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -175,7 +175,7 @@ std::string Manager::addBookFromPathAndGetId(const std::string& pathToOpen,
|
||||
kiwix::Book book;
|
||||
|
||||
if (this->readBookFromPath(pathToOpen, &book)) {
|
||||
if (!pathToSave.empty() && pathToSave != pathToOpen) {
|
||||
if (pathToSave != pathToOpen) {
|
||||
book.setPath(isRelativePath(pathToSave)
|
||||
? computeAbsolutePath(
|
||||
removeLastPathElement(writableLibraryPath),
|
||||
|
||||
@@ -21,11 +21,8 @@ kiwix_sources = [
|
||||
'tools/otherTools.cpp',
|
||||
'kiwixserve.cpp',
|
||||
'name_mapper.cpp',
|
||||
'server/byte_range.cpp',
|
||||
'server/etag.cpp',
|
||||
'server/request_context.cpp',
|
||||
'server/response.cpp',
|
||||
'server/internalServer.cpp'
|
||||
'server/response.cpp'
|
||||
]
|
||||
kiwix_sources += lib_resources
|
||||
|
||||
@@ -35,13 +32,13 @@ else
|
||||
kiwix_sources += 'subprocess_unix.cpp'
|
||||
endif
|
||||
|
||||
if wrapper.contains('android')
|
||||
if 'android' in wrapper
|
||||
install_dir = 'kiwix-lib/jniLibs/' + meson.get_cross_property('android_abi')
|
||||
else
|
||||
install_dir = get_option('libdir')
|
||||
endif
|
||||
|
||||
if wrapper.contains('android') or wrapper.contains('java')
|
||||
if 'android' in wrapper or 'java' in wrapper
|
||||
subdir('wrapper/java')
|
||||
endif
|
||||
|
||||
@@ -54,7 +51,6 @@ kiwixlib = library('kiwix',
|
||||
kiwix_sources,
|
||||
include_directories : inc,
|
||||
dependencies : all_deps,
|
||||
link_args: extra_libs,
|
||||
version: meson.project_version(),
|
||||
install: true,
|
||||
install_dir: install_dir)
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <microhttpd.h>
|
||||
|
||||
#if MHD_VERSION < 0x00097002
|
||||
typedef int MHD_Result;
|
||||
#endif
|
||||
@@ -78,8 +78,6 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
|
||||
ADD_TEXT_ENTRY(entry_node, "name", book.getName());
|
||||
ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
|
||||
ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
|
||||
ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
|
||||
ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
|
||||
ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
|
||||
|
||||
auto content_node = entry_node.append_child("link");
|
||||
|
||||
@@ -103,16 +103,29 @@ zim::File* Reader::getZimFileHandler() const
|
||||
{
|
||||
return this->zimFileHandler;
|
||||
}
|
||||
|
||||
MimeCounterType Reader::parseCounterMetadata() const
|
||||
std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
|
||||
{
|
||||
std::map<const std::string, unsigned int> counters;
|
||||
string mimeType, item, counterString;
|
||||
unsigned int counter;
|
||||
|
||||
zim::Article article = this->zimFileHandler->getArticle('M', "Counter");
|
||||
|
||||
if (article.good()) {
|
||||
return parseMimetypeCounter(article.getData());
|
||||
stringstream ssContent(article.getData());
|
||||
|
||||
while (getline(ssContent, item, ';')) {
|
||||
stringstream ssItem(item);
|
||||
getline(ssItem, mimeType, '=');
|
||||
getline(ssItem, counterString, '=');
|
||||
if (!counterString.empty() && !mimeType.empty()) {
|
||||
sscanf(counterString.c_str(), "%u", &counter);
|
||||
counters.insert(pair<string, int>(mimeType, counter));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return MimeCounterType();
|
||||
return counters;
|
||||
}
|
||||
|
||||
/* Get the count of articles which can be indexed/displayed */
|
||||
@@ -125,10 +138,9 @@ unsigned int Reader::getArticleCount() const
|
||||
if (counterMap.empty()) {
|
||||
counter = this->nsACount;
|
||||
} else {
|
||||
for(auto &pair:counterMap) {
|
||||
if (startsWith(pair.first, "text/html")) {
|
||||
counter += pair.second;
|
||||
}
|
||||
auto it = counterMap.find("text/html");
|
||||
if (it != counterMap.end()) {
|
||||
counter = it->second;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -697,11 +709,13 @@ bool Reader::hasFulltextIndex() const
|
||||
}
|
||||
|
||||
/* Search titles by prefix */
|
||||
|
||||
bool Reader::searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
const bool reset)
|
||||
{
|
||||
bool retVal = false;
|
||||
zim::File::const_iterator articleItr;
|
||||
|
||||
/* Reset the suggestions otherwise check if the suggestions number is less
|
||||
* than the suggestionsCount */
|
||||
if (reset) {
|
||||
@@ -713,30 +727,15 @@ bool Reader::searchSuggestions(const string& prefix,
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = searchSuggestions(prefix, suggestionsCount, this->suggestions);
|
||||
|
||||
/* Set the cursor to the begining */
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
bool Reader::searchSuggestions(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results)
|
||||
{
|
||||
bool retVal = false;
|
||||
|
||||
/* Return if no prefix */
|
||||
if (prefix.size() == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto articleItr = zimFileHandler->findByTitle('A', prefix);
|
||||
for (articleItr = zimFileHandler->findByTitle('A', prefix);
|
||||
articleItr != zimFileHandler->end()
|
||||
&& articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
|
||||
&& results.size() < suggestionsCount;
|
||||
&& this->suggestions.size() < suggestionsCount;
|
||||
++articleItr) {
|
||||
/* Extract the interesting part of article title & url */
|
||||
std::string normalizedArticleTitle
|
||||
@@ -756,8 +755,8 @@ bool Reader::searchSuggestions(const string& prefix,
|
||||
title) */
|
||||
bool insert = true;
|
||||
std::vector<std::vector<std::string>>::iterator suggestionItr;
|
||||
for (suggestionItr = results.begin();
|
||||
suggestionItr != results.end();
|
||||
for (suggestionItr = this->suggestions.begin();
|
||||
suggestionItr != this->suggestions.end();
|
||||
suggestionItr++) {
|
||||
int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
|
||||
if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
|
||||
@@ -774,13 +773,16 @@ bool Reader::searchSuggestions(const string& prefix,
|
||||
suggestion.push_back(articleItr->getTitle());
|
||||
suggestion.push_back(articleFinalUrl);
|
||||
suggestion.push_back(normalizedArticleTitle);
|
||||
results.insert(suggestionItr, suggestion);
|
||||
this->suggestions.insert(suggestionItr, suggestion);
|
||||
}
|
||||
|
||||
/* Suggestions where found */
|
||||
retVal = true;
|
||||
}
|
||||
|
||||
/* Set the cursor to the begining */
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
@@ -795,28 +797,15 @@ std::vector<std::string> Reader::getTitleVariants(
|
||||
return variants;
|
||||
}
|
||||
|
||||
|
||||
bool Reader::searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount)
|
||||
{
|
||||
this->suggestions.clear();
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
auto ret = searchSuggestionsSmart(prefix, suggestionsCount, this->suggestions);
|
||||
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Try also a few variations of the prefix to have better results */
|
||||
bool Reader::searchSuggestionsSmart(const string& prefix,
|
||||
unsigned int suggestionsCount,
|
||||
SuggestionsList_t& results)
|
||||
unsigned int suggestionsCount)
|
||||
{
|
||||
std::vector<std::string> variants = this->getTitleVariants(prefix);
|
||||
bool retVal = false;
|
||||
|
||||
this->suggestions.clear();
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
/* Try to search in the title using fulltext search database */
|
||||
const auto suggestionSearch
|
||||
= this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
|
||||
@@ -832,14 +821,15 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
|
||||
suggestion.push_back(current->getTitle());
|
||||
suggestion.push_back("/A/" + current->getUrl());
|
||||
suggestion.push_back(kiwix::normalize(current->getTitle()));
|
||||
results.push_back(suggestion);
|
||||
this->suggestions.push_back(suggestion);
|
||||
}
|
||||
this->suggestionsOffset = this->suggestions.begin();
|
||||
retVal = true;
|
||||
} else {
|
||||
for (std::vector<std::string>::iterator variantsItr = variants.begin();
|
||||
variantsItr != variants.end();
|
||||
variantsItr++) {
|
||||
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
|
||||
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
|
||||
|| retVal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,30 +93,30 @@ std::string SearchRenderer::getHtml()
|
||||
kainjow::mustache::data pages{kainjow::mustache::data::type::list};
|
||||
|
||||
auto resultStart = mp_searcher->getResultStart();
|
||||
auto resultEnd = 0U;
|
||||
auto resultEnd = mp_searcher->getResultEnd();
|
||||
auto resultCountPerPage = resultEnd - resultStart;
|
||||
auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
|
||||
auto currentPage = 0U;
|
||||
auto pageStart = 0U;
|
||||
auto pageEnd = 0U;
|
||||
auto lastPageStart = 0U;
|
||||
if (pageLength) {
|
||||
currentPage = resultStart/pageLength;
|
||||
if (resultCountPerPage) {
|
||||
currentPage = resultStart/resultCountPerPage;
|
||||
pageStart = currentPage > 4 ? currentPage-4 : 0;
|
||||
pageEnd = currentPage + 5;
|
||||
if (pageEnd > estimatedResultCount / pageLength) {
|
||||
pageEnd = (estimatedResultCount + pageLength - 1) / pageLength;
|
||||
if (pageEnd > estimatedResultCount / resultCountPerPage) {
|
||||
pageEnd = estimatedResultCount / resultCountPerPage;
|
||||
}
|
||||
if (estimatedResultCount > pageLength) {
|
||||
lastPageStart = ((estimatedResultCount-1)/pageLength)*pageLength;
|
||||
if (estimatedResultCount > resultCountPerPage) {
|
||||
lastPageStart = static_cast<int>(round(estimatedResultCount/resultCountPerPage)) * resultCountPerPage;
|
||||
}
|
||||
}
|
||||
|
||||
resultEnd = resultStart+pageLength; //setting result end
|
||||
|
||||
for (unsigned int i = pageStart; i < pageEnd; i++) {
|
||||
kainjow::mustache::data page;
|
||||
page.set("label", to_string(i + 1));
|
||||
page.set("start", to_string(i * pageLength));
|
||||
page.set("start", to_string(i * resultCountPerPage));
|
||||
page.set("end", to_string((i + 1) * resultCountPerPage));
|
||||
|
||||
if (i == currentPage) {
|
||||
page.set("selected", true);
|
||||
@@ -137,8 +137,9 @@ std::string SearchRenderer::getHtml()
|
||||
allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
|
||||
allData.set("resultStart", to_string(resultStart + 1));
|
||||
allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
|
||||
allData.set("pageLength", to_string(pageLength));
|
||||
allData.set("resultRange", to_string(resultCountPerPage));
|
||||
allData.set("resultLastPageStart", to_string(lastPageStart));
|
||||
allData.set("lastResult", to_string(estimatedResultCount));
|
||||
allData.set("protocolPrefix", this->protocolPrefix);
|
||||
allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
|
||||
allData.set("contentId", this->searchContent);
|
||||
@@ -148,4 +149,4 @@ std::string SearchRenderer::getHtml()
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
839
src/server.cpp
839
src/server.cpp
@@ -19,15 +19,125 @@
|
||||
|
||||
#include "server.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
# if !defined(__MINGW32__) && (_MSC_VER < 1600)
|
||||
# include "stdint4win.h"
|
||||
# endif
|
||||
# include <winsock2.h>
|
||||
# include <ws2tcpip.h>
|
||||
# ifdef __GNUC__
|
||||
// inet_pton is not declared in mingw, even if the function exists.
|
||||
extern "C" {
|
||||
WINSOCK_API_LINKAGE INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
|
||||
}
|
||||
# endif
|
||||
typedef UINT64 uint64_t;
|
||||
typedef UINT16 uint16_t;
|
||||
#endif
|
||||
|
||||
extern "C" {
|
||||
#include <microhttpd.h>
|
||||
}
|
||||
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
#include "entry.h"
|
||||
#include "searcher.h"
|
||||
#include "search_renderer.h"
|
||||
#include "opds_dumper.h"
|
||||
|
||||
#include <zim/uuid.h>
|
||||
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include <pthread.h>
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <chrono>
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#include "server/internalServer.h"
|
||||
#ifndef _WIN32
|
||||
# include <arpa/inet.h>
|
||||
#endif
|
||||
|
||||
#include "server/request_context.h"
|
||||
#include "server/response.h"
|
||||
|
||||
#define MAX_SEARCH_LEN 140
|
||||
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
static IdNameMapper defaultNameMapper;
|
||||
|
||||
static int staticHandlerCallback(void* cls,
|
||||
struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls);
|
||||
|
||||
|
||||
class InternalServer {
|
||||
public:
|
||||
InternalServer(Library* library,
|
||||
NameMapper* nameMapper,
|
||||
std::string addr,
|
||||
int port,
|
||||
std::string root,
|
||||
int nbThreads,
|
||||
bool verbose,
|
||||
bool withTaskbar,
|
||||
bool withLibraryButton);
|
||||
virtual ~InternalServer() = default;
|
||||
|
||||
int handlerCallback(struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls);
|
||||
bool start();
|
||||
void stop();
|
||||
|
||||
private:
|
||||
Response handle_request(const RequestContext& request);
|
||||
Response build_500(const std::string& msg);
|
||||
Response build_404(const RequestContext& request, const std::string& zimName);
|
||||
Response build_homepage(const RequestContext& request);
|
||||
Response handle_skin(const RequestContext& request);
|
||||
Response handle_catalog(const RequestContext& request);
|
||||
Response handle_meta(const RequestContext& request);
|
||||
Response handle_search(const RequestContext& request);
|
||||
Response handle_suggest(const RequestContext& request);
|
||||
Response handle_random(const RequestContext& request);
|
||||
Response handle_content(const RequestContext& request);
|
||||
|
||||
kainjow::mustache::data get_default_data();
|
||||
Response get_default_response();
|
||||
|
||||
std::string m_addr;
|
||||
int m_port;
|
||||
std::string m_root;
|
||||
int m_nbThreads;
|
||||
std::atomic_bool m_verbose;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
struct MHD_Daemon* mp_daemon;
|
||||
|
||||
Library* mp_library;
|
||||
NameMapper* mp_nameMapper;
|
||||
};
|
||||
|
||||
|
||||
Server::Server(Library* library, NameMapper* nameMapper) :
|
||||
mp_library(library),
|
||||
mp_nameMapper(nameMapper),
|
||||
@@ -47,16 +157,13 @@ bool Server::start() {
|
||||
m_nbThreads,
|
||||
m_verbose,
|
||||
m_withTaskbar,
|
||||
m_withLibraryButton,
|
||||
m_blockExternalLinks));
|
||||
m_withLibraryButton));
|
||||
return mp_server->start();
|
||||
}
|
||||
|
||||
void Server::stop() {
|
||||
if (mp_server) {
|
||||
mp_server->stop();
|
||||
mp_server.reset(nullptr);
|
||||
}
|
||||
mp_server->stop();
|
||||
mp_server.reset(nullptr);
|
||||
}
|
||||
|
||||
void Server::setRoot(const std::string& root)
|
||||
@@ -70,4 +177,722 @@ void Server::setRoot(const std::string& root)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
InternalServer::InternalServer(Library* library,
|
||||
NameMapper* nameMapper,
|
||||
std::string addr,
|
||||
int port,
|
||||
std::string root,
|
||||
int nbThreads,
|
||||
bool verbose,
|
||||
bool withTaskbar,
|
||||
bool withLibraryButton) :
|
||||
m_addr(addr),
|
||||
m_port(port),
|
||||
m_root(root),
|
||||
m_nbThreads(nbThreads),
|
||||
m_verbose(verbose),
|
||||
m_withTaskbar(withTaskbar),
|
||||
m_withLibraryButton(withLibraryButton),
|
||||
mp_daemon(nullptr),
|
||||
mp_library(library),
|
||||
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
|
||||
{}
|
||||
|
||||
bool InternalServer::start() {
|
||||
#ifdef _WIN32
|
||||
int flags = MHD_USE_SELECT_INTERNALLY;
|
||||
#else
|
||||
int flags = MHD_USE_POLL_INTERNALLY;
|
||||
#endif
|
||||
if (m_verbose.load())
|
||||
flags |= MHD_USE_DEBUG;
|
||||
|
||||
|
||||
struct sockaddr_in sockAddr;
|
||||
memset(&sockAddr, 0, sizeof(sockAddr));
|
||||
sockAddr.sin_family = AF_INET;
|
||||
sockAddr.sin_port = htons(m_port);
|
||||
if (m_addr.empty()) {
|
||||
if (0 != INADDR_ANY)
|
||||
sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
|
||||
} else {
|
||||
if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
|
||||
std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
mp_daemon = MHD_start_daemon(flags,
|
||||
m_port,
|
||||
NULL,
|
||||
NULL,
|
||||
&staticHandlerCallback,
|
||||
this,
|
||||
MHD_OPTION_SOCK_ADDR, &sockAddr,
|
||||
MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
|
||||
MHD_OPTION_END);
|
||||
if (mp_daemon == nullptr) {
|
||||
std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
|
||||
<< " is maybe already occupied or need more permissions to be open. "
|
||||
"Please try as root or with a port number higher or equal to 1024."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void InternalServer::stop()
|
||||
{
|
||||
MHD_stop_daemon(mp_daemon);
|
||||
}
|
||||
|
||||
static int staticHandlerCallback(void* cls,
|
||||
struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls)
|
||||
{
|
||||
InternalServer* _this = static_cast<InternalServer*>(cls);
|
||||
|
||||
return _this->handlerCallback(connection,
|
||||
url,
|
||||
method,
|
||||
version,
|
||||
upload_data,
|
||||
upload_data_size,
|
||||
cont_cls);
|
||||
}
|
||||
|
||||
int InternalServer::handlerCallback(struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls)
|
||||
{
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
if (m_verbose.load() ) {
|
||||
printf("======================\n");
|
||||
printf("Requesting : \n");
|
||||
printf("full_url : %s\n", url);
|
||||
}
|
||||
RequestContext request(connection, m_root, url, method, version);
|
||||
|
||||
if (m_verbose.load() ) {
|
||||
request.print_debug_info();
|
||||
}
|
||||
/* Unexpected method */
|
||||
if (request.get_method() != RequestMethod::GET
|
||||
&& request.get_method() != RequestMethod::POST) {
|
||||
printf("Reject request because of unhandled request method.\n");
|
||||
printf("----------------------\n");
|
||||
return MHD_NO;
|
||||
}
|
||||
|
||||
auto response = handle_request(request);
|
||||
|
||||
if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
|
||||
printf("========== INTERNAL ERROR !! ============\n");
|
||||
if (!m_verbose.load()) {
|
||||
printf("Requesting : \n");
|
||||
printf("full_url : %s\n", url);
|
||||
request.print_debug_info();
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = response.send(request, connection);
|
||||
auto end_time = std::chrono::steady_clock::now();
|
||||
auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
|
||||
if (m_verbose.load()) {
|
||||
printf("Request time : %fs\n", time_span.count());
|
||||
printf("----------------------\n");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
Response InternalServer::handle_request(const RequestContext& request)
|
||||
{
|
||||
try {
|
||||
if (! request.is_valid_url())
|
||||
return build_404(request, "");
|
||||
|
||||
if (kiwix::startsWith(request.get_url(), "/skin/"))
|
||||
return handle_skin(request);
|
||||
|
||||
if (startsWith(request.get_url(), "/catalog"))
|
||||
return handle_catalog(request);
|
||||
|
||||
if (request.get_url() == "/meta")
|
||||
return handle_meta(request);
|
||||
|
||||
if (request.get_url() == "/search")
|
||||
return handle_search(request);
|
||||
|
||||
if (request.get_url() == "/suggest")
|
||||
return handle_suggest(request);
|
||||
|
||||
if (request.get_url() == "/random")
|
||||
return handle_random(request);
|
||||
|
||||
return handle_content(request);
|
||||
} catch (std::exception& e) {
|
||||
fprintf(stderr, "===== Unhandled error : %s\n", e.what());
|
||||
return build_500(e.what());
|
||||
} catch (...) {
|
||||
fprintf(stderr, "===== Unhandled unknown error\n");
|
||||
return build_500("Unknown error");
|
||||
}
|
||||
}
|
||||
|
||||
kainjow::mustache::data InternalServer::get_default_data()
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("root", m_root);
|
||||
return data;
|
||||
}
|
||||
|
||||
Response InternalServer::get_default_response()
|
||||
{
|
||||
return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton);
|
||||
}
|
||||
|
||||
|
||||
Response InternalServer::build_404(const RequestContext& request,
|
||||
const std::string& bookName)
|
||||
{
|
||||
kainjow::mustache::data results;
|
||||
results.set("url", request.get_full_url());
|
||||
|
||||
auto response = get_default_response();
|
||||
response.set_template(RESOURCE::templates::_404_html, results);
|
||||
response.set_mimeType("text/html");
|
||||
response.set_code(MHD_HTTP_NOT_FOUND);
|
||||
response.set_compress(true);
|
||||
response.set_taskbar(bookName, "");
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::build_500(const std::string& msg)
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("error", msg);
|
||||
Response response(m_root, true, false, false);
|
||||
response.set_template(RESOURCE::templates::_500_html, data);
|
||||
response.set_mimeType("text/html");
|
||||
response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::build_homepage(const RequestContext& request)
|
||||
{
|
||||
auto data = get_default_data();
|
||||
|
||||
kainjow::mustache::data books{kainjow::mustache::data::type::list};
|
||||
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
|
||||
auto& currentBook = mp_library->getBookById(bookId);
|
||||
|
||||
kainjow::mustache::data book;
|
||||
book.set("name", mp_nameMapper->getNameForId(bookId));
|
||||
book.set("title", currentBook.getTitle());
|
||||
book.set("description", currentBook.getDescription());
|
||||
book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
|
||||
book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
|
||||
books.push_back(book);
|
||||
}
|
||||
|
||||
data.set("books", books);
|
||||
|
||||
auto response = get_default_response();
|
||||
response.set_template(RESOURCE::templates::index_html, data);
|
||||
response.set_mimeType("text/html; charset=utf-8");
|
||||
response.set_compress(true);
|
||||
response.set_taskbar("", "");
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_meta(const RequestContext& request)
|
||||
{
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::string meta_name;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
meta_name = request.get_argument("name");
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range& e) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
if (reader == nullptr) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
std::string content;
|
||||
std::string mimeType = "text";
|
||||
|
||||
if (meta_name == "title") {
|
||||
content = reader->getTitle();
|
||||
} else if (meta_name == "description") {
|
||||
content = reader->getDescription();
|
||||
} else if (meta_name == "language") {
|
||||
content = reader->getLanguage();
|
||||
} else if (meta_name == "name") {
|
||||
content = reader->getName();
|
||||
} else if (meta_name == "tags") {
|
||||
content = reader->getTags();
|
||||
} else if (meta_name == "date") {
|
||||
content = reader->getDate();
|
||||
} else if (meta_name == "creator") {
|
||||
content = reader->getCreator();
|
||||
} else if (meta_name == "publisher") {
|
||||
content = reader->getPublisher();
|
||||
} else if (meta_name == "favicon") {
|
||||
reader->getFavicon(content, mimeType);
|
||||
} else {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
auto response = get_default_response();
|
||||
response.set_content(content);
|
||||
response.set_mimeType(mimeType);
|
||||
response.set_compress(false);
|
||||
response.set_cache(true);
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_suggest(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_suggest\n");
|
||||
}
|
||||
|
||||
std::string content;
|
||||
std::string mimeType;
|
||||
unsigned int maxSuggestionCount = 10;
|
||||
unsigned int suggestionCount = 0;
|
||||
std::string suggestion;
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::string term;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
term = request.get_argument("term");
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
if (m_verbose.load()) {
|
||||
printf("Searching suggestions for: \"%s\"\n", term.c_str());
|
||||
}
|
||||
|
||||
kainjow::mustache::data results{kainjow::mustache::data::type::list};
|
||||
|
||||
bool first = true;
|
||||
if (reader != nullptr) {
|
||||
/* Get the suggestions */
|
||||
reader->searchSuggestionsSmart(term, maxSuggestionCount);
|
||||
while (reader->getNextSuggestion(suggestion)) {
|
||||
kainjow::mustache::data result;
|
||||
result.set("label", suggestion);
|
||||
result.set("value", suggestion);
|
||||
result.set("first", first);
|
||||
first = false;
|
||||
results.push_back(result);
|
||||
suggestionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Propose the fulltext search if possible */
|
||||
if (reader->hasFulltextIndex()) {
|
||||
kainjow::mustache::data result;
|
||||
result.set("label", "containing '" + term + "'...");
|
||||
result.set("value", term + " ");
|
||||
result.set("first", first);
|
||||
results.push_back(result);
|
||||
}
|
||||
|
||||
auto data = get_default_data();
|
||||
data.set("suggestions", results);
|
||||
|
||||
auto response = get_default_response();
|
||||
response.set_template(RESOURCE::templates::suggestion_json, data);
|
||||
response.set_mimeType("application/json; charset=utf-8");
|
||||
response.set_compress(true);
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_skin(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_skin\n");
|
||||
}
|
||||
|
||||
auto response = get_default_response();
|
||||
auto resourceName = request.get_url().substr(1);
|
||||
try {
|
||||
response.set_content(getResource(resourceName));
|
||||
} catch (const ResourceNotFound& e) {
|
||||
return build_404(request, "");
|
||||
}
|
||||
response.set_mimeType(getMimeTypeForFile(resourceName));
|
||||
response.set_compress(true);
|
||||
response.set_cache(true);
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_search(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_search\n");
|
||||
}
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
std::string patternString;
|
||||
try {
|
||||
patternString = request.get_argument("pattern");
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
/* Retrive geo search */
|
||||
bool has_geo_query = false;
|
||||
float latitude = 0;
|
||||
float longitude = 0;
|
||||
float distance = 0;
|
||||
try {
|
||||
latitude = request.get_argument<float>("latitude");
|
||||
longitude = request.get_argument<float>("longitude");
|
||||
distance = request.get_argument<float>("distance");
|
||||
has_geo_query = true;
|
||||
} catch(const std::out_of_range&) {}
|
||||
catch(const std::invalid_argument&) {}
|
||||
|
||||
std::shared_ptr<Reader> reader(nullptr);
|
||||
try {
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
/* Try first to load directly the article */
|
||||
if (reader != nullptr && !patternString.empty()) {
|
||||
std::string patternCorrespondingUrl;
|
||||
auto variants = reader->getTitleVariants(patternString);
|
||||
auto variantsItr = variants.begin();
|
||||
|
||||
while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
|
||||
try {
|
||||
auto entry = reader->getEntryFromTitle(*variantsItr);
|
||||
entry = entry.getFinalEntry();
|
||||
patternCorrespondingUrl = entry.getPath();
|
||||
break;
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
variantsItr++;
|
||||
}
|
||||
}
|
||||
|
||||
/* If article found then redirect directly to it */
|
||||
if (!patternCorrespondingUrl.empty()) {
|
||||
auto response = get_default_response();
|
||||
response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/* Make the search */
|
||||
auto response = get_default_response();
|
||||
response.set_mimeType("text/html; charset=utf-8");
|
||||
response.set_taskbar(bookName, reader ? reader->getTitle() : "");
|
||||
response.set_compress(true);
|
||||
|
||||
if ( (!reader && !bookName.empty())
|
||||
|| (patternString.empty() && ! has_geo_query) ) {
|
||||
auto data = get_default_data();
|
||||
data.set("pattern", encodeDiples(patternString));
|
||||
response.set_template(RESOURCE::templates::no_search_result_html, data);
|
||||
response.set_code(MHD_HTTP_NOT_FOUND);
|
||||
return response;
|
||||
}
|
||||
|
||||
Searcher searcher;
|
||||
if (reader) {
|
||||
searcher.add_reader(reader.get());
|
||||
} else {
|
||||
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
|
||||
auto currentReader = mp_library->getReaderById(bookId);
|
||||
if (currentReader) {
|
||||
searcher.add_reader(currentReader.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto start = 0;
|
||||
try {
|
||||
start = request.get_argument<unsigned int>("start");
|
||||
} catch (const std::exception&) {}
|
||||
auto end = 25;
|
||||
try {
|
||||
end = request.get_argument<unsigned int>("end");
|
||||
} catch (const std::exception&) {}
|
||||
if (start>end) {
|
||||
auto tmp = start;
|
||||
start = end;
|
||||
end = tmp;
|
||||
}
|
||||
if (end > start + MAX_SEARCH_LEN) {
|
||||
end = start + MAX_SEARCH_LEN;
|
||||
}
|
||||
|
||||
/* Get the results */
|
||||
try {
|
||||
if (patternString.empty()) {
|
||||
searcher.geo_search(latitude, longitude, distance,
|
||||
start, end, m_verbose.load());
|
||||
} else {
|
||||
searcher.search(patternString,
|
||||
start, end, m_verbose.load());
|
||||
}
|
||||
SearchRenderer renderer(&searcher, mp_nameMapper);
|
||||
renderer.setSearchPattern(patternString);
|
||||
renderer.setSearchContent(bookName);
|
||||
renderer.setProtocolPrefix(m_root + "/");
|
||||
renderer.setSearchProtocolPrefix(m_root + "/search?");
|
||||
response.set_content(renderer.getHtml());
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << e.what() << std::endl;
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_random(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_random\n");
|
||||
}
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
if (reader == nullptr) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
try {
|
||||
auto entry = reader->getRandomPage();
|
||||
entry = entry.getFinalEntry();
|
||||
auto response = get_default_response();
|
||||
response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
|
||||
return response;
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
}
|
||||
|
||||
Response InternalServer::handle_catalog(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_catalog");
|
||||
}
|
||||
|
||||
std::string host;
|
||||
std::string url;
|
||||
try {
|
||||
host = request.get_header("Host");
|
||||
url = request.get_url_part(1);
|
||||
} catch (const std::out_of_range&) {
|
||||
return build_404(request, "");
|
||||
}
|
||||
|
||||
if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
|
||||
return build_404(request, "");
|
||||
}
|
||||
|
||||
auto response = get_default_response();
|
||||
response.set_compress(true);
|
||||
if (url == "searchdescription.xml") {
|
||||
response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
|
||||
response.set_mimeType("application/opensearchdescription+xml");
|
||||
return response;
|
||||
}
|
||||
|
||||
zim::Uuid uuid;
|
||||
kiwix::OPDSDumper opdsDumper;
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
|
||||
opdsDumper.setLibrary(mp_library);
|
||||
response.set_mimeType("application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8");
|
||||
std::vector<std::string> bookIdsToDump;
|
||||
if (url == "root.xml") {
|
||||
opdsDumper.setTitle("All zims");
|
||||
uuid = zim::Uuid::generate(host);
|
||||
bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
|
||||
} else if (url == "search") {
|
||||
auto filter = kiwix::Filter().valid(true).local(true).remote(true);
|
||||
string query("<Empty query>");
|
||||
size_t count(10);
|
||||
size_t startIndex(0);
|
||||
try {
|
||||
query = request.get_argument("q");
|
||||
filter.query(query);
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.name(request.get_argument("name"));
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
filter.lang(request.get_argument("lang"));
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
count = extractFromString<unsigned long>(request.get_argument("count"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
startIndex = extractFromString<unsigned long>(request.get_argument("start"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
|
||||
} catch (...) {}
|
||||
opdsDumper.setTitle("Search result for " + query);
|
||||
uuid = zim::Uuid::generate();
|
||||
bookIdsToDump = mp_library->filter(filter);
|
||||
auto totalResults = bookIdsToDump.size();
|
||||
bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
|
||||
if (count>0 && bookIdsToDump.size() > count) {
|
||||
bookIdsToDump.resize(count);
|
||||
}
|
||||
opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
|
||||
}
|
||||
|
||||
opdsDumper.setId(kiwix::to_string(uuid));
|
||||
response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
|
||||
return response;
|
||||
}
|
||||
|
||||
Response InternalServer::handle_content(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_content\n");
|
||||
}
|
||||
|
||||
std::string baseUrl;
|
||||
std::string content;
|
||||
std::string mimeType;
|
||||
|
||||
kiwix::Entry entry;
|
||||
|
||||
std::string bookName;
|
||||
try {
|
||||
bookName = request.get_url_part(0);
|
||||
} catch (const std::out_of_range& e) {
|
||||
return build_homepage(request);
|
||||
}
|
||||
if (bookName.empty())
|
||||
return build_homepage(request);
|
||||
|
||||
std::string bookId;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range& e) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
if (reader == nullptr) {
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
auto urlStr = request.get_url().substr(bookName.size()+1);
|
||||
if (urlStr[0] == '/') {
|
||||
urlStr = urlStr.substr(1);
|
||||
}
|
||||
|
||||
try {
|
||||
entry = reader->getEntryFromPath(urlStr);
|
||||
if (entry.isRedirect() || urlStr.empty()) {
|
||||
// If urlStr is empty, we want to mainPage.
|
||||
// We must do a redirection to the real page.
|
||||
entry = entry.getFinalEntry();
|
||||
auto response = get_default_response();
|
||||
response.set_redirection(m_root + "/" + bookName + "/" +
|
||||
kiwix::urlEncode(entry.getPath()));
|
||||
return response;
|
||||
}
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
if (m_verbose.load())
|
||||
printf("Failed to find %s\n", urlStr.c_str());
|
||||
|
||||
return build_404(request, bookName);
|
||||
}
|
||||
|
||||
try {
|
||||
mimeType = entry.getMimetype();
|
||||
} catch (exception& e) {
|
||||
mimeType = "application/octet-stream";
|
||||
}
|
||||
|
||||
if (m_verbose.load()) {
|
||||
printf("Found %s\n", urlStr.c_str());
|
||||
printf("mimeType: %s\n", mimeType.c_str());
|
||||
}
|
||||
|
||||
if (mimeType.find("text/") != string::npos
|
||||
|| mimeType.find("application/javascript") != string::npos
|
||||
|| mimeType.find("application/json") != string::npos) {
|
||||
zim::Blob raw_content = entry.getBlob();
|
||||
content = string(raw_content.data(), raw_content.size());
|
||||
auto response = get_default_response();
|
||||
|
||||
if (mimeType.find("text/html") != string::npos)
|
||||
response.set_taskbar(bookName, reader->getTitle());
|
||||
|
||||
response.set_mimeType(mimeType);
|
||||
response.set_content(content);
|
||||
response.set_compress(true);
|
||||
response.set_cache(true);
|
||||
return response;
|
||||
} else {
|
||||
int range_len;
|
||||
if (request.get_range().second == -1) {
|
||||
range_len = entry.getSize() - request.get_range().first;
|
||||
} else {
|
||||
range_len = request.get_range().second - request.get_range().first;
|
||||
}
|
||||
auto response = get_default_response();
|
||||
response.set_entry(entry);
|
||||
response.set_mimeType(mimeType);
|
||||
response.set_range_first(request.get_range().first);
|
||||
response.set_range_len(range_len);
|
||||
response.set_cache(true);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#include "byte_range.h"
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <algorithm>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
namespace {
|
||||
|
||||
ByteRange parseByteRange(const std::string& rangeStr)
|
||||
{
|
||||
std::istringstream iss(rangeStr);
|
||||
|
||||
int64_t start, end = INT64_MAX;
|
||||
if (iss >> start) {
|
||||
if ( start < 0 ) {
|
||||
if ( iss.eof() )
|
||||
return ByteRange(-start);
|
||||
} else {
|
||||
char c;
|
||||
if (iss >> c && c=='-') {
|
||||
iss >> end; // if this fails, end is not modified, which is OK
|
||||
if (iss.eof() && start <= end)
|
||||
return ByteRange(ByteRange::PARSED, start, end);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ByteRange(ByteRange::INVALID, 0, INT64_MAX);
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
ByteRange::ByteRange()
|
||||
: kind_(NONE)
|
||||
, first_(0)
|
||||
, last_(INT64_MAX)
|
||||
{}
|
||||
|
||||
ByteRange::ByteRange(Kind kind, int64_t first, int64_t last)
|
||||
: kind_(kind)
|
||||
, first_(first)
|
||||
, last_(last)
|
||||
{
|
||||
assert(kind != NONE);
|
||||
assert(first >= 0);
|
||||
assert(last >= first || (first == 0 && last == -1));
|
||||
}
|
||||
|
||||
ByteRange::ByteRange(int64_t suffix_length)
|
||||
: kind_(PARSED)
|
||||
, first_(-suffix_length)
|
||||
, last_(INT64_MAX)
|
||||
{
|
||||
assert(suffix_length > 0);
|
||||
}
|
||||
|
||||
int64_t ByteRange::first() const
|
||||
{
|
||||
assert(kind_ > PARSED);
|
||||
return first_;
|
||||
}
|
||||
|
||||
int64_t ByteRange::last() const
|
||||
{
|
||||
assert(kind_ > PARSED);
|
||||
return last_;
|
||||
}
|
||||
|
||||
int64_t ByteRange::length() const
|
||||
{
|
||||
assert(kind_ > PARSED);
|
||||
return last_ + 1 - first_;
|
||||
}
|
||||
|
||||
ByteRange ByteRange::parse(const std::string& rangeStr)
|
||||
{
|
||||
const std::string byteUnitSpec("bytes=");
|
||||
if ( ! kiwix::startsWith(rangeStr, byteUnitSpec) )
|
||||
return ByteRange(INVALID, 0, INT64_MAX);
|
||||
|
||||
return parseByteRange(rangeStr.substr(byteUnitSpec.size()));
|
||||
}
|
||||
|
||||
ByteRange ByteRange::resolve(int64_t contentSize) const
|
||||
{
|
||||
if ( kind() == NONE )
|
||||
return ByteRange(RESOLVED_FULL_CONTENT, 0, contentSize-1);
|
||||
|
||||
if ( kind() == INVALID )
|
||||
return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
|
||||
|
||||
const int64_t resolved_first = first_ < 0
|
||||
? std::max(int64_t(0), contentSize + first_)
|
||||
: first_;
|
||||
|
||||
const int64_t resolved_last = std::min(contentSize-1, last_);
|
||||
|
||||
if ( resolved_first > resolved_last )
|
||||
return ByteRange(RESOLVED_UNSATISFIABLE, 0, contentSize-1);
|
||||
|
||||
return ByteRange(RESOLVED_PARTIAL_CONTENT, resolved_first, resolved_last);
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -1,86 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef KIWIXLIB_SERVER_BYTE_RANGE_H
|
||||
#define KIWIXLIB_SERVER_BYTE_RANGE_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
class ByteRange
|
||||
{
|
||||
public: // types
|
||||
// ByteRange is parsed in a request, then it must be resolved (taking
|
||||
// into account the actual size of the requested resource) before
|
||||
// being applied in the response.
|
||||
// The Kind enum represents possible states in such a lifecycle.
|
||||
enum Kind {
|
||||
// The request is not a range request (no Range header)
|
||||
NONE,
|
||||
|
||||
// The value of the Range header is not a valid continuous
|
||||
// range. Note that a valid (according to RFC7233) sequence of multiple
|
||||
// byte ranges is considered invalid in the current implementation
|
||||
// (i.e. only single-range partial requests are supported).
|
||||
INVALID,
|
||||
|
||||
// This byte-range has been successfully parsed from the request
|
||||
PARSED,
|
||||
|
||||
// This is a response to a regular (non-range) request
|
||||
RESOLVED_FULL_CONTENT,
|
||||
|
||||
// The range request is invalid or unsatisfiable
|
||||
RESOLVED_UNSATISFIABLE,
|
||||
|
||||
// This is a response to a (satisfiable) range request
|
||||
RESOLVED_PARTIAL_CONTENT,
|
||||
};
|
||||
|
||||
public: // functions
|
||||
// Constructs a ByteRange object of NONE kind
|
||||
ByteRange();
|
||||
|
||||
// Constructs a ByteRange object of the given kind (except NONE)
|
||||
ByteRange(Kind kind, int64_t first, int64_t last);
|
||||
|
||||
// Constructs a ByteRange object of PARSED kind corresponding to a
|
||||
// range request of the form "Range: bytes=-suffix_length"
|
||||
explicit ByteRange(int64_t suffix_length);
|
||||
|
||||
Kind kind() const { return kind_; }
|
||||
int64_t first() const;
|
||||
int64_t last() const;
|
||||
int64_t length() const;
|
||||
|
||||
static ByteRange parse(const std::string& rangeStr);
|
||||
ByteRange resolve(int64_t contentSize) const;
|
||||
|
||||
private: // data
|
||||
Kind kind_;
|
||||
int64_t first_;
|
||||
int64_t last_;
|
||||
};
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
#endif //KIWIXLIB_SERVER_BYTE_RANGE_H
|
||||
@@ -1,135 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#include "etag.h"
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <sstream>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
namespace {
|
||||
|
||||
// Characters in the options part of the ETag could in principle be picked up
|
||||
// from the latin alphabet in natural order (the character corresponding to
|
||||
// ETag::Option opt would be 'a'+opt; that would somewhat simplify the code in
|
||||
// this file). However it is better to have some mnemonics in the option names,
|
||||
// hence below variable: all_options[opt] corresponds to the character going
|
||||
// into the ETag for ETag::Option opt.
|
||||
// IMPORTANT: The characters in all_options must come in sorted order (so that
|
||||
// IMPORTANT: isValidOptionsString() works correctly).
|
||||
const char all_options[] = "cz";
|
||||
|
||||
static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
|
||||
|
||||
bool isValidServerId(const std::string& s)
|
||||
{
|
||||
return !s.empty() && s.find_first_of("\"/") == std::string::npos;
|
||||
}
|
||||
|
||||
bool isSubsequenceOf(const std::string& s, const std::string& sortedString)
|
||||
{
|
||||
std::string::size_type i = 0;
|
||||
for ( const char c : s )
|
||||
{
|
||||
const std::string::size_type j = sortedString.find(c, i);
|
||||
if ( j == std::string::npos )
|
||||
return false;
|
||||
i = j+1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool isValidOptionsString(const std::string& s)
|
||||
{
|
||||
return isSubsequenceOf(s, all_options);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
void ETag::set_option(Option opt)
|
||||
{
|
||||
if ( ! get_option(opt) )
|
||||
{
|
||||
m_options.push_back(all_options[opt]);
|
||||
std::sort(m_options.begin(), m_options.end());
|
||||
}
|
||||
}
|
||||
|
||||
bool ETag::get_option(Option opt) const
|
||||
{
|
||||
return m_options.find(all_options[opt]) != std::string::npos;
|
||||
}
|
||||
|
||||
std::string ETag::get_etag() const
|
||||
{
|
||||
if ( m_serverId.empty() )
|
||||
return std::string();
|
||||
|
||||
return "\"" + m_serverId + "/" + m_options + "\"";
|
||||
}
|
||||
|
||||
ETag::ETag(const std::string& serverId, const std::string& options)
|
||||
{
|
||||
if ( isValidServerId(serverId) && isValidOptionsString(options) )
|
||||
{
|
||||
m_serverId = serverId;
|
||||
m_options = options;
|
||||
}
|
||||
}
|
||||
|
||||
ETag ETag::parse(std::string s)
|
||||
{
|
||||
if ( kiwix::startsWith("W/", s) )
|
||||
s = s.substr(2);
|
||||
|
||||
if ( s.front() != '"' || s.back() != '"' )
|
||||
return ETag();
|
||||
|
||||
s = s.substr(1, s.size()-2);
|
||||
|
||||
const std::string::size_type i = s.find('/');
|
||||
if ( i == std::string::npos )
|
||||
return ETag();
|
||||
|
||||
return ETag(s.substr(0, i), s.substr(i+1));
|
||||
}
|
||||
|
||||
ETag ETag::match(const std::string& etags, const std::string& server_id)
|
||||
{
|
||||
std::istringstream ss(etags);
|
||||
std::string etag_str;
|
||||
while ( ss >> etag_str )
|
||||
{
|
||||
if ( etag_str.back() == ',' )
|
||||
etag_str.pop_back();
|
||||
|
||||
const ETag etag = parse(etag_str);
|
||||
if ( etag && etag.m_serverId == server_id )
|
||||
return etag;
|
||||
}
|
||||
|
||||
return ETag();
|
||||
}
|
||||
|
||||
} // namespace kiwix
|
||||
@@ -1,85 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef KIWIXLIB_SERVER_ETAG_H
|
||||
#define KIWIXLIB_SERVER_ETAG_H
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
// The ETag string used by Kiwix server (more precisely, its value inside the
|
||||
// double quotes) consists of two parts:
|
||||
//
|
||||
// 1. ServerId - The string obtained on server start up
|
||||
//
|
||||
// 2. Options - Zero or more characters encoding the values of some of the
|
||||
// headers of the response
|
||||
//
|
||||
// The two parts are separated with a slash (/) symbol (which is always present,
|
||||
// even when the the options part is empty). Neither portion of a Kiwix ETag
|
||||
// may contain the slash symbol.
|
||||
// Examples of valid Kiwix server ETags (including the double quotes):
|
||||
//
|
||||
// "abcdefghijklmn/"
|
||||
// "1234567890/z"
|
||||
// "1234567890/cz"
|
||||
//
|
||||
// The options part of the Kiwix ETag allows to correctly set the required
|
||||
// headers when responding to a conditional If-None-Match request with a 304
|
||||
// (Not Modified) response without following the full code path that would
|
||||
// discover the necessary options.
|
||||
|
||||
class ETag
|
||||
{
|
||||
public: // types
|
||||
enum Option {
|
||||
CACHEABLE_ENTITY,
|
||||
COMPRESSED_CONTENT,
|
||||
OPTION_COUNT
|
||||
};
|
||||
|
||||
public: // functions
|
||||
ETag() {}
|
||||
|
||||
void set_server_id(const std::string& id) { m_serverId = id; }
|
||||
void set_option(Option opt);
|
||||
|
||||
explicit operator bool() const { return !m_serverId.empty(); }
|
||||
|
||||
bool get_option(Option opt) const;
|
||||
std::string get_etag() const;
|
||||
|
||||
|
||||
static ETag match(const std::string& etags, const std::string& server_id);
|
||||
|
||||
private: // functions
|
||||
ETag(const std::string& serverId, const std::string& options);
|
||||
|
||||
static ETag parse(std::string s);
|
||||
|
||||
private: // data
|
||||
std::string m_serverId;
|
||||
std::string m_options;
|
||||
};
|
||||
|
||||
} // namespace kiwix
|
||||
|
||||
#endif // KIWIXLIB_SERVER_ETAG_H
|
||||
@@ -1,798 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include "internalServer.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
# if !defined(__MINGW32__) && (_MSC_VER < 1600)
|
||||
# include "stdint4win.h"
|
||||
# endif
|
||||
# include <winsock2.h>
|
||||
# include <ws2tcpip.h>
|
||||
# ifdef __GNUC__
|
||||
// inet_pton is not declared in mingw, even if the function exists.
|
||||
extern "C" {
|
||||
WINSOCK_API_LINKAGE INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
|
||||
}
|
||||
# endif
|
||||
typedef UINT64 uint64_t;
|
||||
typedef UINT16 uint16_t;
|
||||
#endif
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
}
|
||||
|
||||
#include "tools/otherTools.h"
|
||||
#include "tools/pathTools.h"
|
||||
#include "tools/regexTools.h"
|
||||
#include "tools/stringTools.h"
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
#include "entry.h"
|
||||
#include "searcher.h"
|
||||
#include "search_renderer.h"
|
||||
#include "opds_dumper.h"
|
||||
|
||||
#include <zim/uuid.h>
|
||||
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include <pthread.h>
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <chrono>
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
# include <arpa/inet.h>
|
||||
#endif
|
||||
|
||||
#include "request_context.h"
|
||||
#include "response.h"
|
||||
|
||||
#define MAX_SEARCH_LEN 140
|
||||
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
static IdNameMapper defaultNameMapper;
|
||||
|
||||
static MHD_Result staticHandlerCallback(void* cls,
|
||||
struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls);
|
||||
|
||||
|
||||
InternalServer::InternalServer(Library* library,
|
||||
NameMapper* nameMapper,
|
||||
std::string addr,
|
||||
int port,
|
||||
std::string root,
|
||||
int nbThreads,
|
||||
bool verbose,
|
||||
bool withTaskbar,
|
||||
bool withLibraryButton,
|
||||
bool blockExternalLinks) :
|
||||
m_addr(addr),
|
||||
m_port(port),
|
||||
m_root(root),
|
||||
m_nbThreads(nbThreads),
|
||||
m_verbose(verbose),
|
||||
m_withTaskbar(withTaskbar),
|
||||
m_withLibraryButton(withLibraryButton),
|
||||
m_blockExternalLinks(blockExternalLinks),
|
||||
mp_daemon(nullptr),
|
||||
mp_library(library),
|
||||
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
|
||||
{}
|
||||
|
||||
bool InternalServer::start() {
|
||||
#ifdef _WIN32
|
||||
int flags = MHD_USE_SELECT_INTERNALLY;
|
||||
#else
|
||||
int flags = MHD_USE_POLL_INTERNALLY;
|
||||
#endif
|
||||
if (m_verbose.load())
|
||||
flags |= MHD_USE_DEBUG;
|
||||
|
||||
|
||||
struct sockaddr_in sockAddr;
|
||||
memset(&sockAddr, 0, sizeof(sockAddr));
|
||||
sockAddr.sin_family = AF_INET;
|
||||
sockAddr.sin_port = htons(m_port);
|
||||
if (m_addr.empty()) {
|
||||
if (0 != INADDR_ANY)
|
||||
sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
|
||||
} else {
|
||||
if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
|
||||
std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
mp_daemon = MHD_start_daemon(flags,
|
||||
m_port,
|
||||
NULL,
|
||||
NULL,
|
||||
&staticHandlerCallback,
|
||||
this,
|
||||
MHD_OPTION_SOCK_ADDR, &sockAddr,
|
||||
MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
|
||||
MHD_OPTION_END);
|
||||
if (mp_daemon == nullptr) {
|
||||
std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
|
||||
<< " is maybe already occupied or need more permissions to be open. "
|
||||
"Please try as root or with a port number higher or equal to 1024."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
|
||||
m_server_id = kiwix::to_string(server_start_time.count());
|
||||
return true;
|
||||
}
|
||||
|
||||
void InternalServer::stop()
|
||||
{
|
||||
MHD_stop_daemon(mp_daemon);
|
||||
}
|
||||
|
||||
static MHD_Result staticHandlerCallback(void* cls,
|
||||
struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls)
|
||||
{
|
||||
InternalServer* _this = static_cast<InternalServer*>(cls);
|
||||
|
||||
return _this->handlerCallback(connection,
|
||||
url,
|
||||
method,
|
||||
version,
|
||||
upload_data,
|
||||
upload_data_size,
|
||||
cont_cls);
|
||||
}
|
||||
|
||||
MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls)
|
||||
{
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
if (m_verbose.load() ) {
|
||||
printf("======================\n");
|
||||
printf("Requesting : \n");
|
||||
printf("full_url : %s\n", url);
|
||||
}
|
||||
RequestContext request(connection, m_root, url, method, version);
|
||||
|
||||
if (m_verbose.load() ) {
|
||||
request.print_debug_info();
|
||||
}
|
||||
/* Unexpected method */
|
||||
if (request.get_method() != RequestMethod::GET
|
||||
&& request.get_method() != RequestMethod::POST
|
||||
&& request.get_method() != RequestMethod::HEAD) {
|
||||
printf("Reject request because of unhandled request method.\n");
|
||||
printf("----------------------\n");
|
||||
return MHD_NO;
|
||||
}
|
||||
|
||||
auto response = handle_request(request);
|
||||
|
||||
if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
|
||||
printf("========== INTERNAL ERROR !! ============\n");
|
||||
if (!m_verbose.load()) {
|
||||
printf("Requesting : \n");
|
||||
printf("full_url : %s\n", url);
|
||||
request.print_debug_info();
|
||||
}
|
||||
}
|
||||
|
||||
if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
|
||||
response->set_server_id(m_server_id);
|
||||
|
||||
auto ret = response->send(request, connection);
|
||||
auto end_time = std::chrono::steady_clock::now();
|
||||
auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
|
||||
if (m_verbose.load()) {
|
||||
printf("Request time : %fs\n", time_span.count());
|
||||
printf("----------------------\n");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
|
||||
{
|
||||
try {
|
||||
if (! request.is_valid_url())
|
||||
return Response::build_404(*this, request, "");
|
||||
|
||||
const ETag etag = get_matching_if_none_match_etag(request);
|
||||
if ( etag )
|
||||
return Response::build_304(*this, etag);
|
||||
|
||||
if (kiwix::startsWith(request.get_url(), "/skin/"))
|
||||
return handle_skin(request);
|
||||
|
||||
if (startsWith(request.get_url(), "/catalog"))
|
||||
return handle_catalog(request);
|
||||
|
||||
if (request.get_url() == "/meta")
|
||||
return handle_meta(request);
|
||||
|
||||
if (request.get_url() == "/search")
|
||||
return handle_search(request);
|
||||
|
||||
if (request.get_url() == "/suggest")
|
||||
return handle_suggest(request);
|
||||
|
||||
if (request.get_url() == "/random")
|
||||
return handle_random(request);
|
||||
|
||||
if (request.get_url() == "/catch/external")
|
||||
return handle_captured_external(request);
|
||||
|
||||
return handle_content(request);
|
||||
} catch (std::exception& e) {
|
||||
fprintf(stderr, "===== Unhandled error : %s\n", e.what());
|
||||
return Response::build_500(*this, e.what());
|
||||
} catch (...) {
|
||||
fprintf(stderr, "===== Unhandled unknown error\n");
|
||||
return Response::build_500(*this, "Unknown error");
|
||||
}
|
||||
}
|
||||
|
||||
MustacheData InternalServer::get_default_data() const
|
||||
{
|
||||
MustacheData data;
|
||||
data.set("root", m_root);
|
||||
return data;
|
||||
}
|
||||
|
||||
MustacheData InternalServer::homepage_data() const
|
||||
{
|
||||
auto data = get_default_data();
|
||||
|
||||
MustacheData books{MustacheData::type::list};
|
||||
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
|
||||
auto& currentBook = mp_library->getBookById(bookId);
|
||||
|
||||
MustacheData book;
|
||||
book.set("name", mp_nameMapper->getNameForId(bookId));
|
||||
book.set("title", currentBook.getTitle());
|
||||
book.set("description", currentBook.getDescription());
|
||||
book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
|
||||
book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
|
||||
books.push_back(book);
|
||||
}
|
||||
|
||||
data.set("books", books);
|
||||
return data;
|
||||
}
|
||||
|
||||
bool InternalServer::etag_not_needed(const RequestContext& request) const
|
||||
{
|
||||
const std::string url = request.get_url();
|
||||
return kiwix::startsWith(url, "/catalog")
|
||||
|| url == "/search"
|
||||
|| url == "/suggest"
|
||||
|| url == "/random"
|
||||
|| url == "/catch/external";
|
||||
}
|
||||
|
||||
ETag
|
||||
InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
|
||||
{
|
||||
try {
|
||||
const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
|
||||
return ETag::match(etag_list, m_server_id);
|
||||
} catch (const std::out_of_range&) {
|
||||
return ETag();
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
|
||||
{
|
||||
return ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8");
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
|
||||
{
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::string meta_name;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
meta_name = request.get_argument("name");
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range& e) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
if (reader == nullptr) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
std::string content;
|
||||
std::string mimeType = "text";
|
||||
|
||||
if (meta_name == "title") {
|
||||
content = reader->getTitle();
|
||||
} else if (meta_name == "description") {
|
||||
content = reader->getDescription();
|
||||
} else if (meta_name == "language") {
|
||||
content = reader->getLanguage();
|
||||
} else if (meta_name == "name") {
|
||||
content = reader->getName();
|
||||
} else if (meta_name == "tags") {
|
||||
content = reader->getTags();
|
||||
} else if (meta_name == "date") {
|
||||
content = reader->getDate();
|
||||
} else if (meta_name == "creator") {
|
||||
content = reader->getCreator();
|
||||
} else if (meta_name == "publisher") {
|
||||
content = reader->getPublisher();
|
||||
} else if (meta_name == "favicon") {
|
||||
reader->getFavicon(content, mimeType);
|
||||
} else {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
auto response = ContentResponse::build(*this, content, mimeType);
|
||||
response->set_cacheable();
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_suggest\n");
|
||||
}
|
||||
|
||||
std::string content;
|
||||
std::string mimeType;
|
||||
unsigned int maxSuggestionCount = 10;
|
||||
unsigned int suggestionCount = 0;
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::string term;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
term = request.get_argument("term");
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
if (m_verbose.load()) {
|
||||
printf("Searching suggestions for: \"%s\"\n", term.c_str());
|
||||
}
|
||||
|
||||
MustacheData results{MustacheData::type::list};
|
||||
|
||||
bool first = true;
|
||||
if (reader != nullptr) {
|
||||
/* Get the suggestions */
|
||||
SuggestionsList_t suggestions;
|
||||
reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
|
||||
for(auto& suggestion:suggestions) {
|
||||
MustacheData result;
|
||||
result.set("label", suggestion[0]);
|
||||
result.set("value", suggestion[0]);
|
||||
result.set("first", first);
|
||||
first = false;
|
||||
results.push_back(result);
|
||||
suggestionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Propose the fulltext search if possible */
|
||||
if (reader->hasFulltextIndex()) {
|
||||
MustacheData result;
|
||||
result.set("label", "containing '" + term + "'...");
|
||||
result.set("value", term + " ");
|
||||
result.set("first", first);
|
||||
results.push_back(result);
|
||||
}
|
||||
|
||||
auto data = get_default_data();
|
||||
data.set("suggestions", results);
|
||||
|
||||
auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8");
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_skin\n");
|
||||
}
|
||||
|
||||
auto resourceName = request.get_url().substr(1);
|
||||
try {
|
||||
auto response = ContentResponse::build(
|
||||
*this,
|
||||
getResource(resourceName),
|
||||
getMimeTypeForFile(resourceName));
|
||||
response->set_cacheable();
|
||||
return std::move(response);
|
||||
} catch (const ResourceNotFound& e) {
|
||||
return Response::build_404(*this, request, "");
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_search\n");
|
||||
}
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
std::string patternString;
|
||||
try {
|
||||
patternString = request.get_argument("pattern");
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
/* Retrive geo search */
|
||||
bool has_geo_query = false;
|
||||
float latitude = 0;
|
||||
float longitude = 0;
|
||||
float distance = 0;
|
||||
try {
|
||||
latitude = request.get_argument<float>("latitude");
|
||||
longitude = request.get_argument<float>("longitude");
|
||||
distance = request.get_argument<float>("distance");
|
||||
has_geo_query = true;
|
||||
} catch(const std::out_of_range&) {}
|
||||
catch(const std::invalid_argument&) {}
|
||||
|
||||
std::shared_ptr<Reader> reader(nullptr);
|
||||
try {
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
/* Try first to load directly the article */
|
||||
if (reader != nullptr && !patternString.empty()) {
|
||||
std::string patternCorrespondingUrl;
|
||||
auto variants = reader->getTitleVariants(patternString);
|
||||
auto variantsItr = variants.begin();
|
||||
|
||||
while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
|
||||
try {
|
||||
auto entry = reader->getEntryFromTitle(*variantsItr);
|
||||
entry = entry.getFinalEntry();
|
||||
patternCorrespondingUrl = entry.getPath();
|
||||
break;
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
variantsItr++;
|
||||
}
|
||||
}
|
||||
|
||||
/* If article found then redirect directly to it */
|
||||
if (!patternCorrespondingUrl.empty()) {
|
||||
auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl;
|
||||
return Response::build_redirect(*this, redirectUrl);
|
||||
}
|
||||
}
|
||||
|
||||
/* Make the search */
|
||||
if ( (!reader && !bookName.empty())
|
||||
|| (patternString.empty() && ! has_geo_query) ) {
|
||||
auto data = get_default_data();
|
||||
data.set("pattern", encodeDiples(patternString));
|
||||
auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
|
||||
response->set_taskbar(bookName, reader ? reader->getTitle() : "");
|
||||
response->set_code(MHD_HTTP_NOT_FOUND);
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
Searcher searcher;
|
||||
if (reader) {
|
||||
searcher.add_reader(reader.get());
|
||||
} else {
|
||||
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
|
||||
auto currentReader = mp_library->getReaderById(bookId);
|
||||
if (currentReader) {
|
||||
searcher.add_reader(currentReader.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto start = 0;
|
||||
try {
|
||||
start = request.get_argument<unsigned int>("start");
|
||||
} catch (const std::exception&) {}
|
||||
|
||||
auto pageLength = 25;
|
||||
try {
|
||||
pageLength = request.get_argument<unsigned int>("pageLength");
|
||||
} catch (const std::exception&) {}
|
||||
if (pageLength > MAX_SEARCH_LEN) {
|
||||
pageLength = MAX_SEARCH_LEN;
|
||||
}
|
||||
if (pageLength == 0) {
|
||||
pageLength = 25;
|
||||
}
|
||||
|
||||
auto end = start + pageLength;
|
||||
|
||||
/* Get the results */
|
||||
try {
|
||||
if (patternString.empty()) {
|
||||
searcher.geo_search(latitude, longitude, distance,
|
||||
start, end, m_verbose.load());
|
||||
} else {
|
||||
searcher.search(patternString,
|
||||
start, end, m_verbose.load());
|
||||
}
|
||||
SearchRenderer renderer(&searcher, mp_nameMapper);
|
||||
renderer.setSearchPattern(patternString);
|
||||
renderer.setSearchContent(bookName);
|
||||
renderer.setProtocolPrefix(m_root + "/");
|
||||
renderer.setSearchProtocolPrefix(m_root + "/search?");
|
||||
renderer.setPageLength(pageLength);
|
||||
auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
|
||||
response->set_taskbar(bookName, reader ? reader->getTitle() : "");
|
||||
//changing status code if no result obtained
|
||||
if(searcher.getEstimatedResultCount() == 0)
|
||||
{
|
||||
response->set_code(MHD_HTTP_NO_CONTENT);
|
||||
}
|
||||
|
||||
return std::move(response);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << e.what() << std::endl;
|
||||
return Response::build_500(*this, e.what());
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_random\n");
|
||||
}
|
||||
|
||||
std::string bookName;
|
||||
std::string bookId;
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
bookName = request.get_argument("content");
|
||||
bookId = mp_nameMapper->getIdForName(bookName);
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range&) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
if (reader == nullptr) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
try {
|
||||
auto entry = reader->getRandomPage();
|
||||
return build_redirect(bookName, entry.getFinalEntry());
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_captured_external(const RequestContext& request)
|
||||
{
|
||||
std::string source = "";
|
||||
try {
|
||||
source = kiwix::urlDecode(request.get_argument("source"));
|
||||
} catch (const std::out_of_range& e) {}
|
||||
|
||||
if (source.empty())
|
||||
return Response::build_404(*this, request, "");
|
||||
|
||||
auto data = get_default_data();
|
||||
data.set("source", source);
|
||||
return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8");
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_catalog");
|
||||
}
|
||||
|
||||
std::string host;
|
||||
std::string url;
|
||||
try {
|
||||
host = request.get_header("Host");
|
||||
url = request.get_url_part(1);
|
||||
} catch (const std::out_of_range&) {
|
||||
return Response::build_404(*this, request, "");
|
||||
}
|
||||
|
||||
if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
|
||||
return Response::build_404(*this, request, "");
|
||||
}
|
||||
|
||||
if (url == "searchdescription.xml") {
|
||||
auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml");
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
zim::Uuid uuid;
|
||||
kiwix::OPDSDumper opdsDumper;
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
|
||||
opdsDumper.setLibrary(mp_library);
|
||||
std::vector<std::string> bookIdsToDump;
|
||||
if (url == "root.xml") {
|
||||
opdsDumper.setTitle("All zims");
|
||||
uuid = zim::Uuid::generate(host);
|
||||
bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
|
||||
} else if (url == "search") {
|
||||
auto filter = kiwix::Filter().valid(true).local(true).remote(true);
|
||||
string query("<Empty query>");
|
||||
size_t count(10);
|
||||
size_t startIndex(0);
|
||||
try {
|
||||
query = request.get_argument("q");
|
||||
filter.query(query);
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
filter.maxSize(extractFromString<unsigned long>(request.get_argument("maxsize")));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.name(request.get_argument("name"));
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
filter.lang(request.get_argument("lang"));
|
||||
} catch (const std::out_of_range&) {}
|
||||
try {
|
||||
count = extractFromString<unsigned long>(request.get_argument("count"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
startIndex = extractFromString<unsigned long>(request.get_argument("start"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
|
||||
} catch (...) {}
|
||||
try {
|
||||
filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
|
||||
} catch (...) {}
|
||||
opdsDumper.setTitle("Search result for " + query);
|
||||
uuid = zim::Uuid::generate();
|
||||
bookIdsToDump = mp_library->filter(filter);
|
||||
auto totalResults = bookIdsToDump.size();
|
||||
bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
|
||||
if (count>0 && bookIdsToDump.size() > count) {
|
||||
bookIdsToDump.resize(count);
|
||||
}
|
||||
opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
|
||||
}
|
||||
|
||||
opdsDumper.setId(kiwix::to_string(uuid));
|
||||
auto response = ContentResponse::build(
|
||||
*this,
|
||||
opdsDumper.dumpOPDSFeed(bookIdsToDump),
|
||||
"application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8");
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
std::string get_book_name(const RequestContext& request)
|
||||
{
|
||||
try {
|
||||
return request.get_url_part(0);
|
||||
} catch (const std::out_of_range& e) {
|
||||
return std::string();
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::shared_ptr<Reader>
|
||||
InternalServer::get_reader(const std::string& bookName) const
|
||||
{
|
||||
std::shared_ptr<Reader> reader;
|
||||
try {
|
||||
const std::string bookId = mp_nameMapper->getIdForName(bookName);
|
||||
reader = mp_library->getReaderById(bookId);
|
||||
} catch (const std::out_of_range& e) {
|
||||
}
|
||||
return reader;
|
||||
}
|
||||
|
||||
std::unique_ptr<Response>
|
||||
InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
|
||||
{
|
||||
auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
|
||||
return Response::build_redirect(*this, redirectUrl);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
printf("** running handle_content\n");
|
||||
}
|
||||
|
||||
const std::string bookName = get_book_name(request);
|
||||
if (bookName.empty())
|
||||
return build_homepage(request);
|
||||
|
||||
const std::shared_ptr<Reader> reader = get_reader(bookName);
|
||||
if (reader == nullptr) {
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
auto urlStr = request.get_url().substr(bookName.size()+1);
|
||||
if (urlStr[0] == '/') {
|
||||
urlStr = urlStr.substr(1);
|
||||
}
|
||||
|
||||
kiwix::Entry entry;
|
||||
|
||||
try {
|
||||
entry = reader->getEntryFromPath(urlStr);
|
||||
if (entry.isRedirect() || urlStr.empty()) {
|
||||
// If urlStr is empty, we want to mainPage.
|
||||
// We must do a redirection to the real page.
|
||||
return build_redirect(bookName, entry.getFinalEntry());
|
||||
}
|
||||
} catch(kiwix::NoEntry& e) {
|
||||
if (m_verbose.load())
|
||||
printf("Failed to find %s\n", urlStr.c_str());
|
||||
|
||||
return Response::build_404(*this, request, bookName);
|
||||
}
|
||||
|
||||
auto response = EntryResponse::build(*this, request, entry);
|
||||
try {
|
||||
dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
|
||||
} catch (std::bad_cast& e) {}
|
||||
|
||||
if (m_verbose.load()) {
|
||||
printf("Found %s\n", entry.getPath().c_str());
|
||||
printf("mimeType: %s\n", entry.getMimetype().c_str());
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
/*
|
||||
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H
|
||||
#define KIWIXLIB_SERVER_INTERNALSERVER_H
|
||||
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
}
|
||||
|
||||
#include "library.h"
|
||||
#include "name_mapper.h"
|
||||
|
||||
#include <mustache.hpp>
|
||||
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
|
||||
#include "server/request_context.h"
|
||||
#include "server/response.h"
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
typedef kainjow::mustache::data MustacheData;
|
||||
|
||||
class Entry;
|
||||
|
||||
class InternalServer {
|
||||
public:
|
||||
InternalServer(Library* library,
|
||||
NameMapper* nameMapper,
|
||||
std::string addr,
|
||||
int port,
|
||||
std::string root,
|
||||
int nbThreads,
|
||||
bool verbose,
|
||||
bool withTaskbar,
|
||||
bool withLibraryButton,
|
||||
bool blockExternalLinks);
|
||||
virtual ~InternalServer() = default;
|
||||
|
||||
MHD_Result handlerCallback(struct MHD_Connection* connection,
|
||||
const char* url,
|
||||
const char* method,
|
||||
const char* version,
|
||||
const char* upload_data,
|
||||
size_t* upload_data_size,
|
||||
void** cont_cls);
|
||||
bool start();
|
||||
void stop();
|
||||
|
||||
private: // functions
|
||||
std::unique_ptr<Response> handle_request(const RequestContext& request);
|
||||
std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
|
||||
std::unique_ptr<Response> build_homepage(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_skin(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_catalog(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_meta(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_search(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_suggest(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_random(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
|
||||
std::unique_ptr<Response> handle_content(const RequestContext& request);
|
||||
|
||||
MustacheData get_default_data() const;
|
||||
MustacheData homepage_data() const;
|
||||
|
||||
std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
|
||||
bool etag_not_needed(const RequestContext& r) const;
|
||||
ETag get_matching_if_none_match_etag(const RequestContext& request) const;
|
||||
|
||||
private: // data
|
||||
std::string m_addr;
|
||||
int m_port;
|
||||
std::string m_root;
|
||||
int m_nbThreads;
|
||||
std::atomic_bool m_verbose;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
bool m_blockExternalLinks;
|
||||
struct MHD_Daemon* mp_daemon;
|
||||
|
||||
Library* mp_library;
|
||||
NameMapper* mp_nameMapper;
|
||||
|
||||
std::string m_server_id;
|
||||
|
||||
friend std::unique_ptr<Response> Response::build(const InternalServer& server);
|
||||
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype);
|
||||
friend std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry);
|
||||
friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif //KIWIXLIB_SERVER_INTERNALSERVER_H
|
||||
@@ -26,84 +26,106 @@
|
||||
#include <cstdio>
|
||||
#include <atomic>
|
||||
|
||||
#include "tools/stringTools.h"
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
static std::atomic_ullong s_requestIndex(0);
|
||||
|
||||
namespace {
|
||||
|
||||
RequestMethod str2RequestMethod(const std::string& method) {
|
||||
if (method == "GET") return RequestMethod::GET;
|
||||
else if (method == "HEAD") return RequestMethod::HEAD;
|
||||
else if (method == "POST") return RequestMethod::POST;
|
||||
else if (method == "PUT") return RequestMethod::PUT;
|
||||
else if (method == "DELETE") return RequestMethod::DELETE_;
|
||||
else if (method == "CONNECT") return RequestMethod::CONNECT;
|
||||
else if (method == "OPTIONS") return RequestMethod::OPTIONS;
|
||||
else if (method == "TRACE") return RequestMethod::TRACE;
|
||||
else if (method == "PATCH") return RequestMethod::PATCH;
|
||||
else return RequestMethod::OTHER;
|
||||
}
|
||||
|
||||
std::string
|
||||
fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
|
||||
{
|
||||
if (rootLocation.empty()) {
|
||||
// nothing special to handle.
|
||||
return full_url;
|
||||
} else if (full_url == rootLocation) {
|
||||
return "/";
|
||||
} else if (full_url.size() > rootLocation.size() &&
|
||||
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
|
||||
return full_url.substr(rootLocation.size());
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
RequestContext::RequestContext(struct MHD_Connection* connection,
|
||||
std::string rootLocation,
|
||||
const std::string& _url,
|
||||
const std::string& _method,
|
||||
const std::string& method,
|
||||
const std::string& version) :
|
||||
full_url(_url),
|
||||
url(fullURL2LocalURL(_url, rootLocation)),
|
||||
method(str2RequestMethod(_method)),
|
||||
url(_url),
|
||||
valid_url(true),
|
||||
version(version),
|
||||
requestIndex(s_requestIndex++),
|
||||
acceptEncodingDeflate(false),
|
||||
byteRange_()
|
||||
accept_range(false),
|
||||
range_pair(0, -1)
|
||||
{
|
||||
if (method == "GET") {
|
||||
this->method = RequestMethod::GET;
|
||||
} else if (method == "HEAD") {
|
||||
this->method = RequestMethod::HEAD;
|
||||
} else if (method == "POST") {
|
||||
this->method = RequestMethod::POST;
|
||||
} else if (method == "PUT") {
|
||||
this->method = RequestMethod::PUT;
|
||||
} else if (method == "DELETE") {
|
||||
this->method = RequestMethod::DELETE_;
|
||||
} else if (method == "CONNECT") {
|
||||
this->method = RequestMethod::CONNECT;
|
||||
} else if (method == "OPTIONS") {
|
||||
this->method = RequestMethod::OPTIONS;
|
||||
} else if (method == "TRACE") {
|
||||
this->method = RequestMethod::TRACE;
|
||||
} else if (method == "PATCH") {
|
||||
this->method = RequestMethod::PATCH;
|
||||
} else {
|
||||
this->method = RequestMethod::OTHER;
|
||||
}
|
||||
|
||||
MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
|
||||
MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
|
||||
|
||||
valid_url = true;
|
||||
if (rootLocation.empty()) {
|
||||
// nothing special to handle.
|
||||
url = full_url;
|
||||
} else {
|
||||
if (full_url == rootLocation) {
|
||||
url = "/";
|
||||
} else if (full_url.size() > rootLocation.size() &&
|
||||
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
|
||||
url = full_url.substr(rootLocation.size());
|
||||
} else {
|
||||
valid_url = false;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
acceptEncodingDeflate =
|
||||
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
|
||||
} catch (const std::out_of_range&) {}
|
||||
|
||||
/*Check if range is requested. */
|
||||
try {
|
||||
byteRange_ = ByteRange::parse(get_header(MHD_HTTP_HEADER_RANGE));
|
||||
auto range = get_header(MHD_HTTP_HEADER_RANGE);
|
||||
int start = 0;
|
||||
int end = -1;
|
||||
std::istringstream iss(range);
|
||||
char c;
|
||||
|
||||
iss >> start >> c;
|
||||
if (iss.good() && c=='-') {
|
||||
iss >> end;
|
||||
if (iss.fail()) {
|
||||
// Something went wrong will extracting.
|
||||
end = -1;
|
||||
}
|
||||
if (iss.eof()) {
|
||||
accept_range = true;
|
||||
range_pair = std::pair<int, int>(start, end);
|
||||
}
|
||||
}
|
||||
} catch (const std::out_of_range&) {}
|
||||
}
|
||||
|
||||
RequestContext::~RequestContext()
|
||||
{}
|
||||
|
||||
MHD_Result RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char *value)
|
||||
|
||||
int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char *value)
|
||||
{
|
||||
RequestContext *_this = static_cast<RequestContext*>(__this);
|
||||
_this->headers[lcAll(key)] = value;
|
||||
_this->headers[key] = value;
|
||||
return MHD_YES;
|
||||
}
|
||||
|
||||
MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char* value)
|
||||
int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
|
||||
const char *key, const char* value)
|
||||
{
|
||||
RequestContext *_this = static_cast<RequestContext*>(__this);
|
||||
_this->arguments[key] = value == nullptr ? "" : value;
|
||||
@@ -125,11 +147,10 @@ void RequestContext::print_debug_info() const {
|
||||
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
|
||||
}
|
||||
printf("Parsed : \n");
|
||||
printf("full_url: %s\n", full_url.c_str());
|
||||
printf("url : %s\n", url.c_str());
|
||||
printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
|
||||
printf("has_range : %d\n", byteRange_.kind() != ByteRange::NONE);
|
||||
printf("is_valid_url : %d\n", is_valid_url());
|
||||
printf("has_range : %d\n", accept_range);
|
||||
printf("is_valid_url : %d\n", valid_url);
|
||||
printf(".............\n");
|
||||
}
|
||||
|
||||
@@ -167,11 +188,15 @@ std::string RequestContext::get_full_url() const {
|
||||
}
|
||||
|
||||
bool RequestContext::is_valid_url() const {
|
||||
return !url.empty();
|
||||
return valid_url;
|
||||
}
|
||||
|
||||
ByteRange RequestContext::get_range() const {
|
||||
return byteRange_;
|
||||
bool RequestContext::has_range() const {
|
||||
return accept_range;
|
||||
}
|
||||
|
||||
std::pair<int, int> RequestContext::get_range() const {
|
||||
return range_pair;
|
||||
}
|
||||
|
||||
template<>
|
||||
@@ -180,7 +205,7 @@ std::string RequestContext::get_argument(const std::string& name) const {
|
||||
}
|
||||
|
||||
std::string RequestContext::get_header(const std::string& name) const {
|
||||
return headers.at(lcAll(name));
|
||||
return headers.at(name);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,10 +27,8 @@
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "byte_range.h"
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
#include <microhttpd.h>
|
||||
}
|
||||
|
||||
namespace kiwix {
|
||||
@@ -53,7 +51,7 @@ class IndexError: public std::runtime_error {};
|
||||
|
||||
|
||||
class RequestContext {
|
||||
public: // functions
|
||||
public:
|
||||
RequestContext(struct MHD_Connection* connection,
|
||||
std::string rootLocation,
|
||||
const std::string& url,
|
||||
@@ -80,26 +78,28 @@ class RequestContext {
|
||||
std::string get_url_part(int part) const;
|
||||
std::string get_full_url() const;
|
||||
|
||||
ByteRange get_range() const;
|
||||
bool has_range() const;
|
||||
std::pair<int, int> get_range() const;
|
||||
|
||||
bool can_compress() const { return acceptEncodingDeflate; }
|
||||
|
||||
private: // data
|
||||
private:
|
||||
std::string full_url;
|
||||
std::string url;
|
||||
bool valid_url;
|
||||
RequestMethod method;
|
||||
std::string version;
|
||||
unsigned long long requestIndex;
|
||||
|
||||
bool acceptEncodingDeflate;
|
||||
|
||||
ByteRange byteRange_;
|
||||
bool accept_range;
|
||||
std::pair<int, int> range_pair;
|
||||
std::map<std::string, std::string> headers;
|
||||
std::map<std::string, std::string> arguments;
|
||||
|
||||
private: // functions
|
||||
static MHD_Result fill_header(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
static MHD_Result fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
|
||||
};
|
||||
|
||||
template<> std::string RequestContext::get_argument(const std::string& name) const;
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
#include "response.h"
|
||||
#include "request_context.h"
|
||||
#include "internalServer.h"
|
||||
#include "kiwixlib-resources.h"
|
||||
|
||||
#include "tools/regexTools.h"
|
||||
@@ -18,112 +17,25 @@
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
namespace
|
||||
{
|
||||
// some utilities
|
||||
|
||||
std::string render_template(const std::string& template_str, kainjow::mustache::data data)
|
||||
{
|
||||
kainjow::mustache::mustache tmpl(template_str);
|
||||
kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
|
||||
[](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
|
||||
data.set("urlencoded", urlencode);
|
||||
std::stringstream ss;
|
||||
tmpl.render(data, [&ss](const std::string& str) { ss << str; });
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::string get_mime_type(const kiwix::Entry& entry)
|
||||
{
|
||||
try {
|
||||
return entry.getMimetype();
|
||||
} catch (exception& e) {
|
||||
return "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
bool is_compressible_mime_type(const std::string& mimeType)
|
||||
{
|
||||
return mimeType.find("text/") != string::npos
|
||||
|| mimeType.find("application/javascript") != string::npos
|
||||
|| mimeType.find("application/atom") != string::npos
|
||||
|| mimeType.find("application/opensearchdescription") != string::npos
|
||||
|| mimeType.find("application/json") != string::npos;
|
||||
}
|
||||
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
Response::Response(bool verbose)
|
||||
Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton)
|
||||
: m_verbose(verbose),
|
||||
m_returnCode(MHD_HTTP_OK)
|
||||
m_root(root),
|
||||
m_content(""),
|
||||
m_mimeType(""),
|
||||
m_returnCode(MHD_HTTP_OK),
|
||||
m_withTaskbar(withTaskbar),
|
||||
m_withLibraryButton(withLibraryButton),
|
||||
m_useCache(false),
|
||||
m_addTaskbar(false),
|
||||
m_bookName(""),
|
||||
m_startRange(0),
|
||||
m_lenRange(0)
|
||||
{
|
||||
add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build(const InternalServer& server)
|
||||
{
|
||||
return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_304(const InternalServer& server, const ETag& etag)
|
||||
{
|
||||
auto response = Response::build(server);
|
||||
response->set_code(MHD_HTTP_NOT_MODIFIED);
|
||||
response->m_etag = etag;
|
||||
if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
|
||||
response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName)
|
||||
{
|
||||
MustacheData results;
|
||||
results.set("url", request.get_full_url());
|
||||
|
||||
auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
|
||||
response->set_code(MHD_HTTP_NOT_FOUND);
|
||||
response->set_taskbar(bookName, "");
|
||||
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
|
||||
{
|
||||
auto response = Response::build(server);
|
||||
// [FIXME] (compile with recent enough version of libmicrohttpd)
|
||||
// response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE);
|
||||
response->set_code(416);
|
||||
std::ostringstream oss;
|
||||
oss << "bytes */" << resourceLength;
|
||||
response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str());
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
|
||||
{
|
||||
MustacheData data;
|
||||
data.set("error", msg);
|
||||
auto content = render_template(RESOURCE::templates::_500_html, data);
|
||||
std::unique_ptr<Response> response (
|
||||
new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
|
||||
response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
|
||||
{
|
||||
auto response = Response::build(server);
|
||||
response->m_returnCode = MHD_HTTP_FOUND;
|
||||
response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl);
|
||||
return response;
|
||||
}
|
||||
|
||||
static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind,
|
||||
const char *key, const char *value)
|
||||
static int print_key_value (void *cls, enum MHD_ValueKind kind,
|
||||
const char *key, const char *value)
|
||||
{
|
||||
printf (" - %s: '%s'\n", key, value);
|
||||
return MHD_YES;
|
||||
@@ -178,10 +90,23 @@ void print_response_info(int retCode, MHD_Response* response)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
void ContentResponse::introduce_taskbar()
|
||||
std::string render_template(const std::string& template_str, kainjow::mustache::data data)
|
||||
{
|
||||
kainjow::mustache::mustache tmpl(template_str);
|
||||
kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
|
||||
[](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
|
||||
data.set("urlencoded", urlencode);
|
||||
std::stringstream ss;
|
||||
tmpl.render(data, [&ss](const std::string& str) { ss << str; });
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
|
||||
void Response::introduce_taskbar()
|
||||
{
|
||||
if (! m_withTaskbar)
|
||||
// Taskbar is globally disabled.
|
||||
return;
|
||||
kainjow::mustache::data data;
|
||||
data.set("root", m_root);
|
||||
data.set("content", m_bookName);
|
||||
@@ -202,98 +127,88 @@ void ContentResponse::introduce_taskbar()
|
||||
}
|
||||
|
||||
|
||||
void ContentResponse::inject_externallinks_blocker()
|
||||
{
|
||||
kainjow::mustache::data data;
|
||||
data.set("root", m_root);
|
||||
auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
|
||||
m_content = appendToFirstOccurence(
|
||||
m_content,
|
||||
"<head>",
|
||||
script_tag);
|
||||
}
|
||||
|
||||
bool
|
||||
ContentResponse::can_compress(const RequestContext& request) const
|
||||
int Response::send(const RequestContext& request, MHD_Connection* connection)
|
||||
{
|
||||
return request.can_compress()
|
||||
&& is_compressible_mime_type(m_mimeType)
|
||||
&& (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
|
||||
}
|
||||
MHD_Response* response = nullptr;
|
||||
switch (m_mode) {
|
||||
case ResponseMode::RAW_CONTENT : {
|
||||
if (m_addTaskbar) {
|
||||
introduce_taskbar();
|
||||
}
|
||||
|
||||
bool
|
||||
ContentResponse::contentDecorationAllowed() const
|
||||
{
|
||||
return (startsWith(m_mimeType, "text/html")
|
||||
&& m_mimeType.find(";raw=true") == std::string::npos);
|
||||
}
|
||||
bool shouldCompress = m_compress && request.can_compress();
|
||||
shouldCompress &= m_mimeType.find("text/") != string::npos
|
||||
|| m_mimeType.find("application/javascript") != string::npos
|
||||
|| m_mimeType.find("application/json") != string::npos;
|
||||
|
||||
MHD_Response*
|
||||
Response::create_mhd_response(const RequestContext& request)
|
||||
{
|
||||
MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT);
|
||||
return response;
|
||||
}
|
||||
shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
|
||||
|
||||
MHD_Response*
|
||||
ContentResponse::create_mhd_response(const RequestContext& request)
|
||||
{
|
||||
if (contentDecorationAllowed()) {
|
||||
if (m_withTaskbar) {
|
||||
introduce_taskbar();
|
||||
if (shouldCompress) {
|
||||
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
|
||||
uLongf comprLen = compr_buffer.capacity();
|
||||
int err = compress(&compr_buffer[0],
|
||||
&comprLen,
|
||||
(const Bytef*)(m_content.data()),
|
||||
m_content.size());
|
||||
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
|
||||
/* /!\ Internet Explorer has a bug with deflate compression.
|
||||
It can not handle the first two bytes (compression headers)
|
||||
We need to chunk them off (move the content 2bytes)
|
||||
It has no incidence on other browsers
|
||||
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
|
||||
m_content = string((char*)&compr_buffer[2], comprLen - 2);
|
||||
} else {
|
||||
shouldCompress = false;
|
||||
}
|
||||
}
|
||||
|
||||
response = MHD_create_response_from_buffer(
|
||||
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
|
||||
|
||||
if (shouldCompress) {
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
|
||||
}
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
|
||||
break;
|
||||
}
|
||||
if (m_blockExternalLinks) {
|
||||
inject_externallinks_blocker();
|
||||
|
||||
case ResponseMode::REDIRECTION : {
|
||||
response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
|
||||
break;
|
||||
}
|
||||
|
||||
case ResponseMode::ENTRY : {
|
||||
response = MHD_create_response_from_callback(m_entry.getSize(),
|
||||
16384,
|
||||
callback_reader_from_entry,
|
||||
new RunningResponse(m_entry, m_startRange),
|
||||
callback_free_response);
|
||||
MHD_add_response_header(response,
|
||||
MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
|
||||
std::ostringstream oss;
|
||||
oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
|
||||
<< "/" << m_entry.getSize();
|
||||
|
||||
MHD_add_response_header(response,
|
||||
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
|
||||
|
||||
MHD_add_response_header(response,
|
||||
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
bool shouldCompress = can_compress(request);
|
||||
if (shouldCompress) {
|
||||
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
|
||||
uLongf comprLen = compr_buffer.capacity();
|
||||
int err = compress(&compr_buffer[0],
|
||||
&comprLen,
|
||||
(const Bytef*)(m_content.data()),
|
||||
m_content.size());
|
||||
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
|
||||
/* /!\ Internet Explorer has a bug with deflate compression.
|
||||
It can not handle the first two bytes (compression headers)
|
||||
We need to chunk them off (move the content 2bytes)
|
||||
It has no incidence on other browsers
|
||||
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
|
||||
m_content = string((char*)&compr_buffer[2], comprLen - 2);
|
||||
m_etag.set_option(ETag::COMPRESSED_CONTENT);
|
||||
} else {
|
||||
shouldCompress = false;
|
||||
}
|
||||
}
|
||||
|
||||
MHD_Response* response = MHD_create_response_from_buffer(
|
||||
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
|
||||
|
||||
if (shouldCompress) {
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
MHD_add_response_header(
|
||||
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection)
|
||||
{
|
||||
MHD_Response* response = create_mhd_response(request);
|
||||
|
||||
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
|
||||
m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
|
||||
const std::string etag = m_etag.get_etag();
|
||||
if ( ! etag.empty() )
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
|
||||
for(auto& p: m_customHeaders) {
|
||||
MHD_add_response_header(response, p.first.c_str(), p.second.c_str());
|
||||
}
|
||||
m_useCache ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
|
||||
|
||||
if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT)
|
||||
if (m_returnCode == MHD_HTTP_OK && request.has_range())
|
||||
m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
|
||||
|
||||
if (m_verbose)
|
||||
@@ -304,105 +219,32 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
|
||||
return ret;
|
||||
}
|
||||
|
||||
void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
|
||||
void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
|
||||
set_content(render_template(template_str, data));
|
||||
}
|
||||
|
||||
void Response::set_content(const std::string& content) {
|
||||
m_content = content;
|
||||
m_mode = ResponseMode::RAW_CONTENT;
|
||||
}
|
||||
|
||||
void Response::set_redirection(const std::string& url) {
|
||||
m_content = url;
|
||||
m_mode = ResponseMode::REDIRECTION;
|
||||
m_returnCode = MHD_HTTP_FOUND;
|
||||
}
|
||||
|
||||
void Response::set_entry(const Entry& entry) {
|
||||
m_entry = entry;
|
||||
m_mode = ResponseMode::ENTRY;
|
||||
}
|
||||
|
||||
void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
|
||||
{
|
||||
m_addTaskbar = true;
|
||||
m_bookName = bookName;
|
||||
m_bookTitle = bookTitle;
|
||||
}
|
||||
|
||||
|
||||
ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
|
||||
Response(verbose),
|
||||
m_root(root),
|
||||
m_content(content),
|
||||
m_mimeType(mimetype),
|
||||
m_withTaskbar(withTaskbar),
|
||||
m_withLibraryButton(withLibraryButton),
|
||||
m_blockExternalLinks(blockExternalLinks),
|
||||
m_bookName(""),
|
||||
m_bookTitle("")
|
||||
{
|
||||
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
|
||||
}
|
||||
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype)
|
||||
{
|
||||
return std::unique_ptr<ContentResponse>(new ContentResponse(
|
||||
server.m_root,
|
||||
server.m_verbose.load(),
|
||||
server.m_withTaskbar,
|
||||
server.m_withLibraryButton,
|
||||
server.m_blockExternalLinks,
|
||||
content,
|
||||
mimetype));
|
||||
}
|
||||
|
||||
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) {
|
||||
auto content = render_template(template_str, data);
|
||||
return ContentResponse::build(server, content, mimetype);
|
||||
}
|
||||
|
||||
EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) :
|
||||
Response(verbose),
|
||||
m_entry(entry),
|
||||
m_mimeType(mimetype)
|
||||
{
|
||||
m_byteRange = byterange;
|
||||
set_cacheable();
|
||||
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry)
|
||||
{
|
||||
const std::string mimetype = get_mime_type(entry);
|
||||
auto byteRange = request.get_range().resolve(entry.getSize());
|
||||
const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
|
||||
if (noRange && is_compressible_mime_type(mimetype)) {
|
||||
// Return a contentResponse
|
||||
zim::Blob raw_content = entry.getBlob();
|
||||
const std::string content = string(raw_content.data(), raw_content.size());
|
||||
auto response = ContentResponse::build(server, content, mimetype);
|
||||
response->set_cacheable();
|
||||
response->m_byteRange = byteRange;
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
|
||||
auto response = Response::build_416(server, entry.getSize());
|
||||
response->set_cacheable();
|
||||
return response;
|
||||
}
|
||||
|
||||
return std::unique_ptr<Response>(new EntryResponse(
|
||||
server.m_verbose.load(),
|
||||
entry,
|
||||
mimetype,
|
||||
byteRange));
|
||||
}
|
||||
|
||||
MHD_Response*
|
||||
EntryResponse::create_mhd_response(const RequestContext& request)
|
||||
{
|
||||
const auto content_length = m_byteRange.length();
|
||||
MHD_Response* response = MHD_create_response_from_callback(content_length,
|
||||
16384,
|
||||
callback_reader_from_entry,
|
||||
new RunningResponse(m_entry, m_byteRange.first()),
|
||||
callback_free_response);
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
|
||||
if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) {
|
||||
std::ostringstream oss;
|
||||
oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last()
|
||||
<< "/" << m_entry.getSize();
|
||||
|
||||
MHD_add_response_header(response,
|
||||
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
|
||||
}
|
||||
|
||||
MHD_add_response_header(response,
|
||||
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str());
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -22,98 +22,66 @@
|
||||
#define KIWIXLIB_SERVER_RESPONSE_H
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include <mustache.hpp>
|
||||
#include "byte_range.h"
|
||||
#include "entry.h"
|
||||
#include "etag.h"
|
||||
|
||||
extern "C" {
|
||||
#include "microhttpd_wrapper.h"
|
||||
#include <microhttpd.h>
|
||||
}
|
||||
|
||||
namespace kiwix {
|
||||
|
||||
class InternalServer;
|
||||
class RequestContext;
|
||||
enum class ResponseMode {
|
||||
RAW_CONTENT,
|
||||
REDIRECTION,
|
||||
ENTRY
|
||||
};
|
||||
|
||||
class EntryResponse;
|
||||
class RequestContext;
|
||||
|
||||
class Response {
|
||||
public:
|
||||
Response(bool verbose);
|
||||
virtual ~Response() = default;
|
||||
Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton);
|
||||
~Response() = default;
|
||||
|
||||
static std::unique_ptr<Response> build(const InternalServer& server);
|
||||
static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
|
||||
static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName);
|
||||
static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
|
||||
static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
|
||||
static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
|
||||
int send(const RequestContext& request, MHD_Connection* connection);
|
||||
|
||||
MHD_Result send(const RequestContext& request, MHD_Connection* connection);
|
||||
void set_template(const std::string& template_str, kainjow::mustache::data data);
|
||||
void set_content(const std::string& content);
|
||||
void set_redirection(const std::string& url);
|
||||
void set_entry(const Entry& entry);
|
||||
|
||||
|
||||
void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
|
||||
void set_code(int code) { m_returnCode = code; }
|
||||
void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
|
||||
void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
|
||||
void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
|
||||
|
||||
int getReturnCode() const { return m_returnCode; }
|
||||
|
||||
private: // functions
|
||||
virtual MHD_Response* create_mhd_response(const RequestContext& request);
|
||||
MHD_Response* create_error_response(const RequestContext& request) const;
|
||||
|
||||
protected: // data
|
||||
bool m_verbose;
|
||||
int m_returnCode;
|
||||
ByteRange m_byteRange;
|
||||
ETag m_etag;
|
||||
std::map<std::string, std::string> m_customHeaders;
|
||||
|
||||
friend class EntryResponse; // temporary to allow the builder to change m_mode
|
||||
};
|
||||
|
||||
|
||||
class ContentResponse : public Response {
|
||||
public:
|
||||
ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
|
||||
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype);
|
||||
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype);
|
||||
|
||||
void set_cache(bool cache) { m_useCache = cache; }
|
||||
void set_compress(bool compress) { m_compress = compress; }
|
||||
void set_taskbar(const std::string& bookName, const std::string& bookTitle);
|
||||
void set_range_first(uint64_t start) { m_startRange = start; }
|
||||
void set_range_len(uint64_t len) { m_lenRange = len; }
|
||||
|
||||
private:
|
||||
MHD_Response* create_mhd_response(const RequestContext& request);
|
||||
int getReturnCode() { return m_returnCode; }
|
||||
|
||||
void introduce_taskbar();
|
||||
void inject_externallinks_blocker();
|
||||
bool can_compress(const RequestContext& request) const;
|
||||
bool contentDecorationAllowed() const;
|
||||
|
||||
|
||||
private:
|
||||
bool m_verbose;
|
||||
ResponseMode m_mode;
|
||||
std::string m_root;
|
||||
std::string m_content;
|
||||
std::string m_mimeType;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
bool m_blockExternalLinks;
|
||||
std::string m_bookName;
|
||||
std::string m_bookTitle;
|
||||
};
|
||||
|
||||
class EntryResponse : public Response {
|
||||
public:
|
||||
EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange);
|
||||
static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const Entry& entry);
|
||||
|
||||
private:
|
||||
MHD_Response* create_mhd_response(const RequestContext& request);
|
||||
|
||||
Entry m_entry;
|
||||
std::string m_mimeType;
|
||||
int m_returnCode;
|
||||
bool m_withTaskbar;
|
||||
bool m_withLibraryButton;
|
||||
bool m_useCache;
|
||||
bool m_compress;
|
||||
bool m_addTaskbar;
|
||||
std::string m_bookName;
|
||||
std::string m_bookTitle;
|
||||
uint64_t m_startRange;
|
||||
uint64_t m_lenRange;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ UnixImpl::~UnixImpl()
|
||||
#else
|
||||
pthread_cancel(m_waitingThread);
|
||||
#endif
|
||||
pthread_join(m_waitingThread, nullptr);
|
||||
}
|
||||
|
||||
#ifdef __ANDROID__
|
||||
|
||||
@@ -11,8 +11,7 @@
|
||||
WinImpl::WinImpl():
|
||||
m_pid(0),
|
||||
m_running(false),
|
||||
m_subprocessHandle(INVALID_HANDLE_VALUE),
|
||||
m_waitingThreadHandle(INVALID_HANDLE_VALUE)
|
||||
m_handle(INVALID_HANDLE_VALUE)
|
||||
{
|
||||
InitializeCriticalSection(&m_criticalSection);
|
||||
}
|
||||
@@ -20,15 +19,14 @@ WinImpl::WinImpl():
|
||||
WinImpl::~WinImpl()
|
||||
{
|
||||
kill();
|
||||
WaitForSingleObject(m_waitingThreadHandle, INFINITE);
|
||||
CloseHandle(m_subprocessHandle);
|
||||
CloseHandle(m_handle);
|
||||
DeleteCriticalSection(&m_criticalSection);
|
||||
}
|
||||
|
||||
DWORD WINAPI WinImpl::waitForPID(void* _self)
|
||||
{
|
||||
WinImpl* self = static_cast<WinImpl*>(_self);
|
||||
WaitForSingleObject(self->m_subprocessHandle, INFINITE);
|
||||
WaitForSingleObject(self->m_handle, INFINITE);
|
||||
|
||||
EnterCriticalSection(&self->m_criticalSection);
|
||||
self->m_running = false;
|
||||
@@ -81,16 +79,16 @@ void WinImpl::run(commandLine_t& commandLine)
|
||||
&procInfo))
|
||||
{
|
||||
m_pid = procInfo.dwProcessId;
|
||||
m_subprocessHandle = procInfo.hProcess;
|
||||
m_handle = procInfo.hProcess;
|
||||
CloseHandle(procInfo.hThread);
|
||||
m_running = true;
|
||||
m_waitingThreadHandle = CreateThread(NULL, 0, &waitForPID, this, 0, NULL);
|
||||
CreateThread(NULL, 0, &waitForPID, this, 0, NULL );
|
||||
}
|
||||
}
|
||||
|
||||
bool WinImpl::kill()
|
||||
{
|
||||
return TerminateProcess(m_subprocessHandle, 0);
|
||||
return TerminateProcess(m_handle, 0);
|
||||
}
|
||||
|
||||
bool WinImpl::isRunning()
|
||||
|
||||
@@ -11,8 +11,7 @@ class WinImpl : public SubprocessImpl
|
||||
private:
|
||||
int m_pid;
|
||||
bool m_running;
|
||||
HANDLE m_subprocessHandle;
|
||||
HANDLE m_waitingThreadHandle;
|
||||
HANDLE m_handle;
|
||||
CRITICAL_SECTION m_criticalSection;
|
||||
|
||||
public:
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
*/
|
||||
|
||||
#include "tools/otherTools.h"
|
||||
#include <algorithm>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
@@ -281,63 +280,3 @@ bool kiwix::convertStrToBool(const std::string& value)
|
||||
throw std::domain_error(ss.str());
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// The counter metadata format is a list of item separated by a `;` :
|
||||
// item0;item1;item2
|
||||
// Each item is a "tuple" mimetype=number.
|
||||
// However, the mimetype may contains parameters:
|
||||
// text/html;raw=true;foo=bar
|
||||
// So the final format may be complex to parse:
|
||||
// key0=value0;key1;foo=bar=value1;key2=value2
|
||||
|
||||
typedef kiwix::MimeCounterType::value_type MimetypeAndCounter;
|
||||
|
||||
std::string readFullMimetypeAndCounterString(std::istream& in)
|
||||
{
|
||||
std::string mtcStr, params;
|
||||
getline(in, mtcStr, ';');
|
||||
if ( mtcStr.find('=') == std::string::npos )
|
||||
{
|
||||
do
|
||||
{
|
||||
if ( !getline(in, params, ';' ) )
|
||||
return std::string();
|
||||
mtcStr += ";" + params;
|
||||
}
|
||||
while ( std::count(params.begin(), params.end(), '=') != 2 );
|
||||
}
|
||||
return mtcStr;
|
||||
}
|
||||
|
||||
MimetypeAndCounter parseASingleMimetypeCounter(const std::string& s)
|
||||
{
|
||||
const std::string::size_type k = s.find_last_of("=");
|
||||
if ( k != std::string::npos )
|
||||
{
|
||||
const std::string mimeType = s.substr(0, k);
|
||||
std::istringstream counterSS(s.substr(k+1));
|
||||
unsigned int counter;
|
||||
if (counterSS >> counter && counterSS.eof())
|
||||
return MimetypeAndCounter{mimeType, counter};
|
||||
}
|
||||
return MimetypeAndCounter{"", 0};
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterData)
|
||||
{
|
||||
kiwix::MimeCounterType counters;
|
||||
std::istringstream ss(counterData);
|
||||
|
||||
while (ss)
|
||||
{
|
||||
const std::string mtcStr = readFullMimetypeAndCounterString(ss);
|
||||
const MimetypeAndCounter mtc = parseASingleMimetypeCounter(mtcStr);
|
||||
if ( !mtc.first.empty() )
|
||||
counters.insert(mtc);
|
||||
}
|
||||
|
||||
return counters;
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
*/
|
||||
|
||||
#include "tools/pathTools.h"
|
||||
#include <stdexcept>
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <limits.h>
|
||||
@@ -43,16 +42,12 @@
|
||||
#include <algorithm>
|
||||
|
||||
#ifdef _WIN32
|
||||
# define SEPARATOR "\\"
|
||||
# include <io.h>
|
||||
#define SEPARATOR "\\"
|
||||
#else
|
||||
# define SEPARATOR "/"
|
||||
# include <unistd.h>
|
||||
# include <sys/stat.h>
|
||||
#define SEPARATOR "/"
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include <fcntl.h>
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#ifndef PATH_MAX
|
||||
@@ -81,19 +76,13 @@ std::wstring Utf8ToWide(const std::string& str)
|
||||
bool isRelativePath(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
if (path.size() < 3 ) {
|
||||
return true;
|
||||
}
|
||||
if (path.substr(1, 2) == ":\\" || path.substr(0, 2) == "\\\\") {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
|
||||
#else
|
||||
return path.empty() || path.substr(0, 1) == "/" ? false : true;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute)
|
||||
std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute)
|
||||
{
|
||||
std::vector<std::string> ret;
|
||||
#ifdef _WIN32
|
||||
@@ -101,35 +90,10 @@ std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool ab
|
||||
//Starts from there.
|
||||
auto it = find_if(parts.rbegin(), parts.rend(),
|
||||
[](const std::string& p) ->bool
|
||||
{ return ((p.length() == 2 && p[1] == ':')
|
||||
|| (p.length() > 2 && p[0] == '\\' && p[1] == '\\')); });
|
||||
{ return p.length() == 2 && p[1] == ':'; });
|
||||
if (it != parts.rend()) {
|
||||
parts.erase(parts.begin(), it.base()-1);
|
||||
}
|
||||
// Special case for samba mount point starting with two "\\" ("\\\\samba\\foo")
|
||||
if (parts.size() > 2 && parts[0].empty() && parts[1].empty()) {
|
||||
parts.erase(parts.begin(), parts.begin()+2);
|
||||
parts[0] = "\\\\" + parts[0];
|
||||
}
|
||||
// Special case if we have a samba drive not at first.
|
||||
// Path is "..\\\\sambdadrive\\..\\.." So we will have an empty part.
|
||||
auto previous_empty = false;
|
||||
for (it = parts.rbegin(); it!=parts.rend(); it++) {
|
||||
if(it->empty()) {
|
||||
if (previous_empty) {
|
||||
it++;
|
||||
break;
|
||||
} else {
|
||||
previous_empty = true;
|
||||
}
|
||||
} else {
|
||||
previous_empty = false;
|
||||
}
|
||||
}
|
||||
if (it != parts.rend()) {
|
||||
parts.erase(parts.begin(), it.base()-1);
|
||||
parts[0] = "\\\\" + parts[0];
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t index = 0;
|
||||
@@ -175,10 +139,8 @@ std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool ab
|
||||
|
||||
std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
|
||||
{
|
||||
auto parts = kiwix::split(path, SEPARATOR, false);
|
||||
auto pathParts = normalizeParts(parts, false);
|
||||
parts = kiwix::split(absolutePath, SEPARATOR, false);
|
||||
auto absolutePathParts = normalizeParts(parts, true);
|
||||
auto pathParts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
|
||||
auto absolutePathParts = kiwix::split(absolutePath, SEPARATOR, false);
|
||||
|
||||
unsigned int commonCount = 0;
|
||||
while (commonCount < pathParts.size()
|
||||
@@ -205,10 +167,8 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
|
||||
absolutePath = getCurrentDirectory();
|
||||
}
|
||||
|
||||
auto parts = kiwix::split(absolutePath, SEPARATOR, false);
|
||||
auto absoluteParts = normalizeParts(parts, true);
|
||||
parts = kiwix::split(relativePath, SEPARATOR, false);
|
||||
auto relativeParts = normalizeParts(parts, false);
|
||||
auto absoluteParts = normalizeParts(kiwix::split(absolutePath, SEPARATOR, false), true);
|
||||
auto relativeParts = kiwix::split(relativePath, SEPARATOR, false);
|
||||
|
||||
absoluteParts.insert(absoluteParts.end(), relativeParts.begin(), relativeParts.end());
|
||||
auto ret = kiwix::join(normalizeParts(absoluteParts, true), SEPARATOR);
|
||||
@@ -217,8 +177,7 @@ std::string computeAbsolutePath(const std::string& path, const std::string& rela
|
||||
|
||||
std::string removeLastPathElement(const std::string& path)
|
||||
{
|
||||
auto parts_ = kiwix::split(path, SEPARATOR, false);
|
||||
auto parts = normalizeParts(parts_, false);
|
||||
auto parts = normalizeParts(kiwix::split(path, SEPARATOR, false), false);
|
||||
if (!parts.empty()) {
|
||||
parts.pop_back();
|
||||
}
|
||||
@@ -238,8 +197,7 @@ std::string appendToDirectory(const std::string& directoryPath, const std::strin
|
||||
|
||||
std::string getLastPathElement(const std::string& path)
|
||||
{
|
||||
auto parts_ = kiwix::split(path, SEPARATOR);
|
||||
auto parts = normalizeParts(parts_, false);
|
||||
auto parts = normalizeParts(kiwix::split(path, SEPARATOR), false);
|
||||
if (parts.empty()) {
|
||||
return "";
|
||||
}
|
||||
@@ -269,33 +227,14 @@ std::string getFileSizeAsString(const std::string& path)
|
||||
|
||||
std::string getFileContent(const std::string& path)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
auto wpath = Utf8ToWide(path);
|
||||
auto fd = _wopen(wpath.c_str(), _O_RDONLY | _O_BINARY);
|
||||
#else
|
||||
auto fd = open(path.c_str(), O_RDONLY);
|
||||
#endif
|
||||
std::ifstream f(path, std::ios::in|std::ios::ate);
|
||||
std::string content;
|
||||
if (fd != -1) {
|
||||
#ifdef _WIN32
|
||||
auto size = _lseeki64(fd, 0, SEEK_END);
|
||||
#else
|
||||
auto size = lseek(fd, 0, SEEK_END);
|
||||
#endif
|
||||
content.resize(size);
|
||||
#ifdef _WIN32
|
||||
_lseeki64(fd, 0, SEEK_SET);
|
||||
#else
|
||||
lseek(fd, 0, SEEK_SET);
|
||||
#endif
|
||||
auto p = (char*)content.data();
|
||||
while (size) {
|
||||
auto readsize = size > 2048 ? 2048 : size;
|
||||
readsize = ::read(fd, p, readsize);
|
||||
p += readsize;
|
||||
size -= readsize;
|
||||
}
|
||||
close(fd);
|
||||
if (f.is_open()) {
|
||||
auto size = f.tellg();
|
||||
content.reserve(size);
|
||||
f.seekg(0, std::ios::beg);
|
||||
content.assign((std::istreambuf_iterator<char>(f)),
|
||||
std::istreambuf_iterator<char>());
|
||||
}
|
||||
return content;
|
||||
}
|
||||
@@ -348,22 +287,22 @@ std::string makeTmpDirectory()
|
||||
/* Try to create a link and if does not work then make a copy */
|
||||
bool copyFile(const std::string& sourcePath, const std::string& destPath)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return CopyFileW(Utf8ToWide(sourcePath).c_str(), Utf8ToWide(destPath).c_str(), 1);
|
||||
#else
|
||||
try {
|
||||
#ifndef _WIN32
|
||||
if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
|
||||
#endif
|
||||
std::ifstream infile(sourcePath.c_str(), std::ios_base::binary);
|
||||
std::ofstream outfile(destPath.c_str(), std::ios_base::binary);
|
||||
outfile << infile.rdbuf();
|
||||
#ifndef _WIN32
|
||||
}
|
||||
#endif
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << e.what() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string getExecutablePath(bool realPathOnly)
|
||||
@@ -402,21 +341,10 @@ std::string getExecutablePath(bool realPathOnly)
|
||||
|
||||
bool writeTextFile(const std::string& path, const std::string& content)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
auto wpath = Utf8ToWide(path);
|
||||
auto fd = _wopen(wpath.c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC, S_IWRITE);
|
||||
#else
|
||||
auto fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC,
|
||||
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
|
||||
#endif
|
||||
if (fd == -1)
|
||||
return false;
|
||||
|
||||
if (write(fd, content.c_str(), content.size()) != (long)content.size()) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
close(fd);
|
||||
std::ofstream file;
|
||||
file.open(path.c_str());
|
||||
file << content;
|
||||
file.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -436,48 +364,30 @@ std::string getCurrentDirectory()
|
||||
|
||||
std::string getDataDirectory()
|
||||
{
|
||||
// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
|
||||
#ifdef _WIN32
|
||||
wchar_t* cDataDir = ::_wgetenv(L"KIWIX_DATA_DIR");
|
||||
if (cDataDir != nullptr) {
|
||||
return WideToUtf8(cDataDir);
|
||||
}
|
||||
char* cDataDir = ::getenv("APPDATA");
|
||||
#else
|
||||
char* cDataDir = ::getenv("KIWIX_DATA_DIR");
|
||||
if (cDataDir != nullptr) {
|
||||
return cDataDir;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Compute the dataDir from the user directory.
|
||||
std::string dataDir;
|
||||
#ifdef _WIN32
|
||||
cDataDir = ::_wgetenv(L"APPDATA");
|
||||
if (cDataDir == nullptr)
|
||||
cDataDir = ::_wgetenv(L"USERPROFILE");
|
||||
if (cDataDir != nullptr)
|
||||
dataDir = WideToUtf8(cDataDir);
|
||||
#else
|
||||
cDataDir = ::getenv("XDG_DATA_HOME");
|
||||
if (cDataDir != nullptr) {
|
||||
dataDir = cDataDir;
|
||||
} else {
|
||||
cDataDir = ::getenv("HOME");
|
||||
if (cDataDir != nullptr) {
|
||||
dataDir = cDataDir;
|
||||
dataDir = appendToDirectory(dataDir, ".local");
|
||||
dataDir = appendToDirectory(dataDir, "share");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
std::string dataDir = cDataDir==nullptr ? "" : cDataDir;
|
||||
if (!dataDir.empty()) {
|
||||
dataDir = appendToDirectory(dataDir, "kiwix");
|
||||
makeDirectory(dataDir);
|
||||
return dataDir;
|
||||
}
|
||||
|
||||
// Let's use the currentDirectory
|
||||
return getCurrentDirectory();
|
||||
#ifdef _WIN32
|
||||
cDataDir = ::getenv("USERPROFILE");
|
||||
dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
|
||||
#else
|
||||
cDataDir = ::getenv("XDG_DATA_HOME");
|
||||
dataDir = cDataDir==nullptr ? "" : cDataDir;
|
||||
if (dataDir.empty()) {
|
||||
cDataDir = ::getenv("HOME");
|
||||
dataDir = cDataDir==nullptr ? getCurrentDirectory() : cDataDir;
|
||||
dataDir = appendToDirectory(dataDir, ".local");
|
||||
dataDir = appendToDirectory(dataDir, "share");
|
||||
}
|
||||
#endif
|
||||
auto ret = appendToDirectory(dataDir, "kiwix");
|
||||
return ret;
|
||||
}
|
||||
|
||||
static std::map<std::string, std::string> extMimeTypes = {
|
||||
|
||||
@@ -268,8 +268,7 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
|
||||
/* Split string in a token array */
|
||||
std::vector<std::string> kiwix::split(const std::string& str,
|
||||
const std::string& delims,
|
||||
bool trimEmpty,
|
||||
bool keepDelim)
|
||||
bool trimEmpty)
|
||||
{
|
||||
std::string::size_type lastPos = 0;
|
||||
std::string::size_type pos = 0;
|
||||
@@ -280,9 +279,6 @@ std::vector<std::string> kiwix::split(const std::string& str,
|
||||
if (!trimEmpty || !token.empty()) {
|
||||
tokens.push_back(token);
|
||||
}
|
||||
if (keepDelim) {
|
||||
tokens.push_back(str.substr(pos, 1));
|
||||
}
|
||||
lastPos = pos + 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -355,12 +355,9 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
|
||||
unsigned int cCount = jni2c(count, env);
|
||||
|
||||
try {
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||
if (READER->searchSuggestionsSmart(cPrefix, cCount)) {
|
||||
retVal = JNI_TRUE;
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
} catch (std::exception& e) {
|
||||
LOG("Unable to get search results for pattern: %s", cPrefix.c_str());
|
||||
LOG(e.what());
|
||||
@@ -380,14 +377,11 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
|
||||
std::string cUrl;
|
||||
|
||||
try {
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||
if (READER->getNextSuggestion(cTitle, cUrl)) {
|
||||
setStringObjValue(cTitle, titleObj, env);
|
||||
setStringObjValue(cUrl, urlObj, env);
|
||||
retVal = JNI_TRUE;
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
} catch (std::exception& e) {
|
||||
LOG("Unable to get next suggestion");
|
||||
LOG(e.what());
|
||||
|
||||
@@ -85,12 +85,6 @@ Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboo
|
||||
SERVER->setTaskbar(withTaskbar, withLibraryButton);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
|
||||
{
|
||||
SERVER->setBlockExternalLinks(blockExternalLinks);
|
||||
}
|
||||
|
||||
JNIEXPORT jboolean JNICALL
|
||||
Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
|
||||
{
|
||||
|
||||
@@ -34,8 +34,6 @@ public class JNIKiwixServer
|
||||
|
||||
public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
|
||||
|
||||
public native void setBlockExternalLinks(boolean blockExternalLinks);
|
||||
|
||||
public native boolean start();
|
||||
|
||||
public native void stop();
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
#ifndef KIWIX_XMLRPC_H_
|
||||
#define KIWIX_XMLRPC_H_
|
||||
|
||||
#include <stdexcept>
|
||||
#include <tools/otherTools.h>
|
||||
#include <pugixml.hpp>
|
||||
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
resource_files = run_command(find_program('python3'),
|
||||
'-c',
|
||||
'import sys; f=open(sys.argv[1]); print(f.read())',
|
||||
files('resources_list.txt')
|
||||
).stdout().strip().split('\n')
|
||||
|
||||
lib_resources = custom_target('resources',
|
||||
input: 'resources_list.txt',
|
||||
@@ -12,5 +7,5 @@ lib_resources = custom_target('resources',
|
||||
'--hfile', '@OUTPUT1@',
|
||||
'--source_dir', '@OUTDIR@',
|
||||
'@INPUT@'],
|
||||
depend_files: resource_files
|
||||
build_always_stale: true
|
||||
)
|
||||
|
||||
@@ -20,7 +20,6 @@ skin/jquery-ui/jquery-ui.min.css
|
||||
skin/caret.png
|
||||
skin/taskbar.js
|
||||
skin/taskbar.css
|
||||
skin/block_external.js
|
||||
templates/search_result.html
|
||||
templates/no_search_result.html
|
||||
templates/404.html
|
||||
@@ -29,6 +28,4 @@ templates/index.html
|
||||
templates/suggestion.json
|
||||
templates/head_part.html
|
||||
templates/taskbar_part.html
|
||||
templates/external_blocker_part.html
|
||||
templates/captured_external.html
|
||||
opensearchdescription.xml
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
// `block_path` variable used by openzim/warc2zim to detect whether URL blocking is enabled or not
|
||||
var block_path = "/catch/external";
|
||||
// called only on external links
|
||||
function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
|
||||
|
||||
// called on all link clicks. filters external and call capture_event
|
||||
function on_click_event(e) {
|
||||
var target = findParent("a", e.target);
|
||||
if (target !== null && "href" in target) {
|
||||
var href = target.href;
|
||||
if (window.location.pathname.indexOf(block_path) == 0) // already in catch page
|
||||
return;
|
||||
if (href.indexOf(window.location.origin) == 0)
|
||||
return;
|
||||
if (href.substr(0, 2) == "//")
|
||||
return capture_event(e, target);
|
||||
if (href.substr(0, 5) == "http:")
|
||||
return capture_event(e, target);
|
||||
if (href.substr(0, 6) == "https:")
|
||||
return capture_event(e, target);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// script entrypoint (called on document ready)
|
||||
function run() { live('a', 'click', on_click_event); }
|
||||
|
||||
// find first parent with tagname
|
||||
function findParent(tagname, el) {
|
||||
while (el) {
|
||||
if ((el.nodeName || el.tagName).toLowerCase() === tagname.toLowerCase()) {
|
||||
return el;
|
||||
}
|
||||
el = el.parentNode;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// matches polyfill
|
||||
this.Element && function(ElementPrototype) {
|
||||
ElementPrototype.matches = ElementPrototype.matches ||
|
||||
ElementPrototype.matchesSelector ||
|
||||
ElementPrototype.webkitMatchesSelector ||
|
||||
ElementPrototype.msMatchesSelector ||
|
||||
function(selector) {
|
||||
var node = this, nodes = (node.parentNode || node.document).querySelectorAll(selector), i = -1;
|
||||
while (nodes[++i] && nodes[i] != node);
|
||||
return !!nodes[i];
|
||||
}
|
||||
}(Element.prototype);
|
||||
|
||||
// helper for enabling IE 8 event bindings
|
||||
function addEvent(el, type, handler) {
|
||||
if (el.attachEvent) el.attachEvent('on'+type, handler); else el.addEventListener(type, handler);
|
||||
}
|
||||
|
||||
// live binding helper using matchesSelector
|
||||
function live(selector, event, callback, context) {
|
||||
addEvent(context || document, event, function(e) {
|
||||
var found, el = e.target || e.srcElement;
|
||||
while (el && el.matches && el !== context && !(found = el.matches(selector))) el = el.parentElement;
|
||||
if (found) callback.call(el, e);
|
||||
});
|
||||
}
|
||||
|
||||
// in case the document is already rendered
|
||||
if (document.readyState!='loading') run();
|
||||
// modern browsers
|
||||
else if (document.addEventListener) document.addEventListener('DOMContentLoaded', run);
|
||||
// IE <= 8
|
||||
else document.attachEvent('onreadystatechange', function(){
|
||||
if (document.readyState=='complete') run();
|
||||
});
|
||||
16150
static/skin/jquery-ui/jquery-ui.js
vendored
16150
static/skin/jquery-ui/jquery-ui.js
vendored
File diff suppressed because it is too large
Load Diff
@@ -124,7 +124,7 @@ label[for=kiwixsearchbox] {
|
||||
}
|
||||
|
||||
body {
|
||||
padding-top: calc(3em - 5px) !important;
|
||||
padding-top: 3em !important;
|
||||
}
|
||||
|
||||
/* Try to fix buggy stuff in jquery-ui autocomplete */
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>External link blocked</title>
|
||||
</head>
|
||||
<body class="kiwix">
|
||||
<h1>External link blocked</h1>
|
||||
<p>This instance of Kiwix protects you from accidentaly going to external (out-of ZIM) links.</p>
|
||||
<p>If you intend to go to such locations, please click the link below.</p>
|
||||
<p><a href="{{ source }}">Go to {{ source }}</a></p>
|
||||
<div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1 +0,0 @@
|
||||
<script type="text/javascript" src="{{root}}/skin/block_external.js"></script>
|
||||
@@ -29,7 +29,7 @@
|
||||
box-shadow: 2px 2px 5px 0px #ccc;
|
||||
}
|
||||
.book:hover { background-color: #F9F9F9; box-shadow: none;}
|
||||
.book__background { background-repeat: no-repeat; background-size: 48px 48px; background-position: top right; }
|
||||
.book__background { background-repeat: no-repeat; background-size: auto; background-position: top right; }
|
||||
.book__title {
|
||||
padding: 0 55px 0 0;overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
font-size: 18px; color: #0645ad; line-height: 1em;
|
||||
|
||||
@@ -133,7 +133,7 @@
|
||||
<ul>
|
||||
{{#resultLastPageStart}}
|
||||
<li>
|
||||
<a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&pageLength={{pageLength}}">
|
||||
<a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start=0&end={{resultRange}}">
|
||||
◀
|
||||
</a>
|
||||
</li>
|
||||
@@ -141,14 +141,14 @@
|
||||
{{#pages}}
|
||||
<li>
|
||||
<a {{#selected}}class="selected"{{/selected}}
|
||||
href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&pageLength={{pageLength}}">
|
||||
href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{start}}&end={{end}}">
|
||||
{{label}}
|
||||
</a>
|
||||
</li>
|
||||
{{/pages}}
|
||||
{{#resultLastPageStart}}
|
||||
<li>
|
||||
<a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&pageLength={{pageLength}}">
|
||||
<a href="{{searchProtocolPrefix}}pattern={{searchPatternEncoded}}{{#contentId}}&content={{.}}{{/contentId}}&start={{resultLastPageStart}}&end={{lastResult}}">
|
||||
▶
|
||||
</a>
|
||||
</li>
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Matthieu Gautier
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License as
|
||||
* published by the Free Software Foundation; either version 2 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but
|
||||
* is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
|
||||
* warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
|
||||
* NON-INFRINGEMENT. See the GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*
|
||||
*/
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <zim/zim.h>
|
||||
|
||||
namespace kiwix {
|
||||
using CounterType = std::map<const std::string, zim::article_index_type>;
|
||||
CounterType parseMimetypeCounter(const std::string& counterData);
|
||||
};
|
||||
|
||||
using namespace kiwix;
|
||||
#define parse parseMimetypeCounter
|
||||
|
||||
namespace
|
||||
{
|
||||
TEST(ParseCounterTest, simpleMimeType)
|
||||
{
|
||||
{
|
||||
std::string counterStr = "";
|
||||
CounterType counterMap = {};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "foo=1";
|
||||
CounterType counterMap = {{"foo", 1}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "foo=1;text/html=50;";
|
||||
CounterType counterMap = {{"foo", 1}, {"text/html", 50}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ParseCounterTest, paramMimeType)
|
||||
{
|
||||
{
|
||||
std::string counterStr = "text/html;raw=true=1";
|
||||
CounterType counterMap = {{"text/html;raw=true", 1}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "foo=1;text/html;raw=true=50;bar=2";
|
||||
CounterType counterMap = {{"foo", 1}, {"text/html;raw=true", 50}, {"bar", 2}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "foo=1;text/html;raw=true;param=value=50;bar=2";
|
||||
CounterType counterMap = {{"foo", 1}, {"text/html;raw=true;param=value", 50}, {"bar", 2}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "foo=1;text/html;raw=true=50;bar=2";
|
||||
CounterType counterMap = {{"foo", 1}, {"text/html;raw=true", 50}, {"bar", 2}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "application/javascript=8;text/html=3;application/warc-headers=28364;text/html;raw=true=6336;text/css=47;text/javascript=98;image/png=968;image/webp=24;application/json=3694;image/gif=10274;image/jpeg=1582;font/woff2=25;text/plain=284;application/atom+xml=247;application/x-www-form-urlencoded=9;video/mp4=9;application/x-javascript=7;application/xml=1;image/svg+xml=5";
|
||||
CounterType counterMap = {
|
||||
{"application/javascript", 8},
|
||||
{"text/html", 3},
|
||||
{"application/warc-headers", 28364},
|
||||
{"text/html;raw=true", 6336},
|
||||
{"text/css", 47},
|
||||
{"text/javascript", 98},
|
||||
{"image/png", 968},
|
||||
{"image/webp", 24},
|
||||
{"application/json", 3694},
|
||||
{"image/gif", 10274},
|
||||
{"image/jpeg", 1582},
|
||||
{"font/woff2", 25},
|
||||
{"text/plain", 284},
|
||||
{"application/atom+xml", 247},
|
||||
{"application/x-www-form-urlencoded", 9},
|
||||
{"video/mp4", 9},
|
||||
{"application/x-javascript", 7},
|
||||
{"application/xml", 1},
|
||||
{"image/svg+xml", 5}
|
||||
};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ParseCounterTest, wrongType)
|
||||
{
|
||||
CounterType empty = {};
|
||||
{
|
||||
std::string counterStr = "text/html";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html=";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html=foo";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html=123foo";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html=50;foo";
|
||||
CounterType counterMap = {{"text/html", 50}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html;foo=20";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html;foo=20;";
|
||||
ASSERT_EQ(parse(counterStr), empty) << counterStr;
|
||||
}
|
||||
{
|
||||
std::string counterStr = "text/html=50;;foo";
|
||||
CounterType counterMap = {{"text/html", 50}};
|
||||
ASSERT_EQ(parse(counterStr), counterMap) << counterStr;
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
Binary file not shown.
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
rm -f corner_cases.zim
|
||||
zimwriterfs -w empty.html \
|
||||
-f empty.png \
|
||||
-l=en \
|
||||
-t="ZIM corner cases" \
|
||||
-d="" \
|
||||
-c="" \
|
||||
-p="" \
|
||||
corner_cases \
|
||||
corner_cases.zim \
|
||||
&& echo 'corner_cases.zim was successfully created' \
|
||||
|| echo '!!! Failed to create corner_cases.zim !!!' >&2
|
||||
Binary file not shown.
Binary file not shown.
5101
test/httplib.h
5101
test/httplib.h
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
|
||||
* Copyright (C) 2013 Tommi Maekitalo
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License as
|
||||
@@ -262,13 +262,8 @@ TEST_F(LibraryTest, filterCheck)
|
||||
TEST_F(LibraryTest, getBookByPath)
|
||||
{
|
||||
auto& book = lib.getBookById(lib.getBooksIds()[0]);
|
||||
#ifdef _WIN32
|
||||
auto path = "C:\\some\\abs\\path.zim";
|
||||
#else
|
||||
auto path = "/some/abs/path.zim";
|
||||
#endif
|
||||
book.setPath(path);
|
||||
EXPECT_EQ(lib.getBookByPath(path).getId(), book.getId());
|
||||
book.setPath("/some/abs/path.zim");
|
||||
EXPECT_EQ(lib.getBookByPath("/some/abs/path.zim").getId(), book.getId());
|
||||
EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../include/manager.h"
|
||||
#include "../include/library.h"
|
||||
#include "../include/book.h"
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
TEST(ManagerTest, addBookFromPathAndGetIdTest)
|
||||
{
|
||||
kiwix::Library lib;
|
||||
kiwix::Manager manager = kiwix::Manager(&lib);
|
||||
|
||||
auto bookId = manager.addBookFromPathAndGetId("./test/example.zim");
|
||||
EXPECT_NE(bookId, "");
|
||||
kiwix::Book book = lib.getBookById(bookId);
|
||||
EXPECT_EQ(book.getPath(), computeAbsolutePath("", "./test/example.zim"));
|
||||
|
||||
const std::string pathToSave = "./pathToSave";
|
||||
const std::string url = "url";
|
||||
bookId = manager.addBookFromPathAndGetId("./test/example.zim", pathToSave, url, true);
|
||||
book = lib.getBookById(bookId);
|
||||
auto savedPath = computeAbsolutePath(removeLastPathElement(manager.writableLibraryPath), pathToSave);
|
||||
EXPECT_EQ(book.getPath(), savedPath);
|
||||
EXPECT_EQ(book.getUrl(), url);
|
||||
}
|
||||
@@ -1,21 +1,16 @@
|
||||
|
||||
|
||||
tests = [
|
||||
'parseUrl',
|
||||
'library',
|
||||
'regex',
|
||||
'tagParsing',
|
||||
'counterParsing',
|
||||
'stringTools',
|
||||
'pathTools',
|
||||
'kiwixserve',
|
||||
'book',
|
||||
'manager',
|
||||
'book'
|
||||
]
|
||||
|
||||
if build_machine.system() != 'windows'
|
||||
tests += ['server']
|
||||
endif
|
||||
|
||||
|
||||
|
||||
gtest_dep = dependency('gtest',
|
||||
main:true,
|
||||
@@ -23,42 +18,12 @@ gtest_dep = dependency('gtest',
|
||||
required:false)
|
||||
|
||||
if gtest_dep.found() and not meson.is_cross_build()
|
||||
data_files = [
|
||||
'example.zim',
|
||||
'zimfile.zim',
|
||||
'corner_cases.zim'
|
||||
]
|
||||
foreach file : data_files
|
||||
# configure_file(input : 'data/' + file,
|
||||
# output : file,
|
||||
# copy: true )
|
||||
#
|
||||
# Above (commented) command doesn't work with Meson versions below 0.47
|
||||
# (in which the 'copy' keyword was first introduced). We want to keep
|
||||
# compatibility with Ubuntu 18.04 Bionic (which has Meson version 0.45)
|
||||
# until its EOL.
|
||||
#
|
||||
# Below is a python based workaround.
|
||||
configure_file(input : 'data/' + file,
|
||||
output : file,
|
||||
command: [
|
||||
find_program('python3'),
|
||||
'-c',
|
||||
'import sys; import shutil; shutil.copy(sys.argv[1], sys.argv[2])',
|
||||
'@INPUT@',
|
||||
'@OUTPUT@'
|
||||
])
|
||||
endforeach
|
||||
|
||||
foreach test_name : tests
|
||||
# XXX: implicit_include_directories must be set to false, otherwise
|
||||
# XXX: '#include <regex>' includes the regex unit test binary
|
||||
test_exe = executable(test_name, [test_name+'.cpp'],
|
||||
implicit_include_directories: false,
|
||||
link_with : kiwixlib,
|
||||
link_args: extra_link_args,
|
||||
dependencies : all_deps + [gtest_dep],
|
||||
build_rpath : '$ORIGIN')
|
||||
test(test_name, test_exe, timeout : 160)
|
||||
test(test_name, test_exe)
|
||||
endforeach
|
||||
endif
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#ifdef _WIN32
|
||||
# define S "\\"
|
||||
# define AS "c:"
|
||||
# define A_SAMBA "\\\\sambadir"
|
||||
#else
|
||||
# define S "/"
|
||||
# define AS ""
|
||||
@@ -43,10 +42,7 @@
|
||||
#define A4(a, b, c, d) A1(P4(a, b, c, d))
|
||||
#define A5(a, b, c, d, e) A1(P5(a, b, c, d, e))
|
||||
|
||||
std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool absolute);
|
||||
std::vector<std::string> nParts(std::vector<std::string> parts, bool absolute) {
|
||||
return normalizeParts(parts, absolute);
|
||||
}
|
||||
std::vector<std::string> normalizeParts(std::vector<std::string> parts, bool absolute);
|
||||
#ifdef _WIN32
|
||||
std::wstring Utf8ToWide(const std::string& str);
|
||||
std::string WideToUtf8(const std::wstring& wstr);
|
||||
@@ -58,7 +54,7 @@ namespace
|
||||
#define V std::vector<std::string>
|
||||
TEST(pathTools, normalizePartsAbsolute)
|
||||
{
|
||||
#define N(...) nParts(__VA_ARGS__, true)
|
||||
#define N(...) normalizeParts(__VA_ARGS__, true)
|
||||
ASSERT_EQ(N({}), V({}));
|
||||
#ifdef _WIN32
|
||||
ASSERT_EQ(N({"c:"}), V({"c:"}));
|
||||
@@ -79,14 +75,13 @@ TEST(pathTools, normalizePartsAbsolute)
|
||||
|
||||
#ifdef _WIN32
|
||||
ASSERT_EQ(N({"c:", "a", "b", ".", "c", "d:", "..", "foo"}), V({"d:", "foo"}));
|
||||
ASSERT_EQ(N({"","","samba","a","b"}), V({"\\\\samba", "a", "b"}));
|
||||
#endif
|
||||
#undef N
|
||||
}
|
||||
|
||||
TEST(pathTools, normalizePartsRelative)
|
||||
{
|
||||
#define N(...) nParts(__VA_ARGS__, false)
|
||||
#define N(...) normalizeParts(__VA_ARGS__, false)
|
||||
ASSERT_EQ(N({}), V({}));
|
||||
ASSERT_EQ(N({""}), V({}));
|
||||
ASSERT_EQ(N({"a"}), V({"a"}));
|
||||
@@ -108,10 +103,6 @@ TEST(pathTools, isRelativePath)
|
||||
ASSERT_TRUE(isRelativePath(P4("foo","","bar","")));
|
||||
ASSERT_FALSE(isRelativePath(A1("foo")));
|
||||
ASSERT_FALSE(isRelativePath(A2("foo", "bar")));
|
||||
#ifdef _WIN32
|
||||
ASSERT_FALSE(isRelativePath(P2(A_SAMBA, "foo")));
|
||||
ASSERT_FALSE(isRelativePath(P3(A_SAMBA, "foo", "bar")));
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(pathTools, computeAbsolutePath)
|
||||
@@ -130,12 +121,6 @@ TEST(pathTools, computeAbsolutePath)
|
||||
A5("a","b","c","d","foo"));
|
||||
ASSERT_EQ(computeAbsolutePath(A5("a","b","c","d","e"), P5("..","..","..","g","foo")),
|
||||
A4("a","b","g","foo"));
|
||||
#ifdef _WIN32
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b",""), P2("..","foo")),
|
||||
P3(A_SAMBA,"a","foo"));
|
||||
ASSERT_EQ(computeAbsolutePath(P6(A_SAMBA,"a","b","c","d","e"), P5("..","..","..","g","foo")),
|
||||
P5(A_SAMBA,"a","b","g","foo"));
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(pathTools, computeRelativePath)
|
||||
@@ -152,13 +137,6 @@ TEST(pathTools, computeRelativePath)
|
||||
P2("..","foo"));
|
||||
ASSERT_EQ(computeRelativePath(A5("a","b","c","d","e"), A4("a","b","g","foo")),
|
||||
P5("..","..","..","g","foo"));
|
||||
#ifdef _WIN32
|
||||
ASSERT_EQ(computeRelativePath(P3(A_SAMBA,"a","b"), P3(A_SAMBA,"a","foo")),
|
||||
P2("..","foo"));
|
||||
ASSERT_EQ(computeRelativePath(P6(A_SAMBA,"a","b","c","d","e"), P5(A_SAMBA,"a","b","g","foo")),
|
||||
P5("..","..","..","g","foo"));
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(pathTools, removeLastPathElement)
|
||||
@@ -201,15 +179,6 @@ TEST(pathTools, goUp)
|
||||
"c:");
|
||||
ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P4("..","..","..","..")),
|
||||
"c:");
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), ".."),
|
||||
P3(A_SAMBA,"a", "b"));
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P2("..","..")),
|
||||
P2(A_SAMBA,"a"));
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P3("..","..","..")),
|
||||
A_SAMBA);
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","..")),
|
||||
A_SAMBA);
|
||||
|
||||
#else
|
||||
ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P3("..","..","..")),
|
||||
"/");
|
||||
@@ -226,12 +195,6 @@ TEST(pathTools, goUp)
|
||||
A1("foo"));
|
||||
ASSERT_EQ(computeAbsolutePath(A3("a","b","c"), P5("..","..","..","..","foo")),
|
||||
A1("foo"));
|
||||
#ifdef _WIN32
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P4("..","..","..","foo")),
|
||||
P2(A_SAMBA,"foo"));
|
||||
ASSERT_EQ(computeAbsolutePath(P4(A_SAMBA,"a","b","c"), P5("..","..","..","..","foo")),
|
||||
P2(A_SAMBA,"foo"));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -240,22 +203,11 @@ TEST(pathTools, dirChange)
|
||||
{
|
||||
std::string p1("c:\\a\\b\\c");
|
||||
std::string p2("d:\\d\\e\\foo.xml");
|
||||
{
|
||||
std::string relative_path = computeRelativePath(p1, p2);
|
||||
ASSERT_EQ(relative_path, p2);
|
||||
std::string abs_path = computeAbsolutePath(p1, relative_path);
|
||||
ASSERT_EQ(abs_path, p2);
|
||||
ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
|
||||
}
|
||||
std::string ps("\\\\samba\\d\\e\\foo.xml");
|
||||
{
|
||||
std::string relative_path = computeRelativePath(p1, ps);
|
||||
ASSERT_EQ(relative_path, ps);
|
||||
std::string abs_path = computeAbsolutePath(p1, relative_path);
|
||||
ASSERT_EQ(abs_path, ps);
|
||||
// I'm not sure this test is valid on windows :/
|
||||
// ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\\\samba\\d\\e\\foo.xml"), ps);
|
||||
}
|
||||
std::string relative_path = computeRelativePath(p1, p2);
|
||||
ASSERT_EQ(relative_path, "d:\\d\\e\\foo.xml");
|
||||
std::string abs_path = computeAbsolutePath(p1, relative_path);
|
||||
ASSERT_EQ(abs_path, p2);
|
||||
ASSERT_EQ(computeAbsolutePath(p1, "..\\..\\..\\..\\..\\d:\\d\\e\\foo.xml"), p2);
|
||||
}
|
||||
|
||||
TEST(pathTools, Utf8ToWide)
|
||||
|
||||
548
test/server.cpp
548
test/server.cpp
@@ -1,548 +0,0 @@
|
||||
|
||||
#include "./httplib.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
#include "../include/manager.h"
|
||||
#include "../include/server.h"
|
||||
#include "../include/name_mapper.h"
|
||||
|
||||
|
||||
using TestContextImpl = std::vector<std::pair<std::string, std::string> >;
|
||||
struct TestContext : TestContextImpl {
|
||||
TestContext(const std::initializer_list<value_type>& il)
|
||||
: TestContextImpl(il)
|
||||
{}
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream& out, const TestContext& ctx)
|
||||
{
|
||||
out << "Test context:\n";
|
||||
for ( const auto& kv : ctx )
|
||||
out << "\t" << kv.first << ": " << kv.second << "\n";
|
||||
out << std::endl;
|
||||
return out;
|
||||
}
|
||||
|
||||
bool is_valid_etag(const std::string& etag)
|
||||
{
|
||||
return etag.size() >= 2 &&
|
||||
etag.front() == '"' &&
|
||||
etag.back() == '"';
|
||||
}
|
||||
|
||||
template<class T1, class T2>
|
||||
T1 concat(T1 a, const T2& b)
|
||||
{
|
||||
a.insert(a.end(), b.begin(), b.end());
|
||||
return a;
|
||||
}
|
||||
|
||||
typedef httplib::Headers Headers;
|
||||
|
||||
Headers invariantHeaders(Headers headers)
|
||||
{
|
||||
headers.erase("Date");
|
||||
return headers;
|
||||
}
|
||||
|
||||
|
||||
class ZimFileServer
|
||||
{
|
||||
public: // types
|
||||
typedef std::shared_ptr<httplib::Response> Response;
|
||||
typedef std::vector<std::string> FilePathCollection;
|
||||
|
||||
public: // functions
|
||||
ZimFileServer(int serverPort, const FilePathCollection& zimpaths);
|
||||
~ZimFileServer();
|
||||
|
||||
Response GET(const char* path, const Headers& headers = Headers())
|
||||
{
|
||||
return client->Get(path, headers);
|
||||
}
|
||||
|
||||
Response HEAD(const char* path, const Headers& headers = Headers())
|
||||
{
|
||||
return client->Head(path, headers);
|
||||
}
|
||||
|
||||
private: // data
|
||||
kiwix::Library library;
|
||||
kiwix::Manager manager;
|
||||
std::unique_ptr<kiwix::HumanReadableNameMapper> nameMapper;
|
||||
std::unique_ptr<kiwix::Server> server;
|
||||
std::unique_ptr<httplib::Client> client;
|
||||
};
|
||||
|
||||
ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths)
|
||||
: manager(&this->library)
|
||||
{
|
||||
for ( const auto& zimpath : zimpaths ) {
|
||||
if (!manager.addBookFromPath(zimpath, zimpath, "", false))
|
||||
throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'");
|
||||
}
|
||||
|
||||
const std::string address = "127.0.0.1";
|
||||
nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false));
|
||||
server.reset(new kiwix::Server(&library, nameMapper.get()));
|
||||
server->setAddress(address);
|
||||
server->setPort(serverPort);
|
||||
server->setNbThreads(2);
|
||||
server->setVerbose(false);
|
||||
|
||||
if ( !server->start() )
|
||||
throw std::runtime_error("ZimFileServer failed to start");
|
||||
|
||||
client.reset(new httplib::Client(address, serverPort));
|
||||
}
|
||||
|
||||
ZimFileServer::~ZimFileServer()
|
||||
{
|
||||
server->stop();
|
||||
}
|
||||
|
||||
class ServerTest : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
std::unique_ptr<ZimFileServer> zfs1_;
|
||||
|
||||
const int PORT = 8001;
|
||||
const ZimFileServer::FilePathCollection ZIMFILES {
|
||||
"./test/zimfile.zim",
|
||||
"./test/corner_cases.zim"
|
||||
};
|
||||
|
||||
protected:
|
||||
void SetUp() override {
|
||||
zfs1_.reset(new ZimFileServer(PORT, ZIMFILES));
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
zfs1_.reset();
|
||||
}
|
||||
};
|
||||
|
||||
const bool WITH_ETAG = true;
|
||||
const bool NO_ETAG = false;
|
||||
|
||||
struct Resource
|
||||
{
|
||||
bool etag_expected;
|
||||
const char* url;
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream& out, const Resource& r)
|
||||
{
|
||||
out << "url: " << r.url;
|
||||
return out;
|
||||
}
|
||||
|
||||
typedef std::vector<Resource> ResourceCollection;
|
||||
|
||||
const ResourceCollection resources200Compressible{
|
||||
{ WITH_ETAG, "/" },
|
||||
|
||||
{ WITH_ETAG, "/skin/jquery-ui/jquery-ui.structure.min.css" },
|
||||
{ WITH_ETAG, "/skin/jquery-ui/jquery-ui.min.js" },
|
||||
{ WITH_ETAG, "/skin/jquery-ui/external/jquery/jquery.js" },
|
||||
{ WITH_ETAG, "/skin/jquery-ui/jquery-ui.theme.min.css" },
|
||||
{ WITH_ETAG, "/skin/jquery-ui/jquery-ui.min.css" },
|
||||
{ WITH_ETAG, "/skin/taskbar.js" },
|
||||
{ WITH_ETAG, "/skin/taskbar.css" },
|
||||
{ WITH_ETAG, "/skin/block_external.js" },
|
||||
|
||||
{ NO_ETAG, "/catalog/root.xml" },
|
||||
{ NO_ETAG, "/catalog/searchdescription.xml" },
|
||||
{ NO_ETAG, "/catalog/search" },
|
||||
|
||||
{ NO_ETAG, "/search?content=zimfile&pattern=a" },
|
||||
|
||||
{ NO_ETAG, "/suggest?content=zimfile&term=ray" },
|
||||
|
||||
{ NO_ETAG, "/catch/external?source=www.example.com" },
|
||||
|
||||
{ WITH_ETAG, "/zimfile/A/index" },
|
||||
{ WITH_ETAG, "/zimfile/A/Ray_Charles" },
|
||||
};
|
||||
|
||||
const ResourceCollection resources200Uncompressible{
|
||||
{ WITH_ETAG, "/skin/jquery-ui/images/animated-overlay.gif" },
|
||||
{ WITH_ETAG, "/skin/caret.png" },
|
||||
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=title" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=description" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=language" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=name" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=tags" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=date" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=creator" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=publisher" },
|
||||
{ WITH_ETAG, "/meta?content=zimfile&name=favicon" },
|
||||
|
||||
{ WITH_ETAG, "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
|
||||
|
||||
{ WITH_ETAG, "/corner_cases/A/empty.html" },
|
||||
{ WITH_ETAG, "/corner_cases/-/empty.css" },
|
||||
{ WITH_ETAG, "/corner_cases/-/empty.js" },
|
||||
};
|
||||
|
||||
ResourceCollection all200Resources()
|
||||
{
|
||||
return concat(resources200Compressible, resources200Uncompressible);
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, 200)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() )
|
||||
EXPECT_EQ(200, zfs1_->GET(res.url)->status) << "res.url: " << res.url;
|
||||
}
|
||||
|
||||
// seperate test for 204 code
|
||||
|
||||
TEST_F(ServerTest, EmptySearchReturnsA204StatusCode)
|
||||
{
|
||||
const char* url="/search?content=zimfile&pattern=abcd";
|
||||
auto res=zfs1_->GET(url);
|
||||
EXPECT_EQ(204, res->status) << "res.url: " << url;
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CompressibleContentIsCompressedIfAcceptable)
|
||||
{
|
||||
for ( const Resource& res : resources200Compressible ) {
|
||||
const auto x = zfs1_->GET(res.url, { {"Accept-Encoding", "deflate"} });
|
||||
EXPECT_EQ(200, x->status) << res;
|
||||
EXPECT_EQ("deflate", x->get_header_value("Content-Encoding")) << res;
|
||||
EXPECT_EQ("Accept-Encoding", x->get_header_value("Vary")) << res;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, UncompressibleContentIsNotCompressed)
|
||||
{
|
||||
for ( const Resource& res : resources200Uncompressible ) {
|
||||
const auto x = zfs1_->GET(res.url, { {"Accept-Encoding", "deflate"} });
|
||||
EXPECT_EQ(200, x->status) << res;
|
||||
EXPECT_EQ("", x->get_header_value("Content-Encoding")) << res;
|
||||
}
|
||||
}
|
||||
|
||||
const char* urls404[] = {
|
||||
"/non-existent-item",
|
||||
"/skin/non-existent-skin-resource",
|
||||
"/catalog",
|
||||
"/catalog/non-existent-item",
|
||||
"/meta",
|
||||
"/meta?content=zimfile",
|
||||
"/meta?content=zimfile&name=non-existent-item",
|
||||
"/meta?content=non-existent-book&name=title",
|
||||
"/random",
|
||||
"/random?content=non-existent-book",
|
||||
"/search",
|
||||
"/suggest",
|
||||
"/suggest?content=zimfile",
|
||||
"/suggest?content=non-existent-book&term=abcd",
|
||||
"/catch/external",
|
||||
"/zimfile/A/non-existent-article",
|
||||
};
|
||||
|
||||
TEST_F(ServerTest, 404)
|
||||
{
|
||||
for ( const char* url : urls404 )
|
||||
EXPECT_EQ(404, zfs1_->GET(url)->status) << "url: " << url;
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, SuccessfulSearchForAnArticleTitleRedirectsToTheArticle)
|
||||
{
|
||||
auto g = zfs1_->GET("/search?content=zimfile&pattern=ray%20charles" );
|
||||
ASSERT_EQ(302, g->status);
|
||||
ASSERT_TRUE(g->has_header("Location"));
|
||||
ASSERT_EQ("/zimfile/A/Ray_Charles", g->get_header_value("Location"));
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
|
||||
{
|
||||
auto g = zfs1_->GET("/random?content=zimfile");
|
||||
ASSERT_EQ(302, g->status);
|
||||
ASSERT_TRUE(g->has_header("Location"));
|
||||
ASSERT_TRUE(g->get_header_value("Location").find("/zimfile/A/") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, BookMainPageIsRedirectedToArticleIndex)
|
||||
{
|
||||
auto g = zfs1_->GET("/zimfile");
|
||||
ASSERT_EQ(302, g->status);
|
||||
ASSERT_TRUE(g->has_header("Location"));
|
||||
ASSERT_EQ("/zimfile/A/index", g->get_header_value("Location"));
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, HeadMethodIsSupported)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() )
|
||||
EXPECT_EQ(200, zfs1_->HEAD(res.url)->status) << res;
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, TheResponseToHeadRequestHasNoBody)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() )
|
||||
EXPECT_TRUE(zfs1_->HEAD(res.url)->body.empty()) << res;
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, HeadersAreTheSameInResponsesToHeadAndGetRequests)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
httplib::Headers g = zfs1_->GET(res.url)->headers;
|
||||
httplib::Headers h = zfs1_->HEAD(res.url)->headers;
|
||||
EXPECT_EQ(invariantHeaders(g), invariantHeaders(h)) << res;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ETagHeaderIsSetAsNeeded)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
const auto responseToGet = zfs1_->GET(res.url);
|
||||
EXPECT_EQ(res.etag_expected, responseToGet->has_header("ETag")) << res;
|
||||
if ( res.etag_expected ) {
|
||||
EXPECT_TRUE(is_valid_etag(responseToGet->get_header_value("ETag")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ETagIsTheSameInResponsesToDifferentRequestsOfTheSameURL)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
const auto h1 = zfs1_->HEAD(res.url);
|
||||
const auto h2 = zfs1_->HEAD(res.url);
|
||||
EXPECT_EQ(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
const auto g = zfs1_->GET(res.url);
|
||||
const auto h = zfs1_->HEAD(res.url);
|
||||
EXPECT_EQ(h->get_header_value("ETag"), g->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags)
|
||||
{
|
||||
ZimFileServer zfs2(PORT + 1, ZIMFILES);
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( !res.etag_expected ) continue;
|
||||
const auto h1 = zfs1_->HEAD(res.url);
|
||||
const auto h2 = zfs2.HEAD(res.url);
|
||||
EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CompressionInfluencesETag)
|
||||
{
|
||||
for ( const Resource& res : resources200Compressible ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
const auto g1 = zfs1_->GET(res.url);
|
||||
const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
|
||||
const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "deflate"} } );
|
||||
const auto etag = g1->get_header_value("ETag");
|
||||
EXPECT_EQ(etag, g2->get_header_value("ETag"));
|
||||
EXPECT_NE(etag, g3->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ETagOfUncompressibleContentIsNotAffectedByAcceptEncoding)
|
||||
{
|
||||
for ( const Resource& res : resources200Uncompressible ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
const auto g1 = zfs1_->GET(res.url);
|
||||
const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
|
||||
const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "deflate"} } );
|
||||
const auto etag = g1->get_header_value("ETag");
|
||||
EXPECT_EQ(etag, g2->get_header_value("ETag")) << res;
|
||||
EXPECT_EQ(etag, g3->get_header_value("ETag")) << res;
|
||||
}
|
||||
}
|
||||
|
||||
// Pick from the response those headers that are required to be present in the
|
||||
// 304 (Not Modified) response if they would be set in the 200 (OK) response.
|
||||
// NOTE: The "Date" header (which should belong to that list as required
|
||||
// NOTE: by RFC 7232) is not included (since the result of this function
|
||||
// NOTE: will be used to check the equality of headers from the 200 and 304
|
||||
// NOTe: responses).
|
||||
Headers special304Headers(const httplib::Response& r)
|
||||
{
|
||||
Headers result;
|
||||
std::copy_if(
|
||||
r.headers.begin(), r.headers.end(),
|
||||
std::inserter(result, result.end()),
|
||||
[](const Headers::value_type& x) {
|
||||
return x.first == "Cache-Control"
|
||||
|| x.first == "Content-Location"
|
||||
|| x.first == "ETag"
|
||||
|| x.first == "Expires"
|
||||
|| x.first == "Vary";
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
// make a list of three etags with the given one in the middle
|
||||
std::string make_etag_list(const std::string& etag)
|
||||
{
|
||||
return "\"x" + etag.substr(1) + ", "
|
||||
+ etag + ", "
|
||||
+ etag.substr(0, etag.size()-2) + "\"";
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, IfNoneMatchRequestsWithMatchingETagResultIn304Responses)
|
||||
{
|
||||
const char* const encodings[] = { "", "deflate" };
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
for ( const char* enc: encodings ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
const TestContext ctx{ {"url", res.url}, {"encoding", enc} };
|
||||
|
||||
const auto g = zfs1_->GET(res.url, { {"Accept-Encoding", enc} });
|
||||
const auto etag = g->get_header_value("ETag");
|
||||
|
||||
const std::string etags = make_etag_list(etag);
|
||||
const Headers headers{{"If-None-Match", etags}, {"Accept-Encoding", enc}};
|
||||
const auto g2 = zfs1_->GET(res.url, headers );
|
||||
const auto h = zfs1_->HEAD(res.url, headers );
|
||||
EXPECT_EQ(304, h->status) << ctx;
|
||||
EXPECT_EQ(304, g2->status) << ctx;
|
||||
EXPECT_EQ(special304Headers(*g), special304Headers(*g2)) << ctx;
|
||||
EXPECT_EQ(special304Headers(*g2), special304Headers(*h)) << ctx;
|
||||
EXPECT_TRUE(g2->body.empty()) << ctx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
const auto g = zfs1_->GET(res.url);
|
||||
const auto etag = g->get_header_value("ETag");
|
||||
const auto etag2 = etag.substr(0, etag.size() - 1) + "x\"";
|
||||
const auto h = zfs1_->HEAD(res.url, { {"If-None-Match", etag2} } );
|
||||
const auto g2 = zfs1_->GET(res.url, { {"If-None-Match", etag2} } );
|
||||
EXPECT_EQ(200, h->status);
|
||||
EXPECT_EQ(200, g2->status);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ValidSingleRangeByteRangeRequestsAreHandledProperly)
|
||||
{
|
||||
const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
|
||||
const auto full = zfs1_->GET(url);
|
||||
EXPECT_FALSE(full->has_header("Content-Range"));
|
||||
EXPECT_EQ("bytes", full->get_header_value("Accept-Ranges"));
|
||||
|
||||
{
|
||||
const auto p = zfs1_->GET(url, { {"Range", "bytes=0-100000"} } );
|
||||
EXPECT_EQ(206, p->status);
|
||||
EXPECT_EQ(full->body, p->body);
|
||||
EXPECT_EQ("bytes 0-20076/20077", p->get_header_value("Content-Range"));
|
||||
EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
|
||||
}
|
||||
|
||||
{
|
||||
const auto p = zfs1_->GET(url, { {"Range", "bytes=0-10"} } );
|
||||
EXPECT_EQ(206, p->status);
|
||||
EXPECT_EQ("bytes 0-10/20077", p->get_header_value("Content-Range"));
|
||||
EXPECT_EQ(11U, p->body.size());
|
||||
EXPECT_EQ(full->body.substr(0, 11), p->body);
|
||||
EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
|
||||
}
|
||||
|
||||
{
|
||||
const auto p = zfs1_->GET(url, { {"Range", "bytes=123-456"} } );
|
||||
EXPECT_EQ(206, p->status);
|
||||
EXPECT_EQ("bytes 123-456/20077", p->get_header_value("Content-Range"));
|
||||
EXPECT_EQ(334U, p->body.size());
|
||||
EXPECT_EQ(full->body.substr(123, 334), p->body);
|
||||
EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
|
||||
}
|
||||
|
||||
{
|
||||
const auto p = zfs1_->GET(url, { {"Range", "bytes=20000-"} } );
|
||||
EXPECT_EQ(206, p->status);
|
||||
EXPECT_EQ(full->body.substr(20000), p->body);
|
||||
EXPECT_EQ("bytes 20000-20076/20077", p->get_header_value("Content-Range"));
|
||||
EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
|
||||
}
|
||||
|
||||
{
|
||||
const auto p = zfs1_->GET(url, { {"Range", "bytes=-100"} } );
|
||||
EXPECT_EQ(206, p->status);
|
||||
EXPECT_EQ(full->body.substr(19977), p->body);
|
||||
EXPECT_EQ("bytes 19977-20076/20077", p->get_header_value("Content-Range"));
|
||||
EXPECT_EQ("bytes", p->get_header_value("Accept-Ranges"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, InvalidAndMultiRangeByteRangeRequestsResultIn416Responses)
|
||||
{
|
||||
const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
|
||||
|
||||
const char* invalidRanges[] = {
|
||||
"0-10", "bytes=", "bytes=123", "bytes=-10-20", "bytes=10-20xxx",
|
||||
"bytes=10-0", // reversed range
|
||||
"bytes=10-20, 30-40", // multi-range
|
||||
"bytes=1000000-", "bytes=30000-30100" // unsatisfiable ranges
|
||||
};
|
||||
|
||||
for( const char* range : invalidRanges )
|
||||
{
|
||||
const TestContext ctx{ {"Range", range} };
|
||||
const auto p = zfs1_->GET(url, { {"Range", range } } );
|
||||
EXPECT_EQ(416, p->status) << ctx;
|
||||
EXPECT_TRUE(p->body.empty()) << ctx;
|
||||
EXPECT_EQ("bytes */20077", p->get_header_value("Content-Range")) << ctx;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ValidByteRangeRequestsOfZeroSizedEntriesResultIn416Responses)
|
||||
{
|
||||
const char url[] = "/corner_cases/-/empty.js";
|
||||
|
||||
const char* ranges[] = {
|
||||
"bytes=0-",
|
||||
"bytes=-100"
|
||||
};
|
||||
|
||||
for( const char* range : ranges )
|
||||
{
|
||||
const TestContext ctx{ {"Range", range} };
|
||||
const auto p = zfs1_->GET(url, { {"Range", range } } );
|
||||
EXPECT_EQ(416, p->status) << ctx;
|
||||
EXPECT_TRUE(p->body.empty()) << ctx;
|
||||
EXPECT_EQ("bytes */0", p->get_header_value("Content-Range")) << ctx;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, RangeHasPrecedenceOverCompression)
|
||||
{
|
||||
const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
|
||||
|
||||
const Headers onlyRange{ {"Range", "bytes=123-456"} };
|
||||
Headers rangeAndCompression(onlyRange);
|
||||
rangeAndCompression.insert({"Accept-Encoding", "deflate"});
|
||||
|
||||
const auto p1 = zfs1_->GET(url, onlyRange);
|
||||
const auto p2 = zfs1_->GET(url, rangeAndCompression);
|
||||
EXPECT_EQ(p1->status, p2->status);
|
||||
EXPECT_EQ(invariantHeaders(p1->headers), invariantHeaders(p2->headers));
|
||||
EXPECT_EQ(p1->body, p2->body);
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, RangeHeaderIsCaseInsensitive)
|
||||
{
|
||||
const char url[] = "/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg";
|
||||
const auto r0 = zfs1_->GET(url, { {"Range", "bytes=100-200"} } );
|
||||
|
||||
const char* header_variations[] = { "RANGE", "range", "rAnGe", "RaNgE" };
|
||||
for ( const char* header : header_variations ) {
|
||||
const auto r = zfs1_->GET(url, { {header, "bytes=100-200"} } );
|
||||
EXPECT_EQ(206, r->status);
|
||||
EXPECT_EQ("bytes 100-200/20077", r->get_header_value("Content-Range"));
|
||||
EXPECT_EQ(r0->body, r->body);
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@
|
||||
|
||||
namespace kiwix {
|
||||
std::string join(const std::vector<std::string>& list, const std::string& sep);
|
||||
std::vector<std::string> split(const std::string& base, const std::string& sep, bool trimEmpty, bool keepDelim);
|
||||
std::vector<std::string> split(const std::string& base, const std::string& sep, bool trimEmpty);
|
||||
};
|
||||
|
||||
using namespace kiwix;
|
||||
@@ -40,22 +40,17 @@ TEST(stringTools, join)
|
||||
TEST(stringTools, split)
|
||||
{
|
||||
std::vector<std::string> list1 = { "a", "b", "c" };
|
||||
ASSERT_EQ(split("a;b;c", ";", false, false), list1);
|
||||
ASSERT_EQ(split("a;b;c", ";", true, false), list1);
|
||||
ASSERT_EQ(split("a;b;c", ";", false), list1);
|
||||
ASSERT_EQ(split("a;b;c", ";", true), list1);
|
||||
std::vector<std::string> list2 = { "", "a", "b", "c" };
|
||||
ASSERT_EQ(split(";a;b;c", ";", false, false), list2);
|
||||
ASSERT_EQ(split(";a;b;c", ";", true, false), list1);
|
||||
ASSERT_EQ(split(";a;b;c", ";", false), list2);
|
||||
ASSERT_EQ(split(";a;b;c", ";", true), list1);
|
||||
std::vector<std::string> list3 = { "", "a", "b", "c", ""};
|
||||
ASSERT_EQ(split(";a;b;c;", ";", false, false), list3);
|
||||
ASSERT_EQ(split(";a;b;c;", ";", true, false), list1);
|
||||
ASSERT_EQ(split(";a;b;c;", ";", false), list3);
|
||||
ASSERT_EQ(split(";a;b;c;", ";", true), list1);
|
||||
std::vector<std::string> list4 = { "", "a", "b", "", "c", ""};
|
||||
ASSERT_EQ(split(";a;b;;c;", ";", false, false), list4);
|
||||
ASSERT_EQ(split(";a;b;;c;", ";", true, false), list1);
|
||||
|
||||
std::vector<std::string> list5 = { ";", "a", ";", "b", "=", ";", "c", "=", "d", ";"};
|
||||
ASSERT_EQ(split(";a;b=;c=d;", ";=", true, true), list5);
|
||||
std::vector<std::string> list6 = { "", ";", "a", ";", "b", "=", "", ";", "c", "=", "d", ";", ""};
|
||||
ASSERT_EQ(split(";a;b=;c=d;", ";=", false, true), list6);
|
||||
ASSERT_EQ(split(";a;b;;c;", ";", false), list4);
|
||||
ASSERT_EQ(split(";a;b;;c;", ";", true), list1);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user