Compare commits

..

1 Commits

Author SHA1 Message Date
Matthieu Gautier
e2e42ac65c Patch httplib.h for gcc maybe-uninitialized warning.
Recent version of gcc warn about the use `buf` where it may be
uninitialized.
But here, buf is used as a output buffer to initialize it.

I'm not sure we can fix this differently than ignoring the warning.
2021-06-30 17:30:30 +02:00
109 changed files with 1751 additions and 9480 deletions

View File

@@ -8,10 +8,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v1
- name: Setup python 3.10
uses: actions/setup-python@v2
- name: Setup python 3.5
uses: actions/setup-python@v1
with:
python-version: '3.10'
python-version: '3.5'
- name: Install packages
run: |
brew update
@@ -21,7 +21,7 @@ jobs:
- name: Install deps
shell: bash
run: |
ARCHIVE_NAME=deps2_osx_native_dyn_libkiwix.tar.xz
ARCHIVE_NAME=deps2_osx_native_dyn_kiwix-lib.tar.xz
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C $HOME
- name: Compile
shell: bash
@@ -37,8 +37,17 @@ jobs:
export LD_LIBRARY_PATH=$HOME/BUILD_native_dyn/INSTALL/lib:$HOME/BUILD_native_dyn/INSTALL/lib64
cd build
meson test --verbose
ninja coverage
env:
SKIP_BIG_MEMORY_TEST: 1
- name: Publish coverage
shell: bash
run: |
curl https://codecov.io/bash -o codecov.sh
bash codecov.sh -n osx_native_dyn -Z
rm codecov.sh
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
Linux:
strategy:
@@ -85,7 +94,7 @@ jobs:
HOME: /home/runner
runs-on: ubuntu-latest
container:
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
steps:
- name: Extract branch name
shell: bash
@@ -106,7 +115,7 @@ jobs:
- name: Install deps
shell: bash
run: |
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_libkiwix.tar.xz
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-lib.tar.xz
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
- name: Compile
shell: bash

View File

@@ -7,11 +7,7 @@ jobs:
strategy:
fail-fast: false
matrix:
distro:
- ubuntu-jammy
- ubuntu-impish
- ubuntu-focal
- ubuntu-bionic
distro: [ubuntu-hirsute, ubuntu-groovy, ubuntu-focal, ubuntu-bionic]
steps:
- uses: actions/checkout@v2
@@ -34,18 +30,18 @@ jobs:
email: release+launchpad@kiwix.org
distro: ${{ matrix.distro }}
- uses: legoktm/gh-action-build-deb@ubuntu-jammy
if: matrix.distro == 'ubuntu-jammy'
name: Build package for ubuntu-jammy
id: build-ubuntu-jammy
- uses: legoktm/gh-action-build-deb@ubuntu-hirsute
if: matrix.distro == 'ubuntu-hirsute'
name: Build package for ubuntu-hirsute
id: build-ubuntu-hirsute
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}
- uses: legoktm/gh-action-build-deb@ubuntu-impish
if: matrix.distro == 'ubuntu-impish'
name: Build package for ubuntu-impish
id: build-ubuntu-impish
- uses: legoktm/gh-action-build-deb@ubuntu-groovy
if: matrix.distro == 'ubuntu-groovy'
name: Build package for ubuntu-groovy
id: build-ubuntu-groovy
with:
args: --no-sign
ppa: ${{ steps.ppa.outputs.ppa }}

2
.gitignore vendored
View File

@@ -5,5 +5,3 @@ subprojects/googletest-release*
build/
.vscode/
builddir/
.cache/
.clangd/

104
ChangeLog
View File

@@ -1,109 +1,7 @@
libkiwix 10.1.1
===============
* Correctly detect the number of article for older zims (<=6) (@mgautier #743)
* [server] Fix fulltext search (@mgautierfr #724)
* [server][internal] New way to build Error message (@veloman-yunkan #732 #738 #744)
* Fix CI (@mgautierfr #736)
libkiwix 10.1.0
===============
This release is an important one as it fixes a Xss vulnerability introduced
in libkiwix 10.0.0
* [SECURITY] Fix a Xss attack vulnerability (introduced in 10.0.0) (@juuz0 #721)
* [server] Add a option to set a limit on the number of connexion per IP (@kelson42 #700)
* [server] Do not display a lang tag in the UI if the book has no language (@juuz0 #706)
* [server] Add the book title associated to a search results (@thavelick #705, @mgautierfr #718)
* Add `dc:issued` to opds output stream (@veloman-yunkan #715)
* Add handling of several languages not provided by ICU (@juuz0 #701)
* [server] Add a caching system for search and suggestion (@maneeshpm #620)
* Fix cross-compilation (@kelson42 #703)
* Add unit-testing of suggestions and error pages (@veloman-yunkan #709 #710 #727)
* Better testing system of html response (@veloman-yunkan #725)
libkiwix 10.0.1
===============
* [server] The catalog search interpret `count=0` as no limit.
This was the case for a long time. This was changed unintentionally
(@veloman-yunkan #686)
* [server] Correctly generere a human friendly title in the server frontend.
(@juuz0 #687, @kelson42 #689)
* [server] Fix download button if there is no url do download from.
(@juuz0 #691)
* Add non-minified isotope.pkdg.js
Needed for debian packaging as we need the source and minified version is
not the source (@legoktm #693)
* [server] Add a tooltip with the full language for the lang tag.
* CI fixes (@kelson42 @legoktm)
libkiwix 10.0.0
===============
This release is huge release.
The project has been renamed to libkiwix, it is more coherent with the library name.
* Server front page :
- Use js in the front page to display the available book,
using the OPDS stream as source. The front page is now populated only with
the visible books and user can search for books. (@MananJethwany #530, #541, #534)
(@kelson42 #628)
- Revamp css (@MananJethwany #559)
- Correctly Convert 3iso language code to 2iso (@juuz0 #672)
* Server suggestions search :
- Add pagination for suggestion search (@maneeshpm #591)
- Fix suggestion system (@MananJethwany #498)
- Provide the kind and path (when adapted) to the suggestion answer (@MananJethwany #464)
- The displayed suggestion have now highligth on the searched terms (@maneeshpm #505)
- Properly handle html encoding of suggestions (@veloman-yunkan #458)
* Server improvements :
- Remove meta endpoints (@mgautier #669)
- Add raw endpoints to get the raw content of a zim (@mgautierfr #646)
- Add details on 404 error pages (@soumyankar #490)
- Fix headbar insertion when `<head>` tag has attributes (@kelson42 #440)
- Better headbar insertion (after charset definition) (@kelson42 #442)
* New OPDS Stream v2 :
- Add a list of categories (@veloman-yunkan)
- Support for partial entries (@veloman-yunkan #602)
- Support multiple icons size in the OPDS stream (@veloman-yunkan #577 #630)
- Add language endpoint to catalog (@veloman-yunkan #553)
- Add illustration API to get the illustration of a book (@mgautierfr #645)
- OPDS search can now filter books by category (@veloman-yunkan #459)
* Library improvements :
- Allow the libray to be live reloaded when the library.xml changes (@veloman-yunkan #636)
- Properly handle removing of book from the library (@veloman-yunkan #485)
- Use xapian to search for books in the library (@veloman-yunkan #460, #488)
* Added methods/functions :
- Fix `fileExist` and introduce `fileReadable` (@juuz0 #668)
- Add `getVersions` and `printVersions` functions (@kelson42 #665)
- Add `getNetworkInterfaces()` and `getBestPublicIP()` functions (@juuz0 #622)
- Add `get_zimid()` method to the search result (@maneeshpm #510)
* Various improvements :
- Better secret value for aria2c rpc (@juuz0 #666)
- Avoid duplicated Archive/Reader in the Searcher (@veloman-yunkan #648)
- Add basic documentation (@mgautierfr #640)
- Do not use Reader internally (@maneeshpm #536 #576)
- Remove dependency headers from our public headers (@mgautierfr #574)
- Downloader now don't write metalink on the filesystem (@kelson42 #502)
- Support opening a zim file using a fd (@veloman-yukan #429)
- Use C++11 std::thread instead of pthread (@mgautierfr #445)
- [READER] Do not crash if zim file has no `Counter` metadata (@mgautierfr #449)
- Ensure libzim dependency is compiled with xapian (@mgautierfr #434)
- Support video and audio mimetype in `getMediaCount` (@kelson42 #439)
- Better parsing of the counterMap (@kelson42 #437)
- Adapt libkiwix to libzim 7.0.0 (@mgautierfr #428)
- Remove deprecated methods (@mgautierfr)
- CI: Build package for Ubuntu Hirsute, Impish and Jammy (@legoktm #431 #568) and remove Groovy
- Fix compilation for FreeBSD (@swills g#432)
- Many fixes and improvement (@MananJethwany, @maneeshpm, @veloman-yunkan, @mgautierfr)
* ...
kiwix-lib 9.4.1
===============

View File

@@ -5,7 +5,6 @@ The Libkiwix provides the [Kiwix](https://kiwix.org) software suite
core. It contains the code shared by all Kiwix ports (Windows,
GNU/Linux, macOS, Android, iOS, ...).
[![Release](https://img.shields.io/github/v/tag/kiwix/libkiwix?label=release&sort=semver)](https://download.kiwix.org/release/libkiwix/)
[![Repositories](https://img.shields.io/repology/repositories/libkiwix?label=repositories)](https://github.com/kiwix/libkiwix/wiki/Repology)
[![Build Status](https://github.com/kiwix/libkiwix/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/libkiwix/actions?query=branch%3Amaster)
[![CodeFactor](https://www.codefactor.io/repository/github/kiwix/libkiwix/badge)](https://www.codefactor.io/repository/github/kiwix/libkiwix)
@@ -92,14 +91,6 @@ Meson. If you want statically linked libraries, you can add
Depending of you system, `ninja` may be called `ninja-build`.
The android wrapper uses deprecated methods of libkiwix so it cannot be compiled
with `werror=true` (the default). So you must pass `-Dwerror=false` to meson:
```bash
meson . build -Dwrapper=android -Dwerror=false
ninja -C build
```
Testing
-------
@@ -155,53 +146,6 @@ cp ninja ../bin
cd ..
```
Custom Index Page
-----------------
to use custom welcome page mention `customIndexPage` argument in `kiwix::internalServer()` or use `kiwix::server->setCustomIndexTemplate()`.
(note - while using custom html file please mention all external links as absolute path.)
to create a HTML template with custom JS you need to have a look at various OPDS based endpoints as mentioned [here](https://wiki.kiwix.org/wiki/OPDS) to load books.
To use JS provided by kiwix-serve you can use the following template to start with ->
```
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title><-- Custom Tittle --></title>
<script
type="text/javascript"
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
></script>
<script
type="text/javascript"
src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
></script>
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
<script src="{{root}}/skin/iso6391To3.js"></script>
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
</head>
<body>
</body>
</html>
```
- To get books listed using `index.js` add - `<div class="book__list"></div>` under body tag.
- To get number of books listed add - `<h3 class="kiwixHomeBody__results"></h3>` under body tag.
- To add language select box add - `<select id="languageFilter"></select>` under body tag.
- To add language select box add - `<select id="categoryFilter"></select>` under body tag.
- To add search box for books use following form -
```
<form id='kiwixSearchForm'>
<input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
<input type="submit" class="searchButton" value="Search"/>
</form>
```
If you compile manually Libmicrohttpd, you might need to compile it
without GNU TLS, a bug here will empeach further compilation
otherwise.

View File

@@ -26,7 +26,7 @@ task writePom {
project {
groupId 'org.kiwix.kiwixlib'
artifactId 'kiwixlib'
version '10.1.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
version '10.0.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
packaging 'aar'
name 'kiwixlib'
url 'https://github.com/kiwix/libkiwix'

4
debian/control vendored
View File

@@ -4,7 +4,7 @@ Maintainer: Kiwix team <kiwix@kiwix.org>
Build-Depends: debhelper-compat (= 13),
meson,
pkg-config,
libzim-dev (>= 7.2.0~),
libzim-dev (>= 6.1.8),
libcurl4-gnutls-dev,
libicu-dev,
libgtest-dev,
@@ -23,7 +23,7 @@ Section: libdevel
Architecture: any
Multi-Arch: same
Depends: libkiwix10 (= ${binary:Version}), ${misc:Depends}, python3,
libzim-dev (>= 7.2.0~),
libzim-dev (>= 6.0.0),
libicu-dev,
libpugixml-dev,
libcurl4-gnutls-dev,

2
docs/.gitignore vendored
View File

@@ -1,2 +0,0 @@
api
xml

View File

@@ -1,72 +0,0 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'libkiwix'
copyright = '2022, libkiwix-team'
author = 'libzim-team'
# -- General configuration ---------------------------------------------------
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'breathe',
'exhale'
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
if not on_rtd:
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
breathe_projects = {
"libkiwix": "./xml"
}
breathe_default_project = 'libkiwix'
exhale_args = {
"containmentFolder": "./api",
"rootFileName": "ref_api.rst",
"rootFileTitle": "Reference API",
"doxygenStripFromPath":"..",
"treeViewIsBootstrap": True,
"createTreeView" : True,
"exhaleExecutesDoxygen": True,
"exhaleDoxygenStdin": "INPUT = ../include"
}
primary_domain = 'cpp'
highlight_language = 'cpp'

View File

@@ -1,14 +0,0 @@
.. libkiwix documentation master file, created by
sphinx-quickstart on Fri Jul 24 15:40:50 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to libzim's documentation!
==================================
.. toctree::
:maxdepth: 2
:caption: Contents:
usage
api/ref_api

View File

@@ -1,7 +0,0 @@
sphinx = find_program('sphinx-build', native:true)
sphinx_target = run_target('doc',
command: [sphinx, '-bhtml',
meson.current_source_dir(),
meson.current_build_dir()])

View File

@@ -1,2 +0,0 @@
breathe
exhale

View File

@@ -1,17 +0,0 @@
Libkiwix programming
====================
Introduction
------------
libkiwix is written in C++. To use the library, you need the include files of libkiwix have
to link against libzim.
Errors are handled with exceptions. When something goes wrong, libzim throws an error,
which is always derived from std::exception.
All classes are defined in the namespace kiwix.
libkiwix is a set of tools to manage zim files and provide some common functionnality.
While libkiwix has some wrappers around libzim classes, they are deprecated and will be removed
in the future.

View File

@@ -7,7 +7,6 @@ files=(
"include/common/otherTools.h"
"include/common/regexTools.h"
"include/common/networkTools.h"
"include/common/archiveTools.h"
"include/manager.h"
"include/reader.h"
"include/kiwix.h"
@@ -23,7 +22,6 @@ files=(
"src/common/pathTools.cpp"
"src/common/regexTools.cpp"
"src/common/otherTools.cpp"
"src/common/archiveTools.cpp"
"src/common/networkTools.cpp"
"src/common/stringTools.cpp"
"src/xapianSearcher.cpp"

View File

@@ -21,19 +21,11 @@
#define KIWIX_BOOK_H
#include <string>
#include <vector>
#include <memory>
#include <mutex>
#include "common.h"
namespace pugi {
class xml_node;
}
namespace zim {
class Archive;
}
namespace kiwix
{
@@ -45,32 +37,12 @@ class Reader;
*/
class Book
{
public: // types
class Illustration
{
friend class Book;
public:
uint16_t width = 48;
uint16_t height = 48;
std::string mimeType;
std::string url;
const std::string& getData() const;
private:
mutable std::string data;
mutable std::mutex mutex;
};
typedef std::vector<std::shared_ptr<const Illustration>> Illustrations;
public: // functions
public:
Book();
~Book();
bool update(const Book& other);
DEPRECATED void update(const Reader& reader);
void update(const zim::Archive& archive);
void update(const Reader& reader);
void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
std::string getHumanReadableIdFromPath() const;
@@ -96,13 +68,9 @@ class Book
const uint64_t& getArticleCount() const { return m_articleCount; }
const uint64_t& getMediaCount() const { return m_mediaCount; }
const uint64_t& getSize() const { return m_size; }
DEPRECATED const std::string& getFavicon() const;
DEPRECATED const std::string& getFaviconUrl() const;
DEPRECATED const std::string& getFaviconMimeType() const;
Illustrations getIllustrations() const;
std::shared_ptr<const Illustration> getIllustration(unsigned int size) const;
const std::string& getFavicon() const;
const std::string& getFaviconUrl() const { return m_faviconUrl; }
const std::string& getFaviconMimeType() const { return m_faviconMimeType; }
const std::string& getDownloadId() const { return m_downloadId; }
void setReadOnly(bool readOnly) { m_readOnly = readOnly; }
@@ -123,13 +91,14 @@ class Book
void setArticleCount(uint64_t articleCount) { m_articleCount = articleCount; }
void setMediaCount(uint64_t mediaCount) { m_mediaCount = mediaCount; }
void setSize(uint64_t size) { m_size = size; }
void setFavicon(const std::string& favicon) { m_favicon = favicon; }
void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; }
void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; }
private: // functions
private:
std::string getCategoryFromTags() const;
const Illustration& getDefaultIllustration() const;
protected: // data
protected:
std::string m_id;
std::string m_downloadId;
std::string m_path;
@@ -150,11 +119,9 @@ class Book
uint64_t m_mediaCount = 0;
bool m_readOnly = false;
uint64_t m_size = 0;
Illustrations m_illustrations;
// Used as the return value of getDefaultIllustration() when no default
// illustration is found in the book
static const Illustration missingDefaultIllustration;
mutable std::string m_favicon;
std::string m_faviconUrl;
std::string m_faviconMimeType;
};
}

View File

@@ -47,7 +47,7 @@ class Entry
*
* @param article a zim::Article object
*/
DEPRECATED Entry(zim::Entry entry) : Entry(entry, true) {};
Entry(zim::Entry entry);
virtual ~Entry() = default;
/**
@@ -176,16 +176,6 @@ class Entry
private:
zim::Entry entry;
private:
// Entry is deprecated, so we've marked the constructor as deprecated.
// But we still need to construct the entry (in our deprecated code)
// To avoid warning because we use deprecated function, we create a second
// constructor not deprecated. The `bool marker` is unused, it sole purpose
// is to change the signature to have two different constructor.
// This one is not deprecated and we must use it in our private code.
Entry(zim::Entry entry, bool marker);
friend class Reader;
};
}

View File

@@ -22,4 +22,4 @@
#include "library.h"
#endif
#endif

View File

@@ -24,8 +24,6 @@
#include <vector>
#include <map>
#include <memory>
#include <mutex>
#include <zim/archive.h>
#include "book.h"
#include "bookmark.h"
@@ -140,53 +138,20 @@ private: // functions
bool accept(const Book& book) const;
};
/**
* This class is not part of the libkiwix API. Its only purpose is
* to simplify the implementation of the Library's move operations
* and avoid bugs should new data members be added to Library.
*/
class LibraryBase
{
protected: // types
typedef uint64_t LibraryRevision;
struct Entry : Book
{
LibraryRevision lastUpdatedRevision = 0;
// May also keep the Archive and Reader pointers here and get
// rid of the m_readers and m_archives data members in Library
};
protected: // data
LibraryRevision m_revision;
std::map<std::string, Entry> m_books;
std::map<std::string, std::shared_ptr<Reader>> m_readers;
std::map<std::string, std::shared_ptr<zim::Archive>> m_archives;
std::vector<kiwix::Bookmark> m_bookmarks;
class BookDB;
std::unique_ptr<BookDB> m_bookDB;
protected: // functions
LibraryBase();
~LibraryBase();
LibraryBase(LibraryBase&& );
LibraryBase& operator=(LibraryBase&& );
};
/**
* A Library store several books.
*/
class Library : private LibraryBase
class Library
{
// all data fields must be added in LibraryBase
mutable std::mutex m_mutex;
std::map<std::string, kiwix::Book> m_books;
std::map<std::string, std::shared_ptr<Reader>> m_readers;
std::vector<kiwix::Bookmark> m_bookmarks;
class BookDB;
std::unique_ptr<BookDB> m_bookDB;
public:
typedef LibraryRevision Revision;
typedef std::vector<std::string> BookIdCollection;
typedef std::map<std::string, int> AttributeCounts;
public:
Library();
@@ -212,11 +177,6 @@ class Library : private LibraryBase
*/
bool addBook(const Book& book);
/**
* A self-explanatory alias for addBook()
*/
bool addOrUpdateBook(const Book& book) { return addBook(book); }
/**
* Add a bookmark to the library.
*
@@ -233,15 +193,11 @@ class Library : private LibraryBase
*/
bool removeBookmark(const std::string& zimId, const std::string& url);
// XXX: This is a non-thread-safe operation
const Book& getBookById(const std::string& id) const;
// XXX: This is a non-thread-safe operation
Book& getBookById(const std::string& id);
const Book& getBookByPath(const std::string& path) const;
Book getBookByIdThreadSafe(const std::string& id) const;
DEPRECATED std::shared_ptr<Reader> getReaderById(const std::string& id);
std::shared_ptr<zim::Archive> getArchiveById(const std::string& id);
Book& getBookByPath(const std::string& path);
std::shared_ptr<Reader> getReaderById(const std::string& id);
/**
* Remove a book from the library.
@@ -283,13 +239,6 @@ class Library : private LibraryBase
*/
std::vector<std::string> getBooksLanguages() const;
/**
* Get all languagues of the books in the library with counts.
*
* @return A list of languages with the count of books in each language.
*/
AttributeCounts getBooksLanguagesWithCounts() const;
/**
* Get all categories of the books in the library.
*
@@ -325,6 +274,17 @@ class Library : private LibraryBase
*/
BookIdCollection getBooksIds() const;
/**
* Filter the library and generate a new one with the keep elements.
*
* This is equivalent to `listBookIds(ALL, UNSORTED, search)`.
*
* @param search List only books with search in the title or description.
* @return The list of bookIds corresponding to the query.
*/
DEPRECATED BookIdCollection filter(const std::string& search) const;
/**
* Filter the library and return the id of the keep elements.
*
@@ -344,35 +304,43 @@ class Library : private LibraryBase
void sort(BookIdCollection& bookIds, supportedListSortBy sortBy, bool ascending) const;
/**
* Return the current revision of the library.
* List books in the library.
*
* The revision of the library is updated (incremented by one) only by
* the addBook() operation.
*
* @return Current revision of the library.
* @param mode The mode of listing :
* - LOCAL  : list only local books (with a path).
* - REMOTE : list only remote books (with an url).
* - VALID  : list only valid books (without a path or with a
* path pointing to a valid zim file).
* - NOLOCAL : list only books without valid path.
* - NOREMOTE : list only books without url.
* - NOVALID : list only books not valid.
* - ALL : Do not do any filter (LOCAL or REMOTE)
* - Flags can be combined.
* @param sortBy Attribute to sort by the book list.
* @param search List only books with search in the title, description.
* @param language List only books in this language.
* @param creator List only books of this creator.
* @param publisher List only books of this publisher.
* @param maxSize Do not list book bigger than maxSize.
* Set to 0 to cancel this filter.
* @return The list of bookIds corresponding to the query.
*/
LibraryRevision getRevision() const;
/**
* Remove books that have not been updated since the specified revision.
*
* @param rev the library revision to use
* @return Count of books that were removed by this operation.
*/
uint32_t removeBooksNotUpdatedSince(LibraryRevision rev);
DEPRECATED BookIdCollection listBooksIds(
int supportedListMode = ALL,
supportedListSortBy sortBy = UNSORTED,
const std::string& search = "",
const std::string& language = "",
const std::string& creator = "",
const std::string& publisher = "",
const std::vector<std::string>& tags = {},
size_t maxSize = 0) const;
friend class OPDSDumper;
friend class libXMLDumper;
private: // types
typedef const std::string& (Book::*BookStrPropMemFn)() const;
private: // functions
AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
BookIdCollection filterViaBookDB(const Filter& filter) const;
void updateBookDB(const Book& book);
void dropReader(const std::string& bookId);
};
}

View File

@@ -26,7 +26,6 @@
#include <string>
#include <vector>
#include <memory>
namespace pugi {
class xml_document;
@@ -35,25 +34,26 @@ class xml_document;
namespace kiwix
{
class LibraryManipulator
{
public: // functions
explicit LibraryManipulator(Library* library);
virtual ~LibraryManipulator();
class LibraryManipulator {
public:
virtual ~LibraryManipulator() {}
virtual bool addBookToLibrary(Book book) = 0;
virtual void addBookmarkToLibrary(Bookmark bookmark) = 0;
};
Library& getLibrary() const { return library; }
bool addBookToLibrary(const Book& book);
void addBookmarkToLibrary(const Bookmark& bookmark);
uint32_t removeBooksNotUpdatedSince(Library::Revision rev);
protected: // overrides
virtual void bookWasAddedToLibrary(const Book& book);
virtual void bookmarkWasAddedToLibrary(const Bookmark& bookmark);
virtual void booksWereRemovedFromLibrary();
private: // data
kiwix::Library& library;
class DefaultLibraryManipulator : public LibraryManipulator {
public:
DefaultLibraryManipulator(Library* library) :
library(library) {}
virtual ~DefaultLibraryManipulator() {}
bool addBookToLibrary(Book book) {
return library->addBook(book);
}
void addBookmarkToLibrary(Bookmark bookmark) {
library->addBookmark(bookmark);
}
private:
kiwix::Library* library;
};
/**
@@ -61,12 +61,10 @@ class LibraryManipulator
*/
class Manager
{
public: // types
typedef std::vector<std::string> Paths;
public: // functions
explicit Manager(LibraryManipulator* manipulator);
explicit Manager(Library* library);
public:
Manager(LibraryManipulator* manipulator);
Manager(Library* library);
~Manager();
/**
* Read a `library.xml` and add book in the file to the library.
@@ -74,22 +72,10 @@ class Manager
* @param path The (utf8) path to the `library.xml`.
* @param readOnly Set if the libray path could be overwritten latter with
* updated content.
* @param trustLibrary use book metadata coming from XML.
* @return True if file has been properly parsed.
*/
bool readFile(const std::string& path, bool readOnly = true, bool trustLibrary = true);
/**
* Sync the contents of the library with one or more `library.xml` files.
*
* The metadata of the library files is trusted unconditionally.
* Any books not present in the input library.xml files are removed
* from the library.
*
* @param paths The (utf8) paths to the `library.xml` files.
*/
void reload(const Paths& paths);
/**
* Load a library content store in the string.
*
@@ -164,7 +150,8 @@ class Manager
uint64_t m_itemsPerPage = 0;
protected:
std::shared_ptr<kiwix::LibraryManipulator> manipulator;
kiwix::LibraryManipulator* manipulator;
bool mustDeleteManipulator;
bool readBookFromPath(const std::string& path, Book* book);
bool parseXmlDom(const pugi::xml_document& doc,

View File

@@ -13,9 +13,18 @@ headers = [
'search_renderer.h',
'server.h',
'kiwixserve.h',
'name_mapper.h',
'tools.h',
'version.h'
'name_mapper.h'
]
install_headers(headers, subdir:'kiwix')
install_headers(
'tools/base64.h',
'tools/networkTools.h',
'tools/otherTools.h',
'tools/pathTools.h',
'tools/regexTools.h',
'tools/stringTools.h',
subdir:'kiwix/tools'
)

View File

@@ -22,8 +22,6 @@
#include <string>
#include <map>
#include <memory>
#include <mutex>
namespace kiwix
{
@@ -33,15 +31,15 @@ class Library;
class NameMapper {
public:
virtual ~NameMapper() = default;
virtual std::string getNameForId(const std::string& id) const = 0;
virtual std::string getIdForName(const std::string& name) const = 0;
virtual std::string getNameForId(const std::string& id) = 0;
virtual std::string getIdForName(const std::string& name) = 0;
};
class IdNameMapper : public NameMapper {
public:
virtual std::string getNameForId(const std::string& id) const { return id; };
virtual std::string getIdForName(const std::string& name) const { return name; };
virtual std::string getNameForId(const std::string& id) { return id; };
virtual std::string getIdForName(const std::string& name) { return name; };
};
class HumanReadableNameMapper : public NameMapper {
@@ -52,29 +50,11 @@ class HumanReadableNameMapper : public NameMapper {
public:
HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
virtual ~HumanReadableNameMapper() = default;
virtual std::string getNameForId(const std::string& id) const;
virtual std::string getIdForName(const std::string& name) const;
virtual std::string getNameForId(const std::string& id);
virtual std::string getIdForName(const std::string& name);
};
class UpdatableNameMapper : public NameMapper {
typedef std::shared_ptr<NameMapper> NameMapperHandle;
public:
UpdatableNameMapper(Library& library, bool withAlias);
virtual std::string getNameForId(const std::string& id) const;
virtual std::string getIdForName(const std::string& name) const;
void update();
private:
NameMapperHandle currentNameMapper() const;
private:
mutable std::mutex mutex;
Library& library;
NameMapperHandle nameMapper;
const bool withAlias;
};
}

View File

@@ -1,33 +0,0 @@
/*
* Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_OPDS_CATALOG_H
#define KIWIX_OPDS_CATALOG_H
#include "library.h"
namespace kiwix
{
std::string getSearchUrl(const Filter& f);
} // namespace kiwix
#endif // KIWIX_OPDS_CATALOG_H

View File

@@ -26,6 +26,9 @@
#include <pugixml.hpp>
#include "tools/base64.h"
#include "tools/pathTools.h"
#include "tools/regexTools.h"
#include "library.h"
#include "reader.h"
@@ -59,32 +62,17 @@ class OPDSDumper
*
* @param bookIds the ids of the books to include in the feed
* @param query the query used to obtain the list of book ids
* @param partial whether the feed should include partial or complete entries
* @return The OPDS feed.
*/
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
/**
* Dump the OPDS complete entry document.
*
* @param bookId the id of the book
* @return The OPDS complete entry document.
*/
std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const;
/**
* Dump the categories OPDS feed.
*
* @param categories list of category names
* @return The OPDS feed.
*/
std::string categoriesOPDSFeed() const;
/**
* Dump the languages OPDS feed.
*
* @return The OPDS feed.
*/
std::string languagesOPDSFeed() const;
std::string categoriesOPDSFeed(const std::vector<std::string>& categories) const;
/**
* Set the id of the library.

View File

@@ -29,6 +29,8 @@
#include <string>
#include "common.h"
#include "entry.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
using namespace std;
@@ -39,25 +41,26 @@ namespace kiwix
* The SuggestionItem is a helper class that contains the info about a single
* suggestion item.
*/
class SuggestionItem
{
// Functions
public:
private:
// Create a sugggestion item.
explicit SuggestionItem(const std::string& title, const std::string& normalizedTitle,
const std::string& path, const std::string& snippet = "") :
explicit SuggestionItem(std::string title, std::string normalizedTitle,
std::string path, std::string snippet = "") :
title(title),
normalizedTitle(normalizedTitle),
path(path),
snippet(snippet) {}
public:
const std::string& getTitle() const { return title;}
const std::string& getNormalizedTitle() const { return normalizedTitle;}
const std::string& getPath() const { return path;}
const std::string& getSnippet() const { return snippet;}
const std::string getTitle() {return title;}
const std::string getNormalizedTitle() {return normalizedTitle;}
const std::string getPath() {return path;}
const std::string getSnippet() {return snippet;}
bool hasSnippet() const { return !snippet.empty();}
const bool hasSnippet() {return !snippet.empty();}
// Data
private:
@@ -65,13 +68,13 @@ class SuggestionItem
std::string normalizedTitle;
std::string path;
std::string snippet;
friend class Reader;
};
/**
* The Reader class is the class who allow to get an entry content from a zim
* file.
*
* Reader is now deprecated. Directly use `zim::Archive`.
*/
using SuggestionsList_t = std::vector<SuggestionItem>;
@@ -87,18 +90,10 @@ class Reader
* unsplitted path as if the file were not splitted
* (.zim extesion).
*/
explicit DEPRECATED Reader(const string zimFilePath);
/**
* Create a Reader to read a zim file given by the Archive.
*
* @param archive The shared pointer to the Archive object.
*/
explicit DEPRECATED Reader(const std::shared_ptr<zim::Archive> archive)
: Reader(archive, true) {};
explicit Reader(const string zimFilePath);
#ifndef _WIN32
explicit DEPRECATED Reader(int fd);
DEPRECATED Reader(int fd, zim::offset_type offset, zim::size_type size);
explicit Reader(int fd);
Reader(int fd, zim::offset_type offset, zim::size_type size);
#endif
~Reader() = default;
@@ -292,6 +287,16 @@ class Reader
*/
string getScraper() const;
/**
* Get the origId of the zim file.
*
* The origId is only used in the case of patch zim file and is the Id
* of the original zim file.
*
* @return The origId of the zim file as specified in the zim metadata.
*/
string getOrigId() const;
/**
* Get the favicon of the zim file.
*
@@ -483,7 +488,7 @@ class Reader
zim::Archive* getZimArchive() const;
protected:
std::shared_ptr<zim::Archive> zimArchive;
std::unique_ptr<zim::Archive> zimArchive;
std::string zimFilePath;
SuggestionsList_t suggestions;
@@ -491,15 +496,6 @@ class Reader
private:
std::map<const std::string, unsigned int> parseCounterMetadata() const;
// Reader is deprecated, so we've marked the constructor as deprecated.
// But we still need to construct the reader (in our deprecated code)
// To avoid warning because we use deprecated function, we create a
// constructor not deprecated. The `bool marker` is unused, it sole purpose
// is to change the signature to have a different constructor.
// This one is not deprecated and we must use it in our private code.
Reader(const std::shared_ptr<zim::Archive> archive, bool marker);
friend class Library;
};
}

View File

@@ -21,8 +21,6 @@
#define KIWIX_SEARCH_RENDERER_H
#include <string>
#include <zim/search.h>
#include "library.h"
namespace kiwix
{
@@ -36,42 +34,12 @@ class SearchRenderer
{
public:
/**
* Construct a SearchRenderer from a Searcher.
* The default constructor.
*
* This method is now deprecated. Construct the renderer from a
* `zim::SearchResultSet`
*
* @param searcher The `Searcher` to render.
* @param mapper The `NameMapper` to use to do the rendering.
* @param humanReadableName The global zim's humanReadableName.
* Used to generate pagination links.
*/
DEPRECATED SearchRenderer(Searcher* searcher, NameMapper* mapper);
/**
* Construct a SearchRenderer from a SearchResultSet.
*
* The constructed version of the SearchRenderer will not introduce
* the book name for each result. It is better to use the other constructor
* with a Library pointer to have a better html page.
*
* @param srs The `SearchResultSet` to render.
* @param mapper The `NameMapper` to use to do the rendering.
* @param start The start offset used for the srs.
* @param estimatedResultCount The estimatedResultCount of the whole search
*/
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
unsigned int start, unsigned int estimatedResultCount);
/**
* Construct a SearchRenderer from a SearchResultSet.
*
* @param srs The `SearchResultSet` to render.
* @param mapper The `NameMapper` to use to do the rendering.
* @param library The `Library` to use to look up book details for search results.
* @param start The start offset used for the srs.
* @param estimatedResultCount The estimatedResultCount of the whole search
*/
SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
unsigned int start, unsigned int estimatedResultCount);
SearchRenderer(Searcher* searcher, NameMapper* mapper);
~SearchRenderer();
@@ -106,9 +74,8 @@ class SearchRenderer
protected:
std::string beautifyInteger(const unsigned int number);
zim::SearchResultSet m_srs;
Searcher* mp_searcher;
NameMapper* mp_nameMapper;
Library* mp_library;
std::string searchContent;
std::string searchPattern;
std::string protocolPrefix;

View File

@@ -29,8 +29,8 @@
#include <string>
#include <memory>
#include <vector>
#include <zim/search.h>
#include "tools/pathTools.h"
#include "tools/stringTools.h"
using namespace std;
@@ -52,12 +52,9 @@ class Result
};
struct SearcherInternal;
struct SuggestionInternal;
/**
* The Searcher class is reponsible to do different kind of search using the
* fulltext index.
*
* The Searcher is now deprecated. Use libzim search feature.
*/
class Searcher
{
@@ -65,7 +62,7 @@ class Searcher
/**
* The default constructor.
*/
DEPRECATED Searcher();
Searcher();
~Searcher();
@@ -88,12 +85,12 @@ class Searcher
*
* @param search The search query.
* @param resultStart the start offset of the search results (used for pagination).
* @param maxResultCount Maximum results to get from start (used for pagination).
* @param resultEnd the end offset of the search results (used for pagination).
* @param verbose print some info on stdout if true.
*/
void search(const std::string& search,
unsigned int resultStart,
unsigned int maxResultCount,
unsigned int resultEnd,
const bool verbose = false);
/**
@@ -107,12 +104,12 @@ class Searcher
* @param longitude The longitude of the center point.
* @param distance The radius of the disc.
* @param resultStart the start offset of the search results (used for pagination).
* @param maxResultCount Maximum number of results to get from start (used for pagination).
* @param resultEnd the end offset of the search results (used for pagination).
* @param verbose print some info on stdout if true.
*/
void geo_search(float latitude, float longitude, float distance,
unsigned int resultStart,
unsigned int maxResultCount,
unsigned int resultEnd,
const bool verbose = false);
/**
@@ -145,29 +142,23 @@ class Searcher
*/
unsigned int getEstimatedResultCount();
/**
* Get a SearchResultSet object for current search
*/
zim::SearchResultSet getSearchResultSet();
unsigned int getResultStart() { return resultStart; }
unsigned int getMaxResultCount() { return maxResultCount; }
unsigned int getResultEnd() { return resultEnd; }
protected:
std::string beautifyInteger(const unsigned int number);
void closeIndex();
void searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int maxResultCount,
const unsigned int resultEnd,
const bool verbose = false);
std::vector<Reader*> readers;
std::unique_ptr<SearcherInternal> internal;
std::unique_ptr<SuggestionInternal> suggestionInternal;
std::string searchPattern;
unsigned int estimatedResultCount;
unsigned int resultStart;
unsigned int maxResultCount;
unsigned int resultEnd;
private:
void reset();

View File

@@ -54,29 +54,23 @@ namespace kiwix
void setAddress(const std::string& addr) { m_addr = addr; }
void setPort(int port) { m_port = port; }
void setNbThreads(int threads) { m_nbThreads = threads; }
void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
void setVerbose(bool verbose) { m_verbose = verbose; }
void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
void setTaskbar(bool withTaskbar, bool withLibraryButton)
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
void setBlockExternalLinks(bool blockExternalLinks)
{ m_blockExternalLinks = blockExternalLinks; }
int getPort();
std::string getAddress();
protected:
Library* mp_library;
NameMapper* mp_nameMapper;
std::string m_root = "";
std::string m_addr = "";
std::string m_indexTemplateString = "";
int m_port = 80;
int m_nbThreads = 1;
bool m_verbose = false;
bool m_withTaskbar = true;
bool m_withLibraryButton = true;
bool m_blockExternalLinks = false;
int m_ipConnectionLimit = 0;
std::unique_ptr<InternalServer> mp_server;
};
}

View File

@@ -1,220 +0,0 @@
/*
* Copyright 2021 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_TOOLS_H
#define KIWIX_TOOLS_H
#include <string>
#include <vector>
#include <map>
namespace kiwix {
/**
* Return the current directory.
*
* @return the current directory (utf8 encoded)
*/
std::string getCurrentDirectory();
/**
* Return the data directory.
*
* The data directory is a directory where to put data (zim files, ...)
* It depends of the platform and it may be changed by user using environment variable.
*
* The resolution order is :
* - `KIWIX_DATA_DIR` env variable (if set).
* - On Windows :
* . `$APPDATA/kiwix` if $APPDATA is set
* . `$USERPROFILE/kiwix` if $USERPROFILE is set
* - Else :
* . `$XDG_DATA_HOME/kiwix`if $XDG_DATA_HOME is set
* . `$HOME/.local/share/kiwx` if $HOWE is set
* - current directory
*
* @return the path of the data directory (utf8 encoded)
*/
std::string getDataDirectory();
/** Return the path of the executable
*
* Some application may be packaged in auto extractible archive (Appimage) and the
* real executable is different of the path of the archive.
* If `realPathOnly` is true, return the path of the real executable instead of the
* archive launched by the user.
*
* @param realPathOnly If we must return the real path of the executable.
* @return the path of the executable (utf8 encoded)
*/
std::string getExecutablePath(bool realPathOnly = false);
/** Tell if the path is a relative path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path A utf8 encoded path.
* @return true if the path is relative.
*/
bool isRelativePath(const std::string& path);
/** Append a path to another one.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param basePath the base path.
* @param relativePath a path to add to the base path, must be a relative path.
* @return The concatenation of the paths, using the right separator.
*/
std::string appendToDirectory(const std::string& basePath, const std::string& relativePath);
/** Remove the last element of a path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path a path.
* @return The parent directory (or empty string if none).
*/
std::string removeLastPathElement(const std::string& path);
/** Get the last element of a path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path a path.
* @return The base name of the path or empty string if none (ending with a separator).
*/
std::string getLastPathElement(const std::string& path);
/** Compute the absolute path of a relative path based on another one
*
* Equivalent to appendToDirectory followed by a normalization of the path.
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path the base path (if empty, current directory is taken).
* @param relativePath the relative path.
* @return a absolute path.
*/
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
/** Compute the relative path of a path relative to another one
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate paths.
*
* @param path the base path.
* @param absolutePath the absolute path to find the relative path for.
* @return a relative path (pointing to absolutePath, relative to path).
*/
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
/** Sleep the current thread.
*
* This function is provided as a small helper. It is probably better to use native tools.
*
* @param milliseconds The number of milliseconds to wait for.
*/
void sleep(unsigned int milliseconds);
/** Split a string
*
* This function is provided as a small helper. It is probably better to use native tools.
*
* Assuming text = "foo:;bar;baz,oups;"
*
* split(text, ":;", true, true) => ["foo", ":", ";", "bar", ";", "baz,oups", ";"]
* split(text, ":;", true, false) => ["foo", "bar", "baz,oups"] (default)
* split(text, ":;", false, true) => ["foo", ":", "", ";", "bar", ";", "baz,oups", ";", ""]
* split(text, ":;", false, false) => ["foo", "", "bar", "baz,oups", ""]
*
* @param str The string to split.
* @param delims A string of potential delimiters.
* Each charater in the string can be a individual delimiters.
* @param dropEmpty true if empty part must be dropped from the result.
* @param keepDelim true if delimiter must be included from the result.
* @return a list of part (potentially containing delimiters)
*/
std::vector<std::string> split(const std::string& str, const std::string& delims, bool dropEmpty=true, bool keepDelim = false);
/** Convert language code from iso2 code to iso3
*
* This function is provided as a small helper. It is probably better to use native tools
* to manipulate locales.
*
* @param a2code a iso2 code string.
* @return the corresponding iso3 code.
* @throw std::out_of_range if iso2 code is not known.
*/
std::string converta2toa3(const std::string& a2code);
/** Extracts content from given file.
*
* This function provides content of a file provided it's path.
*
* @param path The absolute path provided in string format.
* @return Content of corresponding file in string format.
*/
std::string getFileContent(const std::string& path);
/** Checks if file exists.
*
* This function returns boolean stating if file exists.
*
* @param path The absolute path provided in string format.
* @return Boolean representing if file exists or not.
*/
bool fileExists(const std::string& path);
/** Checks if file is readable.
*
* This function returns boolean stating if file is readable.
*
* @param path The absolute path provided in string format.
* @return Boolean representing if file is readale or not.
*/
bool fileReadable(const std::string& path);
/** Provides mimetype from filename.
*
* This function provides mimetype from file-name.
*
* @param filename string containing filename.
* @return mimetype from filename in string format.
*/
std::string getMimeTypeForFile(const std::string& filename);
/** Provides all available network interfaces
*
* This function provides the available IPv4 network interfaces
*/
std::map<std::string, std::string> getNetworkInterfaces();
/** Provides the best IP address
* This function provides the best IP address from the list given by getNetworkInterfaces
*/
std::string getBestPublicIp();
}
#endif // KIWIX_TOOLS_H

View File

@@ -32,7 +32,9 @@ namespace pugi {
namespace kiwix
{
void sleep(unsigned int milliseconds);
std::string nodeToString(const pugi::xml_node& node);
std::string converta2toa3(const std::string& a2code);
/*
* Convert all format tag string to new format
@@ -48,10 +50,6 @@ namespace kiwix
std::string gen_date_str();
std::string gen_uuid(const std::string& s);
// if s is empty then returns kainjow::mustache::data(false)
// otherwise kainjow::mustache::data(value)
kainjow::mustache::data onlyAsNonEmptyMustacheValue(const std::string& s);
std::string render_template(const std::string& template_str, kainjow::mustache::data data);
}

View File

@@ -26,13 +26,23 @@
std::string WideToUtf8(const std::wstring& wstr);
std::wstring Utf8ToWide(const std::string& str);
#endif
bool isRelativePath(const std::string& path);
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
std::string removeLastPathElement(const std::string& path);
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
unsigned int getFileSize(const std::string& path);
std::string getFileSizeAsString(const std::string& path);
std::string getFileContent(const std::string& path);
bool fileExists(const std::string& path);
bool makeDirectory(const std::string& path);
std::string makeTmpDirectory();
bool copyFile(const std::string& sourcePath, const std::string& destPath);
std::string getLastPathElement(const std::string& path);
std::string getExecutablePath(bool realPathOnly = false);
std::string getCurrentDirectory();
std::string getDataDirectory();
bool writeTextFile(const std::string& path, const std::string& content);
std::string getMimeTypeForFile(const std::string& filename);
#endif

View File

@@ -43,6 +43,7 @@ void loadICUExternalTables();
std::string urlEncode(const std::string& value, bool encodeReserved = false);
std::string urlDecode(const std::string& value, bool component = false);
std::vector<std::string> split(const std::string& str, const std::string& delims, bool trimEmpty = true, bool keepDelim = false);
std::string join(const std::vector<std::string>& list, const std::string& sep);
std::string ucAll(const std::string& word);
@@ -69,7 +70,5 @@ T extractFromString(const std::string& str) {
}
bool startsWith(const std::string& base, const std::string& start);
std::vector<std::string> getTitleVariants(const std::string& title);
} //namespace kiwix
#endif

View File

@@ -1,33 +0,0 @@
/*
* Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_VERSION_H
#define KIWIX_VERSION_H
#include <string>
#include <vector>
namespace kiwix
{
typedef std::vector<std::pair<std::string, std::string>> LibVersions;
LibVersions getVersions();
void printVersions(std::ostream& out = std::cout);
}
#endif // KIWIX_VERSION_H

View File

@@ -1,5 +1,5 @@
project('libkiwix', 'cpp',
version : '10.1.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
version : '10.0.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
license : 'GPLv3+',
default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
@@ -19,11 +19,11 @@ if wrapper.contains('java')
endif
# See https://github.com/kiwix/libkiwix/issues/371
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(host_machine.cpu_family())
if ['arm', 'mips', 'm68k', 'ppc', 'sh4'].contains(target_machine.cpu_family())
extra_libs += '-latomic'
endif
if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or host_machine.system() == 'freebsd'
if (compiler.get_id() == 'gcc' and build_machine.system() == 'linux') or target_machine.system() == 'freebsd'
# C++ std::thread is implemented using pthread on linux by gcc
thread_dep = dependency('threads')
else
@@ -44,19 +44,19 @@ else
error('Cannot found header mustache.hpp')
endif
libzim_dep = dependency('libzim', version : '>=7.2.0', static:static_deps)
libzim_dep = dependency('libzim', version : '>=7.0.0', static:static_deps)
if not compiler.has_header_symbol('zim/zim.h', 'LIBZIM_WITH_XAPIAN')
error('Libzim seems to be compiled without xapian. Xapian support is mandatory.')
endif
extra_cflags = ''
if host_machine.system() == 'windows' and static_deps
if target_machine.system() == 'windows' and static_deps
add_project_arguments('-DCURL_STATICLIB', language : 'cpp')
extra_cflags += '-DCURL_STATICLIB'
endif
if host_machine.system() == 'windows'
if target_machine.system() == 'windows'
add_project_arguments('-DNOMINMAX', language: 'cpp')
endif
@@ -65,7 +65,7 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microh
inc = include_directories('include', extra_include)
conf = configuration_data()
conf.set('LIBKIWIX_VERSION', '"@0@"'.format(meson.project_version()))
conf.set('VERSION', '"@0@"'.format(meson.project_version()))
if build_machine.system() == 'windows'
extra_link_args = ['-lshlwapi', '-lwinmm']
@@ -78,9 +78,6 @@ subdir('scripts')
subdir('static')
subdir('src')
subdir('test')
if get_option('doc')
subdir('docs')
endif
pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl', 'libmicrohttpd', 'xapian-core']

View File

@@ -1,4 +1,2 @@
option('wrapper', type:'array', choices:['java', 'android'], value:[],
description: 'The wrapper to generate.')
option('doc', type : 'boolean', value : false,
description : 'Build the documentations.')

View File

@@ -3,15 +3,13 @@
#include "aria2.h"
#include "xmlrpc.h"
#include <iostream>
#include <algorithm>
#include <sstream>
#include <thread>
#include <chrono>
#include "tools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "downloader.h" // For AriaError
#include <tools/otherTools.h>
#include <tools/pathTools.h>
#include <tools/stringTools.h>
#include <downloader.h> // For AriaError
#ifdef _WIN32
# define ARIA2_CMD "aria2c.exe"
@@ -32,7 +30,7 @@ namespace kiwix {
Aria2::Aria2():
mp_aria(nullptr),
m_port(42042),
m_secret(getNewRpcSecret()),
m_secret("kiwixariarpc"),
m_curlErrorBuffer(new char[CURL_ERROR_SIZE]),
mp_curl(nullptr)
{
@@ -70,7 +68,7 @@ Aria2::Aria2():
callCmd.push_back(rpc_secret.c_str());
callCmd.push_back(rpc_port.c_str());
callCmd.push_back(download_dir.c_str());
if (fileReadable(session_file)) {
if (fileExists(session_file)) {
callCmd.push_back(inputFile.c_str());
}
callCmd.push_back(session.c_str());
@@ -197,13 +195,6 @@ std::string Aria2::tellStatus(const std::string& gid, const std::vector<std::str
return doRequest(methodCall);
}
std::string Aria2::getNewRpcSecret()
{
std::string uuid = gen_uuid("");
uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'));
return uuid.substr(0, 9);
}
std::vector<std::string> Aria2::tellActive()
{
MethodCall methodCall("aria2.tellActive", m_secret);

View File

@@ -37,7 +37,6 @@ class Aria2
std::string addUri(const std::vector<std::string>& uri, const std::vector<std::pair<std::string, std::string>>& options = {});
std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
static std::string getNewRpcSecret();
std::vector<std::string> tellActive();
std::vector<std::string> tellWaiting();
void saveSession();

View File

@@ -20,16 +20,10 @@
#include "book.h"
#include "reader.h"
#include "tools.h"
#include "tools/base64.h"
#include "tools/regexTools.h"
#include "tools/networkTools.h"
#include "tools/otherTools.h"
#include "tools/stringTools.h"
#include "tools/pathTools.h"
#include "tools/archiveTools.h"
#include <zim/archive.h>
#include <pugixml.hpp>
@@ -41,17 +35,11 @@ Book::Book() :
m_readOnly(false)
{
}
/* Destructor */
Book::~Book()
{
}
Book::Illustrations Book::getIllustrations() const
{
return m_illustrations;
}
bool Book::update(const kiwix::Book& other)
{
if (m_readOnly)
@@ -60,43 +48,55 @@ bool Book::update(const kiwix::Book& other)
if (m_id != other.m_id)
return false;
*this = other;
m_readOnly = other.m_readOnly;
m_path = other.m_path;
m_pathValid = other.m_pathValid;
m_title = other.m_title;
m_description = other.m_description;
m_language = other.m_language;
m_creator = other.m_creator;
m_publisher = other.m_publisher;
m_date = other.m_date;
m_url = other.m_url;
m_name = other.m_name;
m_flavour = other.m_flavour;
m_tags = other.m_tags;
m_category = other.m_category;
m_origId = other.m_origId;
m_articleCount = other.m_articleCount;
m_mediaCount = other.m_mediaCount;
m_size = other.m_size;
m_favicon = other.m_favicon;
m_faviconMimeType = other.m_faviconMimeType;
m_faviconUrl = other.m_faviconUrl;
m_downloadId = other.m_downloadId;
return true;
}
void Book::update(const kiwix::Reader& reader)
{
update(*reader.getZimArchive());
}
void Book::update(const zim::Archive& archive) {
m_path = archive.getFilename();
m_path = reader.getZimFilePath();
m_pathValid = true;
m_id = getArchiveId(archive);
m_title = getArchiveTitle(archive);
m_description = getMetaDescription(archive);
m_language = getMetaLanguage(archive);
m_creator = getMetaCreator(archive);
m_publisher = getMetaPublisher(archive);
m_date = getMetaDate(archive);
m_name = getMetaName(archive);
m_flavour = getMetaFlavour(archive);
m_tags = getMetaTags(archive);
m_id = reader.getId();
m_title = reader.getTitle();
m_description = reader.getDescription();
m_language = reader.getLanguage();
m_creator = reader.getCreator();
m_publisher = reader.getPublisher();
m_date = reader.getDate();
m_name = reader.getName();
m_flavour = reader.getFlavour();
m_tags = reader.getTags();
m_category = getCategoryFromTags();
m_articleCount = getArchiveArticleCount(archive);
m_mediaCount = getArchiveMediaCount(archive);
m_size = static_cast<uint64_t>(getArchiveFileSize(archive)) << 10;
m_origId = reader.getOrigId();
m_articleCount = reader.getArticleCount();
m_mediaCount = reader.getMediaCount();
m_size = static_cast<uint64_t>(reader.getFileSize()) << 10;
m_pathValid = true;
m_illustrations.clear();
for ( const auto illustrationSize : archive.getIllustrationSizes() ) {
const auto illustration = std::make_shared<Illustration>();
const zim::Item illustrationItem = archive.getIllustrationItem(illustrationSize);
illustration->width = illustration->height = illustrationSize;
illustration->mimeType = illustrationItem.getMimetype();
illustration->data = illustrationItem.getData();
// NOTE: illustration->url is left uninitialized
m_illustrations.push_back(illustration);
}
reader.getFavicon(m_favicon, m_faviconMimeType);
}
#define ATTR(name) node.attribute(name).value()
@@ -108,7 +108,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
path = computeAbsolutePath(baseDir, path);
}
m_path = path;
m_pathValid = fileReadable(path);
m_pathValid = fileExists(path);
m_title = ATTR("title");
m_description = ATTR("description");
m_language = ATTR("language");
@@ -123,14 +123,9 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir)
m_articleCount = strtoull(ATTR("articleCount"), 0, 0);
m_mediaCount = strtoull(ATTR("mediaCount"), 0, 0);
m_size = strtoull(ATTR("size"), 0, 0) << 10;
std::string favicon_mimetype = ATTR("faviconMimeType");
if (! favicon_mimetype.empty()) {
const auto favicon = std::make_shared<Illustration>();
favicon->data = base64_decode(ATTR("favicon"));
favicon->mimeType = favicon_mimetype;
favicon->url = ATTR("faviconUrl");
m_illustrations.assign(1, favicon);
}
m_favicon = base64_decode(ATTR("favicon"));
m_faviconMimeType = ATTR("faviconMimeType");
m_faviconUrl = ATTR("faviconUrl");
try {
m_downloadId = ATTR("downloadId");
} catch(...) {}
@@ -161,9 +156,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
m_language = VALUE("language");
m_creator = node.child("author").child("name").child_value();
m_publisher = node.child("publisher").child("name").child_value();
const std::string dcIssuedDate = VALUE("dc:issued");
m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
m_date = fromOpdsDate(m_date);
m_date = fromOpdsDate(VALUE("updated"));
m_name = VALUE("name");
m_flavour = VALUE("flavour");
m_tags = VALUE("tags");
@@ -180,11 +173,8 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
m_size = strtoull(linkNode.attribute("length").value(), 0, 0);
}
if (rel == "http://opds-spec.org/image/thumbnail") {
const auto favicon = std::make_shared<Illustration>();
favicon->data.clear();
favicon->url = urlHost + linkNode.attribute("href").value();
favicon->mimeType = linkNode.attribute("type").value();
m_illustrations.assign(1, favicon);
m_faviconUrl = urlHost + linkNode.attribute("href").value();
m_faviconMimeType = linkNode.attribute("type").value();
}
}
@@ -195,7 +185,7 @@ std::string Book::getHumanReadableIdFromPath() const
{
std::string id = m_path;
if (!id.empty()) {
id = kiwix::removeAccents(id);
kiwix::removeAccents(id);
#ifdef _WIN32
id = replaceRegex(id, "", "^.*\\\\");
@@ -217,54 +207,15 @@ void Book::setPath(const std::string& path)
: path;
}
const Book::Illustration Book::missingDefaultIllustration;
std::shared_ptr<const Book::Illustration> Book::getIllustration(unsigned int size) const
{
for ( const auto& ilPtr : m_illustrations ) {
if (ilPtr->width == size && ilPtr->height == size) {
return ilPtr;
}
}
throw std::runtime_error("Cannot find illustration");
}
const Book::Illustration& Book::getDefaultIllustration() const
{
try {
return *getIllustration(48);
} catch (...) {
return missingDefaultIllustration;
}
}
const std::string& Book::Illustration::getData() const
{
if (data.empty() && !url.empty()) {
const std::lock_guard<std::mutex> l(mutex);
if ( data.empty() ) {
try {
data = download(url);
} catch(...) {
std::cerr << "Cannot download favicon from " << url;
}
}
}
return data;
}
const std::string& Book::getFavicon() const {
return getDefaultIllustration().getData();
}
const std::string& Book::getFaviconUrl() const
{
return getDefaultIllustration().url;
}
const std::string& Book::getFaviconMimeType() const
{
return getDefaultIllustration().mimeType;
if (m_favicon.empty() && !m_faviconUrl.empty()) {
try {
m_favicon = download(m_faviconUrl);
} catch(...) {
std::cerr << "Cannot download favicon from " << m_faviconUrl;
}
}
return m_favicon;
}
std::string Book::getTagStr(const std::string& tagName) const {

View File

@@ -1,3 +1,3 @@
#mesondefine LIBKIWIX_VERSION
#mesondefine VERSION

View File

@@ -23,7 +23,7 @@
namespace kiwix
{
Entry::Entry(zim::Entry entry, bool _marker)
Entry::Entry(zim::Entry entry)
: entry(entry)
{
}
@@ -53,7 +53,7 @@ Entry Entry::getRedirectEntry() const
throw NoEntry();
}
return Entry(entry.getRedirectEntry(), true);
return entry.getRedirectEntry();
}
Entry Entry::getFinalEntry() const
@@ -67,7 +67,7 @@ Entry Entry::getFinalEntry() const
if (final_entry.isRedirect()) {
throw NoEntry();
}
return Entry(final_entry, true);
return final_entry;
}
}

View File

@@ -9,7 +9,6 @@
# include <unistd.h>
#endif
#include "tools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"

View File

@@ -22,7 +22,6 @@
#include "reader.h"
#include "libxml_dumper.h"
#include "tools.h"
#include "tools/base64.h"
#include "tools/regexTools.h"
#include "tools/pathTools.h"
@@ -49,48 +48,23 @@ std::string normalizeText(const std::string& text)
return removeAccents(text);
}
bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
{
return book1.isPathValid()
&& book2.isPathValid()
&& book1.getPath() == book2.getPath();
}
} // unnamed namespace
class LibraryBase::BookDB : public Xapian::WritableDatabase
class Library::BookDB : public Xapian::WritableDatabase
{
public:
BookDB() : Xapian::WritableDatabase("", Xapian::DB_BACKEND_INMEMORY) {}
};
LibraryBase::LibraryBase()
/* Constructor */
Library::Library()
: m_bookDB(new BookDB)
{
}
LibraryBase::~LibraryBase()
{
}
Library::Library(Library&& ) = default;
LibraryBase::LibraryBase(LibraryBase&& ) = default;
LibraryBase& LibraryBase::operator=(LibraryBase&& ) = default;
/* Constructor */
Library::Library()
{
}
Library::Library(Library&& other)
: LibraryBase(std::move(other))
{
}
Library& Library::operator=(Library&& other)
{
LibraryBase::operator=(std::move(other));
return *this;
}
Library& Library::operator=(Library&& ) = default;
/* Destructor */
Library::~Library()
@@ -100,37 +74,25 @@ Library::~Library()
bool Library::addBook(const Book& book)
{
std::lock_guard<std::mutex> lock(m_mutex);
++m_revision;
/* Try to find it */
updateBookDB(book);
try {
auto& oldbook = m_books.at(book.getId());
if ( ! booksReferToTheSameArchive(oldbook, book) ) {
dropReader(book.getId());
}
oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
// XXX: Then m_bookDB will become out-of-sync with
// XXX: the real contents of the library.
oldbook.lastUpdatedRevision = m_revision;
oldbook.update(book);
return false;
} catch (std::out_of_range&) {
Entry& newEntry = m_books[book.getId()];
static_cast<Book&>(newEntry) = book;
newEntry.lastUpdatedRevision = m_revision;
m_books[book.getId()] = book;
return true;
}
}
void Library::addBookmark(const Bookmark& bookmark)
{
std::lock_guard<std::mutex> lock(m_mutex);
m_bookmarks.push_back(bookmark);
}
bool Library::removeBookmark(const std::string& zimId, const std::string& url)
{
std::lock_guard<std::mutex> lock(m_mutex);
for(auto it=m_bookmarks.begin(); it!=m_bookmarks.end(); it++) {
if (it->getBookId() == zimId && it->getUrl() == url) {
m_bookmarks.erase(it);
@@ -141,63 +103,27 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
}
void Library::dropReader(const std::string& id)
{
m_readers.erase(id);
m_archives.erase(id);
}
bool Library::removeBookById(const std::string& id)
{
std::lock_guard<std::mutex> lock(m_mutex);
m_bookDB->delete_document("Q" + id);
dropReader(id);
m_readers.erase(id);
return m_books.erase(id) == 1;
}
Library::Revision Library::getRevision() const
{
std::lock_guard<std::mutex> lock(m_mutex);
return m_revision;
}
uint32_t Library::removeBooksNotUpdatedSince(LibraryRevision libraryRevision)
{
BookIdCollection booksToRemove;
{
std::lock_guard<std::mutex> lock(m_mutex);
for ( const auto& entry : m_books) {
if ( entry.second.lastUpdatedRevision <= libraryRevision ) {
booksToRemove.push_back(entry.first);
}
}
}
uint32_t countOfRemovedBooks = 0;
for ( const auto& id : booksToRemove ) {
if ( removeBookById(id) )
++countOfRemovedBooks;
}
return countOfRemovedBooks;
}
const Book& Library::getBookById(const std::string& id) const
{
// XXX: Doesn't make sense to lock this operation since it cannot
// XXX: guarantee thread-safety because of its return type
return m_books.at(id);
}
Book Library::getBookByIdThreadSafe(const std::string& id) const
Book& Library::getBookById(const std::string& id)
{
std::lock_guard<std::mutex> lock(m_mutex);
return getBookById(id);
const Library& const_self = *this;
return const_cast<Book&>(const_self.getBookById(id));
}
const Book& Library::getBookByPath(const std::string& path) const
{
// XXX: Doesn't make sense to lock this operation since it cannot
// XXX: guarantee thread-safety because of its return type
for(auto& it: m_books) {
auto& book = it.second;
if (book.getPath() == path)
@@ -208,43 +134,29 @@ const Book& Library::getBookByPath(const std::string& path) const
throw std::out_of_range(ss.str());
}
Book& Library::getBookByPath(const std::string& path)
{
const Library& const_self = *this;
return const_cast<Book&>(const_self.getBookByPath(path));
}
std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
{
try {
std::lock_guard<std::mutex> lock(m_mutex);
return m_readers.at(id);
} catch (std::out_of_range& e) {}
const auto archive = getArchiveById(id);
if ( !archive )
return nullptr;
const shared_ptr<Reader> reader(new Reader(archive, true));
std::lock_guard<std::mutex> lock(m_mutex);
m_readers[id] = reader;
return reader;
}
std::shared_ptr<zim::Archive> Library::getArchiveById(const std::string& id)
{
std::lock_guard<std::mutex> lock(m_mutex);
try {
return m_archives.at(id);
} catch (std::out_of_range& e) {}
auto book = getBookById(id);
if (!book.isPathValid())
return nullptr;
auto sptr = make_shared<zim::Archive>(book.getPath());
m_archives[id] = sptr;
auto sptr = make_shared<Reader>(book.getPath());
m_readers[id] = sptr;
return sptr;
}
unsigned int Library::getBookCount(const bool localBooks,
const bool remoteBooks) const
{
std::lock_guard<std::mutex> lock(m_mutex);
unsigned int result = 0;
for (auto& pair: m_books) {
auto& book = pair.second;
@@ -258,63 +170,39 @@ unsigned int Library::getBookCount(const bool localBooks,
bool Library::writeToFile(const std::string& path) const
{
const auto allBookIds = getBooksIds();
auto baseDir = removeLastPathElement(path);
LibXMLDumper dumper(this);
dumper.setBaseDir(baseDir);
std::string xml;
{
std::lock_guard<std::mutex> lock(m_mutex);
xml = dumper.dumpLibXMLContent(allBookIds);
};
return writeTextFile(path, xml);
return writeTextFile(path, dumper.dumpLibXMLContent(getBooksIds()));
}
bool Library::writeBookmarksToFile(const std::string& path) const
{
LibXMLDumper dumper(this);
// NOTE: LibXMLDumper::dumpLibXMLBookmark uses Library in a thread-safe way
const std::string xml = dumper.dumpLibXMLBookmark();
return writeTextFile(path, xml);
}
Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
{
std::lock_guard<std::mutex> lock(m_mutex);
AttributeCounts propValueCounts;
for (const auto& pair: m_books) {
const auto& book = pair.second;
if (book.getOrigId().empty()) {
propValueCounts[(book.*p)()] += 1;
}
}
return propValueCounts;
}
std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
{
std::vector<std::string> result;
for ( const auto& kv : getBookAttributeCounts(p) ) {
result.push_back(kv.first);
}
return result;
return writeTextFile(path, dumper.dumpLibXMLBookmark());
}
std::vector<std::string> Library::getBooksLanguages() const
{
return getBookPropValueSet(&Book::getLanguage);
}
std::vector<std::string> booksLanguages;
std::map<std::string, bool> booksLanguagesMap;
Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
{
return getBookAttributeCounts(&Book::getLanguage);
for (auto& pair: m_books) {
auto& book = pair.second;
auto& language = book.getLanguage();
if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
if (book.getOrigId().empty()) {
booksLanguagesMap[language] = true;
booksLanguages.push_back(language);
}
}
}
return booksLanguages;
}
std::vector<std::string> Library::getBooksCategories() const
{
std::lock_guard<std::mutex> lock(m_mutex);
std::set<std::string> categories;
for (const auto& pair: m_books) {
@@ -330,12 +218,40 @@ std::vector<std::string> Library::getBooksCategories() const
std::vector<std::string> Library::getBooksCreators() const
{
return getBookPropValueSet(&Book::getCreator);
std::vector<std::string> booksCreators;
std::map<std::string, bool> booksCreatorsMap;
for (auto& pair: m_books) {
auto& book = pair.second;
auto& creator = book.getCreator();
if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
if (book.getOrigId().empty()) {
booksCreatorsMap[creator] = true;
booksCreators.push_back(creator);
}
}
}
return booksCreators;
}
std::vector<std::string> Library::getBooksPublishers() const
{
return getBookPropValueSet(&Book::getPublisher);
std::vector<std::string> booksPublishers;
std::map<std::string, bool> booksPublishersMap;
for (auto& pair:m_books) {
auto& book = pair.second;
auto& publisher = book.getPublisher();
if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
if (book.getOrigId().empty()) {
booksPublishersMap[publisher] = true;
booksPublishers.push_back(publisher);
}
}
}
return booksPublishers;
}
const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const
@@ -345,7 +261,6 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
}
std::vector<kiwix::Bookmark> validBookmarks;
auto booksId = getBooksIds();
std::lock_guard<std::mutex> lock(m_mutex);
for(auto& bookmark:m_bookmarks) {
if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
validBookmarks.push_back(bookmark);
@@ -356,7 +271,6 @@ const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks
Library::BookIdCollection Library::getBooksIds() const
{
std::lock_guard<std::mutex> lock(m_mutex);
BookIdCollection bookIds;
for (auto& pair: m_books) {
@@ -366,6 +280,15 @@ Library::BookIdCollection Library::getBooksIds() const
return bookIds;
}
Library::BookIdCollection Library::filter(const std::string& search) const
{
if (search.empty()) {
return getBooksIds();
}
return filter(Filter().query(search));
}
void Library::updateBookDB(const Book& book)
{
@@ -537,7 +460,6 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
BookIdCollection bookIds;
std::lock_guard<std::mutex> lock(m_mutex);
Xapian::Enquire enquire(*m_bookDB);
enquire.set_query(query);
const auto results = enquire.get_mset(0, m_books.size());
@@ -551,9 +473,7 @@ Library::BookIdCollection Library::filterViaBookDB(const Filter& filter) const
Library::BookIdCollection Library::filter(const Filter& filter) const
{
BookIdCollection result;
const auto preliminaryResult = filterViaBookDB(filter);
std::lock_guard<std::mutex> lock(m_mutex);
for(auto id : preliminaryResult) {
for(auto id : filterViaBookDB(filter)) {
if(filter.accept(m_books.at(id))) {
result.push_back(id);
}
@@ -622,11 +542,6 @@ std::string Comparator<PUBLISHER>::get_key(const std::string& id)
void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool ascending) const
{
// NOTE: Can reimplement this method in a way that doesn't require locking
// NOTE: for the entire duration of the sort. Will need to obtain (under a
// NOTE: lock) the required atributes from the books once, and then the
// NOTE: sorting will run on a copy of data without locking.
std::lock_guard<std::mutex> lock(m_mutex);
switch(sort) {
case TITLE:
std::sort(bookIds.begin(), bookIds.end(), Comparator<TITLE>(this, ascending));
@@ -649,6 +564,48 @@ void Library::sort(BookIdCollection& bookIds, supportedListSortBy sort, bool asc
}
Library::BookIdCollection Library::listBooksIds(
int mode,
supportedListSortBy sortBy,
const std::string& search,
const std::string& language,
const std::string& creator,
const std::string& publisher,
const std::vector<std::string>& tags,
size_t maxSize) const {
Filter _filter;
if (mode & LOCAL)
_filter.local(true);
if (mode & NOLOCAL)
_filter.local(false);
if (mode & VALID)
_filter.valid(true);
if (mode & NOVALID)
_filter.valid(false);
if (mode & REMOTE)
_filter.remote(true);
if (mode & NOREMOTE)
_filter.remote(false);
if (!tags.empty())
_filter.acceptTags(tags);
if (maxSize != 0)
_filter.maxSize(maxSize);
if (!language.empty())
_filter.lang(language);
if (!publisher.empty())
_filter.publisher(publisher);
if (!creator.empty())
_filter.creator(creator);
if (!search.empty())
_filter.query(search);
auto bookIds = filter(_filter);
sort(bookIds, sortBy, true);
return bookIds;
}
Filter::Filter()
: activeFilters(0),
_maxSize(0)

View File

@@ -20,10 +20,10 @@
#include "libxml_dumper.h"
#include "book.h"
#include "tools.h"
#include "tools/base64.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "tools/pathTools.h"
namespace kiwix
{
@@ -60,13 +60,10 @@ void LibXMLDumper::handleBook(Book book, pugi::xml_node root_node) {
ADD_ATTR_NOT_EMPTY(entry_node, "name", book.getName());
ADD_ATTR_NOT_EMPTY(entry_node, "flavour", book.getFlavour());
ADD_ATTR_NOT_EMPTY(entry_node, "tags", book.getTags());
try {
auto defaultIllustration = book.getIllustration(48);
ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", defaultIllustration->mimeType);
ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", defaultIllustration->url);
if (!defaultIllustration->getData().empty())
ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(defaultIllustration->getData()));
} catch(...) {}
ADD_ATTR_NOT_EMPTY(entry_node, "faviconMimeType", book.getFaviconMimeType());
ADD_ATTR_NOT_EMPTY(entry_node, "faviconUrl", book.getFaviconUrl());
if (!book.getFavicon().empty())
ADD_ATTRIBUTE(entry_node, "favicon", base64_encode(book.getFavicon()));
} else {
ADD_ATTRIBUTE(entry_node, "origId", book.getOrigId());
}
@@ -94,7 +91,7 @@ void LibXMLDumper::handleBookmark(Bookmark bookmark, pugi::xml_node root_node) {
auto book_node = entry_node.append_child("book");
try {
auto book = library->getBookByIdThreadSafe(bookmark.getBookId());
auto book = library->getBookById(bookmark.getBookId());
ADD_TEXT_ENTRY(book_node, "id", book.getId());
ADD_TEXT_ENTRY(book_node, "title", book.getTitle());
ADD_TEXT_ENTRY(book_node, "language", book.getLanguage());

View File

@@ -19,88 +19,34 @@
#include "manager.h"
#include "tools.h"
#include "tools/pathTools.h"
#include <pugixml.hpp>
namespace kiwix
{
namespace
{
struct NoDelete
{
template<class T> void operator()(T*) {}
};
} // unnamed namespace
////////////////////////////////////////////////////////////////////////////////
// LibraryManipulator
////////////////////////////////////////////////////////////////////////////////
LibraryManipulator::LibraryManipulator(Library* library)
: library(*library)
{}
LibraryManipulator::~LibraryManipulator()
{}
bool LibraryManipulator::addBookToLibrary(const Book& book)
{
const auto ret = library.addBook(book);
if ( ret ) {
bookWasAddedToLibrary(book);
}
return ret;
}
void LibraryManipulator::addBookmarkToLibrary(const Bookmark& bookmark)
{
library.addBookmark(bookmark);
bookmarkWasAddedToLibrary(bookmark);
}
uint32_t LibraryManipulator::removeBooksNotUpdatedSince(Library::Revision rev)
{
const auto n = library.removeBooksNotUpdatedSince(rev);
if ( n != 0 ) {
booksWereRemovedFromLibrary();
}
return n;
}
void LibraryManipulator::bookWasAddedToLibrary(const Book& book)
{
}
void LibraryManipulator::bookmarkWasAddedToLibrary(const Bookmark& bookmark)
{
}
void LibraryManipulator::booksWereRemovedFromLibrary()
{
}
////////////////////////////////////////////////////////////////////////////////
// Manager
////////////////////////////////////////////////////////////////////////////////
/* Constructor */
Manager::Manager(LibraryManipulator* manipulator):
writableLibraryPath(""),
manipulator(manipulator, NoDelete())
manipulator(manipulator),
mustDeleteManipulator(false)
{
}
Manager::Manager(Library* library) :
writableLibraryPath(""),
manipulator(new LibraryManipulator(library))
manipulator(new DefaultLibraryManipulator(library)),
mustDeleteManipulator(true)
{
}
/* Destructor */
Manager::~Manager()
{
if (mustDeleteManipulator) {
delete manipulator;
}
}
bool Manager::parseXmlDom(const pugi::xml_document& doc,
bool readOnly,
const std::string& libraryPath,
@@ -268,8 +214,8 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
tmp_path = computeAbsolutePath(getCurrentDirectory(), path);
}
try {
zim::Archive archive(tmp_path);
book->update(archive);
kiwix::Reader reader(tmp_path);
book->update(reader);
book->setPathValid(true);
} catch (const std::exception& e) {
book->setPathValid(false);
@@ -302,21 +248,4 @@ bool Manager::readBookmarkFile(const std::string& path)
return true;
}
void Manager::reload(const Paths& paths)
{
const auto libRevision = manipulator->getLibrary().getRevision();
for (std::string path : paths) {
if (!path.empty()) {
if ( kiwix::isRelativePath(path) )
path = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), path);
if (!readFile(path, false, true)) {
throw std::runtime_error("Failed to load the XML library file '" + path + "'.");
}
}
}
manipulator->removeBooksNotUpdatedSince(libRevision);
}
}

View File

@@ -19,7 +19,6 @@ kiwix_sources = [
'tools/stringTools.cpp',
'tools/networkTools.cpp',
'tools/otherTools.cpp',
'tools/archiveTools.cpp',
'kiwixserve.cpp',
'name_mapper.cpp',
'server/byte_range.cpp',
@@ -27,9 +26,7 @@ kiwix_sources = [
'server/request_context.cpp',
'server/response.cpp',
'server/internalServer.cpp',
'server/internalServer_catalog_v2.cpp',
'opds_catalog.cpp',
'version.cpp'
'server/internalServer_catalog_v2.cpp'
]
kiwix_sources += lib_resources

View File

@@ -51,54 +51,12 @@ HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool w
}
}
std::string HumanReadableNameMapper::getNameForId(const std::string& id) const {
std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
return m_idToName.at(id);
}
std::string HumanReadableNameMapper::getIdForName(const std::string& name) const {
std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
return m_nameToId.at(name);
}
////////////////////////////////////////////////////////////////////////////////
// UpdatableNameMapper
////////////////////////////////////////////////////////////////////////////////
UpdatableNameMapper::UpdatableNameMapper(Library& lib, bool withAlias)
: library(lib)
, withAlias(withAlias)
{
update();
}
void UpdatableNameMapper::update()
{
const auto newNameMapper = new HumanReadableNameMapper(library, withAlias);
std::lock_guard<std::mutex> lock(mutex);
nameMapper.reset(newNameMapper);
}
UpdatableNameMapper::NameMapperHandle
UpdatableNameMapper::currentNameMapper() const
{
// Return a copy of the handle to the current NameMapper object. It will
// ensure that the object survives any call to UpdatableNameMapper::update()
// made before the completion of any pending operation on that object.
std::lock_guard<std::mutex> lock(mutex);
return nameMapper;
}
std::string UpdatableNameMapper::getNameForId(const std::string& id) const
{
// Ensure that the current nameMapper object survives a concurrent call
// to UpdatableNameMapper::update()
return currentNameMapper()->getNameForId(id);
}
std::string UpdatableNameMapper::getIdForName(const std::string& name) const
{
// Ensure that the current nameMapper object survives a concurrent call
// to UpdatableNameMapper::update()
return currentNameMapper()->getIdForName(name);
}
}

View File

@@ -1,74 +0,0 @@
/*
* Copyright 2021 Veloman Yunkan <veloman.yunkan@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "opds_catalog.h"
#include "tools/stringTools.h"
#include <sstream>
namespace kiwix
{
namespace
{
const char opdsSearchEndpoint[] = "/catalog/v2/entries";
enum Separator { AMP };
std::ostringstream& operator<<(std::ostringstream& oss, Separator sep)
{
if ( oss.tellp() > 0 )
oss << "&";
return oss;
}
std::string buildSearchString(const Filter& f)
{
std::ostringstream oss;
if ( f.hasQuery() )
oss << AMP << "q=" << urlEncode(f.getQuery());
if ( f.hasCategory() )
oss << AMP << "category=" << urlEncode(f.getCategory());
if ( f.hasLang() )
oss << AMP << "lang=" << urlEncode(f.getLang());
if ( f.hasName() )
oss << AMP << "name=" << urlEncode(f.getName());
if ( !f.getAcceptTags().empty() )
oss << AMP << "tag=" << urlEncode(join(f.getAcceptTags(), ";"));
return oss.str();
}
} // unnamed namespace
std::string getSearchUrl(const Filter& f)
{
const std::string searchString = buildSearchString(f);
if ( searchString.empty() )
return opdsSearchEndpoint;
else
return opdsSearchEndpoint + ("?" + searchString);
}
} // namespace kiwix

View File

@@ -20,12 +20,10 @@
#include "opds_dumper.h"
#include "book.h"
#include "tools/otherTools.h"
#include "kiwixlib-resources.h"
#include <mustache.hpp>
#include <unicode/locid.h>
#include "tools/stringTools.h"
#include "tools/otherTools.h"
namespace kiwix
{
@@ -51,37 +49,24 @@ namespace
{
typedef kainjow::mustache::data MustacheData;
typedef kainjow::mustache::list BooksData;
typedef kainjow::mustache::list IllustrationInfo;
typedef kainjow::mustache::list BookData;
IllustrationInfo getBookIllustrationInfo(const Book& book)
BookData getBookData(const Library* library, const std::vector<std::string>& bookIds)
{
kainjow::mustache::list illustrations;
if ( book.isPathValid() ) {
for ( const auto& illustration : book.getIllustrations() ) {
// For now, we are handling only sizexsize@1 illustration.
// So we can simply pass one size to mustache.
illustrations.push_back(kainjow::mustache::object{
{"icon_size", to_string(illustration->width)},
{"icon_mimetype", illustration->mimeType}
});
}
}
return illustrations;
}
kainjow::mustache::object getSingleBookData(const Book& book)
{
const auto bookDate = book.getDate() + "T00:00:00Z";
return kainjow::mustache::object{
{"id", book.getId()},
BookData bookData;
for ( const auto& bookId : bookIds ) {
const Book& book = library->getBookById(bookId);
const MustacheData bookUrl = book.getUrl().empty()
? MustacheData(false)
: MustacheData(book.getUrl());
bookData.push_back(kainjow::mustache::object{
{"id", "urn:uuid:"+book.getId()},
{"name", book.getName()},
{"title", book.getTitle()},
{"description", book.getDescription()},
{"language", book.getLanguage()},
{"content_id", urlEncode(book.getHumanReadableIdFromPath(), true)},
{"updated", bookDate}, // XXX: this should be the entry update datetime
{"book_date", bookDate},
{"content_id", book.getHumanReadableIdFromPath()},
{"updated", book.getDate() + "T00:00:00Z"},
{"category", book.getCategory()},
{"flavour", book.getFlavour()},
{"tags", book.getTags()},
@@ -89,150 +74,57 @@ kainjow::mustache::object getSingleBookData(const Book& book)
{"media_count", to_string(book.getMediaCount())},
{"author_name", book.getCreator()},
{"publisher_name", book.getPublisher()},
{"url", onlyAsNonEmptyMustacheValue(book.getUrl())},
{"url", bookUrl},
{"size", to_string(book.getSize())},
{"icons", getBookIllustrationInfo(book)},
};
}
std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
{
auto data = getSingleBookData(book);
data["with_xml_header"] = MustacheData(withXMLHeader);
data["dump_partial_entries"] = MustacheData(partial);
data["endpoint_root"] = endpointRoot;
data["root"] = rootLocation;
return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
}
BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& rootLocation, const std::string& endpointRoot, bool partial)
{
BooksData booksData;
for ( const auto& bookId : bookIds ) {
try {
const Book book = library->getBookByIdThreadSafe(bookId);
booksData.push_back(kainjow::mustache::object{
{"entry", getSingleBookEntryXML(book, false, rootLocation, endpointRoot, partial)}
});
} catch ( const std::out_of_range& ) {
// the book was removed from the library since its id was obtained
// ignore it
}
});
}
return booksData;
return bookData;
}
std::map<std::string, std::string> iso639_3 = {
{"atj", "atikamekw"},
{"azb", "آذربایجان دیلی"},
{"bcl", "central bikol"},
{"bgs", "tagabawa"},
{"bxr", "буряад хэлэн"},
{"cbk", "chavacano"},
{"cdo", "閩東語"},
{"dag", "Dagbani"},
{"diq", "dimli"},
{"dty", "डोटेली"},
{"eml", "emiliân-rumagnōl"},
{"fbs", "српскохрватски"},
{"ido", "ido"},
{"kbp", "kabɩ"},
{"kld", "Gamilaraay"},
{"lbe", "лакку маз"},
{"lbj", "ལ་དྭགས་སྐད་"},
{"map", "Austronesian"},
{"mhr", "марий йылме"},
{"mnw", "ဘာသာမန်"},
{"myn", "mayan"},
{"nah", "nahuatl"},
{"nai", "north American Indian"},
{"nds", "plattdütsch"},
{"nrm", "bhasa narom"},
{"olo", "livvi"},
{"pih", "Pitcairn-Norfolk"},
{"pnb", "Western Panjabi"},
{"rmr", "Caló"},
{"rmy", "romani shib"},
{"roa", "romance languages"},
{"twi", "twi"}
};
std::once_flag fillLanguagesFlag;
void fillLanguagesMap()
{
for (auto icuLangPtr = icu::Locale::getISOLanguages(); *icuLangPtr != NULL; ++icuLangPtr) {
auto lang = *icuLangPtr;
const icu::Locale locale(lang);
icu::UnicodeString ustring;
locale.getDisplayLanguage(locale, ustring);
std::string displayLanguage;
ustring.toUTF8String(displayLanguage);
std::string iso3LangCode = locale.getISO3Language();
iso639_3.insert({iso3LangCode, displayLanguage});
}
}
std::string getLanguageSelfName(const std::string& lang) {
const auto itr = iso639_3.find(lang);
if (itr != iso639_3.end()) {
return itr->second;
}
return lang;
};
} // unnamed namespace
string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
{
const auto booksData = getBooksData(library, bookIds, rootLocation, "", false);
const auto bookData = getBookData(library, bookIds);
const kainjow::mustache::object template_data{
{"date", gen_date_str()},
{"root", rootLocation},
{"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)},
{"filter", onlyAsNonEmptyMustacheValue(query)},
{"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
{"totalResults", to_string(m_totalResults)},
{"startIndex", to_string(m_startIndex)},
{"itemsPerPage", to_string(m_count)},
{"books", booksData }
{"books", bookData }
};
return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
}
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const
{
const auto endpointRoot = rootLocation + "/catalog/v2";
const auto booksData = getBooksData(library, bookIds, rootLocation, endpointRoot, partial);
const auto bookData = getBookData(library, bookIds);
const char* const endpoint = partial ? "/partial_entries" : "/entries";
const kainjow::mustache::object template_data{
{"date", gen_date_str()},
{"endpoint_root", endpointRoot},
{"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
{"filter", onlyAsNonEmptyMustacheValue(query)},
{"endpoint_root", rootLocation + "/catalog/v2"},
{"feed_id", gen_uuid(libraryId + "/entries?"+query)},
{"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
{"query", query.empty() ? "" : "?" + urlEncode(query)},
{"totalResults", to_string(m_totalResults)},
{"startIndex", to_string(m_startIndex)},
{"itemsPerPage", to_string(m_count)},
{"books", booksData },
{"dump_partial_entries", MustacheData(partial)}
{"books", bookData }
};
return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
}
std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
{
return getSingleBookEntryXML(library->getBookById(bookId), true, rootLocation, "", false);
}
std::string OPDSDumper::categoriesOPDSFeed() const
std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categories) const
{
const auto now = gen_date_str();
kainjow::mustache::list categoryData;
for ( const auto& category : library->getBooksCategories() ) {
for ( const auto& category : categories ) {
const auto urlencodedCategoryName = urlEncode(category);
categoryData.push_back(kainjow::mustache::object{
{"name", category},
@@ -253,33 +145,4 @@ std::string OPDSDumper::categoriesOPDSFeed() const
);
}
std::string OPDSDumper::languagesOPDSFeed() const
{
const auto now = gen_date_str();
kainjow::mustache::list languageData;
std::call_once(fillLanguagesFlag, fillLanguagesMap);
for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
const std::string languageCode = langAndBookCount.first;
const int bookCount = langAndBookCount.second;
const auto languageSelfName = getLanguageSelfName(languageCode);
languageData.push_back(kainjow::mustache::object{
{"lang_code", languageCode},
{"lang_self_name", languageSelfName},
{"book_count", to_string(bookCount)},
{"updated", now},
{"id", gen_uuid(libraryId + "/languages/" + languageCode)}
});
}
return render_template(
RESOURCE::templates::catalog_v2_languages_xml,
kainjow::mustache::object{
{"date", now},
{"endpoint_root", rootLocation + "/catalog/v2"},
{"feed_id", gen_uuid(libraryId + "/languages")},
{"languages", languageData }
}
);
}
}

View File

@@ -21,14 +21,48 @@
#include <time.h>
#include <zim/search.h>
#include <zim/suggestion.h>
#include <zim/item.h>
#include <zim/error.h>
#include "tools.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "tools/archiveTools.h"
inline char hi(char v)
{
char hex[] = "0123456789abcdef";
return hex[(v >> 4) & 0xf];
}
inline char lo(char v)
{
char hex[] = "0123456789abcdef";
return hex[v & 0xf];
}
std::string hexUUID(std::string in)
{
std::ostringstream out;
for (unsigned n = 0; n < 4; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 4; n < 6; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 6; n < 8; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 8; n < 10; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 10; n < 16; ++n) {
out << hi(in[n]) << lo(in[n]);
}
std::string op = out.str();
return op;
}
namespace kiwix
{
@@ -52,11 +86,6 @@ Reader::Reader(const string zimFilePath)
srand(time(nullptr));
}
Reader::Reader(const std::shared_ptr<zim::Archive> archive, bool _marker)
: zimArchive(archive),
zimFilePath(archive->getFilename())
{}
#ifndef _WIN32
Reader::Reader(int fd)
: zimArchive(new zim::Archive(fd)),
@@ -82,7 +111,12 @@ zim::Archive* Reader::getZimArchive() const
MimeCounterType Reader::parseCounterMetadata() const
{
return kiwix::parseArchiveCounter(*zimArchive);
try {
auto counterContent = zimArchive->getMetadata("Counter");
return parseMimetypeCounter(counterContent);
} catch (zim::EntryNotFound& e) {
return {};
}
}
/* Get the count of articles which can be indexed/displayed */
@@ -104,7 +138,19 @@ unsigned int Reader::getArticleCount() const
/* Get the count of medias content in the ZIM file */
unsigned int Reader::getMediaCount() const
{
return kiwix::getArchiveMediaCount(*zimArchive);
std::map<const std::string, unsigned int> counterMap
= this->parseCounterMetadata();
unsigned int counter = 0;
for (auto &pair:counterMap) {
if (startsWith(pair.first, "image/") ||
startsWith(pair.first, "video/") ||
startsWith(pair.first, "audio/")) {
counter += pair.second;
}
}
return counter;
}
/* Get the total of all items of a ZIM file, redirects included */
@@ -116,13 +162,15 @@ unsigned int Reader::getGlobalCount() const
/* Return the UID of the ZIM file */
string Reader::getId() const
{
return kiwix::getArchiveId(*zimArchive);
std::ostringstream s;
s << zimArchive->getUuid();
return s.str();
}
Entry Reader::getRandomPage() const
{
try {
return Entry(zimArchive->getRandomEntry(), true);
return zimArchive->getRandomEntry();
} catch(...) {
throw NoEntry();
}
@@ -130,12 +178,19 @@ Entry Reader::getRandomPage() const
Entry Reader::getMainPage() const
{
return Entry(zimArchive->getMainEntry(), true);
return zimArchive->getMainEntry();
}
bool Reader::getFavicon(string& content, string& mimeType) const
{
return kiwix::getArchiveFavicon(*zimArchive, 48, content, mimeType);
try {
auto item = zimArchive->getIllustrationItem();
content = item.getData();
mimeType = item.getMimetype();
return true;
} catch(zim::EntryNotFound& e) {};
return false;
}
string Reader::getZimFilePath() const
@@ -157,32 +212,47 @@ bool Reader::getMetadata(const string& name, string& value) const
string Reader::getName() const
{
return kiwix::getMetaName(*zimArchive);
METADATA("Name")
}
string Reader::getTitle() const
{
return kiwix::getArchiveTitle(*zimArchive);
string value = zimArchive->getMetadata("Title");
if (value.empty()) {
value = getLastPathElement(zimFilePath);
std::replace(value.begin(), value.end(), '_', ' ');
size_t pos = value.find(".zim");
value = value.substr(0, pos);
}
return value;
}
string Reader::getCreator() const
{
return kiwix::getMetaCreator(*zimArchive);
METADATA("Creator")
}
string Reader::getPublisher() const
{
return kiwix::getMetaPublisher(*zimArchive);
METADATA("Publisher")
}
string Reader::getDate() const
{
return kiwix::getMetaDate(*zimArchive);
METADATA("Date")
}
string Reader::getDescription() const
{
return kiwix::getMetaDescription(*zimArchive);
string value;
this->getMetadata("Description", value);
/* Mediawiki Collection tends to use the "Subtitle" name */
if (value.empty()) {
this->getMetadata("Subtitle", value);
}
return value;
}
string Reader::getLongDescription() const
@@ -192,7 +262,7 @@ string Reader::getLongDescription() const
string Reader::getLanguage() const
{
return kiwix::getMetaLanguage(*zimArchive);
METADATA("Language")
}
string Reader::getLicense() const
@@ -202,7 +272,13 @@ string Reader::getLicense() const
string Reader::getTags(bool original) const
{
return kiwix::getMetaTags(*zimArchive, original);
string tags_str;
getMetadata("Tags", tags_str);
if (original) {
return tags_str;
}
auto tags = convertTags(tags_str);
return join(tags, ";");
}
@@ -225,7 +301,7 @@ string Reader::getRelation() const
string Reader::getFlavour() const
{
return kiwix::getMetaFlavour(*zimArchive);
METADATA("Flavour")
}
string Reader::getSource() const
@@ -239,10 +315,39 @@ string Reader::getScraper() const
}
#undef METADATA
string Reader::getOrigId() const
{
string value;
this->getMetadata("startfileuid", value);
if (value.empty()) {
return "";
}
std::string id = value;
std::string origID;
std::string temp = "";
unsigned int k = 0;
char tempArray[16] = "";
for (unsigned int i = 0; i < id.size(); i++) {
if (id[i] == '\n') {
tempArray[k] = atoi(temp.c_str());
temp = "";
k++;
} else {
temp += id[i];
}
}
origID = hexUUID(tempArray);
return origID;
}
Entry Reader::getEntryFromPath(const std::string& path) const
{
if (path.empty() || path == "/") {
return getMainPage();
}
try {
return Entry(kiwix::getEntryFromPath(*zimArchive, path), true);
return zimArchive->getEntryByPath(path);
} catch (zim::EntryNotFound& e) {
throw NoEntry();
}
@@ -256,7 +361,7 @@ Entry Reader::getEntryFromEncodedPath(const std::string& path) const
Entry Reader::getEntryFromTitle(const std::string& title) const
{
try {
return Entry(zimArchive->getEntryByTitle(title), true);
return zimArchive->getEntryByTitle(title);
} catch(zim::EntryNotFound& e) {
throw NoEntry();
}
@@ -355,7 +460,12 @@ bool Reader::searchSuggestions(const string& prefix,
std::vector<std::string> Reader::getTitleVariants(
const std::string& title) const
{
return kiwix::getTitleVariants(title);
std::vector<std::string> variants;
variants.push_back(title);
variants.push_back(kiwix::ucFirst(title));
variants.push_back(kiwix::lcFirst(title));
variants.push_back(kiwix::toTitle(title));
return variants;
}
@@ -378,36 +488,35 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
SuggestionsList_t& results)
{
std::vector<std::string> variants = this->getTitleVariants(prefix);
bool retVal = false;
auto suggestionSearcher = zim::SuggestionSearcher(*zimArchive);
/* Try to search in the title using fulltext search database */
if (zimArchive->hasTitleIndex()) {
auto suggestionSearch = suggestionSearcher.suggest(prefix);
auto suggestionSearcher = zim::Searcher(*zimArchive);
zim::Query suggestionQuery;
suggestionQuery.setQuery(prefix, true);
auto suggestionSearch = suggestionSearcher.search(suggestionQuery);
if (suggestionSearch.getEstimatedMatches()) {
const auto suggestions = suggestionSearch.getResults(0, suggestionsCount);
for (auto current : suggestions) {
for (auto current = suggestions.begin();
current != suggestions.end();
current++) {
SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
current.getPath(), current.getSnippet());
results.push_back(suggestion);
}
retVal = true;
} else {
// Check some of the variants of the prefix
for (std::vector<std::string>::iterator variantsItr = variants.begin();
variantsItr != variants.end();
variantsItr++) {
auto suggestionSearch = suggestionSearcher.suggest(*variantsItr);
for (auto current : suggestionSearch.getResults(0, suggestionsCount)) {
if (results.size() >= suggestionsCount) {
break;
}
SuggestionItem suggestion(current.getTitle(), kiwix::normalize(current.getTitle()),
current.getPath(), current.getSnippet());
results.push_back(suggestion);
}
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, results)
|| retVal;
}
}
return results.size() > 0;
return retVal;
}
/* Get next suggestion */
@@ -466,7 +575,7 @@ bool Reader::isCorrupted() const
/* Return the file size, works also for splitted files */
unsigned int Reader::getFileSize() const
{
return kiwix::getArchiveFileSize(*zimArchive);
return zimArchive->getFilesize() / 1024;
}
}

View File

@@ -26,41 +26,21 @@
#include "library.h"
#include "name_mapper.h"
#include "tools/archiveTools.h"
#include <zim/search.h>
#include <mustache.hpp>
#include "kiwixlib-resources.h"
#include "tools/stringTools.h"
namespace kiwix
{
/* Constructor */
SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
: SearchRenderer(
searcher->getSearchResultSet(),
mapper,
nullptr,
searcher->getEstimatedResultCount(),
searcher->getResultStart())
{}
SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper,
unsigned int start, unsigned int estimatedResultCount)
: SearchRenderer(srs, mapper, nullptr, start, estimatedResultCount)
{}
SearchRenderer::SearchRenderer(zim::SearchResultSet srs, NameMapper* mapper, Library* library,
unsigned int start, unsigned int estimatedResultCount)
: m_srs(srs),
: mp_searcher(searcher),
mp_nameMapper(mapper),
mp_library(library),
protocolPrefix("zim://"),
searchProtocolPrefix("search://?"),
estimatedResultCount(estimatedResultCount),
resultStart(start)
searchProtocolPrefix("search://?")
{}
/* Destructor */
@@ -90,30 +70,29 @@ std::string SearchRenderer::getHtml()
{
kainjow::mustache::data results{kainjow::mustache::data::type::list};
for (auto it = m_srs.begin(); it != m_srs.end(); it++) {
mp_searcher->restart_search();
Result* p_result = NULL;
while ((p_result = mp_searcher->getNextResult())) {
kainjow::mustache::data result;
result.set("title", it.getTitle());
result.set("url", it.getPath());
result.set("snippet", it.getSnippet());
std::string zim_id(it.getZimId());
result.set("resultContentId", mp_nameMapper->getNameForId(zim_id));
if (!mp_library) {
result.set("bookTitle", kainjow::mustache::data(false));
} else {
result.set("bookTitle", mp_library->getBookById(zim_id).getTitle());
}
result.set("title", p_result->get_title());
result.set("url", p_result->get_url());
result.set("snippet", p_result->get_snippet());
result.set("resultContentId", mp_nameMapper->getNameForId(p_result->get_zimId()));
if (it.getWordCount() >= 0) {
result.set("wordCount", kiwix::beautifyInteger(it.getWordCount()));
if (p_result->get_wordCount() >= 0) {
result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
}
results.push_back(result);
delete p_result;
}
// pages
kainjow::mustache::data pages{kainjow::mustache::data::type::list};
auto resultStart = mp_searcher->getResultStart();
auto resultEnd = 0U;
auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
auto currentPage = 0U;
auto pageStart = 0U;
auto pageEnd = 0U;
@@ -167,4 +146,4 @@ std::string SearchRenderer::getHtml()
return ss.str();
}
}
}

View File

@@ -18,16 +18,14 @@
*/
#include <cmath>
#include "searcher.h"
#include "reader.h"
#include <zim/search.h>
#include <zim/suggestion.h>
#include <mustache.hpp>
#include <cmath>
#include "tools/stringTools.h"
#include "kiwixlib-resources.h"
#define MAX_SEARCH_LEN 140
@@ -38,7 +36,6 @@ class _Result : public Result
{
public:
_Result(zim::SearchResultSet::iterator iterator);
_Result(SuggestionItem suggestionItem);
virtual ~_Result(){};
virtual std::string get_url();
@@ -52,8 +49,6 @@ class _Result : public Result
private:
zim::SearchResultSet::iterator iterator;
SuggestionItem suggestionItem;
bool isSuggestion;
};
struct SearcherInternal : zim::SearchResultSet {
@@ -66,20 +61,12 @@ struct SearcherInternal : zim::SearchResultSet {
zim::SearchResultSet::iterator current_iterator;
};
struct SuggestionInternal : zim::SuggestionResultSet {
explicit SuggestionInternal(const zim::SuggestionResultSet& srs)
: zim::SuggestionResultSet(srs),
currentIterator(srs.begin()) {}
zim::SuggestionResultSet::iterator currentIterator;
};
/* Constructor */
Searcher::Searcher()
: searchPattern(""),
estimatedResultCount(0),
resultStart(0),
maxResultCount(0)
resultEnd(0)
{
loadICUExternalTables();
}
@@ -94,12 +81,6 @@ bool Searcher::add_reader(Reader* reader)
if (!reader->hasFulltextIndex()) {
return false;
}
for ( const Reader* const existing_reader : readers ) {
if ( existing_reader->getZimArchive()->getUuid() == reader->getZimArchive()->getUuid() )
return false;
}
this->readers.push_back(reader);
return true;
}
@@ -113,7 +94,7 @@ Reader* Searcher::get_reader(int readerIndex)
/* Search strings in the database */
void Searcher::search(const std::string& search,
unsigned int resultStart,
unsigned int maxResultCount,
unsigned int resultEnd,
const bool verbose)
{
this->reset();
@@ -124,9 +105,9 @@ void Searcher::search(const std::string& search,
this->searchPattern = search;
this->resultStart = resultStart;
this->maxResultCount = maxResultCount;
this->resultEnd = resultEnd;
/* Try to find results */
if (maxResultCount != 0) {
if (resultStart != resultEnd) {
/* Perform the search */
string unaccentedSearch = removeAccents(search);
std::vector<zim::Archive> archives;
@@ -137,11 +118,11 @@ void Searcher::search(const std::string& search,
}
}
zim::Searcher searcher(archives);
searcher.setVerbose(verbose);
zim::Query query;
query.setQuery(unaccentedSearch);
query.setQuery(unaccentedSearch, false);
query.setVerbose(verbose);
zim::Search search = searcher.search(query);
internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
this->estimatedResultCount = search.getEstimatedMatches();
}
@@ -151,7 +132,7 @@ void Searcher::search(const std::string& search,
void Searcher::geo_search(float latitude, float longitude, float distance,
unsigned int resultStart,
unsigned int maxResultCount,
unsigned int resultEnd,
const bool verbose)
{
this->reset();
@@ -165,10 +146,10 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
this->searchPattern = oss.str();
this->resultStart = resultStart;
this->maxResultCount = maxResultCount;
this->resultEnd = resultEnd;
/* Try to find results */
if (maxResultCount == 0) {
if (resultStart == resultEnd) {
return;
}
@@ -178,12 +159,12 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
archives.push_back(*(*current)->getZimArchive());
}
zim::Searcher searcher(archives);
searcher.setVerbose(verbose);
zim::Query query;
query.setQuery("");
query.setVerbose(verbose);
query.setQuery("", false);
query.setGeorange(latitude, longitude, distance);
zim::Search search = searcher.search(query);
internal.reset(new SearcherInternal(search.getResults(resultStart, maxResultCount)));
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
this->estimatedResultCount = search.getEstimatedMatches();
}
@@ -197,21 +178,11 @@ void Searcher::restart_search()
Result* Searcher::getNextResult()
{
if (internal.get() && internal->current_iterator != internal->end()) {
if (internal.get() &&
internal->current_iterator != internal->end()) {
Result* result = new _Result(internal->current_iterator);
internal->current_iterator++;
return result;
} else if (suggestionInternal.get() &&
suggestionInternal->currentIterator != suggestionInternal->end()) {
SuggestionItem item(
suggestionInternal->currentIterator->getTitle(),
normalize(suggestionInternal->currentIterator->getTitle()),
suggestionInternal->currentIterator->getPath(),
suggestionInternal->currentIterator->getSnippet()
);
Result* result = new _Result(item);
suggestionInternal->currentIterator++;
return result;
}
return NULL;
}
@@ -234,15 +205,20 @@ void Searcher::suggestions(std::string& searchPattern, const bool verbose)
this->searchPattern = searchPattern;
this->resultStart = 0;
this->maxResultCount = 10;
this->resultEnd = 10;
string unaccentedSearch = removeAccents(searchPattern);
// Multizim suggestion is not supported as of now! taking only one archive
zim::Archive archive = *(*this->readers.begin())->getZimArchive();
zim::SuggestionSearcher searcher(archive);
searcher.setVerbose(verbose);
zim::SuggestionSearch search = searcher.suggest(searchPattern);
suggestionInternal.reset(new SuggestionInternal(search.getResults(resultStart, maxResultCount)));
std::vector<zim::Archive> archives;
for (auto current = this->readers.begin(); current != this->readers.end();
current++) {
archives.push_back(*(*current)->getZimArchive());
}
zim::Searcher searcher(archives);
zim::Query query;
query.setVerbose(verbose);
query.setQuery(unaccentedSearch, true);
zim::Search search = searcher.search(query);
internal.reset(new SearcherInternal(search.getResults(resultStart, resultEnd)));
this->estimatedResultCount = search.getEstimatedMatches();
}
@@ -252,75 +228,41 @@ unsigned int Searcher::getEstimatedResultCount()
return this->estimatedResultCount;
}
zim::SearchResultSet Searcher::getSearchResultSet()
{
return *(this->internal);
}
_Result::_Result(zim::SearchResultSet::iterator iterator)
: iterator(iterator),
suggestionItem("", "", ""),
isSuggestion(false)
{}
_Result::_Result(SuggestionItem item)
: iterator(),
suggestionItem(item.getTitle(), item.getNormalizedTitle(), item.getPath(), item.getSnippet()),
isSuggestion(true)
{}
: iterator(iterator)
{
}
std::string _Result::get_url()
{
if (isSuggestion) {
return suggestionItem.getPath();
}
return iterator.getPath();
}
std::string _Result::get_title()
{
if (isSuggestion) {
return suggestionItem.getTitle();
}
return iterator.getTitle();
}
int _Result::get_score()
{
if (isSuggestion) {
return 0;
}
return iterator.getScore();
}
std::string _Result::get_snippet()
{
if (isSuggestion) {
return suggestionItem.getSnippet();
}
return iterator.getSnippet();
}
std::string _Result::get_content()
{
if (isSuggestion) return "";
return iterator->getItem(true).getData();
}
int _Result::get_size()
{
if (isSuggestion) {
return 0;
}
return iterator.getSize();
}
int _Result::get_wordCount()
{
if (isSuggestion) {
return 0;
}
return iterator.getWordCount();
}
std::string _Result::get_zimId()
{
if (isSuggestion) {
return "";
}
std::ostringstream s;
s << iterator.getZimId();
return s.str();

View File

@@ -48,9 +48,7 @@ bool Server::start() {
m_verbose,
m_withTaskbar,
m_withLibraryButton,
m_blockExternalLinks,
m_indexTemplateString,
m_ipConnectionLimit));
m_blockExternalLinks));
return mp_server->start();
}
@@ -72,14 +70,4 @@ void Server::setRoot(const std::string& root)
}
}
int Server::getPort()
{
return mp_server->getPort();
}
std::string Server::getAddress()
{
return mp_server->getAddress();
}
}

View File

@@ -43,12 +43,10 @@ extern "C" {
#include "microhttpd_wrapper.h"
}
#include "tools.h"
#include "tools/otherTools.h"
#include "tools/pathTools.h"
#include "tools/regexTools.h"
#include "tools/stringTools.h"
#include "tools/archiveTools.h"
#include "tools/networkTools.h"
#include "library.h"
#include "name_mapper.h"
#include "entry.h"
@@ -57,9 +55,6 @@ extern "C" {
#include "opds_dumper.h"
#include <zim/uuid.h>
#include <zim/error.h>
#include <zim/entry.h>
#include <zim/item.h>
#include <mustache.hpp>
@@ -78,7 +73,6 @@ extern "C" {
#define MAX_SEARCH_LEN 140
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
#define DEFAULT_CACHE_SIZE 2
namespace kiwix {
@@ -95,68 +89,8 @@ inline std::string normalizeRootUrl(std::string rootUrl)
return rootUrl.empty() ? rootUrl : "/" + rootUrl;
}
// Returns the value of env var `name` if found, otherwise returns defaultVal
unsigned int getCacheLength(const char* name, unsigned int defaultVal) {
try {
const char* envString = std::getenv(name);
if (envString == nullptr) {
throw std::runtime_error("Environment variable not set");
}
return extractFromString<unsigned int>(envString);
} catch (...) {}
return defaultVal;
}
} // unnamed namespace
SearchInfo::SearchInfo(const std::string& pattern)
: pattern(pattern),
geoQuery()
{}
SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery)
: pattern(pattern),
geoQuery(geoQuery)
{}
SearchInfo::SearchInfo(const RequestContext& request)
: pattern(request.get_optional_param<std::string>("pattern", "")),
geoQuery(),
bookName(request.get_optional_param<std::string>("content", ""))
{
/* Retrive geo search */
try {
auto latitude = request.get_argument<float>("latitude");
auto longitude = request.get_argument<float>("longitude");
auto distance = request.get_argument<float>("distance");
geoQuery = GeoQuery(latitude, longitude, distance);
} catch(const std::out_of_range&) {}
catch(const std::invalid_argument&) {}
if (!geoQuery && pattern.empty()) {
throw std::invalid_argument("No query provided.");
}
}
zim::Query SearchInfo::getZimQuery(bool verbose) const {
zim::Query query;
if (verbose) {
std::cout << "Performing query '" << pattern<< "'";
}
query.setQuery(pattern);
if (geoQuery) {
if (verbose) {
std::cout << " with geo query '" << geoQuery.distance << "&(" << geoQuery.latitude << ";" << geoQuery.longitude << ")'";
}
query.setGeorange(geoQuery.latitude, geoQuery.longitude, geoQuery.distance);
}
if (verbose) {
std::cout << std::endl;
}
return query;
}
static IdNameMapper defaultNameMapper;
static MHD_Result staticHandlerCallback(void* cls,
@@ -178,9 +112,7 @@ InternalServer::InternalServer(Library* library,
bool verbose,
bool withTaskbar,
bool withLibraryButton,
bool blockExternalLinks,
std::string indexTemplateString,
int ipConnectionLimit) :
bool blockExternalLinks) :
m_addr(addr),
m_port(port),
m_root(normalizeRootUrl(root)),
@@ -189,14 +121,9 @@ InternalServer::InternalServer(Library* library,
m_withTaskbar(withTaskbar),
m_withLibraryButton(withLibraryButton),
m_blockExternalLinks(blockExternalLinks),
m_indexTemplateString(indexTemplateString.empty() ? RESOURCE::templates::index_html : indexTemplateString),
m_ipConnectionLimit(ipConnectionLimit),
mp_daemon(nullptr),
mp_library(library),
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
suggestionSearcherCache(getCacheLength("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
{}
bool InternalServer::start() {
@@ -208,21 +135,21 @@ bool InternalServer::start() {
if (m_verbose.load())
flags |= MHD_USE_DEBUG;
struct sockaddr_in sockAddr;
memset(&sockAddr, 0, sizeof(sockAddr));
sockAddr.sin_family = AF_INET;
sockAddr.sin_port = htons(m_port);
if (m_addr.empty()) {
if (0 != INADDR_ANY) {
if (0 != INADDR_ANY)
sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
}
m_addr = kiwix::getBestPublicIp();
} else {
if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl;
return false;
}
}
mp_daemon = MHD_start_daemon(flags,
m_port,
NULL,
@@ -231,7 +158,6 @@ bool InternalServer::start() {
this,
MHD_OPTION_SOCK_ADDR, &sockAddr,
MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
MHD_OPTION_PER_IP_CONNECTION_LIMIT, m_ipConnectionLimit,
MHD_OPTION_END);
if (mp_daemon == nullptr) {
std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
@@ -326,10 +252,8 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
{
try {
if (! request.is_valid_url()) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
}
if (! request.is_valid_url())
return Response::build_404(*this, request, "", "");
const ETag etag = get_matching_if_none_match_etag(request);
if ( etag )
@@ -341,8 +265,8 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
if (startsWith(request.get_url(), "/catalog/"))
return handle_catalog(request);
if (startsWith(request.get_url(), "/raw/"))
return handle_raw(request);
if (request.get_url() == "/meta")
return handle_meta(request);
if (request.get_url() == "/search")
return handle_search(request);
@@ -359,12 +283,10 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
return handle_content(request);
} catch (std::exception& e) {
fprintf(stderr, "===== Unhandled error : %s\n", e.what());
return HTTP500HtmlResponse(*this, request)
+ e.what();
return Response::build_500(*this, e.what());
} catch (...) {
fprintf(stderr, "===== Unhandled unknown error\n");
return HTTP500HtmlResponse(*this, request)
+ "Unknown error";
return Response::build_500(*this, "Unknown error");
}
}
@@ -398,124 +320,116 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
{
return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8", true);
return ContentResponse::build(*this, RESOURCE::templates::index_html, get_default_data(), "text/html; charset=utf-8", true);
}
/**
* Archive and Zim handlers begin
**/
SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Archive* const archive,
const std::string& bookId, const std::string& queryString, int start, int suggestionCount)
std::unique_ptr<Response> InternalServer::handle_meta(const RequestContext& request)
{
SuggestionsList_t suggestions;
std::shared_ptr<zim::SuggestionSearcher> searcher;
searcher = cache.getOrPut(bookId, [=](){ return make_shared<zim::SuggestionSearcher>(*archive); });
if (archive->hasTitleIndex()) {
auto search = searcher->suggest(queryString);
auto srs = search.getResults(start, suggestionCount);
for (auto it : srs) {
SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
it.getPath(), it.getSnippet());
suggestions.push_back(suggestion);
}
} else {
// TODO: This case should be handled by libzim
std::vector<std::string> variants = getTitleVariants(queryString);
int currCount = 0;
for (auto it = variants.begin(); it != variants.end() && currCount < suggestionCount; it++) {
auto search = searcher->suggest(queryString);
auto srs = search.getResults(0, suggestionCount);
for (auto it : srs) {
SuggestionItem suggestion(it.getTitle(), kiwix::normalize(it.getTitle()),
it.getPath());
suggestions.push_back(suggestion);
currCount++;
}
}
std::string bookName;
std::string bookId;
std::string meta_name;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
meta_name = request.get_argument("name");
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
return Response::build_404(*this, request, bookName, "");
}
return suggestions;
if (reader == nullptr) {
return Response::build_404(*this, request, bookName, "");
}
std::string content;
std::string mimeType = "text";
if (meta_name == "title") {
content = reader->getTitle();
} else if (meta_name == "description") {
content = reader->getDescription();
} else if (meta_name == "language") {
content = reader->getLanguage();
} else if (meta_name == "name") {
content = reader->getName();
} else if (meta_name == "tags") {
content = reader->getTags();
} else if (meta_name == "date") {
content = reader->getDate();
} else if (meta_name == "creator") {
content = reader->getCreator();
} else if (meta_name == "publisher") {
content = reader->getPublisher();
} else if (meta_name == "favicon") {
reader->getFavicon(content, mimeType);
} else {
return Response::build_404(*this, request, bookName, "");
}
auto response = ContentResponse::build(*this, content, mimeType);
response->set_cacheable();
return std::move(response);
}
namespace
{
std::string noSuchBookErrorMsg(const std::string& bookName)
{
return "No such book: " + bookName;
}
std::string noSearchResultsMsg()
{
return "The fulltext search engine is not available for this content.";
}
} // unnamed namespace
std::unique_ptr<Response> InternalServer::handle_suggest(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_suggest\n");
}
std::string bookName, bookId;
std::shared_ptr<zim::Archive> archive;
std::string content;
std::string mimeType;
unsigned int maxSuggestionCount = 10;
unsigned int suggestionCount = 0;
std::string bookName;
std::string bookId;
std::string term;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
archive = mp_library->getArchiveById(bookId);
term = request.get_argument("term");
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {
// error handled by the archive == nullptr check below
}
if (archive == nullptr) {
return HTTP404HtmlResponse(*this, request)
+ noSuchBookErrorMsg(bookName)
+ TaskbarInfo(bookName);
}
const auto queryString = request.get_optional_param("term", std::string());
const auto start = request.get_optional_param<unsigned int>("start", 0);
unsigned int count = request.get_optional_param<unsigned int>("count", 10);
if (count == 0) {
count = 10;
return Response::build_404(*this, request, bookName, "");
}
if (m_verbose.load()) {
printf("Searching suggestions for: \"%s\"\n", queryString.c_str());
printf("Searching suggestions for: \"%s\"\n", term.c_str());
}
MustacheData results{MustacheData::type::list};
bool first = true;
if (reader != nullptr) {
/* Get the suggestions */
SuggestionsList_t suggestions;
reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions);
for(auto& suggestion:suggestions) {
MustacheData result;
result.set("label", suggestion.getTitle());
/* Get the suggestions */
SuggestionsList_t suggestions = getSuggestions(suggestionSearcherCache, archive.get(),
bookId, queryString, start, count);
for(auto& suggestion:suggestions) {
MustacheData result;
result.set("label", suggestion.getTitle());
if (suggestion.hasSnippet()) {
result.set("label", suggestion.getSnippet());
}
if (suggestion.hasSnippet()) {
result.set("label", suggestion.getSnippet());
result.set("value", suggestion.getTitle());
result.set("kind", "path");
result.set("path", suggestion.getPath());
result.set("first", first);
first = false;
results.push_back(result);
suggestionCount++;
}
result.set("value", suggestion.getTitle());
result.set("kind", "path");
result.set("path", suggestion.getPath());
result.set("first", first);
first = false;
results.push_back(result);
}
/* Propose the fulltext search if possible */
if (archive->hasFulltextIndex()) {
if (reader->hasFulltextIndex()) {
MustacheData result;
result.set("label", "containing '" + queryString + "'...");
result.set("value", queryString + " ");
result.set("label", "containing '" + term + "'...");
result.set("value", term + " ");
result.set("kind", "pattern");
result.set("first", first);
results.push_back(result);
@@ -543,8 +457,7 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
response->set_cacheable();
return std::move(response);
} catch (const ResourceNotFound& e) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
return Response::build_404(*this, request, "", "");
}
}
@@ -554,92 +467,99 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
printf("** running handle_search\n");
}
std::string bookName;
std::string bookId;
try {
auto searchInfo = SearchInfo(request);
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
} catch (const std::out_of_range&) {}
std::string bookId;
std::shared_ptr<zim::Archive> archive;
if (!searchInfo.bookName.empty()) {
try {
bookId = mp_nameMapper->getIdForName(searchInfo.bookName);
archive = mp_library->getArchiveById(bookId);
} catch (const std::out_of_range&) {
throw std::invalid_argument("The requested book doesn't exist.");
std::string patternString;
try {
patternString = request.get_argument("pattern");
} catch (const std::out_of_range&) {}
/* Retrive geo search */
bool has_geo_query = false;
float latitude = 0;
float longitude = 0;
float distance = 0;
try {
latitude = request.get_argument<float>("latitude");
longitude = request.get_argument<float>("longitude");
distance = request.get_argument<float>("distance");
has_geo_query = true;
} catch(const std::out_of_range&) {}
catch(const std::invalid_argument&) {}
std::shared_ptr<Reader> reader(nullptr);
try {
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {}
/* Make the search */
if ( (!reader && !bookName.empty())
|| (patternString.empty() && ! has_geo_query) ) {
auto data = get_default_data();
data.set("pattern", encodeDiples(patternString));
auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8");
response->set_taskbar(bookName, reader ? reader->getTitle() : "");
response->set_code(MHD_HTTP_NOT_FOUND);
return std::move(response);
}
Searcher searcher;
if (reader) {
searcher.add_reader(reader.get());
} else {
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto currentReader = mp_library->getReaderById(bookId);
if (currentReader) {
searcher.add_reader(currentReader.get());
}
}
}
auto start = 0;
try {
start = request.get_argument<unsigned int>("start");
} catch (const std::exception&) {}
/* Make the search */
// Try to get a search from the searchInfo, else build it
std::shared_ptr<zim::Search> search;
try {
search = searchCache.getOrPut(searchInfo,
[=](){
std::shared_ptr<zim::Searcher> searcher;
if (archive) {
searcher = searcherCache.getOrPut(bookId, [=](){ return std::make_shared<zim::Searcher>(*archive);});
} else {
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto currentArchive = mp_library->getArchiveById(bookId);
if (currentArchive) {
if (! searcher) {
searcher = std::make_shared<zim::Searcher>(*currentArchive);
} else {
searcher->addArchive(*currentArchive);
}
}
}
}
return make_shared<zim::Search>(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
}
);
} catch(std::runtime_error& e) {
// Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
// (in case of zim file not containing a index)
return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
"Fulltext search unavailable",
"Not Found",
m_root + "/skin/search_results.css")
+ noSearchResultsMsg()
+ TaskbarInfo(searchInfo.bookName, archive.get());
auto pageLength = 25;
try {
pageLength = request.get_argument<unsigned int>("pageLength");
} catch (const std::exception&) {}
if (pageLength > MAX_SEARCH_LEN) {
pageLength = MAX_SEARCH_LEN;
}
if (pageLength == 0) {
pageLength = 25;
}
auto end = start + pageLength;
/* Get the results */
try {
if (patternString.empty()) {
searcher.geo_search(latitude, longitude, distance,
start, end, m_verbose.load());
} else {
searcher.search(patternString,
start, end, m_verbose.load());
}
auto start = 0;
try {
start = request.get_argument<unsigned int>("start");
} catch (const std::exception&) {}
auto pageLength = 25;
try {
pageLength = request.get_argument<unsigned int>("pageLength");
} catch (const std::exception&) {}
if (pageLength > MAX_SEARCH_LEN) {
pageLength = MAX_SEARCH_LEN;
}
if (pageLength == 0) {
pageLength = 25;
}
/* Get the results */
SearchRenderer renderer(search->getResults(start, pageLength), mp_nameMapper, mp_library, start,
search->getEstimatedMatches());
renderer.setSearchPattern(searchInfo.pattern);
renderer.setSearchContent(searchInfo.bookName);
SearchRenderer renderer(&searcher, mp_nameMapper);
renderer.setSearchPattern(patternString);
renderer.setSearchContent(bookName);
renderer.setProtocolPrefix(m_root + "/");
renderer.setSearchProtocolPrefix(m_root + "/search?");
renderer.setPageLength(pageLength);
auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
response->set_taskbar(searchInfo.bookName, archive.get());
response->set_taskbar(bookName, reader ? reader->getTitle() : "");
return std::move(response);
} catch (const std::invalid_argument& e) {
return HTTP400HtmlResponse(*this, request)
+ invalidUrlMsg
+ std::string(e.what());
} catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
return HTTP500HtmlResponse(*this, request)
+ e.what();
return Response::build_500(*this, e.what());
}
}
@@ -650,29 +570,25 @@ std::unique_ptr<Response> InternalServer::handle_random(const RequestContext& re
}
std::string bookName;
std::shared_ptr<zim::Archive> archive;
std::string bookId;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
const std::string bookId = mp_nameMapper->getIdForName(bookName);
archive = mp_library->getArchiveById(bookId);
bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {
// error handled by the archive == nullptr check below
return Response::build_404(*this, request, bookName, "");
}
if (archive == nullptr) {
return HTTP404HtmlResponse(*this, request)
+ noSuchBookErrorMsg(bookName)
+ TaskbarInfo(bookName);
if (reader == nullptr) {
return Response::build_404(*this, request, bookName, "");
}
try {
auto entry = archive->getRandomEntry();
return build_redirect(bookName, getFinalItem(*archive, entry));
} catch(zim::EntryNotFound& e) {
const std::string error_details = "Oops! Failed to pick a random article :(";
return HTTP404HtmlResponse(*this, request)
+ error_details
+ TaskbarInfo(bookName, archive.get());
auto entry = reader->getRandomPage();
return build_redirect(bookName, entry.getFinalEntry());
} catch(kiwix::NoEntry& e) {
return Response::build_404(*this, request, bookName, "");
}
}
@@ -683,10 +599,8 @@ std::unique_ptr<Response> InternalServer::handle_captured_external(const Request
source = kiwix::urlDecode(request.get_argument("source"));
} catch (const std::out_of_range& e) {}
if (source.empty()) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
}
if (source.empty())
return Response::build_404(*this, request, "", "");
auto data = get_default_data();
data.set("source", source);
@@ -705,8 +619,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
host = request.get_header("Host");
url = request.get_url_part(1);
} catch (const std::out_of_range&) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
return Response::build_404(*this, request, "", "");
}
if (url == "v2") {
@@ -714,8 +627,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
}
if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
return Response::build_404(*this, request, "", "");
}
if (url == "searchdescription.xml") {
@@ -794,8 +706,7 @@ InternalServer::search_catalog(const RequestContext& request,
const auto totalResults = bookIdsToDump.size();
const size_t count = request.get_optional_param("count", 10UL);
const size_t startIndex = request.get_optional_param("start", 0UL);
const size_t intendedCount = count > 0 ? count : bookIdsToDump.size();
bookIdsToDump = subrange(bookIdsToDump, startIndex, intendedCount);
bookIdsToDump = subrange(bookIdsToDump, startIndex, count);
opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
return bookIdsToDump;
}
@@ -823,10 +734,22 @@ std::string searchSuggestionHTML(const std::string& searchURL, const std::string
} // unnamed namespace
std::unique_ptr<Response>
InternalServer::build_redirect(const std::string& bookName, const zim::Item& item) const
std::shared_ptr<Reader>
InternalServer::get_reader(const std::string& bookName) const
{
auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(item.getPath());
std::shared_ptr<Reader> reader;
try {
const std::string bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
}
return reader;
}
std::unique_ptr<Response>
InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
{
auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath());
return Response::build_redirect(*this, redirectUrl);
}
@@ -842,18 +765,12 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
if (bookName.empty())
return build_homepage(request);
std::shared_ptr<zim::Archive> archive;
try {
const std::string bookId = mp_nameMapper->getIdForName(bookName);
archive = mp_library->getArchiveById(bookId);
} catch (const std::out_of_range& e) {}
const std::shared_ptr<Reader> reader = get_reader(bookName);
if (reader == nullptr) {
std::string searchURL = m_root+"/search?pattern="+pattern; // Make a full search on the entire library.
const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
if (archive == nullptr) {
const std::string searchURL = m_root + "/search?pattern=" + kiwix::urlEncode(pattern, true);
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+ TaskbarInfo(bookName);
return Response::build_404(*this, request, bookName, "", details);
}
auto urlStr = request.get_url().substr(bookName.size()+1);
@@ -862,96 +779,31 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
}
try {
auto entry = getEntryFromPath(*archive, urlStr);
auto entry = reader->getEntryFromPath(urlStr);
if (entry.isRedirect() || urlStr.empty()) {
// If urlStr is empty, we want to mainPage.
// We must do a redirection to the real page.
return build_redirect(bookName, getFinalItem(*archive, entry));
return build_redirect(bookName, entry.getFinalEntry());
}
auto response = ItemResponse::build(*this, request, entry.getItem());
auto response = ItemResponse::build(*this, request, entry.getZimEntry().getItem());
try {
dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, archive.get());
dynamic_cast<ContentResponse&>(*response).set_taskbar(bookName, reader->getTitle());
} catch (std::bad_cast& e) {}
if (m_verbose.load()) {
printf("Found %s\n", entry.getPath().c_str());
printf("mimeType: %s\n", entry.getItem(true).getMimetype().c_str());
printf("mimeType: %s\n", entry.getMimetype().c_str());
}
return response;
} catch(zim::EntryNotFound& e) {
} catch(kiwix::NoEntry& e) {
if (m_verbose.load())
printf("Failed to find %s\n", urlStr.c_str());
std::string searchURL = m_root + "/search?content=" + bookName + "&pattern=" + kiwix::urlEncode(pattern, true);
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern))
+ TaskbarInfo(bookName, archive.get());
}
}
std::string searchURL = m_root+"/search?content="+bookName+"&pattern="+pattern; // Make a search on this specific book only.
const std::string details = searchSuggestionHTML(searchURL, kiwix::urlDecode(pattern));
std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_raw\n");
}
std::string bookName;
std::string kind;
try {
bookName = request.get_url_part(1);
kind = request.get_url_part(2);
} catch (const std::out_of_range& e) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
}
if (kind != "meta" && kind!= "content") {
const std::string error_details = kind + " is not a valid request for raw content.";
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ error_details;
}
std::shared_ptr<zim::Archive> archive;
try {
const std::string bookId = mp_nameMapper->getIdForName(bookName);
archive = mp_library->getArchiveById(bookId);
} catch (const std::out_of_range& e) {}
if (archive == nullptr) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ noSuchBookErrorMsg(bookName);
}
// Remove the beggining of the path:
// /raw/<bookName>/<kind>/foo
// ^^^^^ ^ ^
// 5 + 1 + 1 = 7
auto itemPath = request.get_url().substr(bookName.size()+kind.size()+7);
try {
if (kind == "meta") {
auto item = archive->getMetadataItem(itemPath);
return ItemResponse::build(*this, request, item, /*raw=*/true);
} else {
auto entry = archive->getEntryByPath(itemPath);
if (entry.isRedirect()) {
return build_redirect(bookName, entry.getItem(true));
}
return ItemResponse::build(*this, request, entry.getItem(), /*raw=*/true);
}
} catch (zim::EntryNotFound& e ) {
if (m_verbose.load()) {
printf("Failed to find %s\n", itemPath.c_str());
}
const std::string error_details = "Cannot find " + kind + " entry " + itemPath;
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg
+ error_details;
return Response::build_404(*this, request, bookName, reader->getTitle(), details);
}
}

View File

@@ -28,9 +28,6 @@ extern "C" {
#include "library.h"
#include "name_mapper.h"
#include <zim/search.h>
#include <zim/suggestion.h>
#include <mustache.hpp>
#include <atomic>
@@ -39,58 +36,9 @@ extern "C" {
#include "server/request_context.h"
#include "server/response.h"
#include "tools/concurrent_cache.h"
namespace kiwix {
struct GeoQuery {
GeoQuery()
: GeoQuery(0, 0, -1)
{}
GeoQuery(float latitude, float longitude, float distance)
: latitude(latitude), longitude(longitude), distance(distance)
{}
float latitude;
float longitude;
float distance;
explicit operator bool() const {
return distance >= 0;
}
friend bool operator<(const GeoQuery& l, const GeoQuery& r)
{
return std::tie(l.latitude, l.longitude, l.distance)
< std::tie(r.latitude, r.longitude, r.distance); // keep the same order
}
};
class SearchInfo {
public:
SearchInfo(const std::string& pattern);
SearchInfo(const std::string& pattern, GeoQuery geoQuery);
SearchInfo(const RequestContext& request);
zim::Query getZimQuery(bool verbose) const;
friend bool operator<(const SearchInfo& l, const SearchInfo& r)
{
return std::tie(l.bookName, l.pattern, l.geoQuery)
< std::tie(r.bookName, r.pattern, r.geoQuery); // keep the same order
}
public: //data
std::string pattern;
GeoQuery geoQuery;
std::string bookName;
};
typedef kainjow::mustache::data MustacheData;
typedef ConcurrentCache<string, std::shared_ptr<zim::Searcher>> SearcherCache;
typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
typedef ConcurrentCache<string, std::shared_ptr<zim::SuggestionSearcher>> SuggestionSearcherCache;
class Entry;
class OPDSDumper;
@@ -106,9 +54,7 @@ class InternalServer {
bool verbose,
bool withTaskbar,
bool withLibraryButton,
bool blockExternalLinks,
std::string indexTemplateString,
int ipConnectionLimit);
bool blockExternalLinks);
virtual ~InternalServer() = default;
MHD_Result handlerCallback(struct MHD_Connection* connection,
@@ -120,34 +66,30 @@ class InternalServer {
void** cont_cls);
bool start();
void stop();
std::string getAddress() { return m_addr; }
int getPort() { return m_port; }
private: // functions
std::unique_ptr<Response> handle_request(const RequestContext& request);
std::unique_ptr<Response> build_redirect(const std::string& bookName, const zim::Item& item) const;
std::unique_ptr<Response> build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
std::unique_ptr<Response> build_homepage(const RequestContext& request);
std::unique_ptr<Response> handle_skin(const RequestContext& request);
std::unique_ptr<Response> handle_catalog(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_illustration(const RequestContext& request);
std::unique_ptr<Response> handle_meta(const RequestContext& request);
std::unique_ptr<Response> handle_search(const RequestContext& request);
std::unique_ptr<Response> handle_suggest(const RequestContext& request);
std::unique_ptr<Response> handle_random(const RequestContext& request);
std::unique_ptr<Response> handle_captured_external(const RequestContext& request);
std::unique_ptr<Response> handle_content(const RequestContext& request);
std::unique_ptr<Response> handle_raw(const RequestContext& request);
std::vector<std::string> search_catalog(const RequestContext& request,
kiwix::OPDSDumper& opdsDumper);
MustacheData get_default_data() const;
std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
bool etag_not_needed(const RequestContext& r) const;
ETag get_matching_if_none_match_etag(const RequestContext& request) const;
@@ -160,23 +102,19 @@ class InternalServer {
bool m_withTaskbar;
bool m_withLibraryButton;
bool m_blockExternalLinks;
std::string m_indexTemplateString;
int m_ipConnectionLimit;
struct MHD_Daemon* mp_daemon;
Library* mp_library;
NameMapper* mp_nameMapper;
SearcherCache searcherCache;
SearchCache searchCache;
SuggestionSearcherCache suggestionSearcherCache;
std::string m_server_id;
std::string m_library_id;
friend std::unique_ptr<Response> Response::build(const InternalServer& server);
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage, bool raw);
friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw);
friend std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage);
friend std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
friend std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg);
};
}

View File

@@ -43,8 +43,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
try {
url = request.get_url_part(2);
} catch (const std::out_of_range&) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
return Response::build_404(*this, request, "", "");
}
if (url == "root.xml") {
@@ -56,22 +55,12 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
"application/opensearchdescription+xml"
);
} else if (url == "entry") {
const std::string entryId = request.get_url_part(3);
return handle_catalog_v2_complete_entry(request, entryId);
} else if (url == "entries") {
return handle_catalog_v2_entries(request, /*partial=*/false);
} else if (url == "partial_entries") {
return handle_catalog_v2_entries(request, /*partial=*/true);
return handle_catalog_v2_entries(request);
} else if (url == "categories") {
return handle_catalog_v2_categories(request);
} else if (url == "languages") {
return handle_catalog_v2_languages(request);
} else if (url == "illustration") {
return handle_catalog_v2_illustration(request);
} else {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
return Response::build_404(*this, request, "", "");
}
}
@@ -85,21 +74,19 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
{"endpoint_root", m_root + "/catalog/v2"},
{"feed_id", gen_uuid(m_library_id)},
{"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
{"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
{"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")}
},
"application/atom+xml;profile=opds-catalog;kind=navigation"
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request)
{
OPDSDumper opdsDumper(mp_library);
opdsDumper.setRootLocation(m_root);
opdsDumper.setLibraryId(m_library_id);
const auto bookIds = search_catalog(request, opdsDumper);
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query());
return ContentResponse::build(
*this,
opdsFeed,
@@ -107,26 +94,6 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId)
{
try {
mp_library->getBookById(entryId);
} catch (const std::out_of_range&) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
}
OPDSDumper opdsDumper(mp_library);
opdsDumper.setRootLocation(m_root);
opdsDumper.setLibraryId(m_library_id);
const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
return ContentResponse::build(
*this,
opdsFeed,
"application/atom+xml;type=entry;profile=opds-catalog"
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
{
OPDSDumper opdsDumper(mp_library);
@@ -134,36 +101,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
opdsDumper.setLibraryId(m_library_id);
return ContentResponse::build(
*this,
opdsDumper.categoriesOPDSFeed(),
opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()),
"application/atom+xml;profile=opds-catalog;kind=navigation"
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
{
OPDSDumper opdsDumper(mp_library);
opdsDumper.setRootLocation(m_root);
opdsDumper.setLibraryId(m_library_id);
return ContentResponse::build(
*this,
opdsDumper.languagesOPDSFeed(),
"application/atom+xml;profile=opds-catalog;kind=navigation"
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_illustration(const RequestContext& request)
{
try {
const auto bookName = request.get_url_part(3);
const auto bookId = mp_nameMapper->getIdForName(bookName);
auto book = mp_library->getBookByIdThreadSafe(bookId);
auto size = request.get_argument<unsigned int>("size");
auto illustration = book.getIllustration(size);
return ContentResponse::build(*this, illustration->getData(), illustration->mimeType);
} catch(...) {
return HTTP404HtmlResponse(*this, request)
+ urlNotFoundMsg;
}
}
} // namespace kiwix

View File

@@ -25,7 +25,6 @@
#include "tools/regexTools.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "tools/archiveTools.h"
#include "string.h"
#include <mustache.hpp>
@@ -84,112 +83,17 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
return response;
}
const UrlNotFoundMsg urlNotFoundMsg;
const InvalidUrlMsg invalidUrlMsg;
std::unique_ptr<ContentResponse> ContentResponseBlueprint::generateResponseObject() const
std::unique_ptr<Response> Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details)
{
auto r = ContentResponse::build(m_server, m_template, m_data, m_mimeType);
r->set_code(m_httpStatusCode);
if ( m_taskbarInfo ) {
r->set_taskbar(m_taskbarInfo->bookName, m_taskbarInfo->archive);
}
return r;
}
MustacheData results;
results.set("url", request.get_full_url());
results.set("details", details);
HTTPErrorHtmlResponse::HTTPErrorHtmlResponse(const InternalServer& server,
const RequestContext& request,
int httpStatusCode,
const std::string& pageTitleMsg,
const std::string& headingMsg,
const std::string& cssUrl)
: ContentResponseBlueprint(&server,
&request,
httpStatusCode,
"text/html; charset=utf-8",
RESOURCE::templates::error_html)
{
kainjow::mustache::list emptyList;
this->m_data = kainjow::mustache::object{
{"CSS_URL", onlyAsNonEmptyMustacheValue(cssUrl) },
{"PAGE_TITLE", pageTitleMsg},
{"PAGE_HEADING", headingMsg},
{"details", emptyList}
};
}
auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html");
response->set_code(MHD_HTTP_NOT_FOUND);
response->set_taskbar(bookName, bookTitle);
HTTP404HtmlResponse::HTTP404HtmlResponse(const InternalServer& server,
const RequestContext& request)
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_NOT_FOUND,
"Content not found",
"Not Found")
{
}
HTTPErrorHtmlResponse& HTTP404HtmlResponse::operator+(UrlNotFoundMsg /*unused*/)
{
const std::string requestUrl = m_request.get_full_url();
kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{url}}" was not found on this server.)");
return *this + msgTmpl.render({"url", requestUrl});
}
HTTPErrorHtmlResponse& HTTPErrorHtmlResponse::operator+(const std::string& msg)
{
m_data["details"].push_back({"p", msg});
return *this;
}
HTTP400HtmlResponse::HTTP400HtmlResponse(const InternalServer& server,
const RequestContext& request)
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_BAD_REQUEST,
"Invalid request",
"Invalid request")
{
}
HTTPErrorHtmlResponse& HTTP400HtmlResponse::operator+(InvalidUrlMsg /*unused*/)
{
std::string requestUrl = m_request.get_full_url();
const auto query = m_request.get_query();
if (!query.empty()) {
requestUrl += "?" + encodeDiples(query);
}
kainjow::mustache::mustache msgTmpl(R"(The requested URL "{{{url}}}" is not a valid request.)");
return *this + msgTmpl.render({"url", requestUrl});
}
HTTP500HtmlResponse::HTTP500HtmlResponse(const InternalServer& server,
const RequestContext& request)
: HTTPErrorHtmlResponse(server,
request,
MHD_HTTP_INTERNAL_SERVER_ERROR,
"Internal Server Error",
"Internal Server Error")
{
// operator+() is a state-modifying operator (akin to operator+=)
*this + "An internal server error occured. We are sorry about that :/";
}
std::unique_ptr<ContentResponse> HTTP500HtmlResponse::generateResponseObject() const
{
// We want a 500 response to be a minimalistic one (so that the server doesn't
// have to provide additional resources required for its proper rendering)
// ";raw=true" in the MIME-type below disables response decoration
// (see ContentResponse::contentDecorationAllowed())
const std::string mimeType = "text/html;charset=utf-8;raw=true";
auto r = ContentResponse::build(m_server, m_template, m_data, mimeType);
r->set_code(m_httpStatusCode);
return r;
}
ContentResponseBlueprint& ContentResponseBlueprint::operator+(const TaskbarInfo& taskbarInfo)
{
this->m_taskbarInfo.reset(new TaskbarInfo(taskbarInfo));
return *this;
return std::move(response);
}
std::unique_ptr<Response> Response::build_416(const InternalServer& server, size_t resourceLength)
@@ -205,6 +109,17 @@ std::unique_ptr<Response> Response::build_416(const InternalServer& server, size
return response;
}
std::unique_ptr<Response> Response::build_500(const InternalServer& server, const std::string& msg)
{
MustacheData data;
data.set("error", msg);
auto content = render_template(RESOURCE::templates::_500_html, data);
std::unique_ptr<Response> response (
new ContentResponse(server.m_root, true, false, false, false, content, "text/html"));
response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
return response;
}
std::unique_ptr<Response> Response::build_redirect(const InternalServer& server, const std::string& redirectUrl)
{
@@ -321,11 +236,8 @@ ContentResponse::can_compress(const RequestContext& request) const
bool
ContentResponse::contentDecorationAllowed() const
{
if (m_raw) {
return false;
}
return (startsWith(m_mimeType, "text/html")
&& m_mimeType.find(";raw=true") == std::string::npos);
return (startsWith(m_mimeType, "text/html")
&& m_mimeType.find(";raw=true") == std::string::npos);
}
MHD_Response*
@@ -406,19 +318,18 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
return ret;
}
void ContentResponse::set_taskbar(const std::string& bookName, const zim::Archive* archive)
void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle)
{
m_bookName = bookName;
m_bookTitle = archive ? getArchiveTitle(*archive) : "";
m_bookTitle = bookTitle;
}
ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) :
Response(verbose),
m_root(root),
m_content(content),
m_mimeType(mimetype),
m_raw(raw),
m_withTaskbar(withTaskbar),
m_withLibraryButton(withLibraryButton),
m_blockExternalLinks(blockExternalLinks),
@@ -428,17 +339,11 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool raw
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
}
std::unique_ptr<ContentResponse> ContentResponse::build(
const InternalServer& server,
const std::string& content,
const std::string& mimetype,
bool isHomePage,
bool raw)
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage)
{
return std::unique_ptr<ContentResponse>(new ContentResponse(
server.m_root,
server.m_verbose.load(),
raw,
server.m_withTaskbar && !isHomePage,
server.m_withLibraryButton,
server.m_blockExternalLinks,
@@ -446,13 +351,7 @@ std::unique_ptr<ContentResponse> ContentResponse::build(
mimetype));
}
std::unique_ptr<ContentResponse> ContentResponse::build(
const InternalServer& server,
const std::string& template_str,
kainjow::mustache::data data,
const std::string& mimetype,
bool isHomePage)
{
std::unique_ptr<ContentResponse> ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage) {
auto content = render_template(template_str, data);
return ContentResponse::build(server, content, mimetype, isHomePage);
}
@@ -467,14 +366,14 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
}
std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw)
std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, const RequestContext& request, const zim::Item& item)
{
const std::string mimetype = get_mime_type(item);
auto byteRange = request.get_range().resolve(item.getSize());
const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT;
if (noRange && is_compressible_mime_type(mimetype)) {
// Return a contentResponse
auto response = ContentResponse::build(server, item.getData(), mimetype, /*isHomePage=*/false, raw);
auto response = ContentResponse::build(server, item.getData(), mimetype);
response->set_cacheable();
response->m_byteRange = byteRange;
return std::move(response);

View File

@@ -33,15 +33,13 @@ extern "C" {
#include "microhttpd_wrapper.h"
}
namespace zim {
class Archive;
} // namespace zim
namespace kiwix {
class InternalServer;
class RequestContext;
class EntryResponse;
class Response {
public:
Response(bool verbose);
@@ -49,7 +47,9 @@ class Response {
static std::unique_ptr<Response> build(const InternalServer& server);
static std::unique_ptr<Response> build_304(const InternalServer& server, const ETag& etag);
static std::unique_ptr<Response> build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName, const std::string& bookTitle, const std::string& details="");
static std::unique_ptr<Response> build_416(const InternalServer& server, size_t resourceLength);
static std::unique_ptr<Response> build_500(const InternalServer& server, const std::string& msg);
static std::unique_ptr<Response> build_redirect(const InternalServer& server, const std::string& redirectUrl);
MHD_Result send(const RequestContext& request, MHD_Connection* connection);
@@ -78,29 +78,11 @@ class Response {
class ContentResponse : public Response {
public:
ContentResponse(
const std::string& root,
bool verbose,
bool raw,
bool withTaskbar,
bool withLibraryButton,
bool blockExternalLinks,
const std::string& content,
const std::string& mimetype);
static std::unique_ptr<ContentResponse> build(
const InternalServer& server,
const std::string& content,
const std::string& mimetype,
bool isHomePage = false,
bool raw = false);
static std::unique_ptr<ContentResponse> build(
const InternalServer& server,
const std::string& template_str,
kainjow::mustache::data data,
const std::string& mimetype,
bool isHomePage = false);
ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype);
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage = false);
static std::unique_ptr<ContentResponse> build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype, bool isHomePage = false);
void set_taskbar(const std::string& bookName, const zim::Archive* archive);
void set_taskbar(const std::string& bookName, const std::string& bookTitle);
private:
MHD_Response* create_mhd_response(const RequestContext& request);
@@ -116,7 +98,6 @@ class ContentResponse : public Response {
std::string m_root;
std::string m_content;
std::string m_mimeType;
bool m_raw;
bool m_withTaskbar;
bool m_withLibraryButton;
bool m_blockExternalLinks;
@@ -124,114 +105,10 @@ class ContentResponse : public Response {
std::string m_bookTitle;
};
struct TaskbarInfo
{
const std::string bookName;
const zim::Archive* const archive;
TaskbarInfo(const std::string& bookName, const zim::Archive* a = nullptr)
: bookName(bookName)
, archive(a)
{}
};
class ContentResponseBlueprint
{
public: // functions
ContentResponseBlueprint(const InternalServer* server,
const RequestContext* request,
int httpStatusCode,
const std::string& mimeType,
const std::string& templateStr)
: m_server(*server)
, m_request(*request)
, m_httpStatusCode(httpStatusCode)
, m_mimeType(mimeType)
, m_template(templateStr)
{}
virtual ~ContentResponseBlueprint() = default;
operator std::unique_ptr<ContentResponse>() const
{
return generateResponseObject();
}
operator std::unique_ptr<Response>() const
{
return operator std::unique_ptr<ContentResponse>();
}
ContentResponseBlueprint& operator+(const TaskbarInfo& taskbarInfo);
protected: // functions
virtual std::unique_ptr<ContentResponse> generateResponseObject() const;
public: //data
const InternalServer& m_server;
const RequestContext& m_request;
const int m_httpStatusCode;
const std::string m_mimeType;
const std::string m_template;
kainjow::mustache::data m_data;
std::unique_ptr<TaskbarInfo> m_taskbarInfo;
};
struct HTTPErrorHtmlResponse : ContentResponseBlueprint
{
HTTPErrorHtmlResponse(const InternalServer& server,
const RequestContext& request,
int httpStatusCode,
const std::string& pageTitleMsg,
const std::string& headingMsg,
const std::string& cssUrl = "");
using ContentResponseBlueprint::operator+;
HTTPErrorHtmlResponse& operator+(const std::string& msg);
};
class UrlNotFoundMsg {};
extern const UrlNotFoundMsg urlNotFoundMsg;
struct HTTP404HtmlResponse : HTTPErrorHtmlResponse
{
HTTP404HtmlResponse(const InternalServer& server,
const RequestContext& request);
using HTTPErrorHtmlResponse::operator+;
HTTPErrorHtmlResponse& operator+(UrlNotFoundMsg /*unused*/);
};
class InvalidUrlMsg {};
extern const InvalidUrlMsg invalidUrlMsg;
struct HTTP400HtmlResponse : HTTPErrorHtmlResponse
{
HTTP400HtmlResponse(const InternalServer& server,
const RequestContext& request);
using HTTPErrorHtmlResponse::operator+;
HTTPErrorHtmlResponse& operator+(InvalidUrlMsg /*unused*/);
};
struct HTTP500HtmlResponse : HTTPErrorHtmlResponse
{
HTTP500HtmlResponse(const InternalServer& server,
const RequestContext& request);
private: // overrides
// generateResponseObject() is overriden in order to produce a minimal
// response without any need for additional resources from the server
std::unique_ptr<ContentResponse> generateResponseObject() const override;
};
class ItemResponse : public Response {
public:
ItemResponse(bool verbose, const zim::Item& item, const std::string& mimetype, const ByteRange& byterange);
static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item, bool raw = false);
static std::unique_ptr<Response> build(const InternalServer& server, const RequestContext& request, const zim::Item& item);
private:
MHD_Response* create_mhd_response(const RequestContext& request);

View File

@@ -1,182 +0,0 @@
/*
* Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "archiveTools.h"
#include "tools.h"
#include "pathTools.h"
#include "otherTools.h"
#include "stringTools.h"
#include <zim/error.h>
#include <zim/item.h>
namespace kiwix
{
std::string getMetadata(const zim::Archive& archive, const std::string& name) {
try {
return archive.getMetadata(name);
} catch (zim::EntryNotFound& e) {
return "";
}
}
std::string getArchiveTitle(const zim::Archive& archive) {
std::string value = getMetadata(archive, "Title");
if (value.empty()) {
value = getLastPathElement(archive.getFilename());
std::replace(value.begin(), value.end(), '_', ' ');
size_t pos = value.find(".zim");
value = value.substr(0, pos);
}
return value;
}
std::string getMetaDescription(const zim::Archive& archive) {
std::string value;
value = getMetadata(archive, "Description");
/* Mediawiki Collection tends to use the "Subtitle" name */
if (value.empty()) {
value = getMetadata(archive, "Subtitle");
}
return value;
}
std::string getMetaTags(const zim::Archive& archive, bool original) {
std::string tags_str = getMetadata(archive, "Tags");
if (original) {
return tags_str;
}
auto tags = convertTags(tags_str);
return join(tags, ";");
}
std::string getMetaLanguage(const zim::Archive& archive) {
return getMetadata(archive, "Language");
}
std::string getMetaName(const zim::Archive& archive) {
return getMetadata(archive, "Name");
}
std::string getMetaDate(const zim::Archive& archive) {
return getMetadata(archive, "Date");
}
std::string getMetaCreator(const zim::Archive& archive) {
return getMetadata(archive, "Creator");
}
std::string getMetaPublisher(const zim::Archive& archive) {
return getMetadata(archive, "Publisher");
}
std::string getMetaFlavour(const zim::Archive& archive) {
return getMetadata(archive, "Flavour");
}
std::string getArchiveId(const zim::Archive& archive) {
return (std::string) archive.getUuid();
}
bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
std::string& content, std::string& mimeType){
try {
auto item = archive.getIllustrationItem(size);
content = item.getData();
mimeType = item.getMimetype();
return true;
} catch(zim::EntryNotFound& e) {};
return false;
}
// should this be in libzim
unsigned int getArchiveMediaCount(const zim::Archive& archive) {
std::map<const std::string, unsigned int> counterMap = parseArchiveCounter(archive);
unsigned int counter = 0;
for (auto &pair:counterMap) {
if (startsWith(pair.first, "image/") ||
startsWith(pair.first, "video/") ||
startsWith(pair.first, "audio/")) {
counter += pair.second;
}
}
return counter;
}
unsigned int getArchiveArticleCount(const zim::Archive& archive) {
// [HACK]
// getArticleCount() returns different things depending of the "version" of the zim.
// On old zim (<=6), it returns the number of entry in `A` namespace
// On recent zim (>=7), it returns:
// - the number of entry in `C` namespace (==getEntryCount) if no frontArticleIndex is present
// - the number of front article if a frontArticleIndex is present
// The use case >=7 without frontArticleIndex is pretty rare so we don't care
// We can detect if we are reading a zim <= 6 by checking if we have a newNamespaceScheme.
if (archive.hasNewNamespaceScheme()) {
//The articleCount is "good"
return archive.getArticleCount();
} else {
// We have to parse the `M/Counter` metadata
unsigned int counter = 0;
for(const auto& pair:parseArchiveCounter(archive)) {
if (startsWith(pair.first, "text/html")) {
counter += pair.second;
}
}
return counter;
}
}
unsigned int getArchiveFileSize(const zim::Archive& archive) {
return archive.getFilesize() / 1024;
}
zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry)
{
return entry.getItem(true);
}
zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path)
{
try {
return archive.getEntryByPath(path);
} catch (zim::EntryNotFound& e) {
if (path.empty() || path == "/") {
return archive.getMainEntry();
}
}
throw zim::EntryNotFound("Cannot find entry for non empty path");
}
MimeCounterType parseArchiveCounter(const zim::Archive& archive) {
try {
auto counterContent = archive.getMetadata("Counter");
return parseMimetypeCounter(counterContent);
} catch (zim::EntryNotFound& e) {
return {};
}
}
} // kiwix

View File

@@ -1,60 +0,0 @@
/*
* Copyright 2021 Maneesh P M <manu.pm55@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_ARCHIVETOOLS_H
#define KIWIX_ARCHIVETOOLS_H
#include <zim/archive.h>
#include <tools/otherTools.h>
/**
* This file contains all the functions that would make handling data related to
* an archive easier.
**/
namespace kiwix
{
std::string getMetadata(const zim::Archive& archive, const std::string& name);
std::string getArchiveTitle(const zim::Archive& archive);
std::string getMetaDescription(const zim::Archive& archive);
std::string getMetaTags(const zim::Archive& archive, bool original = false);
std::string getMetaLanguage(const zim::Archive& archive);
std::string getMetaName(const zim::Archive& archive);
std::string getMetaDate(const zim::Archive& archive);
std::string getMetaCreator(const zim::Archive& archive);
std::string getMetaPublisher(const zim::Archive& archive);
std::string getMetaFlavour(const zim::Archive& archive);
std::string getArchiveId(const zim::Archive& archive);
bool getArchiveFavicon(const zim::Archive& archive, unsigned size,
std::string& content, std::string& mimeType);
unsigned int getArchiveMediaCount(const zim::Archive& archive);
unsigned int getArchiveArticleCount(const zim::Archive& archive);
unsigned int getArchiveFileSize(const zim::Archive& archive);
zim::Item getFinalItem(const zim::Archive& archive, const zim::Entry& entry);
zim::Entry getEntryFromPath(const zim::Archive& archive, const std::string& path);
MimeCounterType parseArchiveCounter(const zim::Archive& archive);
}
#endif

View File

@@ -1,95 +0,0 @@
/*
* Copyright (C) 2021 Matthieu Gautier <mgautier@kymeria.fr>
* Copyright (C) 2020 Veloman Yunkan
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
* NON-INFRINGEMENT. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#ifndef ZIM_CONCURRENT_CACHE_H
#define ZIM_CONCURRENT_CACHE_H
#include "lrucache.h"
#include <future>
#include <mutex>
namespace kiwix
{
/**
ConcurrentCache implements a concurrent thread-safe cache
Compared to kiwix::lru_cache, each access operation is slightly more expensive.
However, different slots of the cache can be safely accessed concurrently
with minimal blocking. Concurrent access to the same element is also
safe, and, in case of a cache miss, will block until that element becomes
available.
*/
template <typename Key, typename Value>
class ConcurrentCache
{
private: // types
typedef std::shared_future<Value> ValuePlaceholder;
typedef lru_cache<Key, ValuePlaceholder> Impl;
public: // types
explicit ConcurrentCache(size_t maxEntries)
: impl_(maxEntries)
{}
// Gets the entry corresponding to the given key. If the entry is not in the
// cache, it is obtained by calling f() (without any arguments) and the
// result is put into the cache.
//
// The cache as a whole is locked only for the duration of accessing
// the respective slot. If, in the case of the a cache miss, the generation
// of the missing element takes a long time, only attempts to access that
// element will block - the rest of the cache remains open to concurrent
// access.
template<class F>
Value getOrPut(const Key& key, F f)
{
std::promise<Value> valuePromise;
std::unique_lock<std::mutex> l(lock_);
const auto x = impl_.getOrPut(key, valuePromise.get_future().share());
l.unlock();
if ( x.miss() ) {
try {
valuePromise.set_value(f());
} catch (std::exception& e) {
drop(key);
throw;
}
}
return x.value().get();
}
bool drop(const Key& key)
{
std::unique_lock<std::mutex> l(lock_);
return impl_.drop(key);
}
private: // data
Impl impl_;
std::mutex lock_;
};
} // namespace kiwix
#endif // ZIM_CONCURRENT_CACHE_H

View File

@@ -1,160 +0,0 @@
/*
* Copyrigth (c) 2021, Matthieu Gautier <mgautier@kymeria.fr>
* Copyright (c) 2020, Veloman Yunkan
* Copyright (c) 2014, lamerman
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of lamerman nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* File: lrucache.hpp
* Author: Alexander Ponomarev
*
* Created on June 20, 2013, 5:09 PM
*/
#ifndef _LRUCACHE_HPP_INCLUDED_
#define _LRUCACHE_HPP_INCLUDED_
#include <map>
#include <list>
#include <cstddef>
#include <stdexcept>
#include <cassert>
namespace kiwix {
template<typename key_t, typename value_t>
class lru_cache {
public: // types
typedef typename std::pair<key_t, value_t> key_value_pair_t;
typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
enum AccessStatus {
HIT, // key was found in the cache
PUT, // key was not in the cache but was created by the getOrPut() access
MISS // key was not in the cache; get() access failed
};
class AccessResult
{
const AccessStatus status_;
const value_t val_;
public:
AccessResult(const value_t& val, AccessStatus status)
: status_(status), val_(val)
{}
AccessResult() : status_(MISS), val_() {}
bool hit() const { return status_ == HIT; }
bool miss() const { return !hit(); }
const value_t& value() const
{
if ( status_ == MISS )
throw std::range_error("There is no such key in cache");
return val_;
}
operator const value_t& () const { return value(); }
};
public: // functions
explicit lru_cache(size_t max_size) :
_max_size(max_size) {
}
// If 'key' is present in the cache, returns the associated value,
// otherwise puts the given value into the cache (and returns it with
// a status of a cache miss).
AccessResult getOrPut(const key_t& key, const value_t& value) {
auto it = _cache_items_map.find(key);
if (it != _cache_items_map.end()) {
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
return AccessResult(it->second->second, HIT);
} else {
putMissing(key, value);
return AccessResult(value, PUT);
}
}
void put(const key_t& key, const value_t& value) {
auto it = _cache_items_map.find(key);
if (it != _cache_items_map.end()) {
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
it->second->second = value;
} else {
putMissing(key, value);
}
}
AccessResult get(const key_t& key) {
auto it = _cache_items_map.find(key);
if (it == _cache_items_map.end()) {
return AccessResult();
} else {
_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
return AccessResult(it->second->second, HIT);
}
}
bool drop(const key_t& key) {
try {
auto list_it = _cache_items_map.at(key);
_cache_items_list.erase(list_it);
_cache_items_map.erase(key);
return true;
} catch (std::out_of_range& e) {
return false;
}
}
bool exists(const key_t& key) const {
return _cache_items_map.find(key) != _cache_items_map.end();
}
size_t size() const {
return _cache_items_map.size();
}
private: // functions
void putMissing(const key_t& key, const value_t& value) {
assert(_cache_items_map.find(key) == _cache_items_map.end());
_cache_items_list.push_front(key_value_pair_t(key, value));
_cache_items_map[key] = _cache_items_list.begin();
if (_cache_items_map.size() > _max_size) {
_cache_items_map.erase(_cache_items_list.back().first);
_cache_items_list.pop_back();
}
}
private: // data
std::list<key_value_pair_t> _cache_items_list;
std::map<key_t, list_iterator_t> _cache_items_map;
size_t _max_size;
};
} // namespace kiwix
#endif /* _LRUCACHE_HPP_INCLUDED_ */

View File

@@ -1,6 +1,5 @@
/*
* Copyright 2012 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright 2021 Nikhil Tanwar <2002nikhiltanwar@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -18,7 +17,6 @@
* MA 02110-1301, USA.
*/
#include "tools.h"
#include <tools/networkTools.h>
#include <stdio.h>
@@ -31,17 +29,6 @@
#include <iostream>
#include <stdexcept>
#ifdef _WIN32
#include <winsock2.h>
#include <ws2tcpip.h>
#include <iostream>
#else
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <net/if.h>
#include <netdb.h>
#endif
size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
{
@@ -70,104 +57,3 @@ std::string kiwix::download(const std::string& url) {
}
return ss.str();
}
std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
std::map<std::string, std::string> interfaces;
#ifdef _WIN32
SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
if (sd == INVALID_SOCKET) {
std::cerr << "Failed to get a socket. Error " << WSAGetLastError() << std::endl;
return interfaces;
}
INTERFACE_INFO InterfaceList[20];
unsigned long nBytesReturned;
if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() << std::endl;
return interfaces;
}
int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
for (int i = 0; i < nNumInterfaces; ++i) {
sockaddr_in *pAddress;
pAddress = (sockaddr_in *) & (InterfaceList[i].iiAddress.AddressIn);
if(pAddress->sin_family == AF_INET) {
/* Add to the map */
std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
interfaces[interfaceName] = interfaceName;
}
}
#else
/* Get Network interfaces information */
char buf[16384];
struct ifconf ifconf;
int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
ifconf.ifc_len = sizeof(buf);
ifconf.ifc_buf=buf;
if(ioctl(fd, SIOCGIFCONF, &ifconf)!=0) {
perror("ioctl(SIOCGIFCONF)");
}
/* Go through each interface */
struct ifreq *ifreq;
ifreq = ifconf.ifc_req;
for (int i = 0; i < ifconf.ifc_len; ) {
if (ifreq->ifr_addr.sa_family == AF_INET) {
/* Get the network interface ip */
char host[128] = { 0 };
const int error = getnameinfo(&(ifreq->ifr_addr), sizeof(ifreq->ifr_addr),
host, sizeof(host),
0, 0, NI_NUMERICHOST);
if (!error) {
std::string interfaceName = std::string(ifreq->ifr_name);
std::string interfaceIp = std::string(host);
/* Add to the map */
interfaces[interfaceName] = interfaceIp;
} else {
perror("getnameinfo()");
}
}
/* some systems have ifr_addr.sa_len and adjust the length that
* way, but not mine. weird */
size_t len;
#ifndef __linux__
len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
#else
len = sizeof(*ifreq);
#endif
ifreq = (struct ifreq*)((char*)ifreq+len);
i += len;
}
#endif
return interfaces;
}
std::string kiwix::getBestPublicIp() {
auto interfaces = getNetworkInterfaces();
#ifndef _WIN32
const char* const prioritizedNames[] =
{ "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
for(auto name: prioritizedNames) {
auto it = interfaces.find(name);
if(it != interfaces.end()) {
return it->second;
}
}
#endif
const char* const prefixes[] = { "192.168", "172.16.", "10.0" };
for(auto prefix : prefixes){
for(auto& itr : interfaces) {
auto interfaceIp = itr.second;
if (interfaceIp.find(prefix) == 0) {
return interfaceIp;
}
}
}
return "127.0.0.1";
}

View File

@@ -17,11 +17,7 @@
* MA 02110-1301, USA.
*/
// Implement function declared in tools.h and tools/otherTools.h
#include "tools.h"
#include "tools/otherTools.h"
#include <algorithm>
#include <iomanip>
@@ -370,13 +366,6 @@ std::string kiwix::gen_uuid(const std::string& s)
return kiwix::to_string(zim::Uuid::generate(s));
}
kainjow::mustache::data kiwix::onlyAsNonEmptyMustacheValue(const std::string& s)
{
return s.empty()
? kainjow::mustache::data(false)
: kainjow::mustache::data(s);
}
std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data)
{
kainjow::mustache::mustache tmpl(template_str);

View File

@@ -17,10 +17,7 @@
* MA 02110-1301, USA.
*/
// Implement method defined in <kiwix/tools.h> and "tools/pathTools.h"
#include "tools.h"
#include "tools/pathTools.h"
#include <stdexcept>
#ifdef __APPLE__
@@ -62,6 +59,7 @@
#define PATH_MAX 1024
#endif
#ifdef _WIN32
std::string WideToUtf8(const std::wstring& wstr)
{
@@ -80,7 +78,7 @@ std::wstring Utf8ToWide(const std::string& str)
}
#endif
bool kiwix::isRelativePath(const std::string& path)
bool isRelativePath(const std::string& path)
{
#ifdef _WIN32
if (path.size() < 3 ) {
@@ -175,7 +173,7 @@ std::vector<std::string> normalizeParts(std::vector<std::string>& parts, bool ab
return ret;
}
std::string kiwix::computeRelativePath(const std::string& path, const std::string& absolutePath)
std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
{
auto parts = kiwix::split(path, SEPARATOR, false);
auto pathParts = normalizeParts(parts, false);
@@ -200,11 +198,11 @@ std::string kiwix::computeRelativePath(const std::string& path, const std::strin
return ret;
}
std::string kiwix::computeAbsolutePath(const std::string& path, const std::string& relativePath)
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
{
std::string absolutePath = path;
if (path.empty()) {
absolutePath = kiwix::getCurrentDirectory();
absolutePath = getCurrentDirectory();
}
auto parts = kiwix::split(absolutePath, SEPARATOR, false);
@@ -217,7 +215,7 @@ std::string kiwix::computeAbsolutePath(const std::string& path, const std::strin
return ret;
}
std::string kiwix::removeLastPathElement(const std::string& path)
std::string removeLastPathElement(const std::string& path)
{
auto parts_ = kiwix::split(path, SEPARATOR, false);
auto parts = normalizeParts(parts_, false);
@@ -228,7 +226,7 @@ std::string kiwix::removeLastPathElement(const std::string& path)
return ret;
}
std::string kiwix::appendToDirectory(const std::string& directoryPath, const std::string& filename)
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
{
std::string newPath = directoryPath;
if (!directoryPath.empty() && directoryPath.back() != SEPARATOR[0]) {
@@ -238,7 +236,7 @@ std::string kiwix::appendToDirectory(const std::string& directoryPath, const std
return newPath;
}
std::string kiwix::getLastPathElement(const std::string& path)
std::string getLastPathElement(const std::string& path)
{
auto parts_ = kiwix::split(path, SEPARATOR);
auto parts = normalizeParts(parts_, false);
@@ -269,7 +267,7 @@ std::string getFileSizeAsString(const std::string& path)
return convert.str();
}
std::string kiwix::getFileContent(const std::string& path)
std::string getFileContent(const std::string& path)
{
#ifdef _WIN32
auto wpath = Utf8ToWide(path);
@@ -302,21 +300,19 @@ std::string kiwix::getFileContent(const std::string& path)
return content;
}
bool kiwix::fileExists(const std::string& path)
bool fileExists(const std::string& path)
{
#ifdef _WIN32
return (_waccess_s(Utf8ToWide(path).c_str(), 0) == 0);
return PathFileExistsW(Utf8ToWide(path).c_str());
#else
return (access(path.c_str(), F_OK) == 0);
#endif
}
bool kiwix::fileReadable(const std::string& path)
{
#ifdef _WIN32
return (_waccess_s(Utf8ToWide(path).c_str(), 4) == 0);
#else
return (access(path.c_str(), R_OK) == 0);
bool flag = false;
std::fstream fin;
fin.open(path.c_str(), std::ios::in);
if (fin.is_open()) {
flag = true;
}
fin.close();
return flag;
#endif
}
@@ -370,7 +366,7 @@ bool copyFile(const std::string& sourcePath, const std::string& destPath)
#endif
}
std::string kiwix::getExecutablePath(bool realPathOnly)
std::string getExecutablePath(bool realPathOnly)
{
if (!realPathOnly) {
char* cAppImage = ::getenv("APPIMAGE");
@@ -424,7 +420,7 @@ bool writeTextFile(const std::string& path, const std::string& content)
return true;
}
std::string kiwix::getCurrentDirectory()
std::string getCurrentDirectory()
{
#ifdef _WIN32
wchar_t* a_cwd = _wgetcwd(NULL, 0);
@@ -438,7 +434,7 @@ std::string kiwix::getCurrentDirectory()
return ret;
}
std::string kiwix::getDataDirectory()
std::string getDataDirectory()
{
// Try to get the dataDir from the `KIWIX_DATA_DIR` env var
#ifdef _WIN32
@@ -507,7 +503,7 @@ static std::map<std::string, std::string> extMimeTypes = {
};
/* Try to get the mimeType from the file extension */
std::string kiwix::getMimeTypeForFile(const std::string& filename)
std::string getMimeTypeForFile(const std::string& filename)
{
std::string mimeType = "text/plain";
auto pos = filename.find_last_of(".");
@@ -528,3 +524,4 @@ std::string kiwix::getMimeTypeForFile(const std::string& filename)
return mimeType;
}

View File

@@ -17,11 +17,9 @@
* MA 02110-1301, USA.
*/
// Implement function declared in tools.h and tools/stringTools.h
#include "tools.h"
#include "tools/stringTools.h"
#include <tools/stringTools.h>
#include "tools/pathTools.h"
#include <tools/pathTools.h>
#include <unicode/normlzr.h>
#include <unicode/rep.h>
#include <unicode/translit.h>
@@ -270,7 +268,7 @@ std::string kiwix::urlDecode(const std::string& value, bool component)
/* Split string in a token array */
std::vector<std::string> kiwix::split(const std::string& str,
const std::string& delims,
bool dropEmpty,
bool trimEmpty,
bool keepDelim)
{
std::string::size_type lastPos = 0;
@@ -279,7 +277,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
while( (pos = str.find_first_of(delims, lastPos)) < str.length() )
{
auto token = str.substr(lastPos, pos - lastPos);
if (!dropEmpty || !token.empty()) {
if (!trimEmpty || !token.empty()) {
tokens.push_back(token);
}
if (keepDelim) {
@@ -289,7 +287,7 @@ std::vector<std::string> kiwix::split(const std::string& str,
}
auto token = str.substr(lastPos);
if (!dropEmpty || !token.empty()) {
if (!trimEmpty || !token.empty()) {
tokens.push_back(token);
}
return tokens;
@@ -397,11 +395,3 @@ bool kiwix::startsWith(const std::string& base, const std::string& start)
&& std::equal(start.begin(), start.end(), base.begin());
}
std::vector<std::string> kiwix::getTitleVariants(const std::string& title) {
std::vector<std::string> variants;
variants.push_back(title);
variants.push_back(kiwix::ucFirst(title));
variants.push_back(kiwix::lcFirst(title));
variants.push_back(kiwix::toTitle(title));
return variants;
}

View File

@@ -1,77 +0,0 @@
/*
* Copyright 2021 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <iostream>
#include <sstream>
#include <version.h>
#include <zim/zim.h>
#include <kiwix_config.h>
#include <unicode/uversion.h>
#include <pugixml.hpp>
#include <curl/curl.h>
#include <microhttpd.h>
#include <xapian.h>
#include <mustache.hpp>
#include <zlib.h>
namespace kiwix
{
LibVersions getVersions() {
LibVersions versions = {
{ "libkiwix", LIBKIWIX_VERSION },
{ "libzim", LIBZIM_VERSION },
{ "libxapian", XAPIAN_VERSION },
{ "libcurl", LIBCURL_VERSION },
{ "libmicrohttpd", MHD_get_version() },
{ "libz", ZLIB_VERSION }
};
// U_ICU_VERSION does not include the patch level if 0
std::ostringstream libicu_version;
libicu_version << U_ICU_VERSION_MAJOR_NUM << "." << U_ICU_VERSION_MINOR_NUM << "." << U_ICU_VERSION_PATCHLEVEL_NUM;
versions.push_back({ "libicu", libicu_version.str() });
// No human readable version string for pugixml
const unsigned pugixml_major = (PUGIXML_VERSION - PUGIXML_VERSION % 1000) / 1000;
const unsigned pugixml_minor = (PUGIXML_VERSION - pugixml_major * 1000 - PUGIXML_VERSION % 10) / 10;
const unsigned pugixml_patch = PUGIXML_VERSION - pugixml_major * 1000 - pugixml_minor * 10;
std::ostringstream libpugixml_version;
libpugixml_version << pugixml_major << "." << pugixml_minor << "." << pugixml_patch;
versions.push_back({ "libpugixml", libpugixml_version.str() });
// Needs version 5.0 of Mustache
#if defined(KAINJOW_MUSTACHE_VERSION_MAJOR)
std::ostringstream libmustache_version;
libmustache_version << KAINJOW_MUSTACHE_VERSION_MAJOR << "." <<
KAINJOW_MUSTACHE_VERSION_MINOR << "." << KAINJOW_MUSTACHE_VERSION_PATCH;
versions.push_back({ "libmustache", libmustache_version.str() });
#endif
return versions;
}
void printVersions(std::ostream& out) {
LibVersions versions = getVersions();
for (const auto& iter : versions) {
out << (iter != versions.front() ? "+ " : "")
<< iter.first << " " << iter.second << std::endl;
}
}
} //namespace kiwix

View File

@@ -18,23 +18,18 @@ skin/jquery-ui/images/ui-bg_glass_95_fef1ec_1x400.png
skin/jquery-ui/jquery-ui.theme.min.css
skin/jquery-ui/jquery-ui.min.css
skin/caret.png
skin/bittorrent.png
skin/magnet.png
skin/download.png
skin/hash.png
skin/search-icon.svg
skin/taskbar.js
skin/langList.js
skin/categoryList.js
skin/iso6391To3.js
skin/isotope.pkgd.min.js
skin/index.js
skin/taskbar.css
skin/index.css
skin/fonts/Poppins.ttf
skin/fonts/Roboto.ttf
skin/block_external.js
skin/search_results.css
templates/search_result.html
templates/error.html
templates/no_search_result.html
templates/404.html
templates/500.html
templates/index.html
templates/suggestion.json
templates/head_taskbar.html
@@ -44,8 +39,6 @@ templates/captured_external.html
templates/catalog_entries.xml
templates/catalog_v2_root.xml
templates/catalog_v2_entries.xml
templates/catalog_v2_entry.xml
templates/catalog_v2_categories.xml
templates/catalog_v2_languages.xml
opensearchdescription.xml
catalog_v2_searchdescription.xml

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

View File

@@ -0,0 +1,19 @@
// eslint-disable-next-line no-unused-vars
const categoryList = {
"other": "Other",
"gutenberg": "Gutenberg",
"mooc": "Mooc",
"phet": "Phet",
"psiram": "Psiram",
"stack_exchange": "Stack Exchange",
"ted": "Ted",
"vikidia": "Vikidia",
"wikibooks": "Wikibooks",
"wikinews": "Wikinews",
"wikipedia": "Wikipedia",
"wikiquote": "Wikiquote",
"wikisource": "Wikisource",
"wikiversity": "Wikiversity",
"wikivoyage": "Wikivoyage",
"wiktionary": "Wiktionary"
}

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 722 B

View File

Binary file not shown.

View File

Binary file not shown.

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 705 B

View File

File diff suppressed because one or more lines are too long

View File

@@ -4,10 +4,10 @@
start: 0,
count: viewPortToCount()
};
const filterTypes = ['lang', 'category', 'q'];
const bookOrderMap = new Map();
const filterCookieName = 'filters';
const oneDayDelta = 86400000;
let loader;
let footer;
let fadeOutDiv;
let iso;
@@ -16,7 +16,6 @@
let filters = getCookie(filterCookieName);
let params = new URLSearchParams(window.location.search || filters || '');
let timer;
let languages = {};
function queryUrlBuilder() {
let url = `${root}/catalog/search?`;
@@ -42,95 +41,40 @@
return result;
}
const humanFriendlySize = (fileSize) => {
if (fileSize === 0) {
return '';
}
const units = ['bytes', 'kB', 'MB', 'GB', 'TB'];
let quotient = Math.floor(Math.log10(fileSize) / 3);
quotient = quotient < units.length ? quotient : units.length - 1;
fileSize /= (1000 ** quotient);
return `${+fileSize.toFixed(2)} ${units[quotient]}`;
};
const humanFriendlyTitle = (title) => {
if (typeof title === 'string' && title.length > 0) {
title = title.replace(/_/g, ' ');
if (title.length > 0) {
return htmlEncode(title[0].toUpperCase() + title.slice(1));
}
}
return '';
}
function htmlEncode(str) {
return str.replace(/[\u00A0-\u9999<>\&]/gim, (i) => `&#${i.charCodeAt(0)};`);
}
function viewPortToCount(){
const zoom = Math.floor((( window.outerWidth - 10 ) / window.innerWidth) * 100);
return Math.floor(window.innerHeight/(3*zoom) + 1)*(window.innerWidth/(2.5*zoom) + 1);
return Math.floor(window.innerHeight/100 + 1)*(window.innerWidth>1000 ? 3 : 2);
}
function getInnerHtml(node, query) {
const queryNode = node.querySelector(query);
return queryNode != null ? queryNode.innerHTML : "";
return node.querySelector(query).innerHTML;
}
function generateBookHtml(book, sort = false) {
const link = book.querySelector('link[type="text/html"]').getAttribute('href');
let iconUrl;
book.querySelectorAll('link[rel="http://opds-spec.org/image/thumbnail"]').forEach(link => {
if (link.getAttribute('type').split(';')[1] == 'width=48' && !iconUrl) {
iconUrl = link.getAttribute('href');
}
});
const link = book.querySelector('link').getAttribute('href');
const title = getInnerHtml(book, 'title');
const description = getInnerHtml(book, 'summary');
const id = getInnerHtml(book, 'id');
const langCode = getInnerHtml(book, 'language');
const language = languages[langCode];
const tags = getInnerHtml(book, 'tags');
let tagHtml = tags.split(';').filter(tag => {return !(tag.split(':')[0].startsWith('_'))})
.map((tag) => {return tag.charAt(0).toUpperCase() + tag.slice(1)})
.join(' | ').replace(/_/g, ' ');
let downloadLink;
let zimSize = 0;
try {
const downloadBookLink = book.querySelector('link[type="application/x-zim"]')
zimSize = parseInt(downloadBookLink.getAttribute('length'));
downloadLink = downloadBookLink.getAttribute('href').split('.meta4')[0];
} catch {
downloadLink = '';
}
const humanFriendlyZimSize = humanFriendlySize(zimSize);
const iconUrl = getInnerHtml(book, 'icon');
const articleCount = getInnerHtml(book, 'articleCount');
const mediaCount = getInnerHtml(book, 'mediaCount');
const divTag = document.createElement('div');
divTag.setAttribute('class', 'book');
divTag.setAttribute('data-id', id);
const linkTag = document.createElement('a');
linkTag.setAttribute('class', 'book');
linkTag.setAttribute('data-id', id);
linkTag.setAttribute('href', link);
if (sort) {
divTag.setAttribute('data-idx', bookOrderMap.get(id));
linkTag.setAttribute('data-idx', bookOrderMap.get(id));
}
const faviconAttr = iconUrl != undefined ? `style="background-image: url('${iconUrl}')"` : '';
const languageAttr = langCode != '' ? `title="${language}" aria-label="${language}"` : 'style="background-color: transparent"';
divTag.innerHTML = `<a class="book__link" href="${link}" data-hover="Preview">
<div class="book__wrapper">
<div class="book__icon" ${faviconAttr}></div>
<div class="book__header">
<div id="book__title">${title}</div>
${downloadLink ? `<div class="book__download"><span data-link="${downloadLink}">Download ${humanFriendlyZimSize ? ` - ${humanFriendlyZimSize}</span></div>`: ''}` : ''}
</div>
<div class="book__description" title="${description}">${description}</div>
<div class="book__languageTag" ${languageAttr}>${getLanguageCodeToDisplay(langCode)}</div>
<div class="book__tags"><div class="book__tags--wrapper">${tagHtml}</div></div>
</div></div></a>`;
return divTag;
}
function getLanguageCodeToDisplay(langCode3Letter) {
const langCode2Letter = (Object.keys(iso6391To3).find(key => iso6391To3[key] === langCode3Letter));
const res = (langCode2Letter != undefined) ? langCode2Letter : langCode3Letter;
return res.toUpperCase();
linkTag.innerHTML = `<div class='book__background' style="background-image: url('${iconUrl}');">
<div class='book__title' title='${title}'>${title}</div>
<div class='book__description' title='${description}'>${description}</div>
<div class='book__info'>${articleCount} articles, ${mediaCount} medias</div>
</div>`;
return linkTag;
}
function toggleFooter(show=false) {
@@ -142,72 +86,8 @@
}
}
function insertModal(button) {
const downloadLink = button.getAttribute('data-link');
button.addEventListener('click', (event) => {
event.preventDefault();
document.body.insertAdjacentHTML('beforeend', `<div class="modal-wrapper">
<div class="modal">
<div class="modal-heading">
<div class="modal-title">
<div>
Download
</div>
</div>
<div onclick="closeModal()" class="modal-close-button">
<div>
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 14 14" fill="none">
<path fill-rule="evenodd" clip-rule="evenodd" d="M13.7071 1.70711C14.0976 1.31658 14.0976
0.683417 13.7071 0.292893C13.3166 -0.0976311 12.6834 -0.0976311 12.2929 0.292893L7 5.58579L1.70711
0.292893C1.31658 -0.0976311 0.683417 -0.0976311 0.292893 0.292893C-0.0976311 0.683417
-0.0976311 1.31658 0.292893 1.70711L5.58579 7L0.292893 12.2929C-0.0976311 12.6834
-0.0976311 13.3166 0.292893 13.7071C0.683417 14.0976 1.31658 14.0976 1.70711 13.7071L7
8.41421L12.2929 13.7071C12.6834 14.0976 13.3166 14.0976 13.7071 13.7071C14.0976 13.3166
14.0976 12.6834 13.7071 12.2929L8.41421 7L13.7071 1.70711Z" fill="black" />
</svg>
</div>
</div>
</div>
<div class="modal-content">
<div class="modal-regular-download">
<a href="${downloadLink}" download>
<img src="../skin/download.png" alt="direct download" />
<div>Direct</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.sha256" download>
<img src="../skin/hash.png" alt="download hash" />
<div>Sha256 hash</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.magnet" target="_blank">
<img src="../skin/magnet.png" alt="download magnet" />
<div>Magnet link</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.torrent" download>
<img src="../skin/bittorrent.png" alt="download torrent" />
<div>Torrent file</div>
</a>
</div>
</div>
</div>
</div>`);
})
}
async function getBookCount(query) {
const url = `${root}/catalog/search?${query}`;
return await fetch(url).then(async (resp) => {
const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
return parseInt(data.querySelector('totalResults').innerHTML);
});
}
async function loadBooks() {
const loader = document.querySelector('.loader');
loader.style.display = 'block';
return await fetch(queryUrlBuilder()).then(async (resp) => {
const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
@@ -216,82 +96,47 @@
bookOrderMap.set(getInnerHtml(book, 'id'), idx);
});
incrementalLoadingParams.start += books.length;
const results = parseInt(data.querySelector('totalResults').innerHTML)
if (results === bookOrderMap.size) {
if (parseInt(data.querySelector('totalResults').innerHTML) === bookOrderMap.size) {
incrementalLoadingParams.count = 0;
toggleFooter(true);
} else {
toggleFooter();
}
const kiwixResultText = document.querySelector('.kiwixHomeBody__results')
if (results) {
let resultText = `${results} books`;
if (results === 1) {
resultText = `${results} book`;
}
kiwixResultText.innerHTML = resultText;
} else {
kiwixResultText.innerHTML = ``;
}
loader.style.display = 'none';
return books;
});
}
async function loadAndDisplayOptions(nodeQuery, query, valueEntryNode) {
await fetch(query).then(async (resp) => {
const data = new window.DOMParser().parseFromString(await resp.text(), 'application/xml');
let optionStr = '';
data.querySelectorAll('entry').forEach(entry => {
const title = getInnerHtml(entry, 'title');
const value = getInnerHtml(entry, valueEntryNode);
const hfTitle = humanFriendlyTitle(title);
if (valueEntryNode == 'language') {
languages[value] = hfTitle;
}
optionStr += (hfTitle != '') ? `<option value="${value}">${hfTitle}</option>` : '';
});
document.querySelector(nodeQuery).innerHTML += optionStr;
});
async function loadAndDisplayOptions(nodeQuery, query) {
// currently taking an object in place of query, will replace it with query while fetching data from backend later on.
document.querySelector(nodeQuery).innerHTML += Object.keys(query)
.map((option) => {return `<option value='${option}'>${htmlEncode(query[option])}</option>`})
.join('');
}
function checkAndInjectEmptyMessage() {
const kiwixHomeBody = document.querySelector('.kiwixHomeBody');
if (!bookOrderMap.size) {
if (!noResultInjected) {
noResultInjected = true;
iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('div'));
iso.remove(document.getElementsByClassName('book__list')[0].getElementsByTagName('a'));
iso.layout();
setTimeout(() => {
const divTag = document.createElement('div');
divTag.setAttribute('class', 'noResults');
divTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter</a>?`;
kiwixHomeBody.append(divTag);
kiwixHomeBody.setAttribute('style', 'display: flex; justify-content: center; align-items: center');
divTag.getElementsByTagName('a')[0].onclick = (event) => {
event.preventDefault();
window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
setCookie(filterCookieName, 'lang=');
resetAndFilter();
document.querySelectorAll('.filter').forEach(filter => {
filter.value = params.get(filter.name) || '';
if (filter.value) {
filter.style = 'background-color: #858585; color: #fff';
} else {
filter.style = 'background-color: #ffffff; color: black';
}
})
};
loader.setAttribute('style', 'position: absolute; top: 50%');
}, 300);
const spanTag = document.createElement('span');
spanTag.setAttribute('class', 'noResults');
spanTag.innerHTML = `No result. Would you like to <a href="/?lang=">reset filter?</a>`;
document.querySelector('body').append(spanTag);
spanTag.getElementsByTagName('a')[0].onclick = (event) => {
event.preventDefault();
window.history.pushState({}, null, `${window.location.href.split('?')[0]}?lang=`);
setCookie(filterCookieName, 'lang=');
resetAndFilter();
filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
};
}
return true;
} else if (noResultInjected) {
noResultInjected = false;
document.getElementsByClassName('noResults')[0].remove();
kiwixHomeBody.removeAttribute('style');
}
loader.removeAttribute('style');
return false;
}
@@ -325,13 +170,7 @@
});
books = [...books].filter((book) => {return !booksToFilter.has(getInnerHtml(book, 'id'))});
booksToDelete.forEach(book => {iso.remove(book);});
books.forEach((book) => {
iso.insert(generateBookHtml(book, sort))
const downloadButton = document.querySelector(`[data-id="${getInnerHtml(book, 'id')}"] .book__download span`);
if (downloadButton) {
insertModal(downloadButton);
}
});
books.forEach((book) => {iso.insert(generateBookHtml(book, sort))});
}
async function resetAndFilter(filterType = '', filterValue = '') {
@@ -346,23 +185,16 @@
window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
setCookie(filterCookieName, params.toString());
}
document.querySelectorAll('.filter').forEach(filter => {
if (filter.value) {
filter.style = 'background-color: #858585; color: #fff';
} else {
filter.style = 'background-color: #ffffff; color: black';
}
});
await loadAndDisplayBooks(true);
}
window.addEventListener('popstate', async () => {
await resetAndFilter();
document.querySelectorAll('.filter').forEach(filter => {filter.value = params.get(filter.name) || ''});
filterTypes.forEach(key => {document.getElementsByName(key)[0].value = params.get(key) || ''});
});
async function loadSubset() {
if (window.innerHeight + window.scrollY >= (document.body.offsetHeight * 0.98)) {
if (window.innerHeight + window.scrollY >= document.body.offsetHeight) {
if (incrementalLoadingParams.count) {
loadAndDisplayBooks();
}
@@ -391,48 +223,28 @@
return index ? parseInt(index) : Infinity;
}
},
sortBy: 'weight',
layoutMode: 'cellsByRow',
cellsByRow: {
columnWidth: '.book',
rowHeight: '.book'
}
sortBy: 'weight'
});
footer = document.getElementById('kiwixfooter');
fadeOutDiv = document.getElementById('fadeOut');
loader = document.querySelector('.loader');
await loadAndDisplayOptions('#languageFilter', `${root}/catalog/v2/languages`, 'language');
await loadAndDisplayOptions('#categoryFilter', `${root}/catalog/v2/categories`, 'title');
await loadAndDisplayBooks();
document.querySelectorAll('.filter').forEach(filter => {
filter.addEventListener('change', () => {resetAndFilter(filter.name, filter.value)});
await loadAndDisplayOptions('#languageFilter', langList);
await loadAndDisplayOptions('#categoryFilter', categoryList);
filterTypes.forEach((filter) => {
const filterTag = document.getElementsByName(filter)[0];
filterTag.addEventListener('change', () => {resetAndFilter(filterTag.name, filterTag.value)});
});
if (filters) {
window.history.pushState({}, null, `${window.location.href.split('?')[0]}?${params.toString()}`);
}
params.forEach((value, key) => {
const selectBox = document.getElementsByName(key)[0];
if (selectBox) {
selectBox.value = value
}
});
params.forEach((value, key) => {document.getElementsByName(key)[0].value = value});
document.getElementById('kiwixSearchForm').onsubmit = (event) => {event.preventDefault()};
if (!window.location.search) {
const browserLang = navigator.language.split('-')[0];
const langFilter = document.getElementById('languageFilter');
const lang = browserLang.length === 3 ? browserLang : iso6391To3[browserLang];
if (await getBookCount(`lang=${lang}`)) {
langFilter.value = lang;
langFilter.dispatchEvent(new Event('change'));
}
langFilter.value = browserLang.length === 3 ? browserLang : iso6391To3[browserLang];
langFilter.dispatchEvent(new Event('change'));
}
document.querySelectorAll('.filter').forEach(filter => {
if (filter.value) {
filter.style = 'background-color: #858585; color: #fff';
} else {
filter.style = 'background-color: #ffffff; color: black';
}
});
setCookie(filterCookieName, params.toString());
}
})();

View File

@@ -1,22 +1,15 @@
// Source: https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
// eslint-disable-next-line no-unused-vars
const iso6391To3 = {
"aa": "aar",
"ab": "abk",
"ae": "ave",
"af": "afr",
"ak": "aka",
"am": "amh",
"an": "arg",
"ar": "ara",
"as": "asm",
"av": "ava",
"ay": "aym",
"az": "aze",
"ba": "bak",
"be": "bel",
"bg": "bul",
"bi": "bis",
"bm": "bam",
"bn": "ben",
"bo": "bod",
@@ -24,33 +17,24 @@ const iso6391To3 = {
"bs": "bos",
"ca": "cat",
"ce": "che",
"ch": "cha",
"co": "cos",
"cr": "cre",
"cs": "ces",
"cu": "chu",
"cv": "chv",
"cy": "cym",
"da": "dan",
"de": "deu",
"dv": "div",
"dz": "dzo",
"el": "ell",
"ee": "ewe",
"en": "eng",
"eo": "epo",
"es": "spa",
"et": "est",
"eu": "eus",
"fa": "fas",
"ff": "ful",
"fi": "fin",
"fj": "fij",
"fo": "fao",
"fr": "fra",
"fy": "fry",
"ga": "gle",
"gd": "gla",
"gl": "glg",
"gn": "grn",
"gu": "guj",
@@ -58,78 +42,44 @@ const iso6391To3 = {
"ha": "hau",
"he": "heb",
"hi": "hin",
"ho": "hmo",
"hr": "hrv",
"ht": "hat",
"hu": "hun",
"hy": "hye",
"hz": "her",
"ia": "ina",
"id": "ind",
"ie": "ile",
"ig": "ibo",
"ii": "iii",
"ik": "ipk",
"io": "ido",
"is": "isl",
"it": "ita",
"iu": "iku",
"ja": "jpn",
"jv": "jav",
"ka": "kat",
"kg": "kon",
"ki": "kik",
"kj": "kua",
"kl": "kal",
"kk": "kaz",
"km": "khm",
"kn": "kan",
"ko": "kor",
"kr": "kau",
"ks": "kas",
"ku": "kur",
"kv": "kom",
"kw": "cor",
"ky": "kir",
"la": "lat",
"lb": "ltz",
"lg": "lug",
"li": "lim",
"ln": "lin",
"lo": "lao",
"lt": "lit",
"lu": "lub",
"lv": "lav",
"mg": "mlg",
"mh": "mah",
"mi": "mri",
"mk": "mkd",
"ml": "mal",
"mn": "mon",
"mr": "mar",
"ms": "msa",
"mt": "mlt",
"my": "mya",
"na": "nau",
"nb": "nob",
"nd": "nde",
"ne": "nep",
"ng": "ndo",
"nl": "nld",
"nn": "nno",
"no": "nor",
"nr": "nbl",
"nv": "nav",
"ny": "nya",
"oc": "oci",
"oj": "oji",
"om": "orm",
"or": "ori",
"os": "oss",
"pa": "pan",
"pi": "pli",
"pl": "pol",
"ps": "pus",
"pt": "por",
"qu": "que",
"rm": "roh",
@@ -138,10 +88,7 @@ const iso6391To3 = {
"ru": "rus",
"rw": "kin",
"sa": "san",
"sc": "srd",
"sd": "snd",
"se": "sme",
"sm": "smo",
"sg": "sag",
"si": "sin",
"sk": "slk",
@@ -151,37 +98,27 @@ const iso6391To3 = {
"sq": "sqi",
"sr": "srp",
"ss": "ssw",
"st": "sot",
"su": "sun",
"sv": "swe",
"sw": "swa",
"ta": "tam",
"te": "tel",
"tg": "tgk",
"th": "tha",
"ti": "tir",
"tk": "tuk",
"tl": "tgl",
"tn": "tsn",
"to": "ton",
"tr": "tur",
"ts": "tso",
"tt": "tat",
"tw": "twi",
"ty": "tah",
"ug": "uig",
"uk": "ukr",
"ur": "urd",
"uz": "uzb",
"ve": "ven",
"vi": "vie",
"vo": "vol",
"wa": "wln",
"wo": "wol",
"xh": "xho",
"yi": "yid",
"yo": "yor",
"za": "zha",
"zh": "zho",
"zu": "zul"
}

View File

File diff suppressed because it is too large Load Diff

View File

File diff suppressed because one or more lines are too long

124
static/skin/langList.js Normal file
View File

@@ -0,0 +1,124 @@
// eslint-disable-next-line no-unused-vars
const langList = {
"aar": "Afaraf",
"afr": "Afrikaans",
"aka": "Akan",
"amh": "አማርኛ",
"ara": "اللغة العربية",
"asm": "অসমীয়া",
"aze": "azərbaycan dili",
"bak": "башҡорт теле",
"bel": "беларуская мова",
"bul": "български език",
"bam": "bamanankan",
"ben": "বাংলা",
"bod": "བོད་ཡིག",
"bre": "brezhoneg",
"bos": "bosanski jezik",
"cat": "Català",
"che": "нохчийн мотт",
"cos": "corsu",
"ces": "čeština",
"chv": "чӑваш чӗлхи",
"cym": "Cymraeg",
"dan": "dansk",
"deu": "Deutsch",
"dzo": "རྫོང་ཁ",
"ewe": "Eʋegbe",
"eng": "English",
"spa": "Español",
"est": "eesti",
"eus": "euskara",
"fas": "فارسی",
"ful": "Fulfulde",
"fin": "suomi",
"fao": "føroyskt",
"fra": "Français",
"gle": "Gaeilge",
"glg": "galego",
"grn": "Avañe'ẽ",
"guj": "ગુજરાતી",
"glv": "Gaelg",
"hau": "هَوُسَ",
"heb": "עברית",
"hin": "हिन्दी",
"hrv": "hrvatski jezik",
"hun": "magyar",
"hye": "Հայերեն",
"ind": "Bahasa Indonesia",
"ibo": "Asụsụ Igbo",
"isl": "Íslenska",
"ita": "Italiano",
"iku": "ᐃᓄᒃᑎᑐᑦ",
"jpn": "日本語",
"jav": "basa Jawa",
"kat": "ქართული",
"kik": "Gĩkũyũ",
"kaz": "қазақ тілі",
"khm": "ខេមរភាសា",
"kan": "ಕನ್ನಡ",
"kor": "한국어",
"kas": "कश्मीरी",
"kur": "Kurdî",
"cor": "Kernewek",
"kir": "Кыргызча",
"ltz": "Lëtzebuergesch",
"lug": "Luganda",
"lin": "Lingála",
"lao": "ພາສາ",
"lit": "lietuvių kalba",
"lav": "latviešu valoda",
"mlg": "fiteny malagasy",
"mri": "te reo Māori",
"mkd": "македонски јазик",
"mal": "മലയാളം",
"mon": "Монгол хэл",
"mar": "मराठी",
"mlt": "Malti",
"mya": "ဗမာစာ",
"nld": "Nederlands",
"nya": "chiCheŵa",
"orm": "Afaan Oromoo",
"pol": "język polski",
"por": "Português",
"que": "Runa Simi",
"roh": "rumantsch grischun",
"run": "Ikirundi",
"ron": "Română",
"rus": "Русский",
"kin": "Ikinyarwanda",
"san": "संस्कृतम्",
"snd": "सिन्धी",
"sag": "yângâ tî sängö",
"sin": "සිංහල",
"slk": "slovenčina",
"slv": "slovenski jezik",
"sna": "chiShona",
"som": "Soomaaliga",
"sqi": "Shqip",
"srp": "српски језик",
"ssw": "SiSwati",
"swe": "svenska",
"tam": "தமிழ்",
"tel": "తెలుగు",
"tgk": "тоҷикӣ",
"tha": "ไทย",
"tir": "ትግርኛ",
"tuk": "Türkmen",
"tsn": "Setswana",
"tur": "Türkçe",
"tso": "Xitsonga",
"tat": "татар теле",
"uig": "ئۇيغۇرچە‎",
"ukr": "Українська",
"urd": "اردو",
"uzb": "Ўзбек",
"ven": "Tshivenḓa",
"vie": "Tiếng Việt",
"wln": "walon",
"wol": "Wollof",
"xho": "isiXhosa",
"yor": "Yorùbá",
"zho": "中文",
"zul": "isiZulu"
}

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 818 B

View File

@@ -1,6 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" version="1.0" width="512.000000" height="512.000000" preserveAspectRatio="xMidYMid meet" style=""><rect id="backgroundrect" width="100%" height="100%" x="0" y="0" fill="none" stroke="none"/>
<g class="currentLayer" style=""><title>Layer 1</title><g transform="translate(0,512) scale(0.10000000149011612,-0.10000000149011612) " fill="#484848" stroke="none" id="svg_1" class="selected" fill-opacity="1">
<path d="M1858 5104 c-341 -50 -678 -192 -963 -408 -104 -78 -304 -278 -384 -384 -124 -164 -256 -409 -315 -583 -311 -929 47 -1926 877 -2442 662 -410 1535 -405 2188 14 l76 49 125 -128 c69 -70 352 -364 630 -653 574 -597 548 -575 688 -567 85 5 123 21 176 75 52 52 77 114 77 193 0 98 -24 142 -151 273 -420 434 -1132 1177 -1132 1182 0 3 22 33 49 68 164 209 291 471 362 746 159 614 28 1255 -363 1774 -78 104 -279 305 -386 386 -288 218 -618 355 -974 406 -130 19 -448 18 -580 -1z m502 -545 c308 -38 607 -184 844 -409 476 -455 606 -1146 327 -1745 -128 -273 -356 -519 -626 -674 -387 -222 -857 -259 -1282 -102 -626 231 -1040 879 -983 1540 13 146 33 241 77 371 214 634 830 1060 1493 1032 36 -1 103 -7 150 -13z" id="svg_2" fill="#484848" fill-opacity="1"/>
</g></g></svg>

Before

Width:  |  Height:  |  Size: 1.2 KiB

View File

@@ -1,87 +0,0 @@
body{
background-color: white;
color: #000000;
font: small/normal Arial,Helvetica,Sans-Serif;
margin-top: 0.5em;
font-size: 90%;
}
a{
color: #04c;
}
a:visited {
color: #639
}
a:hover {
text-decoration: underline
}
h1 {
font-size: 120%;
}
ul {
margin:0;
padding:0
}
.results {
font-size: 110%;
}
.results li {
list-style-type:none;
margin-top: 0.5em;
}
.results a {
font-size: 110%;
text-decoration: underline
}
cite {
font-style:normal;
word-wrap:break-word;
display: block;
font-size: 100%;
}
.informations {
color: #388222;
font-size: 100%;
}
.footer {
padding: 0;
margin-top: 1em;
width: 100%;
float: left
}
.footer a, .footer span {
display: block;
padding: .3em .7em;
margin: 0 .38em 0 0;
text-align:center;
text-decoration: none;
}
.footer a:hover {
background: #ededed;
}
.footer ul, .footer li {
list-style:none;
margin: 0;
padding: 0;
}
.footer li {
float: left;
}
.selected {
background: #ededed;
}

18
static/templates/404.html Normal file
View File

@@ -0,0 +1,18 @@
<!DOCTYPE html>
<html>
<head>
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
<title>Content not found</title>
</head>
<body>
<h1>Not Found</h1>
<p>
The requested URL "{{url}}" was not found on this server.
</p>
{{#details}}
<p>
{{{details}}}
</p>
{{/details}}
</body>
</html>

16
static/templates/500.html Normal file
View File

@@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
<title>Internal Server Error</title>
</head>
<body>
<h1>Internal Server Error</h1>
<p>
An internal server error occured. We are sorry about that :/
</p>
<p>
{{ error }}
</p>
</body>
</html>

View File

@@ -1,6 +1,4 @@
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="http://opds-spec.org/2010/catalog">
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
<id>{{feed_id}}</id>
<title>{{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}}</title>
<updated>{{date}}</updated>
@@ -11,4 +9,30 @@
{{/filter}}
<link rel="self" href="" type="application/atom+xml" />
<link rel="search" type="application/opensearchdescription+xml" href="{{root}}/catalog/searchdescription.xml" />
{{#books}}{{{entry}}}{{/books}}</feed>
{{#books}}
<entry>
<id>{{id}}</id>
<title>{{title}}</title>
<summary>{{description}}</summary>
<language>{{language}}</language>
<updated>{{updated}}</updated>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
<icon>/meta?name=favicon&amp;content={{{content_id}}}</icon>
<link type="text/html" href="/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
</entry>
{{/books}}
</feed>

View File

@@ -1,12 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="https://specs.opds.io/opds-1.2"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">
<id>{{feed_id}}</id>
<link rel="self"
href="{{endpoint_root}}/{{#dump_partial_entries}}partial_{{/dump_partial_entries}}entries{{{query}}}"
href="{{endpoint_root}}/entries{{{query}}}"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<link rel="start"
href="{{endpoint_root}}/root.xml"
@@ -22,4 +21,30 @@
<startIndex>{{startIndex}}</startIndex>
<itemsPerPage>{{itemsPerPage}}</itemsPerPage>
{{/filter}}
{{#books}}{{{entry}}}{{/books}}</feed>
{{#books}}
<entry>
<id>{{id}}</id>
<title>{{title}}</title>
<summary>{{description}}</summary>
<language>{{language}}</language>
<updated>{{updated}}</updated>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
<icon>/meta?name=favicon&amp;content={{{content_id}}}</icon>
<link type="text/html" href="/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
</entry>
{{/books}}
</feed>

View File

@@ -1,33 +0,0 @@
{{#with_xml_header}}<?xml version="1.0" encoding="UTF-8"?>
{{/with_xml_header}} <entry>
<id>urn:uuid:{{id}}</id>
<title>{{title}}</title>
<updated>{{updated}}</updated>
{{#dump_partial_entries}}
<link rel="alternate"
href="{{endpoint_root}}/entry/{{{id}}}"
type="application/atom+xml;type=entry;profile=opds-catalog"/>
{{/dump_partial_entries}}{{^dump_partial_entries}} <summary>{{description}}</summary>
<language>{{language}}</language>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
{{#icons}}<link rel="http://opds-spec.org/image/thumbnail"
href="{{root}}/catalog/v2/illustration/{{{content_id}}}/?size={{icon_size}}"
type="{{icon_mimetype}};width={{icon_size}};height={{icon_size}};scale=1"/>
{{/icons}}<link type="text/html" href="{{root}}/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
<dc:issued>{{book_date}}</dc:issued>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
{{/dump_partial_entries}}
</entry>

View File

@@ -1,28 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="https://specs.opds.io/opds-1.2"
xmlns:thr="http://purl.org/syndication/thread/1.0">
<id>{{feed_id}}</id>
<link rel="self"
href="{{endpoint_root}}/languages"
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
<link rel="start"
href="{{endpoint_root}}/root.xml"
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
<title>List of languages</title>
<updated>{{date}}</updated>
{{#languages}}
<entry>
<title>{{lang_self_name}}</title>
<dc:language>{{{lang_code}}}</dc:language>
<thr:count>{{book_count}}</thr:count>
<link rel="subsection"
href="{{endpoint_root}}/entries?lang={{{lang_code}}}"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<updated>{{updated}}</updated>
<id>{{id}}</id>
</entry>
{{/languages}}
</feed>

View File

@@ -23,15 +23,6 @@
<id>{{all_entries_feed_id}}</id>
<content type="text">All entries from this catalog.</content>
</entry>
<entry>
<title>All entries (partial)</title>
<link rel="subsection"
href="{{endpoint_root}}/partial_entries"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<updated>{{date}}</updated>
<id>{{partial_entries_feed_id}}</id>
<content type="text">All entries from this catalog in partial format.</content>
</entry>
<entry>
<title>List of categories</title>
<link rel="subsection"
@@ -41,13 +32,4 @@
<id>{{category_list_feed_id}}</id>
<content type="text">List of all categories in this catalog.</content>
</entry>
<entry>
<title>List of languages</title>
<link rel="subsection"
href="{{endpoint_root}}/languages"
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
<updated>{{date}}</updated>
<id>{{language_list_feed_id}}</id>
<content type="text">List of all languages in this catalog.</content>
</entry>
</feed>

View File

@@ -1,18 +0,0 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
<title>{{PAGE_TITLE}}</title>
{{#CSS_URL}}
<link type="text/css" href="{{{CSS_URL}}}" rel="Stylesheet" />
{{/CSS_URL}}
</head>
<body>
<h1>{{PAGE_HEADING}}</h1>
{{#details}}
<p>
{{{p}}}
</p>
{{/details}}
</body>
</html>

View File

@@ -2,7 +2,6 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title>Welcome to Kiwix Server</title>
<script
type="text/javascript"
@@ -22,58 +21,186 @@
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css"
rel="Stylesheet"
/>
<link
type="text/css"
href="{{root}}/skin/index.css"
rel="Stylesheet"
/>
<style>
@font-face {
font-family: "poppins";
src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype");
html {
min-height: 100%;
position: relative;
}
@font-face {
font-family: "roboto";
src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype");
body {
background: radial-gradient(#eeeeee 15%, transparent 16%) 0 0,
radial-gradient(#eeeeee 15%, transparent 16%) 8px 8px,
radial-gradient(rgba(255, 255, 255, 0.1) 15%, transparent 20%) 0 1px,
radial-gradient(rgba(255, 255, 255, 0.1) 15%, transparent 20%) 8px 9px;
background-color: #e8e8e8;
background-size: 16px 16px;
margin-left: auto;
margin-right: auto;
max-width: 1100px;
min-height: 100%;
}
.book__list {
text-align: center;
}
.kiwixHomeBody {
position: relative;
text-align: center;
min-height: 100%;
margin: 0 0 15px;
}
.book {
display: inline-block;
vertical-align: bottom;
margin: 8px;
padding: 12px 15px;
width: 300px;
border: 1px solid #ccc;
border-radius: 8px;
text-align: left;
color: #000;
font-family: sans-serif;
font-size: 13px;
background-color: #f1f1f1;
box-shadow: 2px 2px 5px 0 #ccc;
}
#kiwixfooter {
text-align: center;
margin: 0.5em;
position: absolute;
bottom: 0;
left: 46%;
}
.kiwixHomeNavbar {
display: flex;
justify-content: center;
}
.kiwixFilter {
margin: 8px 10px;
}
.kiwixSearch, .searchButton {
margin: 0 13px 0 0;
}
.kiwixSearchForm {
margin: 8px 10px;
float: right;
}
@media (max-width: 1100px) {
.kiwixHomeBody {
padding: 0 125px;
}
}
.book:hover {
background-color: #f9f9f9;
box-shadow: none;
}
.book__background {
background-repeat: no-repeat;
background-size: 48px 48px;
background-position: top right;
}
.book__title {
padding: 0 55px 0 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-size: 18px;
color: #0645ad;
line-height: 1em;
}
.book__description {
padding: 5px 55px 5px 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-size: 15px;
line-height: 1em;
}
.book__info {
color: #777;
font-weight: bold;
font-size: 13px;
line-height: 1em;
}
a:link {
text-decoration: none;
}
a:visited {
text-decoration: none;
}
.noResults {
position: absolute;
top: 48%;
left: 42%;
}
.loader-spinner {
position: absolute;
top: -50%;
left: 50%;
border: 5px solid #f3f3f3;
border-radius: 50%;
border-top: 5px solid #3498db;
width: 40px;
height: 40px;
margin: auto;
-webkit-animation: spin 1s linear infinite; /* Safari */
animation: spin 1s linear infinite;
margin-top: 35px;
margin-bottom: -35px;
z-index: 2;
}
/* Safari */
@-webkit-keyframes spin {
0% { -webkit-transform: rotate(0deg); }
100% { -webkit-transform: rotate(360deg); }
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.loader {
position: relative;
height: 70px;
width: 100%;
}
.fadeOut {
position: fixed;
display: none;
bottom: 0;
left: 0;
z-index: 1;
background: linear-gradient(180deg, rgba(232, 232, 232, 0) 0%, rgb(232, 232, 232) 100%);
height: 80px;
width: 100%;
}
.spacer {
height: 20px;
background: transparent;
}
</style>
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
<script src="{{root}}/skin/categoryList.js"></script>
<script src="{{root}}/skin/langList.js"></script>
<script src="{{root}}/skin/iso6391To3.js"></script>
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
</head>
<body>
<div class='kiwixNav'>
<div class="kiwixNav__filters">
<div class="kiwixNav__select">
<select name="lang" id="languageFilter" class='kiwixNav__kiwixFilter filter'>
<option value="" selected>All languages</option>
</select>
</div>
<div class="kiwixNav__select">
<select name="category" id="categoryFilter" class='kiwixNav__kiwixFilter filter'>
<option value="" selected>All categories</option>
</select>
</div>
</div>
<form id='kiwixSearchForm' class='kiwixNav__SearchForm'>
<input type="text" name="q" placeholder="Search" id="searchFilter" class='kiwixSearch filter'>
<body class="kiwix">
<div class='kiwixHomeNavbar'>
<select name="lang" id="languageFilter" class='kiwixFilter'>
<option value="" selected>All languages</option>
</select>
<select name="category" id="categoryFilter" class='kiwixFilter'>
<option value="" selected>All categories</option>
</select>
<form id='kiwixSearchForm' class='kiwixSearchForm'>
<input type="text" name="q" id="searchFilter" class='kiwixSearch'>
<input type="submit" class="searchButton" value="Search"/>
</form>
</div>
<div class="kiwixHomeBody">
<h3 class="kiwixHomeBody__results"></h3>
<div class="book__list"></div>
<div id="fadeOut" class="fadeOut"></div>
</div>
<div class="loader" style="position: absolute; top: 50%"><div class="loader-spinner"></div></div>
<div id="kiwixfooter" class="kiwixfooter">Powered by&nbsp;<a href="https://kiwix.org">Kiwix</a></div>
<div class="loader"><div class="loader-spinner"></div></div>
<div class="spacer"></div>
<div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>
</body>
<script>
function closeModal() {
for(modal of document.getElementsByClassName('modal-wrapper')) {
modal.remove();
}
}
</script>
</html>

View File

@@ -0,0 +1,103 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta content="text/html; charset=utf-8" http-equiv="content-type" />
<style type="text/css">
body{
color: #000000;
font: small/normal Arial,Helvetica,Sans-Serif;
margin-top: 0.5em;
font-size: 90%;
}
a{
color: #04c;
}
a:visited {
color: #639
}
a:hover {
text-decoration: underline
}
.header {
font-size: 120%;
}
ul {
margin:0;
padding:0
}
.results {
font-size: 110%;
}
.results li {
list-style-type:none;
margin-top: 0.5em;
}
.results a {
font-size: 110%;
text-decoration: underline
}
cite {
font-style:normal;
word-wrap:break-word;
display: block;
font-size: 100%;
}
.informations {
color: #388222;
font-size: 100%;
}
.footer {
padding: 0;
margin-top: 1em;
width: 100%;
float: left
}
.footer a, .footer span {
display: block;
padding: .3em .7em;
margin: 0 .38em 0 0;
text-align:center;
text-decoration: none;
}
.footer a:hover {
background: #ededed;
}
.footer ul, .footer li {
list-style:none;
margin: 0;
padding: 0;
}
.footer li {
float: left;
}
.selected {
background: #ededed;
}
</style>
<title>Fulltext search unavailable</title>
</head>
<body bgcolor="white">
<div class="header">Not found</div>
<p>
There is no article with the title <b> "{{pattern}}"</b>
and the fulltext search engine is not available for this content.
</p>
</body>
</html>

View File

@@ -57,11 +57,6 @@
font-size: 100%;
}
.book-title {
color: #662200;
font-size: 100%;
}
.footer {
padding: 0;
margin-top: 1em;
@@ -107,11 +102,11 @@
</b> of <b>
{{count}}
</b> for <b>
"{{{searchPattern}}}"
{{searchPattern}}
</b>
{{/hasResults}}
{{^hasResults}}
No results were found for <b>"{{{searchPattern}}}"</b>
No results were found for <b>{{searchPattern}}</b>
{{/hasResults}}
</div>
@@ -125,9 +120,6 @@
{{#snippet}}
<cite>{{>snippet}}...</cite>
{{/snippet}}
{{#bookTitle}}
<div class="book-title">from {{bookTitle}}</div>
{{/bookTitle}}
{{#wordCount}}
<div class="informations">{{wordCount}} words</div>
{{/wordCount}}

View File

@@ -5,18 +5,18 @@
<form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
{{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
<label for="kiwixsearchbox">&#x1f50d;</label>
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text" title="Search '{{title}}'" aria-label="Search '{{title}}'">
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
</form>
</div>
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
<div class="kiwix_button_cont">
{{#withlibrarybutton}}
<a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="{{root}}/"><button>&#x1f3e0;</button></a>
<a id="kiwix_serve_taskbar_library_button" href="{{root}}/"><button>&#x1f3e0;</button></a>
{{/withlibrarybutton}}
{{#hascontent}}
<a id="kiwix_serve_taskbar_home_button" title="Go to the main page of '{{title}}'" aria-label="Go to the main page of '{{title}}'" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
<a id="kiwix_serve_taskbar_random_button" title="Go to a randomly selected page" aria-label="Go to a randomly selected page"
<a id="kiwix_serve_taskbar_home_button" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
<a id="kiwix_serve_taskbar_random_button"
href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
{{/hascontent}}
</div>

View File

@@ -2,6 +2,42 @@
#include "../include/book.h"
#include <pugixml.hpp>
TEST(BookTest, updateTest)
{
kiwix::Book book;
book.setId("xyz");
book.setReadOnly(false);
book.setPath("/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim");
book.setPathValid(true);
book.setUrl("book-url");
book.setTags("youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes");
book.setName("skin-of-color-society_en_all");
book.setFavicon("book-favicon");
book.setFaviconMimeType("book-favicon-mimetype");
kiwix::Book newBook;
newBook.setReadOnly(true);
EXPECT_FALSE(newBook.update(book));
newBook.setReadOnly(false);
EXPECT_FALSE(newBook.update(book));
newBook.setId("xyz");
EXPECT_TRUE(newBook.update(book));
EXPECT_EQ(newBook.readOnly(), book.readOnly());
EXPECT_EQ(newBook.getPath(), book.getPath());
EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
EXPECT_EQ(newBook.getUrl(), book.getUrl());
EXPECT_EQ(newBook.getTags(), book.getTags());
EXPECT_EQ(newBook.getCategory(), book.getCategory());
EXPECT_EQ(newBook.getName(), book.getName());
EXPECT_EQ(newBook.getFavicon(), book.getFavicon());
EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType());
}
namespace
{
@@ -32,9 +68,6 @@ TEST(BookTest, updateFromXMLTest)
articleCount="123456"
mediaCount="234567"
size="345678"
favicon="ZmFrZS1ib29rLWZhdmljb24tZGF0YQ=="
faviconMimeType="text/plain"
faviconUrl="http://who.org/zara.fav"
>
</book>
)");
@@ -52,10 +85,6 @@ TEST(BookTest, updateFromXMLTest)
EXPECT_EQ(book.getArticleCount(), 123456U);
EXPECT_EQ(book.getMediaCount(), 234567U);
EXPECT_EQ(book.getSize(), 345678U*1024U);
auto defaultIllustration = book.getIllustration(48);
EXPECT_EQ(defaultIllustration->getData(), "fake-book-favicon-data");
EXPECT_EQ(defaultIllustration->mimeType, "text/plain");
EXPECT_EQ(defaultIllustration->url, "http://who.org/zara.fav");
}
TEST(BookTest, updateFromXMLCategoryHandlingTest)
@@ -137,76 +166,3 @@ TEST(BookTest, updateCopiesCategory)
newBook.update(book);
EXPECT_EQ(newBook.getCategory(), "ted");
}
TEST(BookTest, updateTest)
{
const XMLDoc xml(R"(
<book id="xyz"
path="/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim"
url="book-url"
name="skin-of-color-society_en_all"
tags="youtube;_videos:yes;_ftindex:yes;_ftindex:yes;_pictures:yes;_details:yes"
favicon="Ym9vay1mYXZpY29u"
faviconMimeType="book-favicon-mimetype"
>
</book>
)");
kiwix::Book book;
book.updateFromXml(xml.child("book"), "/data/zim");
book.setReadOnly(false);
book.setPathValid(true);
kiwix::Book newBook;
newBook.setReadOnly(true);
EXPECT_FALSE(newBook.update(book));
newBook.setReadOnly(false);
EXPECT_FALSE(newBook.update(book));
newBook.setId("xyz");
EXPECT_TRUE(newBook.update(book));
EXPECT_EQ(newBook.readOnly(), book.readOnly());
EXPECT_EQ(newBook.getPath(), book.getPath());
EXPECT_EQ(newBook.isPathValid(), book.isPathValid());
EXPECT_EQ(newBook.getUrl(), book.getUrl());
EXPECT_EQ(newBook.getTags(), book.getTags());
EXPECT_EQ(newBook.getCategory(), book.getCategory());
EXPECT_EQ(newBook.getName(), book.getName());
auto defaultIllustration = book.getIllustration(48);
auto newDefaultIllustration = newBook.getIllustration(48);
EXPECT_EQ(newDefaultIllustration->getData(), defaultIllustration->getData());
EXPECT_EQ(newDefaultIllustration->mimeType, defaultIllustration->mimeType);
}
namespace
{
std::string path2HumanReadableId(const std::string& path)
{
const XMLDoc xml("<book id=\"xyz\" path=\"" + path + "\"></book>");
kiwix::Book book;
book.updateFromXml(xml.child("book"), "/data/zim");
return book.getHumanReadableIdFromPath();
}
} // unnamed namespace
TEST(BookTest, getHumanReadableIdFromPath)
{
EXPECT_EQ("abc", path2HumanReadableId("abc.zim"));
EXPECT_EQ("abc", path2HumanReadableId("ABC.zim"));
EXPECT_EQ("abc", path2HumanReadableId("âbç.zim"));
EXPECT_EQ("ancient", path2HumanReadableId("ancient.zimbabwe"));
EXPECT_EQ("ab_cd", path2HumanReadableId("ab cd.zim"));
#ifdef _WIN32
EXPECT_EQ("abc", path2HumanReadableId("C:\\Data\\ZIM\\abc.zim"));
#else
EXPECT_EQ("abc", path2HumanReadableId("/Data/ZIM/abc.zim"));
#endif
EXPECT_EQ("3plus2", path2HumanReadableId("3+2.zim"));
}

View File

@@ -14,8 +14,6 @@
articleCount="284"
mediaCount="2"
size="556"
faviconMimeType="image/png"
favicon="SOME DATA"
></book>
<book
id="raycharles_uncategorized"
@@ -23,11 +21,11 @@
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
title="Ray (uncategorized) Charles"
description="No category is assigned to this library entry."
language="rus"
language="eng"
creator="Wikipedia"
publisher="Kiwix"
date="2020-03-31"
name="wikipedia_ru_ray_charles"
name="wikipedia_en_ray_charles"
tags="unittest;wikipedia;_pictures:no;_videos:no;_details:no"
articleCount="284"
mediaCount="2"
@@ -35,15 +33,15 @@
></book>
<book
id="charlesray"
path="./zimfile&amp;other.zim"
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim"
path="./zimfile.zim"
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
title="Charles, Ray"
description="Wikipedia articles about Ray Charles"
language="fra"
language="eng"
creator="Wikipedia"
publisher="Kiwix"
date="2020-03-31"
name="wikipedia_fr_ray_charles"
name="wikipedia_en_ray_charles"
tags="unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes"
articleCount="284"
mediaCount="2"

View File

Binary file not shown.

View File

@@ -1 +0,0 @@
zimfile.zim

Some files were not shown because too many files have changed in this diff Show More