mirror of
https://github.com/kiwix/kiwix-tools.git
synced 2026-01-10 23:18:59 -05:00
Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62e6c7ffb0 | ||
|
|
1df899ea87 | ||
|
|
3dd357e8ac | ||
|
|
2744879df0 | ||
|
|
19c3ebe629 | ||
|
|
cccd17a394 | ||
|
|
e9676ae66f | ||
|
|
2fd393285f | ||
|
|
a44fd1a4d0 | ||
|
|
a9ff2aebee | ||
|
|
a2b944618f | ||
|
|
d27abb41b7 | ||
|
|
5f12d0349f | ||
|
|
7ec2308053 | ||
|
|
e8b33a7136 | ||
|
|
ca943b9d0b | ||
|
|
c6757dd309 | ||
|
|
23b0ff3d01 | ||
|
|
93116c76b3 | ||
|
|
4aca5ff551 | ||
|
|
4eb38869c1 | ||
|
|
d17fd2af94 | ||
|
|
48db92af09 | ||
|
|
612081ec51 | ||
|
|
4b5eb482b5 | ||
|
|
28997ccd58 | ||
|
|
9dd042d813 | ||
|
|
61580bb738 | ||
|
|
dbae6439b2 | ||
|
|
d6094b4112 | ||
|
|
64f1aa2408 | ||
|
|
ed1bae76c3 | ||
|
|
7418702a07 | ||
|
|
796614fd7d | ||
|
|
2ba9037687 | ||
|
|
22be7c0dcf | ||
|
|
3fc9525754 | ||
|
|
a9d8f77526 | ||
|
|
1109b62570 | ||
|
|
3248c0f233 | ||
|
|
f8f66288ea |
4
.github/FUNDING.yml
vendored
4
.github/FUNDING.yml
vendored
@@ -1,6 +1,6 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: # Replace with a single Ko-fi username
|
||||
@@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
|
||||
liberapay: # Replace with a single Liberapay username
|
||||
issuehunt: # Replace with a single IssueHunt username
|
||||
otechie: # Replace with a single Otechie username
|
||||
custom: https://kiwix.org/support-us/
|
||||
custom: # https://kiwix.org/support-us/
|
||||
|
||||
77
.github/workflows/pr.yml
vendored
Normal file
77
.github/workflows/pr.yml
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
name: CI
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
Linux:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target:
|
||||
- native_static
|
||||
- native_dyn
|
||||
- win32_static
|
||||
- win32_dyn
|
||||
include:
|
||||
- target: native_static
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: native_dyn
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: win32_static
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
- target: win32_dyn
|
||||
image_variant: f31
|
||||
lib_postfix: '64'
|
||||
env:
|
||||
HOME: /home/runner
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: python
|
||||
run: |
|
||||
import json
|
||||
from subprocess import check_call
|
||||
from os import environ
|
||||
with open(environ['GITHUB_EVENT_PATH'], 'r') as f:
|
||||
content = f.read()
|
||||
event_data = json.loads(content)
|
||||
try:
|
||||
branch_ref = event_data['ref'].split('/')[-1]
|
||||
except KeyError:
|
||||
branch_ref = event_data['pull_request']['head']['ref']
|
||||
print("Cloning branch", branch_ref)
|
||||
command = [
|
||||
'git', 'clone',
|
||||
'https://github.com/${{github.repository}}',
|
||||
'--depth=1',
|
||||
'--branch', branch_ref
|
||||
]
|
||||
check_call(command, cwd=environ['HOME'])
|
||||
- name: Install deps
|
||||
shell: bash
|
||||
run: |
|
||||
ARCHIVE_NAME=deps2_${OS_NAME}_${{matrix.target}}_kiwix-tools.tar.xz
|
||||
wget -O- http://tmp.kiwix.org/ci/${ARCHIVE_NAME} | tar -xJ -C /home/runner
|
||||
- name: Compile
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{matrix.target}}" =~ .*_static ]]; then
|
||||
MESON_OPTION="-Dstatic-linkage=true"
|
||||
else
|
||||
MESON_OPTION=""
|
||||
fi
|
||||
if [[ ! "${{matrix.target}}" =~ native_.* ]]; then
|
||||
MESON_OPTION="$MESON_OPTION --cross-file $HOME/BUILD_${{matrix.target}}/meson_cross_file.txt"
|
||||
fi
|
||||
cd $HOME/kiwix-tools
|
||||
meson . build ${MESON_OPTION}
|
||||
cd build
|
||||
ninja
|
||||
env:
|
||||
PKG_CONFIG_PATH: "/home/runner/BUILD_${{matrix.target}}/INSTALL/lib/pkgconfig:/home/runner/BUILD_${{matrix.target}}/INSTALL/lib${{matrix.lib_postfix}}/pkgconfig"
|
||||
CPPFLAGS: "-I/home/runner/BUILD_${{matrix.target}}/INSTALL/include"
|
||||
12
.travis.yml
12
.travis.yml
@@ -1,12 +0,0 @@
|
||||
language: cpp
|
||||
dist: xenial
|
||||
sudo: true
|
||||
cache: ccache
|
||||
install: travis/install_deps.sh
|
||||
script: travis/compile.sh
|
||||
env:
|
||||
matrix:
|
||||
- PLATFORM="native_static"
|
||||
- PLATFORM="native_dyn"
|
||||
- PLATFORM="win32_static"
|
||||
- PLATFORM="win32_dyn"
|
||||
10
Changelog
10
Changelog
@@ -1,3 +1,13 @@
|
||||
kiwix-tools 3.0.2
|
||||
=================
|
||||
|
||||
* New option --version for all tools
|
||||
* Remove benchmark.sh file.
|
||||
* [DOCKER] Add ability to download a file at container start.
|
||||
* [CI] Move to github actions instead of travis.
|
||||
* [SERVER] Trust the given library by default.
|
||||
* [SERVER] Add shortcut alias for option `--address` and `--nodatealias`
|
||||
|
||||
kiwix-tools 3.0.1
|
||||
=================
|
||||
|
||||
|
||||
71
README.md
71
README.md
@@ -1,12 +1,30 @@
|
||||
Kiwix tools
|
||||
===========
|
||||
|
||||
The Kiwix tools is a collection of Kiwix related command line tools:
|
||||
The Kiwix tools is a collection of [Kiwix](https://kiwix.org) related
|
||||
command line tools:
|
||||
* kiwix-manage: Manage XML based library of ZIM files
|
||||
* kiwix-read: Read ZIM file content
|
||||
* kiwix-search: Fulltext search in ZIM files
|
||||
* kiwix-serve: HTTP daemon serving ZIM files
|
||||
|
||||
[](https://travis-ci.com/kiwix/kiwix-tools)
|
||||
[](https://www.codefactor.io/repository/github/kiwix/kiwix-tools)
|
||||
[](https://www.gnu.org/licenses/gpl-3.0)
|
||||
|
||||
Distribution
|
||||
------------
|
||||
|
||||
#### Official
|
||||
|
||||
[](https://download.kiwix.org/release/kiwix-tools/)
|
||||
[](https://hub.docker.com/r/kiwix/kiwix-serve)
|
||||
|
||||
#### Contribution
|
||||
|
||||
[](https://aur.archlinux.org/packages/kiwix-tools/)
|
||||
[](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h)
|
||||
|
||||
Disclaimer
|
||||
----------
|
||||
|
||||
@@ -21,7 +39,9 @@ Preamble
|
||||
Although the Kiwix tools can be compiled/cross-compiled on/for many
|
||||
sytems, the following documentation explains how to do it on POSIX
|
||||
ones. It is primarly thought for GNU/Linux systems and has been tested
|
||||
on recent releases of Ubuntu and Fedora.
|
||||
on recent releases of
|
||||
[Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) and
|
||||
[Fedora](https://getfedora.org).
|
||||
|
||||
Dependencies
|
||||
------------
|
||||
@@ -29,10 +49,10 @@ Dependencies
|
||||
The Kiwix tools rely on a few third party software libraries. They are
|
||||
prerequisites to the Kiwix tools compilation. Therefore, following
|
||||
libraries need to be available:
|
||||
|
||||
* [Kiwix lib](https://github.com/kiwix/kiwix-lib) (no package so far)
|
||||
* [Libmicrohttpd](https://www.gnu.org/software/libmicrohttpd/) (package libmicrohttpd-dev on Ubuntu)
|
||||
* [Zlib](https://www.zlib.net/) (package zlib1g-dev on Ubuntu)
|
||||
* [Libmicrohttpd](https://www.gnu.org/software/libmicrohttpd/)
|
||||
(package `libmicrohttpd-dev` on Debian/Ubuntu)
|
||||
* [Zlib](https://www.zlib.net/) (package `zlib1g-dev` on Debian/Ubuntu)
|
||||
|
||||
These dependencies may or may not be packaged by your operating
|
||||
system. They may also be packaged but only in an older version. They
|
||||
@@ -45,8 +65,8 @@ If you want to install these dependencies locally, then use the
|
||||
kiwix-tools directory as install prefix.
|
||||
|
||||
If you want to compile Kiwix tools statically, the dependencies should
|
||||
be compiled statically (provide a lib...a library), for example by
|
||||
using "--enable-static" with "./configure".
|
||||
be compiled statically (provide a `lib...a` library), for example by
|
||||
using `--enable-static` with `./configure`.
|
||||
|
||||
If you compile manually Libmicrohttpd, you might need to compile it
|
||||
without GNU TLS, a bug here will empeach further compilation of Kiwix
|
||||
@@ -58,18 +78,19 @@ Environment
|
||||
The Kiwix tools build using [Meson](http://mesonbuild.com/) version
|
||||
0.43 or higher. Meson relies itself on Ninja, pkg-config and few other
|
||||
compilation tools. Install them first:
|
||||
* Meson
|
||||
* Ninja
|
||||
* Pkg-config
|
||||
* [Meson](http://mesonbuild.com/)
|
||||
* [Ninja](https://ninja-build.org/)
|
||||
* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
|
||||
|
||||
These tools should be packaged if you use a cutting edge operating
|
||||
system. If not, have a look to the "Troubleshooting" section.
|
||||
system. If not, have a look to the [Troubleshooting](#Troubleshooting)
|
||||
section.
|
||||
|
||||
Compilation
|
||||
-----------
|
||||
|
||||
Once all dependencies are installed, you can compile Kiwix tools with:
|
||||
```
|
||||
```bash
|
||||
meson . build
|
||||
ninja -C build
|
||||
```
|
||||
@@ -84,38 +105,35 @@ Installation
|
||||
------------
|
||||
|
||||
If you want to install the Kiwix tools, here we go:
|
||||
|
||||
```
|
||||
```bash
|
||||
ninja -C build install
|
||||
```
|
||||
|
||||
You might need to run the command as root (or using 'sudo'), depending
|
||||
where you want to install the Kiwix tools. After the installation
|
||||
succeeded, you may need to run ldconfig (as root).
|
||||
You might need to run the command as `root` (or using `sudo`),
|
||||
depending where you want to install the Kiwix tools. After the
|
||||
installation succeeded, you may need to run ldconfig (as `root`).
|
||||
|
||||
Uninstallation
|
||||
------------
|
||||
|
||||
If you want to uninstall the Kiwix tools:
|
||||
|
||||
```
|
||||
```bash
|
||||
ninja -C build uninstall
|
||||
```
|
||||
|
||||
Like for the installation, you might need to run the command as root
|
||||
(or using 'sudo').
|
||||
Like for the installation, you might need to run the command as `root`
|
||||
(or using `sudo`).
|
||||
|
||||
Docker
|
||||
------
|
||||
|
||||
An official Docker image of `kiwix-serve` can be found at
|
||||
https://hub.docker.com/r/kiwix/kiwix-serve.
|
||||
An official Docker image of `kiwix-serve` can be found in the [Docker Hub](https://hub.docker.com/r/kiwix/kiwix-serve).
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
If you need to install Meson "manually":
|
||||
```
|
||||
```bash
|
||||
virtualenv -p python3 ./ # Create virtualenv
|
||||
source bin/activate # Activate the virtualenv
|
||||
pip3 install meson # Install Meson
|
||||
@@ -123,7 +141,7 @@ hash -r # Refresh bash paths
|
||||
```
|
||||
|
||||
If you need to install Ninja "manually":
|
||||
```
|
||||
```bash
|
||||
git clone git://github.com/ninja-build/ninja.git
|
||||
cd ninja
|
||||
git checkout release
|
||||
@@ -142,4 +160,5 @@ repository.
|
||||
License
|
||||
-------
|
||||
|
||||
GPLv3 or later, see COPYING for more details.
|
||||
[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
|
||||
[COPYING](COPYING) for more details.
|
||||
|
||||
@@ -4,7 +4,7 @@ LABEL maintainer Emmanuel Engelhart <kelson@kiwix.org>
|
||||
# Install kiwix-serve
|
||||
WORKDIR /
|
||||
RUN apk add --no-cache curl bzip2
|
||||
RUN curl -kL https://download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-x86_64-1.2.1.tar.gz | tar -xz && \
|
||||
RUN curl -kL https://download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-x86_64.tar.gz | tar -xz && \
|
||||
mv kiwix-tools*/kiwix-serve /usr/local/bin && \
|
||||
rm -r kiwix-tools*
|
||||
|
||||
@@ -12,4 +12,7 @@ RUN curl -kL https://download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-x8
|
||||
EXPOSE 80
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
ENTRYPOINT ["/usr/local/bin/kiwix-serve", "--port", "80"]
|
||||
|
||||
COPY ./start.sh /usr/local/bin/
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/start.sh"]
|
||||
|
||||
@@ -1,13 +1,26 @@
|
||||
kiwix-serve Docker image
|
||||
Kiwix serve Docker image
|
||||
========================
|
||||
|
||||
With local ZIM file
|
||||
-------------------
|
||||
|
||||
* Download a ZIM file from <https://wiki.kiwix.org/wiki/Content>
|
||||
|
||||
* Given `wikipedia.zim` resides in `/tmp/zim/`, execute the following:
|
||||
|
||||
```
|
||||
```bash
|
||||
docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim
|
||||
```
|
||||
|
||||
With remote ZIM file
|
||||
--------------------
|
||||
|
||||
```bash
|
||||
docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:80 kiwix/kiwix-serve
|
||||
```
|
||||
|
||||
Screenshots
|
||||
-----------
|
||||
|
||||

|
||||

|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 187 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 199 KiB After Width: | Height: | Size: 302 KiB |
18
docker/server/start.sh
Executable file
18
docker/server/start.sh
Executable file
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Download if necessary a file
|
||||
if [ ! -z "$DOWNLOAD" ]
|
||||
then
|
||||
ZIM=`basename $DOWNLOAD`
|
||||
wget $DOWNLOAD -O "$ZIM"
|
||||
|
||||
# Set arguments
|
||||
if [ "$#" -eq "0" ]
|
||||
then
|
||||
set -- "$@" $ZIM
|
||||
fi
|
||||
fi
|
||||
|
||||
CMD="/usr/local/bin/kiwix-serve --port=80 $@"
|
||||
echo $CMD
|
||||
$CMD
|
||||
@@ -1,17 +1,19 @@
|
||||
project('kiwix-tools', 'cpp',
|
||||
version : '3.0.1',
|
||||
version : '3.0.2',
|
||||
license : 'GPL',
|
||||
default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
|
||||
|
||||
compiler = meson.get_compiler('cpp')
|
||||
|
||||
add_global_arguments('-DKIWIX_TOOLS_VERSION="@0@"'.format(meson.project_version()), language : 'cpp')
|
||||
|
||||
static_linkage = get_option('static-linkage')
|
||||
if static_linkage
|
||||
add_global_link_arguments('-static-libstdc++', '--static', language:'cpp')
|
||||
endif
|
||||
|
||||
thread_dep = dependency('threads')
|
||||
kiwixlib_dep = dependency('kiwix', version:'>=6.0.0', static:static_linkage)
|
||||
kiwixlib_dep = dependency('kiwix', version:'>=9.0.0', static:static_linkage)
|
||||
microhttpd_dep = dependency('libmicrohttpd', static:static_linkage)
|
||||
z_dep = dependency('zlib', static:static_linkage)
|
||||
|
||||
@@ -24,5 +26,4 @@ if static_linkage
|
||||
endif
|
||||
endif
|
||||
|
||||
#subdir('include')
|
||||
subdir('src')
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
* Copyright 2011-2019 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
@@ -23,6 +23,8 @@
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "../version.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
enum supportedAction { NONE, ADD, SHOW, REMOVE };
|
||||
@@ -78,6 +80,9 @@ void usage()
|
||||
<< "\t\t\t--zimPathToSave=CUSTOM_ZIM_PATH to replace the current ZIM file path" << std::endl
|
||||
<< "\t\t\t--url=HTTP_ZIM_URL to create an \"url\" attribute for the online version of the ZIM file" << std::endl
|
||||
<< std::endl
|
||||
<< "\t\t\tOther options:" << std::endl
|
||||
<< "\t\t\t-v, --version to print the software version" << std::endl
|
||||
<< std::endl
|
||||
|
||||
<< "Examples:" << std::endl
|
||||
<< "\tAdd ZIM files to library: kiwix-manage my_library.xml add first.zim second.zim" << std::endl
|
||||
@@ -144,11 +149,9 @@ int handle_add(kiwix::Library* library, const std::string& libraryPath,
|
||||
case 'u':
|
||||
url = optarg;
|
||||
break;
|
||||
|
||||
case 'o':
|
||||
origID = optarg;
|
||||
break;
|
||||
|
||||
case 'z':
|
||||
zimPathToSave = optarg;
|
||||
break;
|
||||
@@ -217,7 +220,27 @@ int main(int argc, char** argv)
|
||||
supportedAction action = NONE;
|
||||
kiwix::Library library;
|
||||
|
||||
/* Argument parsing */
|
||||
/* General argument parsing */
|
||||
static struct option long_options[] = {
|
||||
{"version", no_argument, 0, 'v'},
|
||||
{0, 0, 0, 0}
|
||||
};
|
||||
|
||||
int option_index = 0;
|
||||
int c;
|
||||
while (true) {
|
||||
c = getopt_long(argc, argv, "v", long_options, &option_index);
|
||||
if (c == -1)
|
||||
break;
|
||||
|
||||
switch (c) {
|
||||
case 'v':
|
||||
version();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Action related argument parsing */
|
||||
if (argc > 2) {
|
||||
libraryPath = argv[1];
|
||||
string actionString = argv[2];
|
||||
|
||||
@@ -22,9 +22,11 @@
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "../version.h"
|
||||
|
||||
void usage()
|
||||
{
|
||||
cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
|
||||
cout << "Usage: kiwix-read [--verbose] [--version] --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
@@ -42,15 +44,19 @@ int main(int argc, char** argv)
|
||||
while (42) {
|
||||
static struct option long_options[]
|
||||
= {{"verbose", no_argument, 0, 'v'},
|
||||
{"version", no_argument, 0, 'V'},
|
||||
{"suggest", required_argument, 0, 's'},
|
||||
{0, 0, 0, 0}};
|
||||
|
||||
if (c != -1) {
|
||||
c = getopt_long(argc, argv, "vs:", long_options, &option_index);
|
||||
c = getopt_long(argc, argv, "Vvs:", long_options, &option_index);
|
||||
|
||||
switch (c) {
|
||||
case 'v':
|
||||
break;
|
||||
case 'V':
|
||||
version();
|
||||
return 0;
|
||||
case 's':
|
||||
pattern = optarg;
|
||||
break;
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
#include <kiwix/reader.h>
|
||||
#include <kiwix/searcher.h>
|
||||
|
||||
#include "../version.h"
|
||||
|
||||
void usage()
|
||||
{
|
||||
@@ -29,7 +30,8 @@ void usage()
|
||||
<< " ZIM is the full path of the ZIM file." << endl
|
||||
<< " PATTERN is/are word(s) - or part of - to search in the ZIM." << endl << endl
|
||||
<< " -s, --suggestion\tSuggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution." << endl
|
||||
<< " -v, --verbose\t\tGive details about the search process" << endl;
|
||||
<< " -v, --verbose\t\tGive details about the search process" << endl
|
||||
<< " -V, --version\t\tPrint software version" << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
@@ -54,15 +56,19 @@ int main(int argc, char** argv)
|
||||
static struct option long_options[]
|
||||
= {{"verbose", no_argument, 0, 'v'},
|
||||
{"suggestion", no_argument, 0, 's'},
|
||||
{"version", no_argument, 0, 'V'},
|
||||
{0, 0, 0, 0}};
|
||||
|
||||
if (c != -1) {
|
||||
c = getopt_long(argc, argv, "vsb:", long_options, &option_index);
|
||||
c = getopt_long(argc, argv, "Vvsb:", long_options, &option_index);
|
||||
|
||||
switch (c) {
|
||||
case 'v':
|
||||
verboseFlag = true;
|
||||
break;
|
||||
case 'V':
|
||||
version();
|
||||
return 0;
|
||||
case 's':
|
||||
suggestionFlag = true;
|
||||
break;
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Parse command line
|
||||
zim=$1
|
||||
serve=$2
|
||||
if (( $# != 2 ))
|
||||
then
|
||||
echo "Usage: ./benchmark ZIM_PATH KIWIX-SERVE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Constants
|
||||
delay=0.01
|
||||
top_log=/tmp/top.log
|
||||
top_calc=/tmp/top.calc
|
||||
|
||||
# Available memory
|
||||
memory_kb=`cat /proc/meminfo | grep "MemTotal" | tr -d " " | cut -f2 -d: | sed -e 's/[^0123456789]//g'`
|
||||
memory_mb=`echo "scale=2;$memory_kb/1024" | bc -l`
|
||||
|
||||
# Start Kiwix-serve
|
||||
echo "Starting kiwix-serve to be tested..."
|
||||
$serve --port=8080 "$zim" &
|
||||
serve_pid=$!
|
||||
|
||||
# Start top to monitor resource usage
|
||||
top -d $delay -b -p $serve_pid | grep $serve_pid > "$top_log" &
|
||||
top_pid=$!
|
||||
|
||||
# Print environnement informations
|
||||
echo "Process to monitor: $serve_pid"
|
||||
echo "Monitoring process: $top_pid"
|
||||
echo "Monitoring delay: $delay s"
|
||||
echo "ZIM file path: $zim"
|
||||
echo "Kiwix-serve path: $serve"
|
||||
echo "Total memory: $memory_mb MB"
|
||||
echo
|
||||
|
||||
# Compute artice list
|
||||
echo "Computing article list snippet..."
|
||||
articles=`zimdump -l $zim | grep ".html" | shuf | head -n 1000`
|
||||
|
||||
# Run wget against kiwix-serve
|
||||
start_date=`date +%s`
|
||||
for LINE in $articles
|
||||
do
|
||||
echo "Scrapping $LINE..."
|
||||
wget --quiet p -P /dev/shm/tmp "http://localhost:8080/wikipedia_en_medicine_2016-09/A/$LINE"
|
||||
rm -rf /dev/shm/tmp
|
||||
done
|
||||
end_date=`date +%s`
|
||||
|
||||
# Kill top instance
|
||||
kill -s STOP $top_pid 2>&1 > /dev/null
|
||||
sed -i '$ d' "$top_log"
|
||||
times=`cat "$top_log" | wc -l`
|
||||
|
||||
# Compute KPI
|
||||
duration=`echo "$end_date-$start_date" | bc -l`
|
||||
cpu_percent=`cat "$top_log" | sed -r -e "s;\s\s*; ;g" -e "s;^ *;;" | cut -d' ' -f9 | tr '\n' '+0' | sed -r -e "s;(.*)[+]$;\1;" -e "s/.*/scale=2\n(&)\/$times\nquit\n/" > "$top_calc" ; bc -q "$top_calc"`
|
||||
memory_percent=`cat "$top_log" | sed -r -e "s;\s\s*; ;g" -e "s;^ *;;" | cut -d' ' -f10 | tr '\n' '+' | sed -r -e "s;(.*)[+]$;\1;" -e "s/.*/scale=2\n(&)\/$times\nquit\n/" > "$top_calc" ; bc -q "$top_calc"`
|
||||
memory_absolut=`echo "scale=2;$memory_mb/100*$memory_percent" | bc -l`
|
||||
|
||||
echo
|
||||
echo "Measure count: $times"
|
||||
echo "Duration: $duration s"
|
||||
echo "CPU (average): $cpu_percent %"
|
||||
echo "Memory (average): $memory_absolut MB"
|
||||
|
||||
# Kill kiwix-serve & top instances
|
||||
kill -s STOP $serve_pid 2>&1 > /dev/null
|
||||
#rm "$top_log" "$top_calc"
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
* Copyright 2009-2019 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
@@ -33,6 +33,8 @@
|
||||
# define MIBSIZE 4
|
||||
#endif
|
||||
|
||||
#include "../version.h"
|
||||
|
||||
#define DEFAULT_THREADS 4
|
||||
|
||||
void usage()
|
||||
@@ -62,7 +64,9 @@ void usage()
|
||||
<< "\t-r, --urlRootLocation\tURL prefix on which the content should be made available (default: /)" << std::endl
|
||||
<< "\t-t, --threads\t\tnumber of threads to run in parallel (default: " << DEFAULT_THREADS << ")" << std::endl
|
||||
<< "\t-v, --verbose\t\tprint debug log to STDOUT" << std::endl
|
||||
<< "\t-V, --version\t\tprint software version" << std::endl
|
||||
<< "\t-z, --nodatealiases\tcreate URL aliases for each content by removing the date" << std::endl
|
||||
<< "\t--donottrustlibrary\tRead the metadata from the zim file instead of trusting the library." << std::endl
|
||||
<< std::endl
|
||||
|
||||
<< "Documentation:" << std::endl
|
||||
@@ -87,12 +91,14 @@ int main(int argc, char** argv)
|
||||
bool noSearchBarFlag = false;
|
||||
bool noDateAliasesFlag = false;
|
||||
bool isVerboseFlag = false;
|
||||
bool trustlibrary = true;
|
||||
string PPIDString;
|
||||
unsigned int PPID = 0;
|
||||
|
||||
static struct option long_options[]
|
||||
= {{"daemon", no_argument, 0, 'd'},
|
||||
{"verbose", no_argument, 0, 'v'},
|
||||
{"version", no_argument, 0, 'V'},
|
||||
{"library", no_argument, 0, 'l'},
|
||||
{"nolibrarybutton", no_argument, 0, 'm'},
|
||||
{"nodatealiases", no_argument, 0, 'z'},
|
||||
@@ -102,13 +108,14 @@ int main(int argc, char** argv)
|
||||
{"address", required_argument, 0, 'i'},
|
||||
{"threads", required_argument, 0, 't'},
|
||||
{"urlRootLocation", required_argument, 0, 'r'},
|
||||
{"donottrustlibrary", no_argument, 0, 'T'},
|
||||
{0, 0, 0, 0}};
|
||||
|
||||
/* Argument parsing */
|
||||
while (true) {
|
||||
int option_index = 0;
|
||||
int c
|
||||
= getopt_long(argc, argv, "mndvla:p:f:t:r:", long_options, &option_index);
|
||||
= getopt_long(argc, argv, "zmndvVla:p:f:t:r:i:", long_options, &option_index);
|
||||
|
||||
if (c != -1) {
|
||||
switch (c) {
|
||||
@@ -118,6 +125,9 @@ int main(int argc, char** argv)
|
||||
case 'v':
|
||||
isVerboseFlag = true;
|
||||
break;
|
||||
case 'V':
|
||||
version();
|
||||
return 0;
|
||||
case 'l':
|
||||
libraryFlag = true;
|
||||
break;
|
||||
@@ -130,6 +140,8 @@ int main(int argc, char** argv)
|
||||
case 'm':
|
||||
noLibraryButtonFlag = true;
|
||||
break;
|
||||
case 'T':
|
||||
trustlibrary = false;
|
||||
case 'p':
|
||||
serverPort = atoi(optarg);
|
||||
break;
|
||||
@@ -181,7 +193,7 @@ int main(int argc, char** argv)
|
||||
= isRelativePath(*itr)
|
||||
? computeAbsolutePath(getCurrentDirectory(), *itr)
|
||||
: *itr;
|
||||
retVal = manager.readFile(libraryPath, true);
|
||||
retVal = manager.readFile(libraryPath, true, trustlibrary);
|
||||
} catch (...) {
|
||||
retVal = false;
|
||||
}
|
||||
|
||||
32
src/version.h
Normal file
32
src/version.h
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
* MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef _KIWIX_TOOLS_VERSION_H_
|
||||
#define _KIWIX_TOOLS_VERSION_H_
|
||||
|
||||
#ifndef KIWIX_TOOLS_VERSION
|
||||
#define KIWIX_TOOLS_VERSION "undefined"
|
||||
#endif
|
||||
|
||||
void version()
|
||||
{
|
||||
std::cout << KIWIX_TOOLS_VERSION << std::endl;
|
||||
}
|
||||
|
||||
#endif //_KIWIX_TOOLs_VERSION_H_
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
BUILD_DIR=${HOME}/BUILD_${PLATFORM}
|
||||
INSTALL_DIR=${BUILD_DIR}/INSTALL
|
||||
|
||||
|
||||
case ${PLATFORM} in
|
||||
"native_static")
|
||||
MESON_OPTION="-Dstatic-linkage=true"
|
||||
;;
|
||||
"native_dyn")
|
||||
MESON_OPTION=""
|
||||
;;
|
||||
"win32_static")
|
||||
MESON_OPTION="-Dstatic-linkage=true --cross-file ${BUILD_DIR}/meson_cross_file.txt"
|
||||
;;
|
||||
"win32_dyn")
|
||||
MESON_OPTION="--cross-file ${BUILD_DIR}/meson_cross_file.txt"
|
||||
;;
|
||||
esac
|
||||
|
||||
cd ${TRAVIS_BUILD_DIR}
|
||||
export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/x86_64-linux-gnu/pkgconfig
|
||||
export PATH=${INSTALL_DIR}/bin:$PATH
|
||||
meson . build ${MESON_OPTION}
|
||||
cd build
|
||||
ninja
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
REPO_NAME=${TRAVIS_REPO_SLUG#*/}
|
||||
ARCHIVE_NAME=deps_linux_xenial_${PLATFORM}_${REPO_NAME}.tar.xz
|
||||
|
||||
# Packages.
|
||||
case ${PLATFORM} in
|
||||
"native_static")
|
||||
PACKAGES="gcc python3.5 cmake libbz2-dev ccache zlib1g-dev uuid-dev"
|
||||
;;
|
||||
"native_dyn")
|
||||
PACKAGES="gcc python3.5 cmake libbz2-dev ccache zlib1g-dev uuid-dev libmicrohttpd-dev"
|
||||
;;
|
||||
"win32_static")
|
||||
PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache python3.5"
|
||||
;;
|
||||
"win32_dyn")
|
||||
PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache python3.5"
|
||||
;;
|
||||
"android_arm")
|
||||
PACKAGES="gcc python3.5 cmake ccache"
|
||||
;;
|
||||
"android_arm64")
|
||||
PACKAGES="gcc python3.5 cmake ccache"
|
||||
;;
|
||||
esac
|
||||
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -qq python3-pip ${PACKAGES}
|
||||
wget https://bootstrap.pypa.io/get-pip.py
|
||||
python3.5 get-pip.py --user
|
||||
python3.5 -m pip install --user --upgrade pip
|
||||
python3.5 -m pip install --user meson==0.49.2
|
||||
|
||||
# Ninja
|
||||
cd $HOME
|
||||
git clone git://github.com/ninja-build/ninja.git
|
||||
cd ninja
|
||||
git checkout release
|
||||
./configure.py --bootstrap
|
||||
sudo cp ninja /bin
|
||||
|
||||
# Dependencies comming from kiwix-build.
|
||||
cd ${HOME}
|
||||
wget http://tmp.kiwix.org/ci/${ARCHIVE_NAME}
|
||||
tar xf ${HOME}/${ARCHIVE_NAME}
|
||||
sudo ln -s travis ../ci_builder
|
||||
Reference in New Issue
Block a user