Merge pull request #90 from kiwix/new_release

Release of kiwix-tools 0.3.0.

.clang-format

@@ -0,0 +1,12 @@
+BasedOnStyle:  Google
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: All
+BreakBeforeBraces: Linux
+DerivePointerAlignment: false
+SpacesInContainerLiterals: false
+Standard: Cpp11
+
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false

.travis.yml

@@ -0,0 +1,11 @@
+language: cpp
+dist: trusty
+sudo: required
+cache: ccache
+install: travis/install_deps.sh
+script: travis/compile.sh
+env:
+- PLATFORM="native_static"
+- PLATFORM="native_dyn"
+- PLATFORM="win32_static"
+- PLATFORM="win32_dyn"

Changelog

@@ -0,0 +1,43 @@
+kiwix-tools 0.3.0
+=================
+
+ * Move to C++11
+
+kiwix-serve
+-----------
+
+ * Add a global taskbar in the welcome page to search in all zims (#49)
+ * Serve the taskbar as css file instead of including it in the html (#68):
+   * Better client caching
+   * The html encoding is now in the first 1024 bytes and firefox correctly
+     detect the encoding
+ * Make kiwix-server multi-threaded (#82)
+ * Correctly return 404 instead of crashing when request inexistant skin file
+   (#83)
+ * Correctly respond to bytes-range requests.(#84)
+ * Directly respond to first request for a url instead of refusing the first
+   connexion
+ * Add support to relative url location. (#86)
+ * Remove caching (on client side) for the welcome page. (#86)
+
+
+kiwix-tools 0.2.0
+=================
+
+ * Remove indexer tools
+
+kiwix-serve
+-----------
+
+ * Correctly fix the deflate data we send over http. (#15)
+ * Update in the taskbar (or topbar):
+    * Taskbar is responsive (github.com/kiwix/kiwix/issues/336)
+    * Force css rules for the taskbar (and not be impacted by content's css)
+ * Add `--nolibrarybutton` to hide the library button from the taskbar.
+ * Rewrite of the welcome page.
+
+
+kiwix-installer
+---------------
+
+ * Remove indexing functionnality

README.md

@@ -30,6 +30,7 @@ libraries need to be available:
 (no package for now)
 * Libmicrohttpd .......... https://www.gnu.org/software/libmicrohttpd/
 (package libmicrohttpd-dev on Ubuntu)
+* CTPP2 ..................................... http://ctpp.havoc.ru/en/
 (package libctpp2-dev on Ubuntu)
 * Zlib .......................................... http://www.zlib.net/
 (package zlib1g-dev on Ubuntu)

format_code.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+
+files=(
+"src/installer/kiwix-install.cpp"
+"src/searcher/kiwix-search.cpp"
+"src/reader/kiwix-read.cpp"
+"src/manager/kiwix-manage.cpp"
+"src/server/kiwix-serve.cpp"
+)
+
+for i in "${files[@]}"
+do
+  echo $i
+  clang-format -i -style=file $i
+done

meson.build

@@ -1,4 +1,7 @@
-project('kiwix-tools', 'cpp')
+project('kiwix-tools', 'cpp',
+  version : '0.3.0',
+  license : 'GPL',
+  default_options: ['c_std=c11', 'cpp_std=c++11'])
 
 compiler = meson.get_compiler('cpp')
 
@@ -8,53 +11,10 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'>=1.0.0', static:static_linkage)
 microhttpd_dep = dependency('libmicrohttpd', static:static_linkage)
 z_dep = dependency('zlib', static:static_linkage)
 
-# Idealy we should not have more dependency, however :
-# We should declare we use ctpp2 in kiwixlib in the pkg-config file.
-# But there is no pkg-config file for ctpp2. Once one exists, no need to
-# declare the dependency here
-ctpp2_prefix_install = get_option('ctpp2-install-prefix')
-find_library_in_compiler = meson.version().version_compare('>=0.31.0')
-if ctpp2_prefix_install == ''
-  if not compiler.has_header('ctpp2/CTPP2Logger.hpp')
-    error('ctpp2/CTPP2Logger.hppnot found')
-  endif
-  if find_library_in_compiler
-    ctpp2_lib = compiler.find_library('ctpp2')
-  else
-    ctpp2_lib = find_library('ctpp2')
-  endif
-  link_args = ['-lctpp2']
-  if meson.is_cross_build()
-    if host_machine.system() == 'windows'
-      link_args += ['-liconv']
-    endif
-  endif
-  ctpp2_dep = declare_dependency(link_args:link_args)
-else
-  if not find_library_in_compiler
-    error('For custom ctpp2_prefix_install you need a meson version >=0.31.0')
-  endif
-  ctpp2_include_path = ctpp2_prefix_install + '/include'
-  ctpp2_include_args = ['-I'+ctpp2_include_path]
-  if not compiler.has_header('ctpp2/CTPP2Logger.hpp', args:ctpp2_include_args)
-    error('ctpp2/CTPP2Logger.hpp not found')
-  endif
-  ctpp2_include_path = include_directories(ctpp2_include_path, is_system:true)
-  ctpp2_lib_path = ctpp2_prefix_install+'/lib'
-  ctpp2_lib = compiler.find_library('ctpp2', dirs:ctpp2_lib_path)
-  link_args = ['-L'+ctpp2_lib_path, '-lctpp2']
-  if meson.is_cross_build()
-    if host_machine.system() == 'windows'
-      link_args += ['-liconv']
-    endif
-  endif
-  ctpp2_dep = declare_dependency(include_directories:ctpp2_include_path, link_args:link_args)
-endif
-
 # This is a temporary workaround until xapian fix it's xapian-core.pc
 # For a correct dependency checking we should test if uuid is really installed
 # but for a workaround, we assume that in xapian is installed, uuid also.
@@ -62,10 +22,15 @@ if meson.is_cross_build() and host_machine.system() == 'windows'
     # xapian doesn't use uuid on windows.
     uuid_dep = declare_dependency()
 else
-    uuid_dep = declare_dependency(link_args:['-luuid', '-lrt'])
+    xapian_dep = dependency('xapian-core', required : false)
+    if xapian_dep.found()
+        uuid_dep = declare_dependency(link_args:['-luuid', '-lrt'])
+    else
+        uuid_dep = declare_dependency()
+    endif
 endif
 
-all_deps = [thread_dep, kiwixlib_dep, microhttpd_dep, ctpp2_dep, z_dep, uuid_dep]
+all_deps = [thread_dep, kiwixlib_dep, microhttpd_dep, z_dep, uuid_dep]
 
 #subdir('include')
 subdir('static')

meson_options.txt

@@ -1,4 +1,2 @@
 option('static-linkage', type : 'boolean', value : false,
   description : 'Create statically linked binaries.')
-option('ctpp2-install-prefix', type : 'string', value : '',
-  description : 'Prefix where ctpp libs has been installed')

src/indexer/kiwix-index.cpp

@@ -1,107 +0,0 @@
-/*
- * Copyright 2009-2013 Emmanuel Engelhart <kelson@kiwix.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU  General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
-
-#include <kiwix/xapianIndexer.h>
-#include <getopt.h>
-
-#ifdef _WIN32
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif
-
-void usage() {
-    cout << "Usage: kiwix-index [--verbose] ZIM_PATH INDEX_PATH" << endl;
-    exit(1);
-}
-
-int main(int argc, char **argv) {
-
-  /* Init the variables */
-  char *zimFilePath = NULL;
-  char *indexPath = NULL;
-  bool verboseFlag = false;
-  int option_index = 0;
-  int c = 0;
-
-  kiwix::XapianIndexer *indexer = NULL;
-
-  /* Argument parsing */
-  while (42) {
-
-    static struct option long_options[] = {
-      {"verbose", no_argument, 0, 'v'},
-      {0, 0, 0, 0}
-    };
-
-    if (c != -1) {
-      c = getopt_long(argc, argv, "v", long_options, &option_index);
-
-      switch (c) {
-	case 'v':
-	  verboseFlag = true;
-	  break;
-      }
-    } else {
-      if (optind < argc) {
-	if (zimFilePath == NULL) {
-	  zimFilePath = argv[optind++];
-	} else if (indexPath == NULL) {
-	  indexPath = argv[optind++];
-	} else {
-	  usage();
-	}
-      } else {
-	break;
-      }
-    }
-  }
-
-  /* Check if we have enough arguments */
-  if (zimFilePath == NULL || indexPath == NULL) {
-    usage();
-  }
-
-  /* Try to prepare the indexing */
-  try {
-    indexer = new kiwix::XapianIndexer();
-  } catch (...) {
-    cerr << "Unable to index '" << zimFilePath << "'." << endl;
-    exit(1);
-  }
-
-  /* Start the indexing */
-  if (indexer != NULL) {
-    indexer->setVerboseFlag(verboseFlag);
-    indexer->start(zimFilePath, indexPath);
-    while (indexer->isRunning()) {
-      if (verboseFlag)
-	cout << indexer->getProgression() << "% of all the articles indexed..." << endl;
-      kiwix::sleep(1000);
-    }
-    if (verboseFlag)
-      cout << "100% of the articles were successfuly indexed..." << endl;
-    delete indexer;
-  } else {
-    cerr << "Unable instanciate the Kiwix indexer." << endl;
-    exit(1);
-  }
-
-  exit(0);
-}

src/indexer/meson.build

@@ -1,3 +0,0 @@
-executable('kiwix-index', ['kiwix-index.cpp'],
-  dependencies:all_deps,
-  install:true)

src/installer/kiwix-install.cpp

@@ -19,64 +19,57 @@
 
 #include <getopt.h>
 #include <kiwix/common/pathTools.h>
-#include <kiwix/xapianIndexer.h>
-#include <kiwix/reader.h>
 #include <kiwix/manager.h>
+#include <kiwix/reader.h>
 
 enum supportedAction { NONE, ADDCONTENT };
 
-void usage() {
-    cout << "Usage: kiwix-install [--verbose] [--buildIndex] addcontent ZIM_PATH KIWIX_PATH" << endl;
-    exit(1);
+void usage()
+{
+  cout << "Usage: kiwix-install [--verbose] addcontent ZIM_PATH KIWIX_PATH"
+       << endl;
+  exit(1);
 }
 
-int main(int argc, char **argv) {
-
+int main(int argc, char** argv)
+{
   /* Init the variables */
-  const char *contentPath = NULL;
-  const char *kiwixPath = NULL;
+  const char* contentPath = NULL;
+  const char* kiwixPath = NULL;
   supportedAction action = NONE;
   bool verboseFlag = false;
-  bool buildIndexFlag = false;
   int option_index = 0;
   int c = 0;
 
   /* Argument parsing */
   while (42) {
-
-    static struct option long_options[] = {
-      {"verbose", no_argument, 0, 'v'},
-      {"buildIndex", no_argument, 0, 'i'},
-      {0, 0, 0, 0}
-    };
+    static struct option long_options[]
+        = {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
 
     if (c != -1) {
       c = getopt_long(argc, argv, "vi", long_options, &option_index);
 
       switch (c) {
-	case 'v':
-	  verboseFlag = true;
-	  break;
-        case 'i':
-	  buildIndexFlag = true;
-	  break;
+        case 'v':
+          verboseFlag = true;
+          break;
       }
     } else {
       if (optind < argc) {
-	if (action == NONE) {
-	  string actionString = argv[optind++];
-	  if (actionString == "addcontent" || actionString == "ADDCONTENT") {
-	    action = ADDCONTENT;
-	  }
-	} else if (contentPath == NULL) {
-	  contentPath = argv[optind++];
-	} else if (kiwixPath == NULL) {
-	  kiwixPath = argv[optind++];
-	} else {
-	  usage();
-	}
+        if (action == NONE) {
+          string actionString = argv[optind++];
+          if (actionString == "addcontent" || actionString == "ADDCONTENT") {
+            action = ADDCONTENT;
+          }
+        } else if (contentPath == NULL) {
+          contentPath = argv[optind++];
+        } else if (kiwixPath == NULL) {
+          kiwixPath = argv[optind++];
+        } else {
+          usage();
+        }
       } else {
-	break;
+        break;
       }
     }
   }
@@ -88,38 +81,48 @@ int main(int argc, char **argv) {
 
   /* Make the action */
   if (action == ADDCONTENT) {
-
     /* Check if the content path exists and is readable */
-    if (verboseFlag) { std::cout << "Check if the ZIM file exists..." << std::endl; }
+    if (verboseFlag) {
+      std::cout << "Check if the ZIM file exists..." << std::endl;
+    }
     if (!fileExists(contentPath)) {
-      cerr << "The content path '" << contentPath << "' does not exist or is not readable." << endl;
+      cerr << "The content path '" << contentPath
+           << "' does not exist or is not readable." << endl;
       exit(1);
     }
 
     /* Check if this is a ZIM file */
     try {
-      if (verboseFlag) { std::cout << "Check if the ZIM file is valid..." << std::endl; }
-      kiwix::Reader *reader = new kiwix::Reader(contentPath);
+      if (verboseFlag) {
+        std::cout << "Check if the ZIM file is valid..." << std::endl;
+      }
+      kiwix::Reader* reader = new kiwix::Reader(contentPath);
       delete reader;
-    } catch (exception &e) {
-      cerr << "The content available at '" << contentPath << "' is not a ZIM file." << endl;
+    } catch (exception& e) {
+      cerr << "The content available at '" << contentPath
+           << "' is not a ZIM file." << endl;
       exit(1);
     }
     string contentFilename = getLastPathElement(contentPath);
 
     /* Check if kiwixPath/kiwix/kiwix.exe exists */
-    if (verboseFlag) { std::cout << "Check if the Kiwix path is valid..." << std::endl; }
+    if (verboseFlag) {
+      std::cout << "Check if the Kiwix path is valid..." << std::endl;
+    }
     string kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix.exe");
     if (!fileExists(kiwixBinaryPath)) {
       kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix");
       if (!fileExists(kiwixBinaryPath)) {
-	cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath << "[.exe]'." << endl;
-	exit(1);
+        cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath
+             << "[.exe]'." << endl;
+        exit(1);
       }
     }
 
     /* Check if the directory "data" structure exists */
-    if (verboseFlag) { std::cout << "Check the target data directory structure..." << std::endl; }
+    if (verboseFlag) {
+      std::cout << "Check the target data directory structure..." << std::endl;
+    }
     string dataPath = computeAbsolutePath(kiwixPath, "data/");
     if (!fileExists(dataPath)) {
       makeDirectory(dataPath);
@@ -137,59 +140,32 @@ int main(int argc, char **argv) {
       makeDirectory(dataLibraryPath);
     }
 
-    /* Check if the directory "data/index" structure exists */
-    string dataIndexPath = computeAbsolutePath(kiwixPath, "data/index/");
-    if (!fileExists(dataIndexPath)) {
-      makeDirectory(dataIndexPath);
-    }
-
     /* Copy the file to the data/content directory */
-    if (verboseFlag) { std::cout << "Copy ZIM file to the target directory..." << std::endl; }
-    string newContentPath = computeAbsolutePath(dataContentPath, contentFilename);
-    if (!fileExists(newContentPath) || getFileSize(contentPath) != getFileSize(newContentPath)) {
+    if (verboseFlag) {
+      std::cout << "Copy ZIM file to the target directory..." << std::endl;
+    }
+    string newContentPath
+        = computeAbsolutePath(dataContentPath, contentFilename);
+    if (!fileExists(newContentPath)
+        || getFileSize(contentPath) != getFileSize(newContentPath)) {
       copyFile(contentPath, newContentPath);
     }
 
-    /* Index the file if necessary */
-    if (verboseFlag) { std::cout << "Check if the index directory exists..." << std::endl; }
-    string indexFilename = contentFilename + ".idx";
-    string indexPath = computeAbsolutePath(dataIndexPath, indexFilename);
-    if (buildIndexFlag && !fileExists(indexPath)) {
-      if (verboseFlag) { std::cout << "Start indexing the ZIM file..." << std::endl; }
-      kiwix::XapianIndexer *indexer = NULL;
-      try {
-	  indexer = new kiwix::XapianIndexer();
-      } catch (...) {
-	cerr << "Unable to index '" << contentPath << "'." << endl;
-	exit(1);
-      }
-
-      if (indexer != NULL) {
-	indexer->setVerboseFlag(verboseFlag);
-	indexer->start(contentPath, indexPath);
-	while (indexer->isRunning()) {
-	  kiwix::sleep(1000);
-	}
-	delete indexer;
-      } else {
-	cerr << "Unable instanciate the Kiwix indexer." << endl;
-	exit(1);
-      }
-    }
-
     /* Add the file to the library.xml */
-    if (verboseFlag) { std::cout << "Create the library XML file..." << std::endl; }
+    if (verboseFlag) {
+      std::cout << "Create the library XML file..." << std::endl;
+    }
     kiwix::Manager libraryManager;
-    string libraryPath = computeAbsolutePath(dataLibraryPath, contentFilename + ".xml");
-    string bookId = libraryManager.addBookFromPathAndGetId(newContentPath, "../content/" + contentFilename, "", false);
+    string libraryPath
+        = computeAbsolutePath(dataLibraryPath, contentFilename + ".xml");
+    string bookId = libraryManager.addBookFromPathAndGetId(
+        newContentPath, "../content/" + contentFilename, "", false);
     if (!bookId.empty()) {
       libraryManager.setCurrentBookId(bookId);
-      if (buildIndexFlag) {
-	libraryManager.setBookIndex(bookId, "../index/" + indexFilename, kiwix::XAPIAN);
-      }
       libraryManager.writeFile(libraryPath);
     } else {
-      cerr << "Unable to build or save library file '" << libraryPath << "'" << endl;
+      cerr << "Unable to build or save library file '" << libraryPath << "'"
+           << endl;
     }
   }
 

src/manager/kiwix-manage.cpp

@@ -21,47 +21,54 @@
 #include <unistd.h>
 #endif
 #include <getopt.h>
-#include <iostream>
-#include <cstdlib>
 #include <kiwix/common/pathTools.h>
 #include <kiwix/manager.h>
+#include <cstdlib>
+#include <iostream>
 
 using namespace std;
 
 enum supportedAction { NONE, ADD, SHOW, REMOVE };
 
-
-void show(kiwix::Library library) {
-    std::vector<kiwix::Book>::iterator itr;
-    unsigned int inc = 1;
-    for ( itr = library.books.begin(); itr != library.books.end(); ++itr ) {
-      std::cout << "#" << inc++
-		<< std::endl << "id:\t\t" << itr->id
-		<< std::endl << "path:\t\t" << itr->path
-		<< std::endl << "indexpath:\t" << itr->indexPath
-		<< std::endl << "url:\t\t" << itr->url
-		<< std::endl << "title:\t\t" << itr->title
-		<< std::endl << "name:\t\t" << itr->name
-		<< std::endl << "tags:\t\t" << itr->tags
-		<< std::endl << "description:\t" << itr->description
-		<< std::endl << "creator:\t" << itr->creator
-		<< std::endl << "date:\t\t" << itr->date
-		<< std::endl << "articleCount:\t" << itr->articleCount
-		<< std::endl << "mediaCount:\t" << itr->mediaCount
-		<< std::endl << "size:\t\t" << itr->size << " KB"
-		<< std::endl << std::endl;
-    }
+void show(kiwix::Library library)
+{
+  std::vector<kiwix::Book>::iterator itr;
+  unsigned int inc = 1;
+  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+    std::cout << "#" << inc++ << std::endl
+              << "id:\t\t" << itr->id << std::endl
+              << "path:\t\t" << itr->path << std::endl
+              << "indexpath:\t" << itr->indexPath << std::endl
+              << "url:\t\t" << itr->url << std::endl
+              << "title:\t\t" << itr->title << std::endl
+              << "name:\t\t" << itr->name << std::endl
+              << "tags:\t\t" << itr->tags << std::endl
+              << "description:\t" << itr->description << std::endl
+              << "creator:\t" << itr->creator << std::endl
+              << "date:\t\t" << itr->date << std::endl
+              << "articleCount:\t" << itr->articleCount << std::endl
+              << "mediaCount:\t" << itr->mediaCount << std::endl
+              << "size:\t\t" << itr->size << " KB" << std::endl
+              << std::endl;
+  }
 }
 
-void usage() {
-    cerr << "Usage:" << endl;
-    cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH [--zimPathToSave=../content/foobar.zim] [--current] [--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] [--url=http://...metalink]" << endl;
-    cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... (show everything if no param.)" << endl;
-    cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
+void usage()
+{
+  cerr << "Usage:" << endl;
+  cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH "
+          "[--zimPathToSave=../content/foobar.zim] [--current] "
+          "[--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] "
+          "[--url=http://...metalink]"
+       << endl;
+  cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... "
+          "(show everything if no param.)"
+       << endl;
+  cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
 }
 
-int main(int argc, char **argv) {
-
+int main(int argc, char** argv)
+{
   string libraryPath = "";
   supportedAction action = NONE;
   string zimPath = "";
@@ -89,9 +96,9 @@ int main(int argc, char **argv) {
   }
 
   /* Try to read the file */
-  libraryPath = isRelativePath(libraryPath) ?
-    computeAbsolutePath(getCurrentDirectory(), libraryPath) :
-    libraryPath;
+  libraryPath = isRelativePath(libraryPath)
+                    ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
+                    : libraryPath;
   libraryManager.readFile(libraryPath, false);
 
   /* SHOW */
@@ -103,81 +110,78 @@ int main(int argc, char **argv) {
     string indexPath;
     kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
     string url;
-    string origID="";
+    string origID = "";
     bool setCurrent = false;
 
-    if (argc>3) {
+    if (argc > 3) {
       zimPath = argv[3];
     }
 
     /* Options parsing */
     optind = 2;
     while (42) {
+      static struct option long_options[]
+          = {{"url", required_argument, 0, 'u'},
+             {"origId", required_argument, 0, 'o'},
+             {"indexPath", required_argument, 0, 'i'},
+             {"indexBackend", required_argument, 0, 'b'},
+             {"zimPathToSave", required_argument, 0, 'z'},
+             {"current", no_argument, 0, 'c'},
+             {0, 0, 0, 0}};
 
-      static struct option long_options[] = {
-	{"url", required_argument, 0, 'u'},
-	{"origId", required_argument, 0, 'o'},
-	{"indexPath", required_argument, 0, 'i'},
-	{"indexBackend", required_argument, 0, 'b'},
-	{"zimPathToSave", required_argument, 0, 'z'},
-	{"current", no_argument, 0, 'c'},
-	{0, 0, 0, 0}
-      };
-      
       c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
-      
+
       if (c != -1) {
+        switch (c) {
+          case 'u':
+            url = optarg;
+            break;
 
-	switch (c) {
-        case 'u':
-	  url = optarg;
-	  break;
+          case 'o':
+            origID = optarg;
+            break;
 
-        case 'o':
-	  origID = optarg;
-	  break;
+          case 'c':
+            setCurrent = true;
+            break;
 
-        case 'c':
-	  setCurrent = true;
-	  break;
+          case 'i':
+            indexPath = optarg;
+            break;
 
-	case 'i':
-	  indexPath = optarg;
-	  break;
+          case 'b':
+            if (!strcmp(optarg, "xapian")) {
+              indexBackend = kiwix::XAPIAN;
+            } else {
+              usage();
+            }
+            break;
 
-	case 'b':
-	  if (!strcmp(optarg, "xapian")) {
-	    indexBackend = kiwix::XAPIAN;
-	  } else {
-	    usage();
-	  }
-	  break;
-
-	case 'z':
-	  zimPathToSave = optarg;
-	  break;
-
-	}
+          case 'z':
+            zimPathToSave = optarg;
+            break;
+        }
       } else {
-	break;
+        break;
       }
     }
 
     if (!zimPath.empty()) {
       zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      string bookId = libraryManager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false);
+      string bookId = libraryManager.addBookFromPathAndGetId(
+          zimPath, zimPathToSave, url, false);
 
       if (!bookId.empty()) {
+        if (setCurrent)
+          libraryManager.setCurrentBookId(bookId);
 
-	if (setCurrent)
-	  libraryManager.setCurrentBookId(bookId);
-
-	/* Save the index infos if necessary */
-	if (!indexPath.empty())
-	  libraryManager.setBookIndex(bookId, indexPath, indexBackend);
+        /* Save the index infos if necessary */
+        if (!indexPath.empty())
+          libraryManager.setBookIndex(bookId, indexPath, indexBackend);
 
       } else {
-	cerr << "Unable to build or save library file '" << libraryPath << "'" << endl;
+        cerr << "Unable to build or save library file '" << libraryPath << "'"
+             << endl;
       }
     } else {
       std::cerr << "Invalid zim file path" << std::endl;
@@ -187,7 +191,7 @@ int main(int argc, char **argv) {
     unsigned int bookIndex = 0;
     const unsigned int totalBookCount = libraryManager.getBookCount(true, true);
 
-    if (argc>3) {
+    if (argc > 3) {
       bookIndex = atoi(argv[3]);
     }
 
@@ -195,9 +199,13 @@ int main(int argc, char **argv) {
       libraryManager.removeBookByIndex(bookIndex - 1);
     } else {
       if (totalBookCount > 0) {
-	std::cerr << "Invalid book index number. Please give a number between 1 and " << totalBookCount << std::endl;
+        std::cerr
+            << "Invalid book index number. Please give a number between 1 and "
+            << totalBookCount << std::endl;
       } else {
-	std::cerr << "Invalid book index number. Library is empty, no book to delete." << std::endl;
+        std::cerr
+            << "Invalid book index number. Library is empty, no book to delete."
+            << std::endl;
       }
     }
   }

src/meson.build

@@ -1,6 +1,4 @@
 
-
-subdir('indexer')
 subdir('installer')
 subdir('manager')
 subdir('reader')

src/reader/kiwix-read.cpp

@@ -18,59 +18,53 @@
  */
 
 #include <getopt.h>
-#include <unistd.h>
-#include <string>
-#include <map>
-#include <kiwix/common/tree.h>
 #include <kiwix/reader.h>
+#include <unistd.h>
+#include <map>
+#include <string>
 
-void usage() {
-    cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
-    exit(1);
+void usage()
+{
+  cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
+  exit(1);
 }
 
-void updateSuggestionTree(tree< pair<string, unsigned> > &suggestionTree, string suggestion) {
-  return;
-}
-
-int main(int argc, char **argv) {
-
+int main(int argc, char** argv)
+{
   /* Init the variables */
-  const char *filePath = NULL;
-  const char *pattern = NULL;
+  const char* filePath = NULL;
+  const char* pattern = NULL;
   int option_index = 0;
   int c = 0;
 
-  kiwix::Reader *reader = NULL;
+  kiwix::Reader* reader = NULL;
 
   /* Argument parsing */
   while (42) {
+    static struct option long_options[]
+        = {{"verbose", no_argument, 0, 'v'},
+           {"suggest", required_argument, 0, 's'},
+           {0, 0, 0, 0}};
 
-    static struct option long_options[] = {
-      {"verbose", no_argument, 0, 'v'},
-      {"suggest", required_argument, 0, 's'},
-      {0, 0, 0, 0}
-    };
-    
     if (c != -1) {
       c = getopt_long(argc, argv, "vs:", long_options, &option_index);
 
       switch (c) {
-	case 'v':
-	  break;
-      case 's':
-	pattern = optarg;
-	break;
+        case 'v':
+          break;
+        case 's':
+          pattern = optarg;
+          break;
       }
     } else {
       if (optind < argc) {
-	if (filePath == NULL) {
-	  filePath = argv[optind++];
-	} else {
-	  usage();
-	}
+        if (filePath == NULL) {
+          filePath = argv[optind++];
+        } else {
+          usage();
+        }
       } else {
-	break;
+        break;
       }
     }
   }
@@ -90,25 +84,23 @@ int main(int argc, char **argv) {
     string contentType;
     unsigned int contentLength = 0;
     string suggestion;
-    
+
     if (pattern != NULL) {
       std::cout << "Searching suggestions for: " << pattern << std::endl;
 
       reader->searchSuggestionsSmart(pattern, 10);
       while (reader->getNextSuggestion(suggestion)) {
-	std::cout << suggestion << std::endl;
+        std::cout << suggestion << std::endl;
       }
     }
 
     /*
-    if (reader->getContentByUrl(mainPageUrl, content, contentLength, contentType)) {
+    if (reader->getContentByUrl(mainPageUrl, content, contentLength,
+    contentType)) {
       cout << content << endl;
     }
     */
 
-    //    tree< pair<string, unsigned> > tree;
-    //updateSuggestionTree(tree, string(suggestion));
-
     delete reader;
   } else {
     cerr << "Unable instanciate the Kiwix reader." << endl;

src/searcher/kiwix-search.cpp

@@ -18,95 +18,97 @@
  */
 
 #include <getopt.h>
+#include <kiwix/reader.h>
+#include <kiwix/searcher.h>
 #include <unistd.h>
-#include <kiwix/xapianSearcher.h>
 
-enum supportedBackend { XAPIAN };
-
-void usage() {
-    cout << "Usage: kiwix-search [--verbose|-v] [--backend|-b=xapian] INDEX_PATH SEARCH" << endl;
-    exit(1);
+void usage()
+{
+  cout << "Usage: kiwix-search [--verbose|-v] ZIM_PATH SEARCH" << endl;
+  exit(1);
 }
 
-int main(int argc, char **argv) {
-
+int main(int argc, char** argv)
+{
   /* Init the variables */
-  //const char *indexPath = "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
-  //const char *indexPath = "/home/itamar/testindex";
-  const char *indexPath = NULL;
-  const char *search = NULL;
+  // const char *indexPath =
+  // "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
+  // const char *indexPath = "/home/itamar/testindex";
+  const char* zimPath = NULL;
+  const char* search = NULL;
   bool verboseFlag = false;
   int option_index = 0;
   int c = 0;
-  supportedBackend backend = XAPIAN;
 
-  kiwix::Searcher *searcher = NULL;
+  kiwix::Searcher* searcher = NULL;
+  kiwix::Reader* reader = NULL;
 
   /* Argument parsing */
   while (42) {
-
-    static struct option long_options[] = {
-      {"verbose", no_argument, 0, 'v'},
-      {"backend", required_argument, 0, 'b'},
-      {0, 0, 0, 0}
-    };
+    static struct option long_options[]
+        = {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
 
     if (c != -1) {
       c = getopt_long(argc, argv, "vb:", long_options, &option_index);
 
       switch (c) {
-	case 'v':
-	  verboseFlag = true;
-	  break;
-	case 'b':
-	  if (!strcmp(optarg, "xapian")) {
-	    backend = XAPIAN;
-	  } else {
-	    usage();
-	  }
-	  break;
+        case 'v':
+          verboseFlag = true;
+          break;
       }
     } else {
       if (optind < argc) {
-	if (indexPath == NULL) {
-	  indexPath = argv[optind++];
-	} else if (search == NULL) {
-	  search = argv[optind++];
-	} else {
-	  cout << indexPath << endl;
-	  usage();
-	}
+        if (zimPath == NULL) {
+          zimPath = argv[optind++];
+        } else if (search == NULL) {
+          search = argv[optind++];
+        } else {
+          cout << zimPath << endl;
+          usage();
+        }
       } else {
-	break;
+        break;
       }
     }
   }
 
   /* Check if we have enough arguments */
-  if (indexPath == NULL || search == NULL) {
+  if (zimPath == NULL || search == NULL) {
     usage();
   }
 
   /* Try to prepare the indexing */
   try {
-      searcher = new kiwix::XapianSearcher(indexPath);
+    reader = new kiwix::Reader(zimPath);
   } catch (...) {
-    cerr << "Unable to search through index '" << indexPath << "'." << endl;
-    exit(1);
+    /* Cannot open the zimPath, maybe it is a plain old xapian database
+     * directory */
+  }
+
+  if (reader) {
+    searcher = new kiwix::Searcher();
+    searcher->add_reader(reader, "");
+  } else {
+    try {
+      searcher = new kiwix::Searcher(zimPath, NULL, "");
+    } catch (...) {
+      cerr << "Unable to search through zim '" << zimPath << "'." << endl;
+      exit(1);
+    }
   }
 
   /* Start the indexing */
   if (searcher != NULL) {
     string searchString(search);
     searcher->search(searchString, 0, 10);
-    string url;
-    string title;
-    unsigned int score;
-    while (searcher->getNextResult(url, title, score)) {
-      cout << title << endl;
+    kiwix::Result* p_result;
+    while ((p_result = searcher->getNextResult())) {
+      cout << p_result->get_title() << endl;
+      delete p_result;
     }
 
     delete searcher;
+    delete reader;
 
     //      kiwix::XapianSearcher::terminate();
   } else {

static/meson.build

@@ -1,4 +1,4 @@
 
-res_compiler = find_program('compile_resources.py')
+res_compiler = find_program('kiwix-compile-resources', 'compile_resources.py')
 
 subdir('server')

static/server/global_taskbar.html.part

@@ -0,0 +1,13 @@
+<span class="kiwix">
+  <span id="kiwixtoolbar" class="ui-widget-header">
+    <div class="kiwix_centered">
+      <div class="kiwix_searchform">
+        <form class="kiwixsearch" method="GET" action="__ROOT_LOCATION__/search" id="kiwixsearchform">
+          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
+          <input type="submit" value="Search">
+        </form>
+      </div>
+    </div>
+  </span>
+</span>
+<div style="display: block; height: 5em;"></div>

static/server/home.html.tmpl

@@ -3,10 +3,43 @@
 <head>
   <meta charset="UTF-8" />
   <title>Welcome to Kiwix Server</title>
-  <script type="text/javascript" src="/skin/jquery-ui/external/jquery/jquery.js"></script>
-  <script type="text/javascript" src="/skin/jquery-ui/jquery-ui.min.js"></script>
-  <link type="text/css" href="/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
-  <link type="text/css" href="/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
+  <script type="text/javascript" src="__ROOT_LOCATION__/skin/jquery-ui/external/jquery/jquery.js"></script>
+  <script type="text/javascript" src="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.min.js"></script>
+  <link type="text/css" href="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
+  <link type="text/css" href="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
+  <style>
+    	body {
+	        background:
+     	        radial-gradient(#EEEEEE 15%, transparent 16%) 0 0,
+	        radial-gradient(#EEEEEE 15%, transparent 16%) 8px 8px,
+	        radial-gradient(rgba(255,255,255,.1) 15%, transparent 20%) 0 1px,
+	        radial-gradient(rgba(255,255,255,.1) 15%, transparent 20%) 8px 9px;
+	        background-color:#E8E8E8;
+	        background-size:16px 16px;
+          	margin-left: auto;
+        	margin-right: auto;
+        	max-width: 65.8em;
+	}
+	.book__list { text-align: center; }
+	.book {
+		display: inline-block; vertical-align: bottom; margin: 8px; padding: 12px 15px; width: 300px;
+		border: 1px solid #ccc; border-radius: 8px;
+	        text-align: left; color: #000; font-family: sans-serif; font-size: 13px;
+	        background-color:#F1F1F1;
+	        box-shadow: 2px 2px 5px 0px #ccc;
+	}
+	.book:hover { background-color: #F9F9F9; box-shadow: none;}
+	.book__background { background-repeat: no-repeat; background-size: auto; background-position: top right; }
+	.book__title {
+		padding: 0 55px 0 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+		font-size: 18px; color: #0645ad;
+	}
+	.book__description {
+		padding: 5px 55px 5px 0px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+		font-size: 15px;
+	}
+	.book__info { line-height: 18px; color: #777; font-weight: bold; font-size: 13px; }
+  </style>
 <script>
 $(function() {
 $( "#accordion" ).accordion();
@@ -15,7 +48,7 @@ $( "#accordion" ).accordion();
 </head>
 <body class="kiwix">
 
-<div id="accordion" class="kiwix">
+<div class="kiwix">
   __BOOKS__
 </div>
 </body>

static/server/include.html.part

@@ -1,11 +1,14 @@
-<script type="text/javascript" src="/skin/jquery-ui/external/jquery/jquery.js"></script>
-<script type="text/javascript" src="/skin/jquery-ui/jquery-ui.min.js"></script>
+<link type="text/css" href="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
+<link type="text/css" href="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
+<link type="text/css" href="__ROOT_LOCATION__/skin/taskbar.css" rel="Stylesheet" />
+<script type="text/javascript" src="__ROOT_LOCATION__/skin/jquery-ui/external/jquery/jquery.js"></script>
+<script type="text/javascript" src="__ROOT_LOCATION__/skin/jquery-ui/jquery-ui.min.js"></script>
 <script>
   var jk = jQuery.noConflict();
   jk(function() {
   jk( "#kiwixsearchbox" ).autocomplete({
 
-  source: "/suggest?content=__CONTENT__",
+  source: "__ROOT_LOCATION__/suggest?content=__CONTENT__",
   dataType: "json",
   cache: false,
 

static/server/resources_list.txt

@@ -21,3 +21,4 @@ home.html.tmpl
 include.html.part
 taskbar.css
 taskbar.html.part
+global_taskbar.html.part

static/server/taskbar.css

@@ -5,16 +5,57 @@
     right:0;
     top: 0;
     z-index:100;
-    max-height: 2.8em;
+}
+
+#kiwixtoolbar > a {
+    float: left;
+}
+
+.height_separator {
+    height: 3em;
 }
 
 #kiwixsearchbox {
     width: 20em;
 }
 
-.kiwixsearch {
-	float: right;
-}s
+.kiwix .kiwix_centered {
+    max-width: 720px;
+    margin: 0 auto;
+}
+.kiwix .kiwix_button_wrapper {
+    float: left;
+}
+.kiwix .kiwix_searchform {
+    float: right;
+}
+.kiwix #kiwixtoolbar button,
+.kiwix #kiwixtoolbar input[type="submit"] {
+    box-sizing: border-box !important;
+    height: 26px !important;
+    line-height: 20px !important;
+    margin-right: 5px !important;
+    padding: 2px 6px !important;
+    border: 1px solid #999 !important;
+    border-radius: 3px !important;
+    background-color: #ededed !important;
+    font-weight: normal !important;
+    cursor: pointer !important;
+    font-size: 16px !important;
+}
+.kiwix #kiwixtoolbar a {
+    margin-bottom: 5px;
+}
+.kiwix #kiwixtoolbar #kiwixsearchform input[type='text'] {
+    box-sizing: border-box !important;
+    height: 26px !important;
+    line-height: 20px !important;
+    border: 1px solid #999 !important;
+    border-radius: 3px !important;
+    background-color: #fff !important;
+    padding: 2px 2px 2px 3px !important;
+    font-size: 16px !important;
+}
 
 /* Try to fix buggy stuff in jquery-ui autocomplete */
 #ui-id-1, .ui-autocomplete {
@@ -26,25 +67,22 @@ li.ui-state-focus {
     font-weight: bold;
 }
 
-/* Specific CSS for Bookeen Cybook device (800x600 B&W ereader) */
-.cybook #kiwixtoolbar button, .cybook #kiwixtoolbar input {
-	font-size: 1.5em;
+@media (max-width: 520px) {
+    .kiwixsearch {
+        margin-top: 5px;
+    }
 }
-.cybook #kiwixsearchbox {
-	width: 7em;
-}
-.cybook a {
-	text-decoration: underline;
-}
-
-@media only screen and (min--moz-device-pixel-ratio: 1.5),
-only screen and (-o-min-device-pixel-ratio: 1.5/1),
-only screen and (-webkit-min-device-pixel-ratio: 1.5),
-only screen and (min-device-pixel-ratio: 1.5) {
-    #kiwixtoolbar button, #kiwixtoolbar input {
-    	font-size: 2em;
+@media (max-width: 645px) {
+    .kiwixsearch {
+        float: none;
     }
     #kiwixsearchbox {
-		width: 7em;
-	}
+      width: 65%;
+    }
+    #kiwixsearchform input[type="submit"] {
+        width: 28%;
+    }
+    .height_separator {
+        height: 6em;
+    }
 }

static/server/taskbar.html.part

@@ -1,15 +1,19 @@
-<link type="text/css" href="/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
-<link type="text/css" href="/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
 <span class="kiwix">
   <span id="kiwixtoolbar" class="ui-widget-header">
-  	<a href="/"><button>Library</button></a>
-  	<a href="/__CONTENT__/"><button>Home</button></a>
-  	<a href="/random?content=__CONTENT__"><button>Random</button></a>
-  	<form class="kiwixsearch" method="GET" action="/search" id="kiwixsearchform">
-  	<input type="hidden" name="content" value="__CONTENT__" />
-  	<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
-  	<input type="submit" value="Search">
-  	</form>
+    <div class="kiwix_centered">
+      <div class="kiwix_button_wrapper">
+        <a id="kiwix_serve_taskbar_library_button" href="__ROOT_LOCATION__/"><button>Library</button></a>
+        <a id="kiwix_serve_taskbar_home_button" href="__ROOT_LOCATION__/__CONTENT__/"><button>Home</button></a>
+        <a id="kiwix_serve_taskbar_random_button" href="__ROOT_LOCATION__/random?content=__CONTENT__"><button>Random</button></a>
+      </div>
+      <div class="kiwix_searchform">
+        <form class="kiwixsearch" method="GET" action="__ROOT_LOCATION__/search" id="kiwixsearchform">
+          <input type="hidden" name="content" value="__CONTENT__" />
+          <input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
+          <input type="submit" value="Search">
+        </form>
+      </div>
+    </div>
   </span>
 </span>
-<div style="display: block; height: 3em;"></div>
+<div style="display: block; height: 5em;"></div>

travis/compile.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_DIR=${HOME}/BUILD_${PLATFORM}
+INSTALL_DIR=${BUILD_DIR}/INSTALL
+
+
+case ${PLATFORM} in
+    "native_static")
+         MESON_OPTION="--default-library=static"
+         ;;
+    "native_dyn")
+         MESON_OPTION="--default-library=shared"
+         ;;
+    "win32_static")
+         MESON_OPTION="--default-library=static --cross-file ${BUILD_DIR}/meson_cross_file.txt"
+         ;;
+    "win32_dyn")
+         MESON_OPTION="--default-library=shared --cross-file ${BUILD_DIR}/meson_cross_file.txt"
+         ;;
+    "android_arm")
+         MESON_OPTION="-Dandroid=true --default-library=shared --cross-file ${BUILD_DIR}/meson_cross_file.txt"
+         ;;
+    "android_arm64")
+         MESON_OPTION="-Dandroid=true --default-library=shared --cross-file ${BUILD_DIR}/meson_cross_file.txt"
+         ;;
+
+esac
+
+cd ${TRAVIS_BUILD_DIR}
+export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/x86_64-linux-gnu/pkgconfig
+export PATH=${INSTALL_DIR}/bin:$PATH
+meson . build -Dctpp2-install-prefix=${INSTALL_DIR} ${MESON_OPTION}
+cd build
+ninja

travis/install_deps.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+
+REPO_NAME=${TRAVIS_REPO_SLUG#*/}
+ARCHIVE_NAME=deps_${PLATFORM}_${REPO_NAME}.tar.gz
+
+# Packages.
+case ${PLATFORM} in
+     "native_static")
+         PACKAGES="gcc cmake libbz2-dev ccache zlib1g-dev uuid-dev libctpp2-dev"
+         ;;
+     "native_dyn")
+         PACKAGES="gcc cmake libbz2-dev ccache zlib1g-dev uuid-dev libctpp2-dev libmicrohttpd-dev"
+         ;;
+     "win32_static")
+         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache"
+         ;;
+     "win32_dyn")
+         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache"
+         ;;
+     "android_arm")
+         PACKAGES="gcc cmake ccache"
+         ;;
+     "android_arm64")
+         PACKAGES="gcc cmake ccache"
+         ;;
+esac
+
+sudo apt-get update -qq
+sudo apt-get install -qq python3-pip ${PACKAGES}
+sudo pip3 install meson
+
+# Ninja
+cd $HOME
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+sudo cp ninja /bin
+
+# Dependencies comming from kiwix-build.
+cd ${HOME}
+wget http://tmp.kiwix.org/ci/${ARCHIVE_NAME}
+mkdir -p BUILD_${PLATFORM}
+cd BUILD_${PLATFORM}
+tar xf ${HOME}/${ARCHIVE_NAME}