Merge pull request #260 from kiwix/new_version

New version 1.0.0

Changelog

@@ -1,3 +1,40 @@
+kiwix-tools 1.0.0
+=================
+
+ * [CI] Use the new deps archive xz
+ * Move version 1.0.0. There is no need to stay in pre 1.0 version.
+
+kiwix-serve
+-----------
+
+ * Correctly implement redirection.
+   kiwix-serve now return a 302 http status code instead of resolving the
+   redirection internally and return the content.
+
+
+kiwix-tools 0.9.0
+=================
+
+* Update README
+* Update man pages
+* Remove support of external indexes (manage, search, serve)
+* Update build system as we don't use ctpp2 anymore
+* Update to last kiwix-lib API.
+
+kiwix-manage
+------------
+
+ * Update usage.
+
+
+kiwix-tools 0.8.0
+=================
+
+kiwix-manage
+------------
+
+ * Be able to remove several books from the library in one command.
+
 kiwix-tools 0.7.0
 =================
 

README.md

@@ -16,7 +16,7 @@ Preamble
 
 Although the Kiwix tools can be compiled/cross-compiled on/for many
 sytems, the following documentation explains how to do it on POSIX
-ones. It is primarly though for GNU/Linux systems and has been tested
+ones. It is primarly thought for GNU/Linux systems and has been tested
 on recent releases of Ubuntu and Fedora.
 
 Dependencies

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-tools', 'cpp',
-  version : '0.7.0',
+  version : '1.0.0',
   license : 'GPL',
   default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -11,7 +11,7 @@ if static_linkage
 endif
 
 thread_dep = dependency('threads')
-kiwixlib_dep = dependency('kiwix', version:'>=3.0.0', static:static_linkage)
+kiwixlib_dep = dependency('kiwix', version:'>=4.0.0', static:static_linkage)
 microhttpd_dep = dependency('libmicrohttpd', static:static_linkage)
 z_dep = dependency('zlib', static:static_linkage)
 

src/man/kiwix-manage.1

@@ -15,12 +15,12 @@ kiwix\-manage LIBRARY_PATH add ZIM_PATH ...
 .PP
 \fBkiwix\-manage\fP tool for managing a kiwix library.
 .PP
-Allows one to manage the content of the kiwix library. Library file is an XML flat file
-listing ZIM files with all necessary information like favicon, date, creator,
-description, indexpath, filepath, title, source/metalink etc.
+Allows one to manage the content of the kiwix library. The library file is a
+flat XML file listing ZIM files with all necessary information like favicon,
+date, creator, description, indexpath, filepath, title, source/metalink, etc.
 .
 .PP
- Example library file can be found at http://www.kiwix.org/library\.xml
+ Example library file can be found at https://www.kiwix.org/library\.xml
 .br
 .TP
 \fBadd\fR
@@ -55,7 +55,7 @@ Path to full-text index for that ZIM file.
 Set the content location of the ZIM file over the network for in\-kiwix download.
 
 .SH SEE ALSO
-kiwix(1) kiwix\-install(1) kiwix\-serve(1)
+kiwix(1) kiwix\-serve(1)
 .SH AUTHOR
 Emmanuel Engelhart <kelson@kiwix.org>
 .br

src/man/kiwix-serve.1

@@ -9,7 +9,7 @@ kiwix\-serve [\-\-index=INDEX_PATH] [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [
 kiwix\-serve \-\-library [\-\-port=PORT] [\-\-verbose] [\-\-daemon] [\-\-attachToProcess=PID] LIBRARY_PATH
 .SH DESCRIPTION
 .PP
-Stand\-alone HTTP server for serving ZIM content over the network.
+Stand\-alone HTTP server for serving ZIM contents over the network.
 
 .TP
 \fB\-\-index=INDEX_PATH\fR
@@ -27,7 +27,7 @@ Enable verbose output.
 
 .TP
 \fB\-\-daemon\fR
-Run the server as a daemon
+Run the server as a daemon.
 
 .TP
 \fB\-\-attachToProcess=PID\fR
@@ -49,12 +49,11 @@ Serves the contents of a library file instead of a single ZIM file.
 \fBLIBRARY_PATH\fR
 Kiwix library file path.
 .br
-Library is XML file created using \fBkiwix-manage\fB.
+Library is an XML file created using \fBkiwix-manage\fB.
 
 .SH SEE ALSO
 kiwix(1) kiwix\-manage(1)
-.br
-kiwix\-install(1)
+
 .SH AUTHOR
 Emmanuel Engelhart <kelson@kiwix.org>
 .br

src/manager/kiwix-manage.cpp

@@ -21,8 +21,8 @@
 #include <unistd.h>
 #endif
 #include <getopt.h>
-#include <kiwix/common/pathTools.h>
-#include <kiwix/common/stringTools.h>
+#include <kiwix/tools/pathTools.h>
+#include <kiwix/tools/stringTools.h>
 #include <kiwix/manager.h>
 #include <kiwix/downloader.h>
 #include <cstdlib>
@@ -44,7 +44,6 @@ void show(kiwix::Library* library)
     std::cout << "#" << inc++ << std::endl
               << "id:\t\t" << book.getId() << std::endl
               << "path:\t\t" << book.getPath() << std::endl
-              << "indexpath:\t" << book.getIndexPath() << std::endl
               << "url:\t\t" << book.getUrl() << std::endl
               << "title:\t\t" << book.getTitle() << std::endl
               << "name:\t\t" << book.getName() << std::endl
@@ -64,7 +63,6 @@ void usage()
   cerr << "Usage:" << endl;
   cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH "
           "[--zimPathToSave=../content/foobar.zim] [--current] "
-          "[--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] "
           "[--url=http://...metalink]"
        << endl;
   cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... "
@@ -86,7 +84,6 @@ bool handle_add(kiwix::Library* library, const std::string& libraryPath,
 {
   string zimPath;
   string zimPathToSave = ".";
-  string indexPath;
   string url;
   string origID = "";
   int option_index = 0;
@@ -103,11 +100,10 @@ bool handle_add(kiwix::Library* library, const std::string& libraryPath,
     static struct option long_options[]
         = {{"url", required_argument, 0, 'u'},
            {"origId", required_argument, 0, 'o'},
-           {"indexPath", required_argument, 0, 'i'},
            {"zimPathToSave", required_argument, 0, 'z'},
            {0, 0, 0, 0}};
 
-    c = getopt_long(argc, argv, "cz:u:i:", long_options, &option_index);
+    c = getopt_long(argc, argv, "cz:u:", long_options, &option_index);
 
     if (c != -1) {
       switch (c) {
@@ -119,10 +115,6 @@ bool handle_add(kiwix::Library* library, const std::string& libraryPath,
           origID = optarg;
           break;
 
-        case 'i':
-          indexPath = optarg;
-          break;
-
         case 'z':
           zimPathToSave = optarg;
           break;
@@ -135,21 +127,7 @@ bool handle_add(kiwix::Library* library, const std::string& libraryPath,
   if (!zimPath.empty()) {
     kiwix::Manager manager(library);
     zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-    string bookId = manager.addBookFromPathAndGetId(
-        zimPath, zimPathToSave, url, false);
-     if (!bookId.empty()) {
-      /* Save the index infos if necessary */
-      if (!indexPath.empty()) {
-        if (isRelativePath(indexPath)) {
-          indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
-        }
-        library->getBookById(bookId).setIndexPath(indexPath);
-      }
-     } else {
-      cerr << "Unable to build or save library file '" << libraryPath << "'"
-           << endl;
-      resultCode = 1;
-    }
+    manager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false);
   } else {
     std::cerr << "Invalid zim file path" << std::endl;
     resultCode = 1;
@@ -165,19 +143,22 @@ bool handle_remove(kiwix::Library* library, const std::string& libraryPath,
   const unsigned int totalBookCount = library->getBookCount(true, true);
   bool exitCode = 0;
 
-  if (argc > 3) {
-    bookId = argv[3];
+  if (argc <= 3) {
+    std::cerr << "BookId to remove missing in the command line" << std::endl;
+    return (-1);
   }
 
-  if (!library->removeBookById(bookId)) {
-    if (totalBookCount > 0) {
-      std::cerr
-          << "Invalid book id." << std::endl;
-      exitCode = 1;
-    } else {
-      std::cerr
-          << "Invalid book id. Library is empty, no book to delete."
-          << std::endl;
+  if (!totalBookCount) {
+    std::cerr << "Library is empty, no book to delete."
+              << std::endl;
+    return 1;
+  }
+
+  for (int i = 3; i<argc; i++) {
+    bookId = argv[i];
+
+    if (!library->removeBookById(bookId)) {
+      std::cerr << "Invalid book id '" << bookId << "'." << std::endl;
       exitCode = 1;
     }
   }

src/searcher/kiwix-search.cpp

@@ -27,8 +27,7 @@ void usage()
 {
   cout << "Usage: kiwix-search [OPTIONS] ZIM PATTERN" << endl << endl
        << "  kiwix-search allows to find articles based on the a fulltext search pattern." << endl << endl
-       << "  ZIM is the full path of the ZIM file. This can also be a disctinct fulltext" << endl
-       << "  index directory (usually distributed with the *.idx extension)." << endl
+       << "  ZIM is the full path of the ZIM file." << endl
        << "  PATTERN is/are word(s) - or part of - to search in the ZIM." << endl << endl
        << "  -s, --suggestion\tSuggest article titles based on the few letters of the PATTERN instead of making a fulltext search. Work a bit like a completion solution." << endl
        << "  -v, --verbose\t\tGive details about the search process" << endl;
@@ -102,12 +101,8 @@ int main(int argc, char** argv)
     searcher = new kiwix::Searcher();
     searcher->add_reader(reader, "");
   } else {
-    try {
-      searcher = new kiwix::Searcher(zimPath, NULL, "");
-    } catch (...) {
-      cerr << "Unable to search through zim '" << zimPath << "'." << endl;
-      exit(1);
-    }
+    cerr << "Unable to search through zim '" << zimPath << "'." << endl;
+    exit(1);
   }
 
   /* Start the indexing */

src/server/kiwix-serve.cpp

@@ -44,10 +44,10 @@ extern "C" {
 #endif
 
 #include <getopt.h>
-#include <kiwix/common/otherTools.h>
-#include <kiwix/common/pathTools.h>
-#include <kiwix/common/regexTools.h>
-#include <kiwix/common/stringTools.h>
+#include <kiwix/tools/otherTools.h>
+#include <kiwix/tools/pathTools.h>
+#include <kiwix/tools/regexTools.h>
+#include <kiwix/tools/stringTools.h>
 #include <kiwix/manager.h>
 #include <kiwix/reader.h>
 #include <kiwix/searcher.h>
@@ -538,9 +538,9 @@ static struct MHD_Response* handle_search(RequestContext* request)
 
   /* Retrive geo search */
   bool has_geo_query = false;
-  float latitude;
-  float longitude;
-  float distance;
+  float latitude = 0;
+  float longitude = 0;
+  float distance = 0;
   try {
     latitude = request->get_argument<float>("latitude");
     longitude = request->get_argument<float>("longitude");
@@ -766,10 +766,22 @@ static struct MHD_Response* handle_content(RequestContext* request)
   }
 
   auto urlStr = request->get_url().substr(humanReadableBookId.size()+1);
+  if (urlStr[0] == '/') {
+    urlStr = urlStr.substr(1);
+  }
 
   try {
     entry = reader->getEntryFromPath(urlStr);
-    entry = entry.getFinalEntry();
+    if (entry.isRedirect() || urlStr.empty()) {
+      // If urlStr is empty, we want to mainPage.
+      // We must do a redirection to the real page.
+      entry = entry.getFinalEntry();
+      std::string httpRedirection = (
+        rootLocation + "/" + humanReadableBookId + "/" +
+        kiwix::urlEncode(entry.getPath()));
+      request->httpResponseCode = MHD_HTTP_FOUND;
+      return build_response("", 0, httpRedirection, "", false, false);
+    }
   } catch(kiwix::NoEntry& e) {
     if (isVerbose.load())
       printf("Failed to find %s\n", urlStr.c_str());
@@ -794,29 +806,8 @@ static struct MHD_Response* handle_content(RequestContext* request)
     zim::Blob raw_content = entry.getBlob();
     content = string(raw_content.data(), raw_content.size());
 
-    /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
-     * /A/Kiwix */
     if (mimeType.find("text/html") != string::npos) {
-      baseUrl = "/" + entry.getPath();
-      pthread_mutex_lock(&regexLock);
-      content = replaceRegex(content,
-                             "$1$2" + rootLocation + "/" + humanReadableBookId + "/$3/",
-                             "(href|src)(=[\"|\']{0,1})/([A-Z|\\-])/");
-      content = replaceRegex(content,
-                             "$1$2" + rootLocation + "/" + humanReadableBookId + "/$3/",
-                             "(@import[ ]+)([\"|\']{0,1})/([A-Z|\\-])/");
-      content = replaceRegex(
-          content,
-          "<head><base href=\"" + rootLocation + "/" + humanReadableBookId + baseUrl + "\" />",
-          "<head>");
-      pthread_mutex_unlock(&regexLock);
       introduceTaskbar(content, humanReadableBookId);
-    } else if (mimeType.find("text/css") != string::npos) {
-      pthread_mutex_lock(&regexLock);
-      content = replaceRegex(content,
-                             "$1$2" + rootLocation + "/" + humanReadableBookId + "/$3/",
-                             "(url|URL)(\\([\"|\']{0,1})/([A-Z|\\-])/");
-      pthread_mutex_unlock(&regexLock);
     }
 
     bool deflated
@@ -914,7 +905,6 @@ int main(int argc, char** argv)
   struct MHD_Daemon* daemon;
   vector<string> zimPathes;
   string libraryPath;
-  string indexPath;
   string rootPath;
   string interface;
   int serverPort = 80;
@@ -930,7 +920,6 @@ int main(int argc, char** argv)
          {"library", no_argument, 0, 'l'},
          {"nolibrarybutton", no_argument, 0, 'm'},
          {"nosearchbar", no_argument, 0, 'n'},
-         {"index", required_argument, 0, 'i'},
          {"attachToProcess", required_argument, 0, 'a'},
          {"port", required_argument, 0, 'p'},
          {"interface", required_argument, 0, 'f'},
@@ -942,7 +931,7 @@ int main(int argc, char** argv)
   while (true) {
     int option_index = 0;
     int c
-        = getopt_long(argc, argv, "mndvli:a:p:f:t:r:", long_options, &option_index);
+        = getopt_long(argc, argv, "mndvla:p:f:t:r:", long_options, &option_index);
 
     if (c != -1) {
       switch (c) {
@@ -961,9 +950,6 @@ int main(int argc, char** argv)
         case 'm':
           noLibraryButtonFlag = true;
           break;
-        case 'i':
-          indexPath = optarg;
-          break;
         case 'p':
           serverPort = atoi(optarg);
           break;
@@ -1024,11 +1010,6 @@ int main(int argc, char** argv)
     exit(1);
   }
 
-  if ((zimPathes.size() > 1) && !indexPath.empty()) {
-    cerr << "You cannot set a indexPath if you also set several zimPathes";
-    exit(1);
-  }
-
   /* Setup the library manager and get the list of books */
   kiwix::Manager manager(&library);
   if (libraryFlag) {
@@ -1071,12 +1052,6 @@ int main(int argc, char** argv)
         exit(1);
       }
     }
-    if (!indexPath.empty()) {
-      if (isRelativePath(indexPath)) {
-        indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
-      }
-      library.getBookById(library.getBooksIds()[0]).setIndexPath(indexPath);
-    }
   }
 
   /* Instance the readers and searcher and build the corresponding maps */
@@ -1087,7 +1062,6 @@ int main(int argc, char** argv)
   for (auto& bookId: booksIds) {
     auto& currentBook = library.getBookById(bookId);
     auto zimPath = currentBook.getPath();
-    auto indexPath = currentBook.getIndexPath();
 
     /* Instanciate the ZIM file handler */
     kiwix::Reader* reader = NULL;
@@ -1108,17 +1082,6 @@ int main(int argc, char** argv)
       searcher->add_reader(reader, humanReadableId);
       globalSearcher->add_reader(reader, humanReadableId);
       searchers[humanReadableId] = searcher;
-    } else if ( !indexPath.empty() ) {
-      try {
-        kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId);
-        searcher->setProtocolPrefix(rootLocation + "/");
-        searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
-        searchers[humanReadableId] = searcher;
-      } catch (...) {
-        cerr << "Unable to open the search index '" << indexPath << "'."
-             << endl;
-        searchers[humanReadableId] = nullptr;
-      }
     } else {
         searchers[humanReadableId] = nullptr;
     }

travis/compile.sh

@@ -24,6 +24,6 @@ esac
 cd ${TRAVIS_BUILD_DIR}
 export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/x86_64-linux-gnu/pkgconfig
 export PATH=${INSTALL_DIR}/bin:$PATH
-meson . build -Dctpp2-install-prefix=${INSTALL_DIR} ${MESON_OPTION}
+meson . build ${MESON_OPTION}
 cd build
 ninja

travis/install_deps.sh

@@ -3,33 +3,36 @@
 set -e
 
 REPO_NAME=${TRAVIS_REPO_SLUG#*/}
-ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz
+ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.xz
 
 # Packages.
 case ${PLATFORM} in
      "native_static")
-         PACKAGES="gcc cmake libbz2-dev ccache zlib1g-dev uuid-dev libctpp2-dev"
+         PACKAGES="gcc python3.5 cmake libbz2-dev ccache zlib1g-dev uuid-dev"
          ;;
      "native_dyn")
-         PACKAGES="gcc cmake libbz2-dev ccache zlib1g-dev uuid-dev libctpp2-dev libmicrohttpd-dev"
+         PACKAGES="gcc python3.5 cmake libbz2-dev ccache zlib1g-dev uuid-dev libmicrohttpd-dev"
          ;;
      "win32_static")
-         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache"
+         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache python3.5"
          ;;
      "win32_dyn")
-         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache"
+         PACKAGES="g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools ccache python3.5"
          ;;
      "android_arm")
-         PACKAGES="gcc cmake ccache"
+         PACKAGES="gcc python3.5 cmake ccache"
          ;;
      "android_arm64")
-         PACKAGES="gcc cmake ccache"
+         PACKAGES="gcc python3.5 cmake ccache"
          ;;
 esac
 
 sudo apt-get update -qq
 sudo apt-get install -qq python3-pip ${PACKAGES}
-sudo pip3 install meson==0.43.0
+wget https://bootstrap.pypa.io/get-pip.py
+python3.5 get-pip.py --user
+python3.5 -m pip install --user --upgrade pip
+python3.5 -m pip install --user meson
 
 # Ninja
 cd $HOME