Merge pull request #227 from kiwix/new_version

New version 5.1.0

.travis.yml

@@ -1,6 +1,6 @@
 language: cpp
-dist: trusty
-sudo: false
+dist: xenial
+sudo: true
 cache: ccache
 before_install:
 - PATH=$PATH:$HOME/bin

ChangeLog

@@ -1,3 +1,19 @@
+kiwix-lib 5.1.0
+===============
+
+ * Add function to pause, resume and stop downloads.
+ * Add zim's tags in the opds stream.
+ * Addapt to new libzim 5.0.0 API.
+
+kiwix-lib 5.0.0
+===============
+
+ * Remove error message when trying to open an wrong zim file.
+ * Rewrite `makeTmpDirectory` to not use uuid functions on windows.
+ * [API break] Remove `getNetworkInterfaces` and `getBestPublicIp`.
+ * Remove rpath
+ * Detect infinite (and too long) redirect loops.
+
 kiwix-lib 4.1.0
 ===============
 

include/downloader.h

@@ -54,6 +54,9 @@ class Download {
       m_status(K_UNKNOWN),
       m_did(did) {};
   void updateStatus(bool follow=false);
+  void pauseDownload();
+  void resumeDownload();
+  void cancelDownload();
   StatusResult getStatus()          { return m_status; }
   std::string  getDid()             { return m_did; }
   std::string  getFollowedBy()      { return m_followedBy; }

include/tools/networkTools.h

@@ -20,13 +20,10 @@
 #ifndef KIWIX_NETWORKTOOLS_H
 #define KIWIX_NETWORKTOOLS_H
 
-#include <map>
 #include <string>
 
 namespace kiwix
 {
-std::map<std::string, std::string> getNetworkInterfaces();
-std::string getBestPublicIp();
 std::string download(const std::string& url);
 }
 

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-lib', 'cpp',
-  version : '4.1.0',
+  version : '5.1.0',
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -14,7 +14,7 @@ endif
 
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 

src/aria2.cpp

@@ -82,7 +82,7 @@ Aria2::Aria2():
 
   int watchdog = 50;
   while(--watchdog) {
-    std::this_thread::sleep_for(std::chrono::microseconds(10000));
+    sleep(10);
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
@@ -171,7 +171,6 @@ std::vector<std::string> Aria2::tellActive()
   MethodCall methodCall("aria2.tellActive", m_secret);
   auto statusArray = methodCall.newParamValue().getArray();
   statusArray.addValue().set(std::string("gid"));
-  statusArray.addValue().set(std::string("following"));
   auto responseContent = doRequest(methodCall);
   MethodResponse response(responseContent);
   std::vector<std::string> activeGID;
@@ -186,6 +185,27 @@ std::vector<std::string> Aria2::tellActive()
   return activeGID;
 }
 
+std::vector<std::string> Aria2::tellWaiting()
+{
+  MethodCall methodCall("aria2.tellWaiting", m_secret);
+  methodCall.newParamValue().set(0);
+  methodCall.newParamValue().set(99); // max number of downloads to be returned, don't know how to set this properly assumed that there will not be more than 99 paused downloads.
+  auto statusArray = methodCall.newParamValue().getArray();
+  statusArray.addValue().set(std::string("gid"));
+  auto responseContent = doRequest(methodCall);
+  MethodResponse response(responseContent);
+  std::vector<std::string> waitingGID;
+  int index = 0;
+  while(true) {
+    try {
+      auto structNode = response.getParamValue(0).getArray().getValue(index++).getStruct();
+      auto gidNode = structNode.getMember("gid");
+      waitingGID.push_back(gidNode.getValue().getAsS());
+    } catch (InvalidRPCNode& e) { break; }
+  }
+  return waitingGID;
+}
+
 void Aria2::saveSession()
 {
   MethodCall methodCall("aria2.saveSession", m_secret);
@@ -199,5 +219,25 @@ void Aria2::shutdown()
   doRequest(methodCall);
 }
 
+void Aria2::pause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.pause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::unpause(const std::string& gid)
+{
+    MethodCall methodCall("aria2.unpause", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
+
+void Aria2::remove(const std::string& gid)
+{
+    MethodCall methodCall("aria2.remove", m_secret);
+    methodCall.newParamValue().set(gid);
+    doRequest(methodCall);
+}
 
 } // end namespace kiwix

src/aria2.h

@@ -37,8 +37,12 @@ class Aria2
     std::string addUri(const std::vector<std::string>& uri);
     std::string tellStatus(const std::string& gid, const std::vector<std::string>& statusKey);
     std::vector<std::string> tellActive();
+    std::vector<std::string> tellWaiting();
     void saveSession();
     void shutdown();
+    void pause(const std::string& gid);
+    void unpause(const std::string& gid);
+    void remove(const std::string& gid);
 };
 
 }; //end namespace kiwix

src/book.cpp

@@ -138,6 +138,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_language = VALUE("language");
   m_date = fromOpdsDate(VALUE("updated"));
   m_creator = node.child("author").child("name").child_value();
+  m_tags = VALUE("tags");
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();

src/downloader.cpp

@@ -36,6 +36,8 @@ namespace kiwix
 
 void Download::updateStatus(bool follow)
 {
+  if (m_status == Download::K_REMOVED)
+    return;
   static std::vector<std::string> statusKey = {"status", "files", "totalLength",
                                                "completedLength", "followedBy",
                                                "downloadSpeed", "verifiedLength"};
@@ -93,6 +95,33 @@ void Download::updateStatus(bool follow)
   }
 }
 
+void Download::resumeDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->unpause(m_followedBy);
+    else
+        mp_aria->unpause(m_did);
+    updateStatus(true);
+}
+
+void Download::pauseDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->pause(m_followedBy);
+    else
+        mp_aria->pause(m_did);
+    updateStatus(true);
+}
+
+void Download::cancelDownload()
+{
+    if (!m_followedBy.empty())
+        mp_aria->remove(m_followedBy);
+    else
+        mp_aria->remove(m_did);
+    m_status = Download::K_REMOVED;
+}
+
 /* Constructor */
 Downloader::Downloader() :
   mp_aria(new Aria2())
@@ -101,6 +130,10 @@ Downloader::Downloader() :
     m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
     m_knownDownloads[gid]->updateStatus();
   }
+  for (auto gid : mp_aria->tellWaiting()) {
+    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+    m_knownDownloads[gid]->updateStatus();
+  }
 }
 
 
@@ -139,14 +172,23 @@ Download* Downloader::startDownload(const std::string& uri)
 Download* Downloader::getDownload(const std::string& did)
 {
   try {
+    m_knownDownloads.at(did).get()->updateStatus(true);
     return m_knownDownloads.at(did).get();
   } catch(exception& e) {
     for (auto gid : mp_aria->tellActive()) {
       if (gid == did) {
         m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
         return m_knownDownloads[gid].get();
       }
     }
+    for (auto gid : mp_aria->tellWaiting()) {
+      if (gid == did) {
+        m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+        m_knownDownloads.at(gid).get()->updateStatus(true);
+        return m_knownDownloads[gid].get();
+      }
+  }
     throw e;
   }
 }

src/entry.cpp

@@ -122,7 +122,6 @@ Entry Entry::getFinalEntry() const
   if (final_article.good()) {
     return final_article;
   }
-
   int loopCounter = 42;
   final_article = article;
   while (final_article.isRedirect() && loopCounter--) {
@@ -131,7 +130,10 @@ Entry Entry::getFinalEntry() const
       throw NoEntry();
     }
   }
-
+  // Prevent infinite loops.
+  if (final_article.isRedirect()) {
+    throw NoEntry();
+  }
   return final_article;
 }
 

src/manager.cpp

@@ -217,7 +217,6 @@ bool Manager::readBookFromPath(const std::string& path, kiwix::Book* book)
     book->update(reader);
     book->setPathValid(true);
   } catch (const std::exception& e) {
-    std::cerr << "Invalid " << tmp_path << " : " << e.what() << std::endl;
     book->setPathValid(false);
     return false;
   }

src/meson.build

@@ -44,5 +44,4 @@ kiwixlib = library('kiwix',
                    dependencies : all_deps,
                    version: meson.project_version(),
                    install: true,
-                   install_dir: install_dir,
-                   install_rpath: '$ORIGIN')
+                   install_dir: install_dir)

src/opds_dumper.cpp

@@ -74,6 +74,7 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
   ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate()));
   ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription());
+  ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
 
   auto content_node = entry_node.append_child("link");
   content_node.append_attribute("type") = "text/html";

src/reader.cpp

@@ -765,7 +765,7 @@ bool Reader::searchSuggestionsSmart(const string& prefix,
   this->suggestions.clear();
   this->suggestionsOffset = this->suggestions.begin();
   /* Try to search in the title using fulltext search database */
-  const zim::Search* suggestionSearch
+  const auto suggestionSearch
       = this->getZimFileHandler()->suggestions(prefix, 0, suggestionsCount);
 
   if (suggestionSearch->get_matches_estimated()) {

src/tools/networkTools.cpp

@@ -19,167 +19,17 @@
 
 #include <tools/networkTools.h>
 
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#else
-#include <net/if.h>
-#include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
 
 #include <curl/curl.h>
 
 #include <sstream>
 #include <iostream>
+#include <stdexcept>
 
 
-std::map<std::string, std::string> kiwix::getNetworkInterfaces()
-{
-  std::map<std::string, std::string> interfaces;
-
-#ifdef _WIN32
-  SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
-  if (sd == (SOCKET)SOCKET_ERROR) {
-    std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  INTERFACE_INFO InterfaceList[20];
-  unsigned long nBytesReturned;
-  if (WSAIoctl(sd,
-               SIO_GET_INTERFACE_LIST,
-               0,
-               0,
-               &InterfaceList,
-               sizeof(InterfaceList),
-               &nBytesReturned,
-               0,
-               0)
-      == SOCKET_ERROR) {
-    std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
-              << std::endl;
-    return interfaces;
-  }
-
-  int nNumInterfaces = nBytesReturned / sizeof(INTERFACE_INFO);
-  for (int i = 0; i < nNumInterfaces; ++i) {
-    sockaddr_in* pAddress;
-    pAddress = (sockaddr_in*)&(InterfaceList[i].iiAddress);
-
-    /* Add to the map */
-    std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
-    std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
-    interfaces.insert(
-        std::pair<std::string, std::string>(interfaceName, interfaceIp));
-  }
-#else
-  /* Get Network interfaces information */
-  char buf[16384];
-  struct ifconf ifconf;
-  int fd = socket(PF_INET, SOCK_DGRAM, 0); /* Only IPV4 */
-  ifconf.ifc_len = sizeof buf;
-  ifconf.ifc_buf = buf;
-  if (ioctl(fd, SIOCGIFCONF, &ifconf) != 0) {
-    perror("ioctl(SIOCGIFCONF)");
-    exit(EXIT_FAILURE);
-  }
-
-  /* Go through each interface */
-  int i;
-  size_t len;
-  struct ifreq* ifreq;
-  ifreq = ifconf.ifc_req;
-  for (i = 0; i < ifconf.ifc_len;) {
-    if (ifreq->ifr_addr.sa_family == AF_INET) {
-      /* Get the network interface ip */
-      char host[128] = {0};
-      const int error = getnameinfo(&(ifreq->ifr_addr),
-                                    sizeof ifreq->ifr_addr,
-                                    host,
-                                    sizeof host,
-                                    0,
-                                    0,
-                                    NI_NUMERICHOST);
-      if (!error) {
-        std::string interfaceName = std::string(ifreq->ifr_name);
-        std::string interfaceIp = std::string(host);
-        /* Add to the map */
-        interfaces.insert(
-            std::pair<std::string, std::string>(interfaceName, interfaceIp));
-      } else {
-        perror("getnameinfo()");
-      }
-    }
-
-/* some systems have ifr_addr.sa_len and adjust the length that
- * way, but not mine. weird */
-#ifndef __linux__
-    len = IFNAMSIZ + ifreq->ifr_addr.sa_len;
-#else
-    len = sizeof *ifreq;
-#endif
-    ifreq = (struct ifreq*)((char*)ifreq + len);
-    i += len;
-  }
-#endif
-  return interfaces;
-}
-
-std::string kiwix::getBestPublicIp()
-{
-  std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
-
-#ifndef _WIN32
-  const char* const prioritizedNames[]
-      = {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
-  const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
-  for (int i = 0; i < count; ++i) {
-    std::map<std::string, std::string>::const_iterator it
-        = interfaces.find(prioritizedNames[i]);
-    if (it != interfaces.end()) {
-      return it->second;
-    }
-  }
-#endif
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
-      return interfaceIp;
-    }
-  }
-
-  for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
-       iter != interfaces.end();
-       ++iter) {
-    std::string interfaceIp = iter->second;
-    if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
-      return interfaceIp;
-    }
-  }
-
-  return "127.0.0.1";
-}
-
 size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
   auto str = static_cast<std::stringstream*>(userdata);

src/tools/pathTools.cpp

@@ -228,25 +228,20 @@ bool makeDirectory(const string& path)
 string makeTmpDirectory()
 {
 #ifdef _WIN32
-  char cbase[MAX_PATH+1];
-  int base_len = GetTempPath(MAX_PATH+1, cbase);
-  UUID uuid;
-  UuidCreate(&uuid);
-  char* dir_name;
-  UuidToString(&uuid, reinterpret_cast<unsigned char**>(&dir_name));
-  string dir(cbase, base_len);
-  dir += dir_name;
-  _mkdir(dir.c_str());
-  RpcStringFree(reinterpret_cast<unsigned char**>(&dir_name));
+  char cbase[MAX_PATH];
+  char ctmp[MAX_PATH];
+  GetTempPath(MAX_PATH-14, cbase);
+  // This create a file for us, ensure it is unique.
+  // So we need to delete it and create the directory using the same name.
+  GetTempFileName(cbase, "kiwix", 0, ctmp);
+  DeleteFile(ctmp);
+  _mkdir(ctmp);
+  return string(ctmp);
 #else
-  string base = "/tmp";
-  auto _template = base + "/kiwix-lib_XXXXXX";
-  char* _template_array = new char[_template.size()+1];
-  memcpy(_template_array, _template.c_str(), _template.size());
+  char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
   string dir = mkdtemp(_template_array);
-  delete[] _template_array;
-#endif
   return dir;
+#endif
 }
 
 /* Try to create a link and if does not work then make a copy */

travis/install_deps.sh

@@ -3,7 +3,6 @@
 set -e
 
 REPO_NAME=${TRAVIS_REPO_SLUG#*/}
-ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.xz
 
 # Ninja
 cd $HOME
@@ -13,6 +12,7 @@ then
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip
   unzip ninja-mac.zip ninja
+  ARCHIVE_NAME=deps_osx_${PLATFORM}_${REPO_NAME}.tar.xz
 else
   wget https://bootstrap.pypa.io/get-pip.py
   python3.5 get-pip.py --user
@@ -22,6 +22,7 @@ else
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
   unzip ninja-linux.zip ninja
+  ARCHIVE_NAME=deps_linux_xenial_${PLATFORM}_${REPO_NAME}.tar.xz
 fi
 
 mkdir -p $HOME/bin
@@ -31,3 +32,4 @@ cp ninja $HOME/bin
 cd ${HOME}
 wget http://tmp.kiwix.org/ci/${ARCHIVE_NAME}
 tar xf ${HOME}/${ARCHIVE_NAME}
+sudo ln -s travis ../ci_builder