Bump-up version to 8.2.2 and update Changelog

Remove trailing spaces

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: https://kiwix.org/support-us/

.github/move.yml

@@ -0,0 +1,27 @@
+# Configuration for Move Issues - https://github.com/dessant/move-issues
+
+# Delete the command comment when it contains no other content
+deleteCommand: true
+
+# Close the source issue after moving
+closeSourceIssue: true
+
+# Lock the source issue after moving
+lockSourceIssue: false
+
+# Mention issue and comment authors
+mentionAuthors: true
+
+# Preserve mentions in the issue content
+keepContentMentions: true
+
+# Move labels that also exist on the target repository
+moveLabels: true
+
+# Set custom aliases for targets
+# aliases:
+#   r: repo
+#   or: owner/repo
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,15 @@
+daysUntilClose: false
+staleLabel: stale
+
+issues:
+  daysUntilStale: 60
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be now be reviewed manually. Thank you
+    for your contributions.
+pulls:
+   daysUntilStale: 7
+   markComment: >
+     This pull request has been automatically marked as stale because it has not had
+     recent activity. It will be now be reviewed manually. Thank you
+     for your contributions.

COPYING

@@ -77,7 +77,7 @@ modification follow.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
- 
+
   "The Program" refers to any copyrightable work licensed under this
 License.  Each licensee is addressed as "you".  "Licensees" and
 "recipients" may be individuals or organizations.
@@ -510,7 +510,7 @@ actual knowledge that, but for the patent license, your conveying the
 covered work in a country, or your recipient's use of the covered work
 in a country, would infringe one or more identifiable patents in that
 country that you have reason to believe are valid.
-  
+
   If, pursuant to or in connection with a single transaction or
 arrangement, you convey, or propagate by procuring conveyance of, a
 covered work, and grant a patent license to some of the parties

ChangeLog

@@ -1,3 +1,20 @@
+kiwix-lib 8.2.2
+===============
+
+ * Improve a few compilation scripts
+
+kiwix-lib 8.2.1
+===============
+
+ * Reintroduce kiwix-serve taskbar
+
+kiwix-lib 8.2.0
+===============
+
+ * More debug information if aria2c command fails
+ * Allow to set kiwix-serve port
+ * Better (dead) bookmarks mgmt
+
 kiwix-lib 8.1.0
 ===============
 

README.md

@@ -7,7 +7,7 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
 [![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
-[![Build Status](https://travis-ci.org/kiwix/kiwix-lib.svg?branch=master)](https://travis-ci.org/kiwix/kiwix-lib)
+[![Build Status](https://travis-ci.com/kiwix/kiwix-lib.svg?branch=master)](https://travis-ci.com/kiwix/kiwix-lib)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
 [![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '8.1.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '8.2.2' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

include/downloader.h

@@ -97,6 +97,7 @@ class Downloader
 
   size_t getNbDownload() { return m_knownDownloads.size(); }
   std::vector<std::string> getDownloadIds();
+  const std::string &getAria2LaunchCmd();
 
  private:
   std::map<std::string, std::unique_ptr<Download>> m_knownDownloads;

include/entry.h

@@ -64,14 +64,14 @@ class Entry
      * @return the path of the entry.
      */
     std::string getPath() const;
-    
+
     /**
      * Get the title of the entry.
      *
      * @return the title of the entry.
      */
     std::string getTitle() const;
-        
+
     /**
      * Get the content of the entry.
      *
@@ -81,7 +81,7 @@ class Entry
      * @return the content of the entry.
      */
     std::string getContent() const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -91,7 +91,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset = 0) const;
-        
+
     /**
      * Get the blob of the entry.
      *
@@ -102,7 +102,7 @@ class Entry
      * @return the blob of the entry.
      */
     zim::Blob   getBlob(offset_type offset, size_type size) const;
-        
+
     /**
      * Get the info for direct access to the content of the entry.
      *
@@ -118,7 +118,7 @@ class Entry
      *         Return <"",0> if is not possible to read directly.
      */
     std::pair<std::string, offset_type> getDirectAccessInfo() const;
-        
+
     /**
      * Get the size of the entry.
      *
@@ -132,8 +132,8 @@ class Entry
      * @return the mime_type of the entry.
      */
     std::string getMimetype() const;
-    
-    
+
+
     /**
      * Get if the entry is a redirect entry.
      *

include/kiwixserve.h

@@ -17,6 +17,7 @@ class KiwixServe
     void shutDown();
     bool isRunning();
     int getPort() { return m_port; }
+    int setPort(int port);
 
   private:
   std::unique_ptr<Subprocess> mp_kiwixServe;

include/library.h

@@ -208,7 +208,7 @@ class Library
    *
    * @return A list of bookmarks
    */
-  const std::vector<kiwix::Bookmark>& getBookmarks() { return m_bookmarks; }
+  const std::vector<kiwix::Bookmark> getBookmarks(bool onlyValidBookmarks = true);
 
   /**
    * Get all book ids of the books in the library.

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-lib', 'cpp',
-  version : '8.1.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '8.2.2', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 

scripts/kiwix-compile-resources

@@ -81,12 +81,12 @@ class Resource:
             data_identifier="_".join([""]+self.identifier),
             resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
             resource_len=len(self.data),
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1],
             env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
         )
-    
+
     def dump_getter(self):
         return resource_getter_template.format(
             common_name=self.filename,
@@ -95,11 +95,11 @@ class Resource:
 
     def dump_decl(self):
         return resource_decl_template.format(
-            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
             namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
             identifier=self.identifier[-1]
         )
-    
+
 
 
 master_c_template = """//This file is automaically generated. Do not modify it.
@@ -113,7 +113,7 @@ static std::string init_resource(const char* name, const unsigned char* content,
     char * resPath = getenv(name);
     if (NULL == resPath)
         return std::string(reinterpret_cast<const char*>(content), len);
-    
+
     std::ifstream ifs(resPath);
     if (!ifs.good())
         return std::string(reinterpret_cast<const char*>(content), len);
@@ -137,7 +137,7 @@ def gen_c_file(resources, basename):
        include_file=basename,
        basename=to_identifier(basename)
     )
- 
+
 
 
 master_h_template = """//This file is automaically generated. Do not modify it.

src/android/kiwixreader.cpp

@@ -353,14 +353,17 @@ Java_org_kiwix_kiwixlib_JNIKiwixReader_searchSuggestions(JNIEnv* env,
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getNextSuggestion(JNIEnv* env,
                                                          jobject obj,
-                                                         jobject titleObj)
+                                                         jobject titleObj,
+                                                         jobject urlObj)
 {
   jboolean retVal = JNI_FALSE;
   std::string cTitle;
+  std::string cUrl;
 
   try {
-    if (READER->getNextSuggestion(cTitle)) {
+    if (READER->getNextSuggestion(cTitle, cUrl)) {
       setStringObjValue(cTitle, titleObj, env);
+      setStringObjValue(cUrl, urlObj, env);
       retVal = JNI_TRUE;
     }
   } catch (std::exception& e) {

src/android/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -102,7 +102,7 @@ public class JNIKiwixReader
 
   public native boolean searchSuggestions(String prefix, int count);
 
-  public native boolean getNextSuggestion(JNIKiwixString title);
+  public native boolean getNextSuggestion(JNIKiwixString title, JNIKiwixString url);
 
   public native boolean getPageUrlFromTitle(String title, JNIKiwixString url);
 

src/aria2.cpp

@@ -78,18 +78,32 @@ Aria2::Aria2():
   callCmd.push_back("--max-concurrent-downloads=42");
   callCmd.push_back("--rpc-max-request-size=6M");
   callCmd.push_back("--file-allocation=none");
+  for (auto &cmd : callCmd) {
+      m_launchCmd.append(cmd).append(" ");
+  }
   mp_aria = Subprocess::run(callCmd);
   mp_curl = curl_easy_init();
+  char errbuf[CURL_ERROR_SIZE];
   curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc");
   curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port);
   curl_easy_setopt(mp_curl, CURLOPT_POST, 1L);
+  curl_easy_setopt(mp_curl, CURLOPT_ERRORBUFFER, errbuf);
 
   int watchdog = 50;
   while(--watchdog) {
     sleep(10);
+    errbuf[0] = 0;
     auto res = curl_easy_perform(mp_curl);
     if (res == CURLE_OK) {
       break;
+    } else if (watchdog == 1) {
+      std::cerr <<" curl_easy_perform() failed." << std::endl;
+      fprintf(stderr, "\nlibcurl: (%d) ", res);
+      if (errbuf[0] != 0) {
+        std::cerr << errbuf << std::endl;
+      } else {
+        std::cerr << curl_easy_strerror(res) << std::endl;
+      }
     }
   }
   if (!watchdog) {

src/aria2.h

@@ -26,6 +26,7 @@ class Aria2
     std::string m_downloadDir;
     CURL* mp_curl;
     pthread_mutex_t m_lock;
+    std::string m_launchCmd;
 
     std::string doRequest(const MethodCall& methodCall);
 
@@ -43,6 +44,7 @@ class Aria2
     void pause(const std::string& gid);
     void unpause(const std::string& gid);
     void remove(const std::string& gid);
+    const std::string &getLaunchCmd() { return m_launchCmd; };
 };
 
 }; //end namespace kiwix

src/downloader.cpp

@@ -127,17 +127,24 @@ void Download::cancelDownload()
 Downloader::Downloader() :
   mp_aria(new Aria2())
 {
-  for (auto gid : mp_aria->tellActive()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellActive()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellActive failed : " << e.what();
   }
-  for (auto gid : mp_aria->tellWaiting()) {
-    m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
-    m_knownDownloads[gid]->updateStatus();
+  try {
+    for (auto gid : mp_aria->tellWaiting()) {
+      m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));
+      m_knownDownloads[gid]->updateStatus();
+    }
+  } catch (std::exception& e) {
+    std::cerr << "aria2 tellWaiting failed : " << e.what();
   }
 }
 
-
 /* Destructor */
 Downloader::~Downloader()
 {
@@ -156,6 +163,11 @@ std::vector<std::string> Downloader::getDownloadIds() {
   return ret;
 }
 
+const std::string &Downloader::getAria2LaunchCmd()
+{
+  return mp_aria->getLaunchCmd();
+}
+
 Download* Downloader::startDownload(const std::string& uri)
 {
   for (auto& p: m_knownDownloads) {

src/kiwixserve.cpp

@@ -68,4 +68,14 @@ bool KiwixServe::isRunning()
     return false;
 }
 
+int KiwixServe::setPort(int port)
+{
+    if (port >= 1 && port <= 65535) {
+        m_port = port;
+    } else {
+        return -1;
+    }
+    return m_port;
+}
+
 }

src/library.cpp

@@ -184,6 +184,21 @@ std::vector<std::string> Library::getBooksPublishers()
   return booksPublishers;
 }
 
+const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks)
+{
+  if (!onlyValidBookmarks) {
+    return m_bookmarks;
+  }
+  std::vector<kiwix::Bookmark> validBookmarks;
+  auto booksId = getBooksIds();
+  for(auto& bookmark:m_bookmarks) {
+    if (std::find(booksId.begin(), booksId.end(), bookmark.getBookId()) != booksId.end()) {
+      validBookmarks.push_back(bookmark);
+    }
+  }
+  return validBookmarks;
+}
+
 std::vector<std::string> Library::getBooksIds()
 {
   std::vector<std::string> bookIds;

src/server.cpp

@@ -870,23 +870,11 @@ Response InternalServer::handle_content(const RequestContext& request)
     zim::Blob raw_content = entry.getBlob();
     content = string(raw_content.data(), raw_content.size());
     auto response = get_default_response();
-    response.set_mimeType(mimeType);
 
-    /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
-     * /A/Kiwix */
-    if (mimeType.find("text/html") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(href|src)(=[\"|\']{0,1})/([A-Z|\\-])/");
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(@import[ ]+)([\"|\']{0,1})/([A-Z|\\-])/");
+    if (mimeType.find("text/html") != string::npos)
       response.set_taskbar(bookName, reader->getTitle());
-    } else if (mimeType.find("text/css") != string::npos) {
-      content = replaceRegex(content,
-                             "$1$2" + m_root + "/" + bookName + "/$3/",
-                             "(url|URL)(\\([\"|\']{0,1})/([A-Z|\\-])/");
-    }
+
+    response.set_mimeType(mimeType);
     response.set_content(content);
     response.set_compress(true);
     response.set_cache(true);

src/server/response.cpp

@@ -140,7 +140,7 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
       bool shouldCompress = m_compress && request.can_compress();
       shouldCompress &= m_mimeType.find("text/") != string::npos
                      || m_mimeType.find("application/javascript") != string::npos
-                     || m_mimeType.find("application/json") != string::npos; 
+                     || m_mimeType.find("application/json") != string::npos;
 
       shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
 

src/tools/base64.cpp

@@ -1,4 +1,4 @@
-/* 
+/*
    base64.cpp and base64.h
 
    Copyright (C) 2004-2008 René Nyffenegger
@@ -27,7 +27,7 @@
 #include <tools/base64.h>
 #include <iostream>
 
-static const std::string base64_chars = 
+static const std::string base64_chars =
              "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
              "abcdefghijklmnopqrstuvwxyz"
   "0123456789+/";

static/opensearchdescription.xml

@@ -2,7 +2,7 @@
 <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
   <ShortName>Zim catalog search</ShortName>
   <Description>Search zim files in the catalog.</Description>
-  <Url type="application/atom+xml;profile=opds-catalog" 
+  <Url type="application/atom+xml;profile=opds-catalog"
        xmlns:atom="http://www.w3.org/2005/Atom"
        indexOffset="0"
        template="/{{root}}/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>

test/kiwixserve.cpp

@@ -0,0 +1,18 @@
+#include "gtest/gtest.h"
+#include "../include/kiwixserve.h"
+
+TEST(KiwixServeTest, PortTest)
+{
+    kiwix::KiwixServe kiwixServe("libraryPath", 8181);
+    EXPECT_EQ(kiwixServe.getPort(), 8181);
+    kiwixServe.setPort(8484);
+    EXPECT_EQ(kiwixServe.getPort(), 8484);
+    EXPECT_EQ(kiwixServe.setPort(0), -1);
+    EXPECT_EQ(kiwixServe.setPort(3456789), -1);
+}
+
+int main(int argc, char** argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

test/library.cpp

@@ -180,6 +180,7 @@ const char * sampleOpdsStream = R"(
 
 #include "../include/library.h"
 #include "../include/manager.h"
+#include "../include/bookmark.h"
 
 namespace
 {
@@ -191,9 +192,34 @@ class LibraryTest : public ::testing::Test {
      manager.readOpds(sampleOpdsStream, "foo.urlHost");
   }
 
+    kiwix::Bookmark createBookmark(const std::string &id) {
+        kiwix::Bookmark bookmark;
+        bookmark.setBookId(id);
+        return bookmark;
+    };
+
   kiwix::Library lib;
 };
 
+TEST_F(LibraryTest, getBookMarksTest)
+{
+    auto bookId1 = lib.getBooksIds()[0];
+    auto bookId2 = lib.getBooksIds()[1];
+
+    lib.addBookmark(createBookmark(bookId1));
+    lib.addBookmark(createBookmark("invalid-bookmark-id"));
+    lib.addBookmark(createBookmark(bookId2));
+    auto onlyValidBookmarks = lib.getBookmarks();
+    auto allBookmarks = lib.getBookmarks(false);
+
+    EXPECT_EQ(onlyValidBookmarks[0].getBookId(), bookId1);
+    EXPECT_EQ(onlyValidBookmarks[1].getBookId(), bookId2);
+
+    EXPECT_EQ(allBookmarks[0].getBookId(), bookId1);
+    EXPECT_EQ(allBookmarks[1].getBookId(), "invalid-bookmark-id");
+    EXPECT_EQ(allBookmarks[2].getBookId(), bookId2);
+}
+
 TEST_F(LibraryTest, sanityCheck)
 {
   EXPECT_EQ(lib.getBookCount(true, true), 10U);

test/meson.build

@@ -6,7 +6,8 @@ tests = [
     'regex',
     'tagParsing',
     'stringTools',
-    'pathTools'
+    'pathTools',
+    'kiwixserve'
 ]
 
 

travis/install_deps.sh

@@ -8,6 +8,8 @@ REPO_NAME=${TRAVIS_REPO_SLUG#*/}
 cd $HOME
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]
 then
+  # upgrade wget as openssl is updated by travis
+  brew upgrade wget
   pip3 install meson==0.49.2
 
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip