Update minimum version of libzim.

Fix book articlecount

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 name: CI
 
-on: [pull_request]
+on: [push]
 
 jobs:
   Macos:
@@ -85,25 +85,20 @@ jobs:
     container:
       image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26"
     steps:
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
     - name: Checkout code
       shell: python
       run: |
-        import json
         from subprocess import check_call
         from os import environ
-        with open(environ['GITHUB_EVENT_PATH'], 'r') as f:
-            content = f.read()
-        event_data = json.loads(content)
-        try:
-            branch_ref = event_data['ref'].split('/')[-1]
-        except KeyError:
-            branch_ref = event_data['pull_request']['head']['ref']
-        print("Cloning branch", branch_ref)
         command = [
           'git', 'clone',
           'https://github.com/${{github.repository}}',
           '--depth=1',
-          '--branch', branch_ref
+          '--branch', '${{steps.extract_branch.outputs.branch}}'
         ]
         check_call(command, cwd=environ['HOME'])
     - name: Install deps

ChangeLog

@@ -1,3 +1,12 @@
+kiwix-lib 9.1.0
+===============
+
+ * [JAVA] Add a method to get the size of an article.
+ * Add a new method to the libray to get the book by path.
+ * Add a option to make the server blocks external link.
+   Links are intercepted by js an redirected to a "portail" page.
+ * [ODPS] Correctly handle book's articleCount and mediaCount.
+
 kiwix-lib 9.0.1
 ===============
 

README.md

@@ -7,7 +7,7 @@ GNU/Linux, macOS, Android, iOS, ...).
 
 [![Download](https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/images/download.svg)](https://bintray.com/kiwix/kiwix/kiwixlib/_latestVersion)
 [![AUR version](https://img.shields.io/aur/version/kiwix-lib)](https://aur.archlinux.org/packages/kiwix-lib/)
-[![Build Status](https://travis-ci.com/kiwix/kiwix-lib.svg?branch=master)](https://travis-ci.com/kiwix/kiwix-lib)
+[![Build Status](https://github.com/kiwix/kiwix-lib/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-lib/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-lib/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-lib)
 [![Codecov](https://codecov.io/gh/kiwix/kiwix-lib/branch/master/graph/badge.svg)](https://codecov.io/gh/kiwix/kiwix-lib)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
@@ -41,7 +41,7 @@ libraries need to be available:
 * [Aria2](https://aria2.github.io/) (package `aria2` on Ubuntu)
 * [Mustache](https://github.com/kainjow/Mustache) (Just copy the
 header `mustache.hpp` somewhere it can be found by the compiler and/or
-set CPPFLAGS with correct `-I` option)
+set CPPFLAGS with correct `-I` option). Use Mustache version 3 only.
 
 These dependencies may or may not be packaged by your operating
 system. They may also be packaged but only in an older version. The

android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle

@@ -26,7 +26,7 @@ task writePom {
     project {
         groupId 'org.kiwix.kiwixlib'
         artifactId 'kiwixlib'
-        version '9.0.1' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
+        version '9.1.0' + (System.env.KIWIXLIB_BUILDVERSION == null ? '' : '-'+System.env.KIWIXLIB_BUILDVERSION)
         packaging 'aar'
         name 'kiwixlib'
         url 'https://github.com/kiwix/kiwix-lib'

include/book.h

@@ -98,7 +98,7 @@ class Book
   std::string m_id;
   std::string m_downloadId;
   std::string m_path;
-  bool m_pathValid;
+  bool m_pathValid = false;
   std::string m_title;
   std::string m_description;
   std::string m_language;
@@ -110,10 +110,10 @@ class Book
   std::string m_flavour;
   std::string m_tags;
   std::string m_origId;
-  uint64_t m_articleCount;
-  uint64_t m_mediaCount;
-  bool m_readOnly;
-  uint64_t m_size;
+  uint64_t m_articleCount = 0;
+  uint64_t m_mediaCount = 0;
+  bool m_readOnly = false;
+  uint64_t m_size = 0;
   mutable std::string m_favicon;
   std::string m_faviconUrl;
   std::string m_faviconMimeType;

include/library.h

@@ -149,6 +149,7 @@ class Library
   bool removeBookmark(const std::string& zimId, const std::string& url);
 
   Book& getBookById(const std::string& id);
+  Book& getBookByPath(const std::string& path);
   std::shared_ptr<Reader> getReaderById(const std::string& id);
 
   /**

include/searcher.h

@@ -89,7 +89,7 @@ class Searcher
    * @param resultEnd the end offset of the search results (used for pagination).
    * @param verbose print some info on stdout if true.
    */
-  void search(std::string& search,
+  void search(const std::string& search,
               unsigned int resultStart,
               unsigned int resultEnd,
               const bool verbose = false);

include/server.h

@@ -56,7 +56,9 @@ namespace kiwix
        void setNbThreads(int threads) { m_nbThreads = threads; }
        void setVerbose(bool verbose) { m_verbose = verbose; }
        void setTaskbar(bool withTaskbar, bool withLibraryButton)
-         { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+        { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
+       void setBlockExternalLinks(bool blockExternalLinks)
+        { m_blockExternalLinks = blockExternalLinks; }
 
      protected:
        Library* mp_library;
@@ -68,6 +70,7 @@ namespace kiwix
        bool m_verbose = false;
        bool m_withTaskbar = true;
        bool m_withLibraryButton = true;
+       bool m_blockExternalLinks = false;
        std::unique_ptr<InternalServer> mp_server;
   };
 }

meson.build

@@ -1,5 +1,5 @@
 project('kiwix-lib', 'cpp',
-  version : '9.0.1', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
+  version : '9.1.0', # Also change this in android-kiwix-lib-publisher/kiwixLibAndroid/build.gradle
   license : 'GPL',
   default_options : ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
@@ -20,7 +20,7 @@ endif
 
 thread_dep = dependency('threads')
 libicu_dep = dependency('icu-i18n', static:static_deps)
-libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
+libzim_dep = dependency('libzim', version : '>=6.1.0', static:static_deps)
 pugixml_dep = dependency('pugixml', static:static_deps)
 libcurl_dep = dependency('libcurl', static:static_deps)
 microhttpd_dep = dependency('libmicrohttpd', static:static_deps)

src/book.cpp

@@ -156,6 +156,8 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
   m_name = VALUE("name");
   m_flavour = VALUE("flavour");
   m_tags = VALUE("tags");
+  m_articleCount = strtoull(VALUE("articleCount"), 0, 0);
+  m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0);
   for(auto linkNode = node.child("link"); linkNode;
            linkNode = linkNode.next_sibling("link")) {
     std::string rel = linkNode.attribute("rel").value();

src/library.cpp

@@ -85,6 +85,18 @@ Book& Library::getBookById(const std::string& id)
   return m_books.at(id);
 }
 
+Book& Library::getBookByPath(const std::string& path)
+{
+  for(auto& it: m_books) {
+    auto& book = it.second;
+    if (book.getPath() == path)
+      return book;
+  }
+  std::ostringstream ss;
+  ss << "No book with path " << path << " in the library." << std::endl;
+  throw std::out_of_range(ss.str());
+}
+
 std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
 {
   try {

src/opds_dumper.cpp

@@ -78,6 +78,8 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) {
   ADD_TEXT_ENTRY(entry_node, "name", book.getName());
   ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour());
   ADD_TEXT_ENTRY(entry_node, "tags", book.getTags());
+  ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount()));
+  ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount()));
   ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath());
 
   auto content_node = entry_node.append_child("link");

src/searcher.cpp

@@ -97,7 +97,7 @@ Reader* Searcher::get_reader(int readerIndex)
 }
 
 /* Search strings in the database */
-void Searcher::search(std::string& search,
+void Searcher::search(const std::string& search,
                       unsigned int resultStart,
                       unsigned int resultEnd,
                       const bool verbose)

src/server.cpp

@@ -95,7 +95,8 @@ class InternalServer {
                    int nbThreads,
                    bool verbose,
                    bool withTaskbar,
-                   bool withLibraryButton);
+                   bool withLibraryButton,
+                   bool blockExternalLinks);
     virtual ~InternalServer() = default;
 
     int handlerCallback(struct MHD_Connection* connection,
@@ -119,6 +120,7 @@ class InternalServer {
     Response handle_search(const RequestContext& request);
     Response handle_suggest(const RequestContext& request);
     Response handle_random(const RequestContext& request);
+    Response handle_captured_external(const RequestContext& request);
     Response handle_content(const RequestContext& request);
 
     kainjow::mustache::data get_default_data();
@@ -131,6 +133,7 @@ class InternalServer {
     std::atomic_bool m_verbose;
     bool m_withTaskbar;
     bool m_withLibraryButton;
+    bool m_blockExternalLinks;
     struct MHD_Daemon* mp_daemon;
 
     Library* mp_library;
@@ -157,7 +160,8 @@ bool Server::start() {
     m_nbThreads,
     m_verbose,
     m_withTaskbar,
-    m_withLibraryButton));
+    m_withLibraryButton,
+    m_blockExternalLinks));
   return mp_server->start();
 }
 
@@ -186,7 +190,8 @@ InternalServer::InternalServer(Library* library,
                                int nbThreads,
                                bool verbose,
                                bool withTaskbar,
-                               bool withLibraryButton) :
+                               bool withLibraryButton,
+                               bool blockExternalLinks) :
   m_addr(addr),
   m_port(port),
   m_root(root),
@@ -194,6 +199,7 @@ InternalServer::InternalServer(Library* library,
   m_verbose(verbose),
   m_withTaskbar(withTaskbar),
   m_withLibraryButton(withLibraryButton),
+  m_blockExternalLinks(blockExternalLinks),
   mp_daemon(nullptr),
   mp_library(library),
   mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
@@ -340,6 +346,9 @@ Response InternalServer::handle_request(const RequestContext& request)
     if (request.get_url() == "/random")
       return handle_random(request);
 
+    if (request.get_url() == "/catch/external")
+      return handle_captured_external(request);
+
     return handle_content(request);
   } catch (std::exception& e) {
     fprintf(stderr, "===== Unhandled error : %s\n", e.what());
@@ -359,7 +368,7 @@ kainjow::mustache::data InternalServer::get_default_data()
 
 Response InternalServer::get_default_response()
 {
-  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton);
+  return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
 }
 
 
@@ -383,7 +392,7 @@ Response InternalServer::build_500(const std::string& msg)
 {
   kainjow::mustache::data data;
   data.set("error", msg);
-  Response response(m_root, true, false, false);
+  Response response(m_root, true, false, false, false);
   response.set_template(RESOURCE::templates::_500_html, data);
   response.set_mimeType("text/html");
   response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
@@ -711,6 +720,26 @@ Response InternalServer::handle_random(const RequestContext& request)
   }
 }
 
+Response InternalServer::handle_captured_external(const RequestContext& request)
+{
+  std::string source = "";
+  try {
+    source = kiwix::urlDecode(request.get_argument("source"));
+  } catch (const std::out_of_range& e) {}
+
+  if (source.empty())
+    return build_404(request, "");
+
+  auto data = get_default_data();
+  data.set("source", source);
+  auto response = get_default_response();
+  response.set_template(RESOURCE::templates::captured_external_html, data);
+  response.set_mimeType("text/html; charset=utf-8");
+  response.set_compress(true);
+  response.set_taskbar("", "");
+  return response;
+}
+
 Response InternalServer::handle_catalog(const RequestContext& request)
 {
   if (m_verbose.load()) {

src/server/response.cpp

@@ -17,7 +17,7 @@
 
 namespace kiwix {
 
-Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton)
+Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
   : m_verbose(verbose),
     m_root(root),
     m_content(""),
@@ -25,6 +25,7 @@ Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool
     m_returnCode(MHD_HTTP_OK),
     m_withTaskbar(withTaskbar),
     m_withLibraryButton(withLibraryButton),
+    m_blockExternalLinks(blockExternalLinks),
     m_useCache(false),
     m_addTaskbar(false),
     m_bookName(""),
@@ -127,6 +128,17 @@ void Response::introduce_taskbar()
 }
 
 
+void Response::inject_externallinks_blocker()
+{
+  kainjow::mustache::data data;
+  data.set("root", m_root);
+  auto script_tag = render_template(RESOURCE::templates::external_blocker_part_html, data);
+  m_content = appendToFirstOccurence(
+    m_content,
+    "<head>",
+    script_tag);
+}
+
 
 int Response::send(const RequestContext& request, MHD_Connection* connection)
 {
@@ -136,6 +148,9 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
       if (m_addTaskbar) {
         introduce_taskbar();
       }
+      if ( m_blockExternalLinks ) {
+        inject_externallinks_blocker();
+      }
 
       bool shouldCompress = m_compress && request.can_compress();
       shouldCompress &= m_mimeType.find("text/") != string::npos

src/server/response.h

@@ -42,7 +42,7 @@ class RequestContext;
 
 class Response {
   public:
-    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton);
+    Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks);
     ~Response() = default;
 
     int send(const RequestContext& request, MHD_Connection* connection);
@@ -64,6 +64,7 @@ class Response {
     int getReturnCode() { return m_returnCode; }
 
     void introduce_taskbar();
+    void inject_externallinks_blocker();
 
   private:
     bool m_verbose;
@@ -75,6 +76,7 @@ class Response {
     int m_returnCode;
     bool m_withTaskbar;
     bool m_withLibraryButton;
+    bool m_blockExternalLinks;
     bool m_useCache;
     bool m_compress;
     bool m_addTaskbar;

src/wrapper/java/kiwixreader.cpp

@@ -21,6 +21,7 @@
 
 #include <jni.h>
 #include <zim/file.h>
+#include <exception>
 #include "org_kiwix_kiwixlib_JNIKiwixReader.h"
 
 #include "tools/base64.h"
@@ -305,6 +306,22 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getContentPa
   return data;
 }
 
+JNIEXPORT jlong JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixReader_getArticleSize(
+    JNIEnv* env, jobject obj, jstring url)
+{
+  std::string cUrl = jni2c(url, env);
+  try {
+    auto entry = READER->getEntryFromEncodedPath(cUrl);
+    entry = entry.getFinalEntry();
+    return c2jni(entry.getSize(), env);
+  } catch(std::exception& e) {
+    LOG("Unable to get size for url : %s", cUrl.c_str());
+    LOG(e.what());
+  }
+  return c2jni(0, env);
+}
+
 JNIEXPORT jobject JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation(
     JNIEnv* env, jobject obj, jstring url)

src/wrapper/java/kiwixserver.cpp

@@ -85,6 +85,12 @@ Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboo
   SERVER->setTaskbar(withTaskbar, withLibraryButton);
 }
 
+JNIEXPORT void JNICALL
+Java_org_kiwix_kiwixlib_JNIKiwixServer_setBlockExternalLinks(JNIEnv* env, jobject obj, jboolean blockExternalLinks)
+{
+  SERVER->setBlockExternalLinks(blockExternalLinks);
+}
+
 JNIEXPORT jboolean JNICALL
 Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
 {

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java

@@ -82,6 +82,16 @@ public class JNIKiwixReader
                                       int len,
                                       JNIKiwixInt size);
 
+  /**
+   *
+   * Get the size of an article.
+   *
+   * @param   url The url of the article.
+   * @return  The size of the final (redirections are resolved) article (in byte).
+   *          Return 0 if the article is not found.
+   */
+  public native long getArticleSize(String url);
+
   /**
    * getDirectAccessInformation.
    *

src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixServer.java

@@ -34,6 +34,8 @@ public class JNIKiwixServer
 
   public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
 
+  public native void setBlockExternalLinks(boolean blockExternalLinks);
+
   public native boolean start();
 
   public native void stop();

static/resources_list.txt

@@ -20,6 +20,7 @@ skin/jquery-ui/jquery-ui.min.css
 skin/caret.png
 skin/taskbar.js
 skin/taskbar.css
+skin/block_external.js
 templates/search_result.html
 templates/no_search_result.html
 templates/404.html
@@ -28,4 +29,6 @@ templates/index.html
 templates/suggestion.json
 templates/head_part.html
 templates/taskbar_part.html
+templates/external_blocker_part.html
+templates/captured_external.html
 opensearchdescription.xml

static/skin/block_external.js

@@ -0,0 +1,72 @@
+var block_path = "/catch/external";
+// called only on external links
+function capture_event(e, target) { target.setAttribute("href", encodeURI(block_path + "?source=" + target.href)); }
+
+// called on all link clicks. filters external and call capture_event
+function on_click_event(e) {
+  var target = findParent("a", e.target);
+  if (target !== null && "href" in target) {
+    var href = target.href;
+    if (window.location.pathname.indexOf(block_path) == 0) // already in catch page
+      return;
+    if (href.indexOf(window.location.origin) == 0)
+      return;
+    if (href.substr(0, 2) == "//")
+      return capture_event(e, target);
+    if (href.substr(0, 5) == "http:")
+      return capture_event(e, target);
+    if (href.substr(0, 6) == "https:")
+      return capture_event(e, target);
+    return;
+  }
+}
+
+// script entrypoint (called on document ready)
+function run() { live('a', 'click', on_click_event); }
+
+// find first parent with tagname
+function findParent(tagname, el) {
+  while (el) {
+    if ((el.nodeName || el.tagName).toLowerCase() === tagname.toLowerCase()) {
+      return el;
+    }
+    el = el.parentNode;
+  }
+  return null;
+}
+
+// matches polyfill
+this.Element && function(ElementPrototype) {
+    ElementPrototype.matches = ElementPrototype.matches ||
+    ElementPrototype.matchesSelector ||
+    ElementPrototype.webkitMatchesSelector ||
+    ElementPrototype.msMatchesSelector ||
+    function(selector) {
+        var node = this, nodes = (node.parentNode || node.document).querySelectorAll(selector), i = -1;
+        while (nodes[++i] && nodes[i] != node);
+        return !!nodes[i];
+    }
+}(Element.prototype);
+
+// helper for enabling IE 8 event bindings
+function addEvent(el, type, handler) {
+    if (el.attachEvent) el.attachEvent('on'+type, handler); else el.addEventListener(type, handler);
+}
+
+// live binding helper using matchesSelector
+function live(selector, event, callback, context) {
+    addEvent(context || document, event, function(e) {
+        var found, el = e.target || e.srcElement;
+        while (el && el.matches && el !== context && !(found = el.matches(selector))) el = el.parentElement;
+        if (found) callback.call(el, e);
+    });
+}
+
+// in case the document is already rendered
+if (document.readyState!='loading') run();
+// modern browsers
+else if (document.addEventListener) document.addEventListener('DOMContentLoaded', run);
+// IE <= 8
+else document.attachEvent('onreadystatechange', function(){
+    if (document.readyState=='complete') run();
+});

static/templates/captured_external.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <meta charset="UTF-8" />
+  <title>External link blocked</title>
+</head>
+<body class="kiwix">
+<h1>External link blocked</h1>
+<p>This instance of Kiwix protects you from accidentaly going to external (out-of ZIM) links.</p>
+<p>If you intend to go to such locations, please click the link below.</p>
+<p><a href="{{ source }}">Go to {{ source }}</a></p>
+<div id="kiwixfooter">Powered by <a href="https://kiwix.org">Kiwix</a></div>
+</body>
+</html>

static/templates/external_blocker_part.html

@@ -0,0 +1 @@
+<script type="text/javascript" src="{{root}}/skin/block_external.js"></script>

test/library.cpp

@@ -258,4 +258,12 @@ TEST_F(LibraryTest, filterCheck)
   EXPECT_EQ(bookIds.size(), 1U);
 
 }
+
+TEST_F(LibraryTest, getBookByPath)
+{
+  auto& book = lib.getBookById(lib.getBooksIds()[0]);
+  book.setPath("/some/abs/path.zim");
+  EXPECT_EQ(lib.getBookByPath("/some/abs/path.zim").getId(), book.getId());
+  EXPECT_THROW(lib.getBookByPath("non/existant/path.zim"), std::out_of_range);
+}
 };