updated version string to "20.0"

Fix for GCC 7 needing option “-fpermissive” to compile it’s own file
“arm_acle.h”.

.gitignore

@@ -26,11 +26,14 @@
 # GNU Autotools
 .deps/
 config.h
+config.h.in~
 config.log
 config.status
 Makefile
 stamp-h1
 autom4te.cache/
+.dirstamp
+*.o-*
 
 # Visual Studio User-specific files
 *.suo
@@ -57,6 +60,10 @@ ipch/
 *.svclog
 *.scc
 *.sln
+.vscode/
+
+# macOS
+.DS_Store
 
 # NZBGet specific
 nzbget
@@ -64,3 +71,4 @@ code_revision.cpp
 *.temp
 *.pyc
 pytest.ini
+.cache

.travis.yml

@@ -4,6 +4,7 @@ language: cpp
 
 matrix:
   include:
+
     - compiler: gcc
       addons:
         apt:
@@ -11,7 +12,12 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             - g++-5
-      env: COMPILER=g++-5
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-5
+
     - compiler: gcc
       addons:
         apt:
@@ -19,7 +25,24 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             - g++-4.9
-      env: COMPILER=g++-4.9
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.9
+
+    - compiler: gcc
+      addons:
+        apt:
+          packages:
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.8
+        - CXXFLAGS="-std=c++11 -O2 -s"
+        - CONFIGUREOPTS="--disable-cpp-check"
+
     - compiler: clang
       addons:
         apt:
@@ -28,9 +51,17 @@ matrix:
             - llvm-toolchain-precise-3.6
           packages:
             - clang-3.6
-      env: COMPILER=clang++-3.6
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=clang++-3.6
+
+install:
+  - sudo pip install -U pytest
 
 script:
   - $COMPILER --version
-  - CXX=$COMPILER ./configure --enable-tests && make
-  - ./nzbget --tests exclude:[DupeMatcher]
+  - CXX=$COMPILER ./configure $CONFIGUREOPTS --enable-tests && make
+  - ./nzbget --tests
+  - cd tests/functional && pytest -v

ChangeLog

@@ -1,3 +1,234 @@
+nzbget-20.0:
+  - massive performance optimisations in downloader:
+        - improved yEnc decoder;
+        - improved CRC32 calculation;
+        - processing data in one pass;
+        - SIMD implementation of decoder and CRC functions on x86 and ARM CPUs;
+          SIMD code relies on node-yencode library by Anime Tosho
+          (https://github.com/animetosho/node-yencode);
+        - overall performance improvement up to +500% on x86 and +250% on ARM
+          (better speed or less CPU usage);
+  - using glibc instead of uClibc in universal installer builds for Linux:
+        - this significantly improves performance;
+        - compatibility with Android and other systems is hopefully also improved;
+        - in universal installer glibc is used on x86 and ARM;
+        - uClibc is still used on MIPS and PPC;
+  - performance optimisations in web-interface:
+        - reduced number of requests when loading webui by combining all
+          javascript-files into one and all css-files into one;
+        - reduced load time by calling multiple API-methods simultaneously;
+        - extensive use of browser caching for static files significantly
+          reduces the amount of data transferred on webui initialisation;
+        - extensive use of browser caching for API requests reduces the amount
+          of data transferred during webui status updates, especially when
+          nzbget is in idle state and there are no changes in download queue or
+          history;
+        - avoid work in browser on status updates if there are no changes in
+          download queue or history;
+        - support for keep alive connections significantly reduces overhead for
+          establishing connections on webui status updates, especially when
+          connecting to nzbget via TLS/SSL (avoiding TLS handshakes);
+  - a number of performance optimisations for large download queue with
+    thousands of items:
+        - much faster loading of queue from disk greatly improves program start
+          up time;
+        - improved queue management for faster download speed;
+  - now offering 64 bit binaries for Windows:
+        - installer includes 32 and 64 bit nzbget binaries;
+        - when updating from older versions the 64 bit binary is installed
+          automatically, although into the old location to keep all your
+          shortcuts intact;
+        - using word "windows" instead of "win32" in the setup file name;
+  - automatic update check:
+        - new option "UpdateCheck" to check for stable or testing versions (or
+          disable);
+        - when a new version is found a notification is shown;
+        - the update check is enabled by default for stable versions;
+  - significantly improved logging performance, especially in debug builds;
+  - par-check prints par2 creator application to help identify creator app
+    issues;
+  - added support for Unix domain sockets (POSIX only);
+  - better error handling when fetching rss feeds;
+  - updated POSIX build files to newer autotools version:
+        - compatibility with newer autotools;
+        - compatibility with newer platforms such as aarch64;
+  - better username/password validation when testing connection on settings
+    page;
+  - improved rar-renamer to better handle certain cases;
+  - new option "SkipWrite" for easier speed tests;
+  - support for redirect codes 303, 307 and 308 in web-client for fetching of
+    rss feeds and nzb-files;
+  - installer for FreeBSD is now built using Clang instead of GCC; this fixes
+    incompatibility with FreeBSD 11;
+  - universal Linux installer can now be used on FreeBSD (via Linux
+    compatibility mode);
+  - updated unrar to v5.50;
+  - more robust news server connection test;
+  - enhancements in NServ:
+        - memory cache to reduce disk load during speed tests - new command line
+          switch "-m";
+        - speed control - new command line switches "-w" and "-r";
+        - show IP address of incoming connection;
+  - changed default location of log-file;
+  - better handling of broken connections;
+  - removed obsolete or less useful options "SaveQueue", "ReloadQueue",
+    "TerminateTimeout", "AccurateRate", "BrokenLog";
+  - renamed option "LogBufferSize" to "LogBuffer";
+  - passwords of failed login attempts are no longer printed to log to improve
+    security;
+  - cookies in web interface are now saved with "httpOnly" attribute to improve
+    security;
+  - titles and duplicate keys in duplicate check are now case insensitive;
+  - added LibreSSL support;
+  - web interface now has a better icon for favorites or home screen of mobile
+    devices;
+  - improved duplicate detection for obfuscated downloads having files with
+    same subjects;
+  - direct rename and direct unpack are now active by default on new
+    installations, except for slow Linux systems;
+  - added advice for letsencrypt in option descriptions;
+  - fixed incorrect renaming in rar-renamer which could cause some downloads to
+    fail;
+  - fixed race condition in queue script coordinator which could cause crashes;
+  - fixed: post-processing parameters were sometimes case sensitive causing
+    issues;
+  - fixed DNS resolving issues on Android;
+  - fixed: backup servers not used on certain article decoding errors;
+  - fixed: when direct rename was active certain downloads with damaged
+    par2-files become paused at near completion and required manual resuming;
+  - fixed: crash when flushing article cache after direct rename;
+  - fixed: deleting active par-job may crash the program;
+  - fixed: functional tests may fail on Windows due to locked files;
+  - fixed: unpack using password file doesn't work on Windows;
+  - fixed: compiler error when building using GnuTLS;
+  - fixed: Linux installer failure on android emulator;
+  - fixed: options formatted as password fields when they shouldn't;
+  - fixed: slightly off article statistics after direct rename;
+  - fixed: NServ terminated if client interrupted connection;
+  - fixed: example pp-scripts may not work properly if nzbget password or
+    username contained special characters;
+  - fix in functional tests to not rely on sizes of externally generated files;
+  - fixed: option AuthorizedIP did not work with IPv6 addresses;
+  - fixed crash on certain malformed articles;
+  - fixed crash which could happen on Windows when reloading or terminating the
+    program;
+  - fixed logging of IPv6 addresses.
+
+nzbget-19.1:
+  - proper handling of changing category (and destination path) during direct
+    unpack; direct unpack now gracefully aborts with cleanup; the files will
+    be unpacked during post-processing stage;
+  - fixed: password protected downloads of certain kind may sometimes end up
+    with no files if direct unpack was active;
+  - fixed: rar-renamer mistakenly renamed some encrypted rar3 files causing
+    unnecessary repair;
+  - fixed: rar-renamer could not process some encrypted rar3-archives.
+
+nzbget-19.0:
+  - unpack during downloading:
+        - downloaded files can now be unpacked as soon as every archive part is
+          downloaded;
+        - new option "DirectUnpack" to activate direct unpacking;
+        - direct unpack works even with obfuscated downloads; option
+         "DirectRename" (see below) must be active for that;
+        - option "ReorderFiles" (see below) should be also active for optimal
+          file download order;
+        - direct unpack works for rar-archives; 7-zip archives and simply
+          splitted files are processed by default unpack module;
+        - direct unpack obviously works only for healthy download; if download
+          is damaged the direct unpack cancels and the download is unpacked
+          during post-processing stage after files are repaired;
+        - direct unpack reduces the time needed to complete download and
+      post-processing;
+        - it also allows to start watching of video files during download
+          (requires compatible video player software);
+  - renaming of obfuscated file names during downloading:
+        - correct file names for obfuscated downloads are now determined during
+          download stage (instead of post-processing stage);
+        - downloaded files are saved into disk directly with correct names;
+        - direct renaming uses par2-files to restore correct file names;
+        - new option "DirectRename" to activate direct renaming;
+        - new queue-event NZB_NAMED, sent after the inner files are renamed;
+  - automatic reordering of files:
+        - inner files within nzb reordered to ensure download of files in
+          archive parts order;
+        - the files are reordered when nzb is added to queue;
+        - if direct renaming is active (option "DirectRename") the files are
+      reordered again after the correct names becomes known;
+        - new option "ReorderFiles";
+        - new command "GroupSortFiles" in api-method "editqueue";
+        - new subcommand "SF" of remote command "-E/--edit";
+  - new option "FileNaming" to control how to name obfuscated files (before they
+    get renamed by par-rename, rar-rename or direct-rename);
+  - TLS certificate verification:
+        - when connecting to a news server (for downloading) or a web server
+         (for fetching of rss feeds and nzb-files) the authenticity of the
+          server is validated using server security certificate. If the check
+          fails that means the connection cannot be trusted and must be closed
+          with an error message explaining the security issue;
+        - new options "CertCheck" and "CertStore";
+        - official NZBGet packages come with activated certificate check;
+        - when updating from an older NZBGet version the option CertCheck will
+          be automatically activated when the settings is saved (switch to
+          Settings page in web-interface and click "Save all changed");
+  - authentication via form in web-interface as alternative to HTTP
+    authentication:
+        - that must help with password tools having issues with HTTP
+          authentication dialog;
+        - new option "FormAuth";
+  - drop-downs (context menus) for priority, category and status columns:
+        - quicker changing of priority and category;
+        - easier access to actions via drop-down (context menu) in status
+          column;
+  - extensions scripts can now be executed from settings page:
+        - script authors define custom buttons;
+        - when clicked the script is executed in a special mode and obtain extra
+          parameters;
+        - example script "Email.py" extended with button "Send test e-mail";
+  - on Windows NZBGet can now associate itself with nzb-files:
+        - use option in Windows installer to register NZBGet for nzb-files;
+  - unrar shipped within Linux package is now compiled with "fallocate" option
+    to improve compatibility with media players when watching videos during
+    downloading and unpacking;
+  - support for HTTP-header "X-Forwarded-For" in IP-logging;
+  - improvements in RSS feed view in phone mode;
+  - set name, password and dupe info when adding via URL by click on a button
+    near URL field in web-interface;
+  - backup-badge for items in history similar to downloads tab;
+  - show backup icon in history in phone theme;
+  - added support for ECC certificates in built-in web-server;
+  - save changes before performing actions in history dialog;
+  - proper exit code on client command success or failure.
+  - added host name to all error messages regarding connection issues;
+  - new button "Volume Statistics" in section "News Servers" of settings page;
+    shows the same volume data as in global statistics dialog;
+  - new option "ServerX.Notes" for user comments on news servers;
+  - new parameters for api-method "servervolumes" as a performance optimization
+    measure to reduce amount of transferred data;
+  - new option to force connection to news servers via ipv4 or ipv6;
+  - removed unnecessary requests to news servers;
+  - updated unrar to v5.40;
+  - clear script execution log before executing script;
+  - added support for crash dumps on Windows:
+        - renamed option "DumpCore" to "CrashDump";
+        - new option "CrashTrace" to make it possible to disable default
+          printing off call stack in order to produce more relevant crash dumps;
+  - fixed: startup scheduler tasks can be executed again;
+  - fixed: "fatal" messages when compiling from sources.
+  - fixed: per-nzb download statistics could be wrong if the program was
+    reloaded during downloading.
+  - fixed crash which may happen between post-processing steps;
+  - fixed: asterix (*) was sometimes passed as parameter to extension scripts
+    (Windows only);
+  - fixed potential crash during update via web-interface.
+
+nzbget-18.1:
+  - fixed: crash during download caused by a race condition;
+  - fixed: sleep mode did not work on Windows;
+  - fixed: queue was not saved after deleting of queued post-jobs;
+  - fixed: possible crash at the end of post-processing;
+  - fixed: "undefined" in reorder extension scripts.
+
 nzbget-18.0:
   - automatic deobfuscation of rar-archives without par-files:
         - obfuscated downloads not having par-files can now be successfully

Makefile.am

@@ -28,6 +28,8 @@ nzbget_SOURCES = \
 	daemon/connect/WebDownloader.h \
 	daemon/extension/FeedScript.cpp \
 	daemon/extension/FeedScript.h \
+	daemon/extension/CommandScript.cpp \
+	daemon/extension/CommandScript.h \
 	daemon/extension/NzbScript.cpp \
 	daemon/extension/NzbScript.h \
 	daemon/extension/PostScript.cpp \
@@ -56,8 +58,8 @@ nzbget_SOURCES = \
 	daemon/frontend/LoggableFrontend.h \
 	daemon/frontend/NCursesFrontend.cpp \
 	daemon/frontend/NCursesFrontend.h \
-    daemon/main/CommandLineParser.cpp \
-    daemon/main/CommandLineParser.h \
+	daemon/main/CommandLineParser.cpp \
+	daemon/main/CommandLineParser.h \
 	daemon/main/DiskService.cpp \
 	daemon/main/DiskService.h \
 	daemon/main/Maintenance.cpp \
@@ -106,6 +108,10 @@ nzbget_SOURCES = \
 	daemon/postprocess/Repair.h \
 	daemon/postprocess/Unpack.cpp \
 	daemon/postprocess/Unpack.h \
+	daemon/postprocess/DirectUnpack.cpp \
+	daemon/postprocess/DirectUnpack.h \
+	daemon/queue/DirectRenamer.cpp \
+	daemon/queue/DirectRenamer.h \
 	daemon/queue/DiskState.cpp \
 	daemon/queue/DiskState.h \
 	daemon/queue/DownloadInfo.cpp \
@@ -208,6 +214,25 @@ nzbget_SOURCES += \
 	lib/par2/verificationpacket.h
 endif
 
+# Simd decoder and Crc32
+nzbget_SOURCES += \
+	lib/yencode/YEncode.h \
+	lib/yencode/SimdInit.cpp \
+	lib/yencode/SimdDecoder.cpp \
+	lib/yencode/ScalarDecoder.cpp \
+	lib/yencode/Sse2Decoder.cpp \
+	lib/yencode/Ssse3Decoder.cpp \
+	lib/yencode/PclmulCrc.cpp \
+	lib/yencode/NeonDecoder.cpp \
+	lib/yencode/AcleCrc.cpp \
+	lib/yencode/SliceCrc.cpp
+
+lib/yencode/Sse2Decoder.$(OBJEXT) : CXXFLAGS+=$(SSE2_CXXFLAGS)
+lib/yencode/Ssse3Decoder.$(OBJEXT) : CXXFLAGS+=$(SSSE3_CXXFLAGS)
+lib/yencode/PclmulCrc.$(OBJEXT) : CXXFLAGS+=$(PCLMUL_CXXFLAGS)
+lib/yencode/NeonDecoder.$(OBJEXT) : CXXFLAGS+=$(NEON_CXXFLAGS)
+lib/yencode/AcleCrc.$(OBJEXT) : CXXFLAGS+=$(ACLECRC_CXXFLAGS)
+
 AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/connect \
 	-I$(srcdir)/daemon/extension \
@@ -220,7 +245,8 @@ AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/remote \
 	-I$(srcdir)/daemon/util \
 	-I$(srcdir)/daemon/nserv \
-	-I$(srcdir)/lib/par2
+	-I$(srcdir)/lib/par2 \
+	-I$(srcdir)/lib/yencode
 
 if WITH_TESTS
 nzbget_SOURCES += \
@@ -235,6 +261,7 @@ nzbget_SOURCES += \
 	tests/postprocess/DupeMatcherTest.cpp \
 	tests/postprocess/RarRenamerTest.cpp \
 	tests/postprocess/RarReaderTest.cpp \
+	tests/postprocess/DirectUnpackTest.cpp \
 	tests/queue/NzbFileTest.cpp \
 	tests/nntp/ServerPoolTest.cpp \
 	tests/util/FileSystemTest.cpp \
@@ -276,9 +303,9 @@ windows_FILES = \
 	windows/resources/trayicon_idle.ico \
 	windows/resources/trayicon_paused.ico \
 	windows/resources/trayicon_working.ico \
-	windows/setup/nzbget-setup.nsi \
-	windows/setup/install.bmp \
-	windows/setup/uninstall.bmp
+	windows/resources/install.bmp \
+	windows/resources/uninstall.bmp \
+	windows/nzbget-setup.nsi
 
 osx_FILES = \
 	osx/App_Prefix.pch \
@@ -362,7 +389,9 @@ webui_FILES = \
 	webui/img/favicon.ico \
 	webui/img/download-anim-green-2x.png \
 	webui/img/download-anim-orange-2x.png \
-	webui/img/transmit-reload-2x.gif
+	webui/img/transmit-reload-2x.gif \
+	webui/img/favicon-256x256-opaque.png \
+	webui/img/favicon-256x256.png
 
 scripts_FILES = \
 	scripts/EMail.py \
@@ -455,21 +484,21 @@ uninstall-conf:
 #    we create new file "code_revision.c" with empty revision number.
 code_revision.cpp: FORCE
 	@ if test -d ./.git ; then \
-		B="$(shell git branch | sed -n -e 's/^\* \(.*\)/\1/p')"; \
-		M="$(shell git status --porcelain)" ; \
+		B=`git branch | sed -n -e 's/^\* \(.*\)/\1/p'`; \
+		M=`git status --porcelain` ; \
 		if test "$$M" != "" ; then \
 			M="M" ; \
 		fi ; \
 		if test "$$B" = "master" ; then \
 			V="$$M" ; \
 		elif test "$$B" = "develop" ; then \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M" ; \
 		else \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M ($$B)" ; \
 		fi ; \
-		H="$(shell test -f ./code_revision.cpp && head -n 1 code_revision.cpp)"; \
+		H=`test -f ./code_revision.cpp && head -n 1 code_revision.cpp`; \
 		if test "/* $$V */" != "$$H" ; then \
 			( \
 			echo "/* $$V */" ;\

README.md

@@ -14,5 +14,5 @@ the program can be compiled from sources.
 
 - [Home page (nzbget.net)](http://nzbget.net) - for first time visitors, learn more about NZBGet;
 - [Downloads](http://nzbget.net/download) - get the binaries and sources;
-- [Documentation](https://github.com/nzbget/nzbget/wiki) - installation manuals, HOW-TOs, API;
+- [Documentation](http://nzbget.net/documentation) - installation manuals, HOW-TOs, API;
 - [Forum](http://forum.nzbget.net) - get support, share your ideas, scripts, add-ons.

config.h.in

@@ -13,6 +13,9 @@
 /* Define to 1 to disable gzip-support */
 #undef DISABLE_GZIP
 
+/* Define to 1 to not use libxml2, only for development purposes */
+#undef DISABLE_LIBXML2
+
 /* Define to 1 to disable par-verification and repair */
 #undef DISABLE_PARCHECK
 
@@ -134,6 +137,9 @@
 /* Define to 1 if variadic macros are supported */
 #undef HAVE_VARIADIC_MACROS
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#undef HAVE_X509_CHECK_HOST
+
 /* Define to 1 to exclude debug-code */
 #undef NDEBUG
 
@@ -152,6 +158,9 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME
 
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
 
@@ -167,9 +176,10 @@
 /* Version number of package */
 #undef VERSION
 
-/* Define to 1 if your processor stores words with the most significant byte
-   first (like Motorola and SPARC, unlike Intel and VAX). */
-#undef WORDS_BIGENDIAN
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
 
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS

configure.ac

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,11 +20,11 @@
 #                                               -*- Autoconf -*-
 # Process this file with autoconf to produce a configure script.
 
-AC_PREREQ(2.59)
-AC_INIT(nzbget, 18.0, hugbug@users.sourceforge.net)
+AC_PREREQ(2.65)
+AC_INIT(nzbget, 20.0, hugbug@users.sourceforge.net)
 AC_CONFIG_AUX_DIR(posix)
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE([foreign])
+AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_CONFIG_SRCDIR([daemon/main/nzbget.cpp])
 AC_CONFIG_HEADERS([config.h])
 AM_MAINTAINER_MODE
@@ -65,8 +65,7 @@ fi
 dnl
 dnl Checks for header files.
 dnl
-AC_CHECK_HEADERS(sys/prctl.h)
-AC_CHECK_HEADERS(regex.h)
+AC_CHECK_HEADERS(sys/prctl.h regex.h endian.h getopt.h)
 
 
 dnl
@@ -146,7 +145,7 @@ if test "$FOUND" = "no"; then
 		[	char* szHost; struct hostent hinfobuf; char* strbuf; int h_errnop;
 			struct hostent* hinfo = gethostbyname_r(szHost, &hinfobuf, strbuf, 1024, &h_errnop); ],
 		AC_MSG_RESULT([[yes, and it takes 5 arguments]])
-		FOUND="yes"                      
+		FOUND="yes"
 		AC_DEFINE([HAVE_GETHOSTBYNAME_R_5], 1, [Define to 1 if gethostbyname_r takes 5 arguments]),
 		FOUND="no")
 	
@@ -196,17 +195,17 @@ AC_TRY_COMPILE([
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (size_t*)NULL)],[
-    AC_MSG_RESULT(size_t)
-    SOCKLEN_T=size_t],[
-    AC_TRY_COMPILE([
+	AC_MSG_RESULT(size_t)
+	SOCKLEN_T=size_t],[
+	AC_TRY_COMPILE([
 #include <stddef.h>
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (int*)NULL)],[
-      AC_MSG_RESULT(int)
-      SOCKLEN_T=int],[
-      AC_MSG_WARN(could not determine)
-      SOCKLEN_T=int])])])
+	AC_MSG_RESULT(int)
+	SOCKLEN_T=int],[
+	AC_MSG_WARN(could not determine)
+	SOCKLEN_T=int])])])
 AC_DEFINE_UNQUOTED(SOCKLEN_T, $SOCKLEN_T, [Determine what socket length (socklen_t) data type is])
 
 
@@ -238,26 +237,36 @@ AC_TRY_COMPILE(
 dnl 
 dnl checks for libxml2 includes and libraries.
 dnl
-AC_ARG_WITH(libxml2_includes,
-	[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
-	[CPPFLAGS="${CPPFLAGS} -I${withval}"]
-	[INCVAL="yes"],
-	[INCVAL="no"])
-AC_ARG_WITH(libxml2_libraries,
-	[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
-	[LDFLAGS="${LDFLAGS} -L${withval}"]
-	[LIBVAL="yes"],
-	[LIBVAL="no"])
-if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
-	PKG_CHECK_MODULES(libxml2, libxml-2.0,
-		[LIBS="${LIBS} $libxml2_LIBS"]
-		[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+AC_MSG_CHECKING(whether to use libxml2)
+AC_ARG_ENABLE(libxml2,
+	[AS_HELP_STRING([--disable-libxml2], [do not use libxml2 (removes dependency from libxml2-library, only for development purposes)])],
+	[USELIBXML2=$enableval],
+	[USELIBXML2=yes] )
+AC_MSG_RESULT($USELIBXML2)
+if test "$USELIBXML2" = "yes"; then
+	AC_ARG_WITH(libxml2_includes,
+		[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
+		[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+		[INCVAL="yes"],
+		[INCVAL="no"])
+	AC_ARG_WITH(libxml2_libraries,
+		[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
+		[LDFLAGS="${LDFLAGS} -L${withval}"]
+		[LIBVAL="yes"],
+		[LIBVAL="no"])
+	if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+		PKG_CHECK_MODULES(libxml2, libxml-2.0,
+			[LIBS="${LIBS} $libxml2_LIBS"]
+			[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+			AC_MSG_ERROR("libxml2 library not found"))
+	fi
+	AC_CHECK_HEADER(libxml/tree.h,,
+		AC_MSG_ERROR("libxml2 header files not found"))
+	AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
 		AC_MSG_ERROR("libxml2 library not found"))
+else
+	AC_DEFINE([DISABLE_LIBXML2],1,[Define to 1 to not use libxml2, only for development purposes])
 fi
-AC_CHECK_HEADER(libxml/tree.h,,
-	AC_MSG_ERROR("libxml2 header files not found"))
-AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
-	AC_MSG_ERROR("libxml2 library not found"))
 
 
 dnl
@@ -327,11 +336,8 @@ AC_MSG_RESULT($ENABLEPARCHECK)
 if test "$ENABLEPARCHECK" = "yes"; then
 	dnl PAR2 checks. 
 	dnl 
-	dnl Checks for header files.
-	AC_CHECK_HEADERS([endian.h] [getopt.h])
 	dnl Checks for typedefs, structures, and compiler characteristics.
 	AC_TYPE_SIZE_T
-	AC_C_BIGENDIAN
 	AC_FUNC_FSEEKO
 	dnl Checks for library functions.
 	AC_CHECK_FUNCS([stricmp])
@@ -396,6 +402,8 @@ if test "$USETLS" = "yes"; then
 			if test "$FOUND" = "yes"; then
 				TLSLIB="OpenSSL"
 				AC_DEFINE([HAVE_OPENSSL],1,[Define to 1 to use OpenSSL library for TLS/SSL-support and decryption.])
+				AC_SEARCH_LIBS([X509_check_host], [crypto],
+					AC_DEFINE([HAVE_X509_CHECK_HOST],1,[Define to 1 if OpenSSL supports function "X509_check_host".]))
 			fi
 		fi
 	fi
@@ -539,6 +547,36 @@ else
 fi
 
 
+dnl
+dnl Determine if CPU supports SIMD instructions
+dnl
+AC_MSG_CHECKING(whether to use SIMD-optimized routines)
+USE_SIMD=no
+case $host_cpu in
+	i?86|x86_64)
+		SSE2_CXXFLAGS="-msse2"
+		SSSE3_CXXFLAGS="-mssse3"
+		PCLMUL_CXXFLAGS="-msse4.1 -mpclmul"
+		USE_SIMD=yes
+		;;
+	arm*)
+		NEON_CXXFLAGS="-mfpu=neon"
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+	aarch64)
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+esac
+AC_MSG_RESULT($USE_SIMD)
+AC_SUBST([SSE2_CXXFLAGS])
+AC_SUBST([SSSE3_CXXFLAGS])
+AC_SUBST([PCLMUL_CXXFLAGS])
+AC_SUBST([NEON_CXXFLAGS])
+AC_SUBST([ACLECRC_CXXFLAGS])
+
+
 dnl 
 dnl Some Linux systems require an empty signal handler for SIGCHLD
 dnl in order for exit codes to be correctly delivered to parent process.
@@ -597,11 +635,10 @@ dnl
 dnl variadic macros
 dnl 
 AC_MSG_CHECKING(for variadic macros)
-AC_COMPILE_IFELSE([
-	#define macro(...)   macrofunc(__VA_ARGS__)
-	int macrofunc(int a, int b) { return a + b; }
-	int test() { return macro(1, 2); }
-	],
+AC_TRY_COMPILE(
+	[ #define macro(...)   macrofunc(__VA_ARGS__) ]
+	[ int macrofunc(int a, int b) { return a + b; } ],
+	[ int a=macro(1, 2); ],
 	AC_MSG_RESULT([yes])
 	AC_DEFINE([HAVE_VARIADIC_MACROS], 1, Define to 1 if variadic macros are supported),
 	AC_MSG_RESULT([no]))

daemon/connect/Connection.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "nzbget.h"
 #include "Connection.h"
 #include "Log.h"
+#include "FileSystem.h"
 
 static const int CONNECTION_READBUFFER_SIZE = 1024;
 #ifndef HAVE_GETADDRINFO
@@ -79,6 +80,10 @@ void closesocket_gracefully(SOCKET socket)
 	closesocket(socket);
 }
 
+#ifdef __linux__
+CString ResolveAndroidHost(const char* host);
+#endif
+
 void Connection::Init()
 {
 	debug("Initializing global connection data");
@@ -117,7 +122,6 @@ void Connection::Final()
 
 Connection::Connection(const char* host, int port, bool tls) :
 	m_host(host), m_port(port), m_tls(tls)
-
 {
 	debug("Creating Connection");
 
@@ -132,6 +136,7 @@ Connection::Connection(SOCKET socket, bool tls)
 	m_tls = tls;
 	m_status = csConnected;
 	m_socket = socket;
+	m_host = GetRemoteAddr();
 	m_bufAvail = 0;
 	m_timeout = 60;
 	m_suppressErrors = true;
@@ -210,91 +215,122 @@ bool Connection::Bind()
 		return true;
 	}
 
-#ifdef HAVE_GETADDRINFO
-	struct addrinfo addr_hints, *addr_list, *addr;
-
-	memset(&addr_hints, 0, sizeof(addr_hints));
-	addr_hints.ai_family = AF_UNSPEC;    // Allow IPv4 or IPv6
-	addr_hints.ai_socktype = SOCK_STREAM,
-	addr_hints.ai_flags = AI_PASSIVE;    // For wildcard IP address
-
-	BString<100> portStr("%d", m_port);
-
-	int res = getaddrinfo(m_host, portStr, &addr_hints, &addr_list);
-	if (res != 0)
-	{
-		ReportError("Could not resolve hostname %s", m_host, true
 #ifndef WIN32
-			, res != EAI_SYSTEM ? res : 0
-			, res != EAI_SYSTEM ? gai_strerror(res) : nullptr
-#endif
-			);
-		return false;
-	}
-
-	m_broken = false;
-	m_socket = INVALID_SOCKET;
-	for (addr = addr_list; addr != nullptr; addr = addr->ai_next)
+	if (m_host && m_host[0] == '/')
 	{
-		m_socket = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
-#ifdef WIN32
-		SetHandleInformation((HANDLE)m_socket, HANDLE_FLAG_INHERIT, 0);
-#endif
-		if (m_socket != INVALID_SOCKET)
+		struct sockaddr_un addr;
+		addr.sun_family = AF_UNIX;
+		if (strlen(m_host) >= sizeof(addr.sun_path))
+		{
+			ReportError("Binding socket failed for %s: name too long\n", m_host, false);
+			return false;
+		}
+		strcpy(addr.sun_path, m_host);
+
+		m_socket = socket(PF_UNIX, SOCK_STREAM, 0);
+		if (m_socket == INVALID_SOCKET)
+		{
+			ReportError("Socket creation failed for %s", m_host, true);
+			return false;
+		}
+
+		unlink(m_host);
+
+		if (bind(m_socket, (struct sockaddr *)&addr, sizeof(addr)) == -1)
 		{
-			int opt = 1;
-			setsockopt(m_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
-			res = bind(m_socket, addr->ai_addr, addr->ai_addrlen);
-			if (res != -1)
-			{
-				// Connection established
-				break;
-			}
 			// Connection failed
 			closesocket(m_socket);
 			m_socket = INVALID_SOCKET;
 		}
 	}
+	else
+#endif
+	{
+#ifdef HAVE_GETADDRINFO
+		struct addrinfo addr_hints, *addr_list, *addr;
 
-	freeaddrinfo(addr_list);
+		memset(&addr_hints, 0, sizeof(addr_hints));
+		addr_hints.ai_family = m_ipVersion == ipV4 ? AF_INET : m_ipVersion == ipV6 ? AF_INET6 : AF_UNSPEC;
+		addr_hints.ai_socktype = SOCK_STREAM;
+		addr_hints.ai_flags = AI_PASSIVE;    // For wildcard IP address
+
+		BString<100> portStr("%d", m_port);
+
+		int res = getaddrinfo(m_host, portStr, &addr_hints, &addr_list);
+		if (res != 0)
+		{
+			ReportError("Could not resolve hostname %s", m_host, true
+#ifndef WIN32
+				, res != EAI_SYSTEM ? res : 0
+				, res != EAI_SYSTEM ? gai_strerror(res) : nullptr
+#endif
+				);
+			return false;
+		}
+
+		m_socket = INVALID_SOCKET;
+		for (addr = addr_list; addr != nullptr; addr = addr->ai_next)
+		{
+			m_socket = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
+#ifdef WIN32
+			SetHandleInformation((HANDLE)m_socket, HANDLE_FLAG_INHERIT, 0);
+#endif
+			if (m_socket != INVALID_SOCKET)
+			{
+				int opt = 1;
+				setsockopt(m_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
+				res = bind(m_socket, addr->ai_addr, addr->ai_addrlen);
+				if (res != -1)
+				{
+					// Connection established
+					break;
+				}
+				// Connection failed
+				closesocket(m_socket);
+				m_socket = INVALID_SOCKET;
+			}
+		}
+
+		freeaddrinfo(addr_list);
 
 #else
 
-	struct sockaddr_in	sSocketAddress;
-	memset(&sSocketAddress, 0, sizeof(sSocketAddress));
-	sSocketAddress.sin_family = AF_INET;
-	if (!m_host || strlen(m_host) == 0)
-	{
-		sSocketAddress.sin_addr.s_addr = htonl(INADDR_ANY);
-	}
-	else
-	{
-		sSocketAddress.sin_addr.s_addr = ResolveHostAddr(m_host);
-		if (sSocketAddress.sin_addr.s_addr == INADDR_NONE)
+		struct sockaddr_in	sSocketAddress;
+		memset(&sSocketAddress, 0, sizeof(sSocketAddress));
+		sSocketAddress.sin_family = AF_INET;
+		if (!m_host || strlen(m_host) == 0)
 		{
+			sSocketAddress.sin_addr.s_addr = htonl(INADDR_ANY);
+		}
+		else
+		{
+			sSocketAddress.sin_addr.s_addr = ResolveHostAddr(m_host);
+			if (sSocketAddress.sin_addr.s_addr == INADDR_NONE)
+			{
+				return false;
+			}
+		}
+		sSocketAddress.sin_port = htons(m_port);
+
+		m_socket = socket(PF_INET, SOCK_STREAM, 0);
+		if (m_socket == INVALID_SOCKET)
+		{
+			ReportError("Socket creation failed for %s", m_host, true);
 			return false;
 		}
-	}
-	sSocketAddress.sin_port = htons(m_port);
 
-	m_socket = socket(PF_INET, SOCK_STREAM, 0);
-	if (m_socket == INVALID_SOCKET)
-	{
-		ReportError("Socket creation failed for %s", m_host, true);
-		return false;
-	}
+		int opt = 1;
+		setsockopt(m_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
 
-	int opt = 1;
-	setsockopt(m_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
-
-	int res = bind(m_socket, (struct sockaddr *) &sSocketAddress, sizeof(sSocketAddress));
-	if (res == -1)
-	{
-		// Connection failed
-		closesocket(m_socket);
-		m_socket = INVALID_SOCKET;
-	}
+		int res = bind(m_socket, (struct sockaddr *) &sSocketAddress, sizeof(sSocketAddress));
+		if (res == -1)
+		{
+			// Connection failed
+			closesocket(m_socket);
+			m_socket = INVALID_SOCKET;
+		}
 #endif
+	}
 
 	if (m_socket == INVALID_SOCKET)
 	{
@@ -325,7 +361,7 @@ int Connection::WriteLine(const char* buffer)
 	int res = send(m_socket, buffer, strlen(buffer), 0);
 	if (res <= 0)
 	{
-		m_broken = true;
+		m_status = csBroken;
 	}
 
 	return res;
@@ -346,7 +382,7 @@ bool Connection::Send(const char* buffer, int size)
 		int res = send(m_socket, buffer + bytesSent, size-bytesSent, 0);
 		if (res <= 0)
 		{
-			m_broken = true;
+			m_status = csBroken;
 			return false;
 		}
 		bytesSent += res;
@@ -374,14 +410,18 @@ char* Connection::ReadLine(char* buffer, int size, int* bytesReadOut)
 			bufAvail = recv(m_socket, m_readBuf, m_readBuf.Size() - 1, 0);
 			if (bufAvail < 0)
 			{
-				ReportError("Could not receive data on socket", nullptr, true);
-				m_broken = true;
+				ReportError("Could not receive data on socket from %s", m_host, true);
+				m_status = csBroken;
 				break;
 			}
 			else if (bufAvail == 0)
 			{
 				break;
 			}
+			else
+			{
+				m_totalBytesRead += bufAvail;
+			}
 			bufPtr = m_readBuf;
 			m_readBuf[bufAvail] = '\0';
 		}
@@ -424,8 +464,6 @@ char* Connection::ReadLine(char* buffer, int size, int* bytesReadOut)
 		*bytesReadOut = bytesRead;
 	}
 
-	m_totalBytesRead += bytesRead;
-
 	if (inpBuffer == buffer)
 	{
 		return nullptr;
@@ -446,19 +484,21 @@ std::unique_ptr<Connection> Connection::Accept()
 	SOCKET socket = accept(m_socket, nullptr, nullptr);
 	if (socket == INVALID_SOCKET && m_status != csCancelled)
 	{
-		ReportError("Could not accept connection", nullptr, true);
+		ReportError("Could not accept connection for %s", m_host, true);
 	}
 	if (socket == INVALID_SOCKET)
 	{
 		return nullptr;
 	}
 
+	InitSocketOpts(socket);
+
 	return std::make_unique<Connection>(socket, m_tls);
 }
 
 int Connection::TryRecv(char* buffer, int size)
 {
-	debug("Receiving data");
+	//debug("Receiving data");
 
 	memset(buffer, 0, size);
 
@@ -466,7 +506,11 @@ int Connection::TryRecv(char* buffer, int size)
 
 	if (received < 0)
 	{
-		ReportError("Could not receive data on socket", nullptr, true);
+		ReportError("Could not receive data on socket from %s", m_host, true);
+	}
+	else
+	{
+		m_totalBytesRead += received;
 	}
 
 	return received;
@@ -474,7 +518,7 @@ int Connection::TryRecv(char* buffer, int size)
 
 bool Connection::Recv(char * buffer, int size)
 {
-	debug("Receiving data (full buffer)");
+	//debug("Receiving data (full buffer)");
 
 	memset(buffer, 0, size);
 
@@ -498,11 +542,12 @@ bool Connection::Recv(char * buffer, int size)
 		// Did the recv succeed?
 		if (received <= 0)
 		{
-			ReportError("Could not receive data on socket", nullptr, true);
+			ReportError("Could not receive data on socket from %s", m_host, true);
 			return false;
 		}
 		bufPtr += received;
 		NeedBytes -= received;
+		m_totalBytesRead += received;
 	}
 	return true;
 }
@@ -512,113 +557,156 @@ bool Connection::DoConnect()
 	debug("Do connecting");
 
 	m_socket = INVALID_SOCKET;
-	m_broken = false;
 
-#ifdef HAVE_GETADDRINFO
-	struct addrinfo addr_hints, *addr_list, *addr;
-
-	memset(&addr_hints, 0, sizeof(addr_hints));
-	addr_hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
-	addr_hints.ai_socktype = SOCK_STREAM;
-
-	BString<100> portStr("%d", m_port);
-
-	int res = getaddrinfo(m_host, portStr, &addr_hints, &addr_list);
-	if (res != 0)
-	{
-		ReportError("Could not resolve hostname %s", m_host, true
 #ifndef WIN32
-					, res != EAI_SYSTEM ? res : 0
-					, res != EAI_SYSTEM ? gai_strerror(res) : nullptr
-#endif
-					);
-		return false;
-	}
-
-	std::vector<SockAddr> triedAddr;
-	bool connected = false;
-
-	for (addr = addr_list; addr != nullptr; addr = addr->ai_next)
+	if (m_host && m_host[0] == '/')
 	{
-		// don't try the same combinations of ai_family, ai_socktype, ai_protocol multiple times
-		SockAddr sa = { addr->ai_family, addr->ai_socktype, addr->ai_protocol };
-		if (std::find(triedAddr.begin(), triedAddr.end(), sa) != triedAddr.end())
+		struct sockaddr_un addr;
+		addr.sun_family = AF_UNIX;
+		if (strlen(m_host) >= sizeof(addr.sun_path))
 		{
-			continue;
+			ReportError("Connection to %s failed: name too long\n", m_host, false);
+			return false;
 		}
-		triedAddr.push_back(sa);
+		strcpy(addr.sun_path, m_host);
 
-		if (m_socket != INVALID_SOCKET)
-		{
-			closesocket(m_socket);
-		}
-
-		m_socket = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
-#ifdef WIN32
-		SetHandleInformation((HANDLE)m_socket, HANDLE_FLAG_INHERIT, 0);
-#endif
+		m_socket = socket(PF_UNIX, SOCK_STREAM, 0);
 		if (m_socket == INVALID_SOCKET)
 		{
-			// try another addr/family/protocol
-			continue;
+			ReportError("Socket creation failed for %s", m_host, true);
+			return false;
 		}
 
-		if (ConnectWithTimeout(addr->ai_addr, addr->ai_addrlen))
+		if (!ConnectWithTimeout(&addr, sizeof(addr)))
 		{
-			// Connection established
-			connected = true;
-			break;
+			ReportError("Connection to %s failed", m_host, true);
+			closesocket(m_socket);
+			m_socket = INVALID_SOCKET;
+			return false;
 		}
 	}
-
-	if (m_socket == INVALID_SOCKET && addr_list)
+	else
+#endif
 	{
-		ReportError("Socket creation failed for %s", m_host, true);
-	}
+#ifdef HAVE_GETADDRINFO
+		struct addrinfo addr_hints, *addr_list, *addr;
 
-	if (!connected && m_socket != INVALID_SOCKET)
-	{
-		ReportError("Connection to %s failed", m_host, true);
-		closesocket(m_socket);
-		m_socket = INVALID_SOCKET;
-	}
+		memset(&addr_hints, 0, sizeof(addr_hints));
+		addr_hints.ai_family = m_ipVersion == ipV4 ? AF_INET : m_ipVersion == ipV6 ? AF_INET6 : AF_UNSPEC;
+		addr_hints.ai_socktype = SOCK_STREAM;
 
-	freeaddrinfo(addr_list);
+		BString<100> portStr("%d", m_port);
 
-	if (m_socket == INVALID_SOCKET)
-	{
-		return false;
-	}
+		int res = getaddrinfo(m_host, portStr, &addr_hints, &addr_list);
+		debug("getaddrinfo for %s: %i", *m_host, res);
+
+#ifdef __linux__
+		if (res != 0)
+		{
+			CString resolvedHost = ResolveAndroidHost(m_host);
+			if (!resolvedHost.Empty())
+			{
+				res = getaddrinfo(resolvedHost, portStr, &addr_hints, &addr_list);
+			}
+		}
+#endif
+
+		if (res != 0)
+		{
+			ReportError("Could not resolve hostname %s", m_host, true
+#ifndef WIN32
+						, res != EAI_SYSTEM ? res : 0
+						, res != EAI_SYSTEM ? gai_strerror(res) : nullptr
+#endif
+						);
+			return false;
+		}
+
+		std::vector<SockAddr> triedAddr;
+		bool connected = false;
+
+		for (addr = addr_list; addr != nullptr; addr = addr->ai_next)
+		{
+			// don't try the same combinations of ai_family, ai_socktype, ai_protocol multiple times
+			SockAddr sa = { addr->ai_family, addr->ai_socktype, addr->ai_protocol };
+			if (std::find(triedAddr.begin(), triedAddr.end(), sa) != triedAddr.end())
+			{
+				continue;
+			}
+			triedAddr.push_back(sa);
+
+			if (m_socket != INVALID_SOCKET)
+			{
+				closesocket(m_socket);
+			}
+
+			m_socket = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
+#ifdef WIN32
+			SetHandleInformation((HANDLE)m_socket, HANDLE_FLAG_INHERIT, 0);
+#endif
+			if (m_socket == INVALID_SOCKET)
+			{
+				// try another addr/family/protocol
+				continue;
+			}
+
+			if (ConnectWithTimeout(addr->ai_addr, addr->ai_addrlen))
+			{
+				// Connection established
+				connected = true;
+				break;
+			}
+		}
+
+		if (m_socket == INVALID_SOCKET && addr_list)
+		{
+			ReportError("Socket creation failed for %s", m_host, true);
+		}
+
+		if (!connected && m_socket != INVALID_SOCKET)
+		{
+			ReportError("Connection to %s failed", m_host, true);
+			closesocket(m_socket);
+			m_socket = INVALID_SOCKET;
+		}
+
+		freeaddrinfo(addr_list);
+
+		if (m_socket == INVALID_SOCKET)
+		{
+			return false;
+		}
 
 #else
 
-	struct sockaddr_in	sSocketAddress;
-	memset(&sSocketAddress, 0, sizeof(sSocketAddress));
-	sSocketAddress.sin_family = AF_INET;
-	sSocketAddress.sin_port = htons(m_port);
-	sSocketAddress.sin_addr.s_addr = ResolveHostAddr(m_host);
-	if (sSocketAddress.sin_addr.s_addr == INADDR_NONE)
-	{
-		return false;
-	}
+		struct sockaddr_in	sSocketAddress;
+		memset(&sSocketAddress, 0, sizeof(sSocketAddress));
+		sSocketAddress.sin_family = AF_INET;
+		sSocketAddress.sin_port = htons(m_port);
+		sSocketAddress.sin_addr.s_addr = ResolveHostAddr(m_host);
+		if (sSocketAddress.sin_addr.s_addr == INADDR_NONE)
+		{
+			return false;
+		}
 
-	m_socket = socket(PF_INET, SOCK_STREAM, 0);
-	if (m_socket == INVALID_SOCKET)
-	{
-		ReportError("Socket creation failed for %s", m_host, true);
-		return false;
-	}
+		m_socket = socket(PF_INET, SOCK_STREAM, 0);
+		if (m_socket == INVALID_SOCKET)
+		{
+			ReportError("Socket creation failed for %s", m_host, true);
+			return false;
+		}
 
-	if (!ConnectWithTimeout(&sSocketAddress, sizeof(sSocketAddress)))
-	{
-		ReportError("Connection to %s failed", m_host, true);
-		closesocket(m_socket);
-		m_socket = INVALID_SOCKET;
-		return false;
-	}
+		if (!ConnectWithTimeout(&sSocketAddress, sizeof(sSocketAddress)))
+		{
+			ReportError("Connection to %s failed", m_host, true);
+			closesocket(m_socket);
+			m_socket = INVALID_SOCKET;
+			return false;
+		}
 #endif
+	}
 
-	if (!InitSocketOpts())
+	if (!InitSocketOpts(m_socket))
 	{
 		return false;
 	}
@@ -633,7 +721,7 @@ bool Connection::DoConnect()
 	return true;
 }
 
-bool Connection::InitSocketOpts()
+bool Connection::InitSocketOpts(SOCKET socket)
 {
 	char* optbuf = nullptr;
 	int optsize = 0;
@@ -648,13 +736,13 @@ bool Connection::InitSocketOpts()
 	optbuf = (char*)&TimeVal;
 	optsize = sizeof(TimeVal);
 #endif
-	int err = setsockopt(m_socket, SOL_SOCKET, SO_RCVTIMEO, optbuf, optsize);
+	int err = setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, optbuf, optsize);
 	if (err != 0)
 	{
 		ReportError("Socket initialization failed for %s", m_host, true);
 		return false;
 	}
-	err = setsockopt(m_socket, SOL_SOCKET, SO_SNDTIMEO, optbuf, optsize);
+	err = setsockopt(socket, SOL_SOCKET, SO_SNDTIMEO, optbuf, optsize);
 	if (err != 0)
 	{
 		ReportError("Socket initialization failed for %s", m_host, true);
@@ -720,7 +808,7 @@ bool Connection::ConnectWithTimeout(void* address, int address_len)
 	//connect succeeded right away?
 	if (ret != 0)
 	{
-		ret = select(m_socket + 1, &rset, &wset, nullptr, m_timeout ? &ts : nullptr);
+		ret = select((int)m_socket + 1, &rset, &wset, nullptr, m_timeout ? &ts : nullptr);
 		//we are waiting for connect to complete now
 		if (ret < 0)
 		{
@@ -781,6 +869,15 @@ bool Connection::DoDisconnect()
 	{
 #ifndef DISABLE_TLS
 		CloseTls();
+#endif
+#ifndef WIN32
+		int is_listening;
+		socklen_t len = sizeof(is_listening);
+		if (m_host && m_host[0] == '/'
+			&& getsockopt(m_socket, SOL_SOCKET, SO_ACCEPTCONN, &is_listening, &len) == 0 && is_listening)
+		{
+			unlink(m_host);
+		}
 #endif
 		if (m_gracefull)
 		{
@@ -810,10 +907,17 @@ void Connection::Cancel()
 	if (m_socket != INVALID_SOCKET)
 	{
 		m_status = csCancelled;
-		int r = shutdown(m_socket, SHUT_RDWR);
-		if (r == -1)
+
+		if (m_forceClose)
 		{
-			ReportError("Could not shutdown connection", nullptr, true);
+			SOCKET socket = m_socket;
+			m_socket = INVALID_SOCKET;
+			shutdown(socket, SHUT_RDWR);
+			closesocket(socket);
+		}
+		else
+		{
+			shutdown(m_socket, SHUT_RDWR);
 		}
 	}
 }
@@ -988,16 +1092,47 @@ in_addr_t Connection::ResolveHostAddr(const char* host)
 
 const char* Connection::GetRemoteAddr()
 {
-	struct sockaddr_in PeerName;
-	int peerNameLength = sizeof(PeerName);
-	if (getpeername(m_socket, (struct sockaddr*)&PeerName, (SOCKLEN_T*) &peerNameLength) >= 0)
+#ifndef WIN32
+	if (m_host && m_host[0] == '/')
+	{
+		return "-";
+	}
+#endif
+
+	m_remoteAddr.Clear();
+
+	char peerName[1024];
+	int peerNameLength = sizeof(peerName);
+	if (getpeername(m_socket, (sockaddr*)&peerName, (SOCKLEN_T*)&peerNameLength) >= 0)
 	{
 #ifdef WIN32
-		m_remoteAddr = inet_ntoa(PeerName.sin_addr);
+		HMODULE module = LoadLibrary("ws2_32.dll");
+		if (module)
+		{
+			using inet_ntop_t = PCTSTR WSAAPI (INT Family, PVOID pAddr, PTSTR pStringBuf, size_t StringBufSize);
+			inet_ntop_t* inet_ntop = (inet_ntop_t*)GetProcAddress(module, "inet_ntop");
+			if (inet_ntop)
+			{
+				inet_ntop(((sockaddr_in*)&peerName)->sin_family,
+					((sockaddr_in*)&peerName)->sin_family == AF_INET6 ?
+					(void*)&((sockaddr_in6*)&peerName)->sin6_addr :
+					(void*)&((sockaddr_in*)&peerName)->sin_addr,
+					m_remoteAddr, m_remoteAddr.Capacity());
+			}
+			FreeLibrary(module);
+		}
+		if (m_remoteAddr.Empty())
+		{
+			m_remoteAddr = inet_ntoa(((sockaddr_in*)&peerName)->sin_addr);
+		}
 #else
-		inet_ntop(AF_INET, &PeerName.sin_addr, m_remoteAddr, m_remoteAddr.Capacity());
-		m_remoteAddr[m_remoteAddr.Capacity() - 1] = '\0';
+		inet_ntop(((sockaddr_in*)&peerName)->sin_family,
+			((sockaddr_in*)&peerName)->sin_family == AF_INET6 ?
+			(void*)&((sockaddr_in6*)&peerName)->sin6_addr :
+			(void*)&((sockaddr_in*)&peerName)->sin_addr,
+			m_remoteAddr, m_remoteAddr.Capacity());
 #endif
+		m_remoteAddr[m_remoteAddr.Capacity() - 1] = '\0';
 	}
 
 	return m_remoteAddr;
@@ -1009,3 +1144,210 @@ int Connection::FetchTotalBytesRead()
 	m_totalBytesRead = 0;
 	return total;
 }
+
+
+#ifdef __linux__
+
+//******************************************************************************
+// Android resolver proxy from AOSP (reworked):
+// https://github.com/aosp-mirror/platform_bionic/blob/6c1d23f059986e4ee6f52c44a4944089aae9181d/libc/dns/net/gethnamaddr.c
+
+#define	MAXALIASES	35
+#define	MAXADDRS	35
+#define ALIGNBYTES (sizeof(uintptr_t) - 1)
+#define ALIGN(p) (((uintptr_t)(p) + ALIGNBYTES) &~ ALIGNBYTES)
+
+// This should be synchronized to ResponseCode.h
+static const int DnsProxyQueryResult = 222;
+
+FILE* android_open_proxy()
+{
+	const char* cache_mode = getenv("ANDROID_DNS_MODE");
+	bool use_proxy = (cache_mode == nullptr || strcmp(cache_mode, "local") != 0);
+	if (!use_proxy) {
+		return nullptr;
+	}
+
+	int s = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
+	if (s == -1) {
+		return nullptr;
+	}
+
+	const int one = 1;
+	setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
+
+	struct sockaddr_un proxy_addr;
+	memset(&proxy_addr, 0, sizeof(proxy_addr));
+	proxy_addr.sun_family = AF_UNIX;
+	strncpy(proxy_addr.sun_path, "/dev/socket/dnsproxyd", sizeof(proxy_addr.sun_path));
+
+	if (connect(s, (const struct sockaddr*) &proxy_addr, sizeof(proxy_addr)) != 0) {
+		close(s);
+		return nullptr;
+	}
+
+	return fdopen(s, "r+");
+}
+
+static struct hostent * android_read_hostent(FILE* proxy, struct hostent* hp, char* hbuf, size_t hbuflen)
+{
+	uint32_t size;
+	char buf[4];
+	if (fread(buf, 1, sizeof(buf), proxy) != sizeof(buf)) return nullptr;
+
+	// This is reading serialized data from system/netd/server/DnsProxyListener.cpp
+	// and changes here need to be matched there.
+	int result_code = strtol(buf, nullptr, 10);
+	if (result_code != DnsProxyQueryResult) {
+		fread(&size, 1, sizeof(size), proxy);
+		return nullptr;
+	}
+
+	if (fread(&size, 1, sizeof(size), proxy) != sizeof(size)) return nullptr;
+	size = ntohl(size);
+
+	memset(hp, 0, sizeof(*hp));
+	char *ptr = hbuf;
+	char *hbuf_end = hbuf + hbuflen;
+
+	if (ptr + size > hbuf_end) {
+		return nullptr;
+	}
+	if (fread(ptr, 1, size, proxy) != size) return nullptr;
+	hp->h_name = ptr;
+	ptr += size;
+
+	char *aliases_ptrs[MAXALIASES];
+	char **aliases = &aliases_ptrs[0];
+
+	while (1) {
+		if (fread(&size, 1, sizeof(size), proxy) != sizeof(size)) return nullptr;
+		size = ntohl(size);
+
+		if (size == 0) {
+			*aliases = nullptr;
+			break;
+		}
+		if (ptr + size > hbuf_end) {
+			return nullptr;
+		}
+		if (fread(ptr, 1, size, proxy) != size) return nullptr;
+		if (aliases < &aliases_ptrs[MAXALIASES - 1]) {
+		  *aliases++ = ptr;
+		}
+		ptr += size;
+	}
+
+	// Fix alignment after variable-length data.
+	ptr = (char*)ALIGN(ptr);
+
+	int aliases_len = ((int)(aliases - aliases_ptrs) + 1) * sizeof(*hp->h_aliases);
+	if (ptr + aliases_len > hbuf_end) {
+		return nullptr;
+	}
+	hp->h_aliases = (char**)ptr;
+	memcpy(ptr, aliases_ptrs, aliases_len);
+	ptr += aliases_len;
+
+	if (fread(&size, 1, sizeof(size), proxy) != sizeof(size)) return nullptr;
+	hp->h_addrtype = ntohl(size);
+
+	if (fread(&size, 1, sizeof(size), proxy) != sizeof(size)) return nullptr;
+	hp->h_length = ntohl(size);
+
+	char *addr_ptrs[MAXADDRS];
+	char **addr_p = &addr_ptrs[0];
+
+	while (1) {
+		if (fread(&size, 1, sizeof(size), proxy) != sizeof(size)) return nullptr;
+		size = ntohl(size);
+		if (size == 0) {
+			*addr_p = nullptr;
+			break;
+		}
+		if (ptr + size > hbuf_end) {
+			return nullptr;
+		}
+		if (fread(ptr, 1, size, proxy) != size) return nullptr;
+		if (addr_p < &addr_ptrs[MAXADDRS - 1]) {
+		  *addr_p++ = ptr;
+		}
+		ptr += size;
+	}
+
+	// Fix alignment after variable-length data.
+	ptr = (char*)ALIGN(ptr);
+
+	int addrs_len = ((int)(addr_p - addr_ptrs) + 1) * sizeof(*hp->h_addr_list);
+	if (ptr + addrs_len > hbuf_end) {
+		return nullptr;
+	}
+	hp->h_addr_list = (char**)ptr;
+	memcpy(ptr, addr_ptrs, addrs_len);
+	return hp;
+}
+
+// very similar in proxy-ness to android_getaddrinfo_proxy
+static struct hostent * android_gethostbyname_internal(const char *name, int af,
+	struct hostent *hp, char *hbuf, size_t hbuflen)
+{
+	FILE* proxy = android_open_proxy();
+	if (proxy == nullptr) {
+		return nullptr;
+	}
+
+	// This is writing to system/netd/server/DnsProxyListener.cpp and changes
+	// here need to be matched there.
+	if (fprintf(proxy, "gethostbyname %s %s %d",
+			"^",
+			name == nullptr ? "^" : name,
+			af) < 0) {
+		fclose(proxy);
+		return nullptr;
+	}
+
+	if (fputc(0, proxy) == EOF || fflush(proxy) != 0) {
+		fclose(proxy);
+		return nullptr;
+	}
+
+	struct hostent* result = android_read_hostent(proxy, hp, hbuf, hbuflen);
+	fclose(proxy);
+	return result;
+}
+
+CString ResolveAndroidHost(const char* host)
+{
+	debug("ResolveAndroidHost");
+
+	bool android = FileSystem::DirectoryExists("/data/dalvik-cache");
+	if (!android)
+	{
+		debug("Not android");
+		return nullptr;
+	}
+
+	struct hostent hinfobuf;
+	char strbuf[1024];
+
+	struct hostent* hinfo = android_gethostbyname_internal(host, AF_INET, &hinfobuf, strbuf, sizeof(strbuf));
+	if (hinfo == nullptr)
+	{
+		// trying IPv6
+		hinfo = android_gethostbyname_internal(host, AF_INET6, &hinfobuf, strbuf, sizeof(strbuf));
+	}
+
+	if (hinfo == nullptr)
+	{
+		debug("android_gethostbyname_r failed");
+		return nullptr;
+	}
+
+	BString<1024> result;
+	inet_ntop(hinfo->h_addrtype, hinfo->h_addr_list[0], result, result.Capacity());
+
+	debug("android_gethostbyname_r returned %s", *result);
+	return *result;
+}
+
+#endif

daemon/connect/Connection.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -41,7 +41,15 @@ public:
 		csConnected,
 		csDisconnected,
 		csListening,
-		csCancelled
+		csCancelled,
+		csBroken
+	};
+
+	enum EIPVersion
+	{
+		ipAuto,
+		ipV4,
+		ipV6
 	};
 
 	Connection(const char* host, int port, bool tls);
@@ -65,12 +73,14 @@ public:
 	const char* GetCipher() { return m_cipher; }
 	void SetCipher(const char* cipher) { m_cipher = cipher; }
 	void SetTimeout(int timeout) { m_timeout = timeout; }
+	void SetIPVersion(EIPVersion ipVersion) { m_ipVersion = ipVersion; }
 	EStatus GetStatus() { return m_status; }
 	void SetSuppressErrors(bool suppressErrors);
 	bool GetSuppressErrors() { return m_suppressErrors; }
 	const char* GetRemoteAddr();
 	bool GetGracefull() { return m_gracefull; }
 	void SetGracefull(bool gracefull) { m_gracefull = gracefull; }
+	void SetForceClose(bool forceClose) { m_forceClose = forceClose; }
 #ifndef DISABLE_TLS
 	bool StartTls(bool isClient, const char* certFile, const char* keyFile);
 #endif
@@ -80,6 +90,7 @@ protected:
 	CString m_host;
 	int m_port;
 	bool m_tls;
+	EIPVersion m_ipVersion = ipAuto;
 	SOCKET m_socket = INVALID_SOCKET;
 	CString m_cipher;
 	CharBuffer m_readBuf;
@@ -90,8 +101,8 @@ protected:
 	bool m_suppressErrors = true;
 	BString<100> m_remoteAddr;
 	int m_totalBytesRead = 0;
-	bool m_broken = false;
 	bool m_gracefull = false;
+	bool m_forceClose = false;
 
 	struct SockAddr
 	{
@@ -129,7 +140,7 @@ protected:
 	virtual void PrintError(const char* errMsg);
 	bool DoConnect();
 	bool DoDisconnect();
-	bool InitSocketOpts();
+	bool InitSocketOpts(SOCKET socket);
 	bool ConnectWithTimeout(void* address, int address_len);
 #ifndef HAVE_GETADDRINFO
 	in_addr_t ResolveHostAddr(const char* host);

daemon/connect/TlsSocket.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,6 +25,8 @@
 #include "TlsSocket.h"
 #include "Thread.h"
 #include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
 
 class TlsSocketFinalizer
 {
@@ -36,6 +38,7 @@ public:
 };
 
 std::unique_ptr<TlsSocketFinalizer> m_tlsSocketFinalizer;
+CString TlsSocket::m_certStore;
 
 #ifdef HAVE_LIBGNUTLS
 #ifdef NEED_GCRYPT_LOCKING
@@ -195,18 +198,43 @@ void TlsSocket::Final()
 #ifdef HAVE_LIBGNUTLS
 	gnutls_global_deinit();
 #endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+#ifndef LIBRESSL_VERSION_NUMBER
+	FIPS_mode_set(0);
+#endif
+#ifdef NEED_CRYPTO_LOCKING
+	CRYPTO_set_locking_callback(nullptr);
+	CRYPTO_set_id_callback(nullptr);
+#endif
+	ERR_remove_state(0);
+#if	OPENSSL_VERSION_NUMBER >= 0x10002000L && ! defined (LIBRESSL_VERSION_NUMBER)
+	SSL_COMP_free_compression_methods();
+#endif
+	//ENGINE_cleanup();
+	CONF_modules_free();
+	CONF_modules_unload(1);
+	COMP_zlib_cleanup();
+	ERR_free_strings();
+	EVP_cleanup();
+	CRYPTO_cleanup_all_ex_data();
+#endif /* HAVE_OPENSSL */
 }
 
 TlsSocket::~TlsSocket()
 {
 	Close();
+
+#ifdef HAVE_OPENSSL
+	ERR_remove_state(0);
+#endif
 }
 
-void TlsSocket::ReportError(const char* errMsg)
+void TlsSocket::ReportError(const char* errMsg, bool suppressable)
 {
 #ifdef HAVE_LIBGNUTLS
 	const char* errstr = gnutls_strerror(m_retCode);
-	if (m_suppressErrors)
+	if (suppressable && m_suppressErrors)
 	{
 		debug("%s: %s", errMsg, errstr);
 	}
@@ -217,16 +245,14 @@ void TlsSocket::ReportError(const char* errMsg)
 #endif /* HAVE_LIBGNUTLS */
 
 #ifdef HAVE_OPENSSL
-	int errcode;
+	int errcode = ERR_get_error();
 	do
 	{
-		errcode = ERR_get_error();
-
 		char errstr[1024];
 		ERR_error_string_n(errcode, errstr, sizeof(errstr));
 		errstr[1024-1] = '\0';
 
-		if (m_suppressErrors)
+		if (suppressable && m_suppressErrors)
 		{
 			debug("%s: %s", errMsg, errstr);
 		}
@@ -238,6 +264,8 @@ void TlsSocket::ReportError(const char* errMsg)
 		{
 			PrintError(errMsg);
 		}
+
+		errcode = ERR_get_error();
 	} while (errcode);
 #endif /* HAVE_OPENSSL */
 }
@@ -254,7 +282,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_certificate_allocate_credentials(&cred);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -266,7 +294,7 @@ bool TlsSocket::Start()
 			m_certFile, m_keyFile, GNUTLS_X509_FMT_PEM);
 		if (m_retCode != 0)
 		{
-			ReportError("Could not load certificate or key file");
+			ReportError("Could not load certificate or key file", false);
 			Close();
 			return false;
 		}
@@ -276,7 +304,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_init(&sess, m_isClient ? GNUTLS_CLIENT : GNUTLS_SERVER);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
@@ -291,7 +319,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_priority_set_direct((gnutls_session_t)m_session, priority, nullptr);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not select cipher for TLS");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
@@ -301,7 +329,7 @@ bool TlsSocket::Start()
 		m_retCode = gnutls_server_name_set((gnutls_session_t)m_session, GNUTLS_NAME_DNS, m_host, m_host.Length());
 		if (m_retCode != 0)
 		{
-			ReportError("Could not set host name for TLS");
+			ReportError("Could not set hostname for TLS");
 			Close();
 			return false;
 		}
@@ -311,7 +339,7 @@ bool TlsSocket::Start()
 		(gnutls_certificate_credentials_t*)m_context);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not initialize TLS session");
+		ReportError("Could not initialize TLS session", false);
 		Close();
 		return false;
 	}
@@ -321,7 +349,13 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_handshake((gnutls_session_t)m_session);
 	if (m_retCode != 0)
 	{
-		ReportError("TLS handshake failed");
+		ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -335,7 +369,7 @@ bool TlsSocket::Start()
 
 	if (!m_context)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -343,47 +377,77 @@ bool TlsSocket::Start()
 	{
 		if (SSL_CTX_use_certificate_chain_file((SSL_CTX*)m_context, m_certFile) != 1)
 		{
-			ReportError("Could not load certificate file");
+			ReportError("Could not load certificate file", false);
 			Close();
 			return false;
 		}
 		if (SSL_CTX_use_PrivateKey_file((SSL_CTX*)m_context, m_keyFile, SSL_FILETYPE_PEM) != 1)
 		{
-			ReportError("Could not load key file");
+			ReportError("Could not load key file", false);
 			Close();
 			return false;
 		}
 		if (!SSL_CTX_set_options((SSL_CTX*)m_context, SSL_OP_NO_SSLv3))
 		{
-			ReportError("Could not select minimum protocol version for TLS");
+			ReportError("Could not select minimum protocol version for TLS", false);
 			Close();
 			return false;
 		}
+
+		// For ECC certificates
+		EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
+		if (!ecdh)
+		{
+			ReportError("Could not generate ecdh parameters for TLS", false);
+			Close();
+			return false;
+		}
+		if (!SSL_CTX_set_tmp_ecdh((SSL_CTX*)m_context, ecdh))
+		{
+			ReportError("Could not set ecdh parameters for TLS", false);
+			EC_KEY_free(ecdh);
+			Close();
+			return false;
+		}
+		EC_KEY_free(ecdh);
+	}
+
+	if (m_isClient && !m_certStore.Empty())
+	{
+		// Enable certificate validation
+		if (SSL_CTX_load_verify_locations((SSL_CTX*)m_context, m_certStore, nullptr) != 1)
+		{
+			ReportError("Could not set certificate store location", false);
+			Close();
+			return false;
+		}
+
+		SSL_CTX_set_verify((SSL_CTX*)m_context, SSL_VERIFY_PEER, nullptr);
 	}
 
 	m_session = SSL_new((SSL_CTX*)m_context);
 	if (!m_session)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
 
 	if (!m_cipher.Empty() && !SSL_set_cipher_list((SSL*)m_session, m_cipher))
 	{
-		ReportError("Could not select cipher for TLS");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
 
-	if (m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host))
+	if (m_isClient && m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host))
 	{
 		ReportError("Could not set host name for TLS");
 		Close();
 		return false;
 	}
 
-	if (!SSL_set_fd((SSL*)m_session, m_socket))
+	if (!SSL_set_fd((SSL*)m_session, (int)m_socket))
 	{
 		ReportError("Could not set the file descriptor for TLS");
 		Close();
@@ -393,7 +457,23 @@ bool TlsSocket::Start()
 	int error_code = m_isClient ? SSL_connect((SSL*)m_session) : SSL_accept((SSL*)m_session);
 	if (error_code < 1)
 	{
-		ReportError("TLS handshake failed");
+		long verifyRes = SSL_get_verify_result((SSL*)m_session);
+		if (verifyRes != X509_V_OK)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification",
+				*m_host, X509_verify_cert_error_string(verifyRes)));
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		}
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -403,6 +483,124 @@ bool TlsSocket::Start()
 #endif /* HAVE_OPENSSL */
 }
 
+bool TlsSocket::ValidateCert()
+{
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER >= 0x030104
+#if	GNUTLS_VERSION_NUMBER >= 0x030306
+	if (FileSystem::DirectoryExists(m_certStore))
+	{
+		if (gnutls_certificate_set_x509_trust_dir((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+	else
+#endif
+	{
+		if (gnutls_certificate_set_x509_trust_file((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+
+	unsigned int status = 0;
+	if (gnutls_certificate_verify_peers3((gnutls_session_t)m_session, m_host, &status) != 0 ||
+		gnutls_certificate_type_get((gnutls_session_t)m_session) != GNUTLS_CRT_X509)
+	{
+		ReportError("Could not verify TLS certificate");
+		return false;
+	}
+
+	if (status != 0)
+	{
+		if (status & GNUTLS_CERT_UNEXPECTED_OWNER)
+		{
+			// Extracting hostname from the certificate
+			unsigned int cert_list_size = 0;
+			const gnutls_datum_t* cert_list = gnutls_certificate_get_peers((gnutls_session_t)m_session, &cert_list_size);
+			if (cert_list_size > 0)
+			{
+				gnutls_x509_crt_t cert;
+				gnutls_x509_crt_init(&cert);
+				gnutls_x509_crt_import(cert, &cert_list[0], GNUTLS_X509_FMT_DER);
+				char dn[256];
+				size_t size = sizeof(dn);
+				if (gnutls_x509_crt_get_dn_by_oid(cert, GNUTLS_OID_X520_COMMON_NAME, 0, 0, dn, &size) == 0)
+				{
+					PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+						" For more info visit http://nzbget.net/certificate-verification", *m_host, dn));
+					gnutls_x509_crt_deinit(cert);
+					return false;
+				}
+				gnutls_x509_crt_deinit(cert);
+			}
+		}
+
+		gnutls_datum_t msgdata;
+		if (gnutls_certificate_verification_status_print(status, GNUTLS_CRT_X509, &msgdata, 0) == 0)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host, msgdata.data));
+			gnutls_free(&msgdata);
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS certificate verification failed for %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		}
+		return false;
+	}
+#endif
+	return true;
+#endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+	// verify a server certificate was presented during the negotiation
+	X509* cert = SSL_get_peer_certificate((SSL*)m_session);
+	if (!cert)
+	{
+		PrintError(BString<1024>("TLS certificate verification failed for %s: no certificate provided by server."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		return false;
+	}
+
+#ifdef HAVE_X509_CHECK_HOST
+	// hostname verification
+	if (!m_host.Empty() && X509_check_host(cert, m_host, m_host.Length(), 0, nullptr) != 1)
+	{
+		char* certHost = nullptr;
+        // Find the position of the CN field in the Subject field of the certificate
+        int common_name_loc = X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1);
+        if (common_name_loc >= 0)
+		{
+			// Extract the CN field
+			X509_NAME_ENTRY* common_name_entry = X509_NAME_get_entry(X509_get_subject_name(cert), common_name_loc);
+			if (common_name_entry != nullptr)
+			{
+				// Convert the CN field to a C string
+				ASN1_STRING* common_name_asn1 = X509_NAME_ENTRY_get_data(common_name_entry);
+				if (common_name_asn1 != nullptr)
+				{
+					certHost = (char*)ASN1_STRING_data(common_name_asn1);
+				}
+			}
+        }
+
+		PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host, certHost));
+		X509_free(cert);
+		return false;
+	}
+#endif
+
+	X509_free(cert);
+	return true;
+#endif /* HAVE_OPENSSL */
+}
+
 void TlsSocket::Close()
 {
 	if (m_session)

daemon/connect/TlsSocket.h

@@ -33,6 +33,7 @@ public:
 		m_certFile(certFile), m_keyFile(keyFile), m_cipher(cipher) {}
 	virtual ~TlsSocket();
 	static void Init();
+	static void InitOptions(const char* certStore) { m_certStore = certStore; }
 	bool Start();
 	void Close();
 	int Send(const char* buffer, int size);
@@ -41,7 +42,7 @@ public:
 
 protected:
 	virtual void PrintError(const char* errMsg);
-	
+
 private:
 	bool m_isClient;
 	CString m_host;
@@ -53,12 +54,14 @@ private:
 	bool m_initialized = false;
 	bool m_connected = false;
 	int m_retCode;
+	static CString m_certStore;
 
 	// using "void*" to prevent the including of GnuTLS/OpenSSL header files into TlsSocket.h
 	void* m_context = nullptr;
 	void* m_session = nullptr;
 
-	void ReportError(const char* errMsg);
+	void ReportError(const char* errMsg, bool suppressable = true);
+	bool ValidateCert();
 
 	static void Final();
 	friend class TlsSocketFinalizer;

daemon/connect/WebDownloader.cpp

@@ -441,7 +441,9 @@ WebDownloader::EStatus WebDownloader::CheckResponse(const char* response)
 		warn("URL %s failed: %s", *m_infoName, hTTPResponse);
 		return adNotFound;
 	}
-	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3))
+	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3) ||
+		 !strncmp(hTTPResponse, "303", 3) || !strncmp(hTTPResponse, "307", 3) ||
+		 !strncmp(hTTPResponse, "308", 3))
 	{
 		m_redirecting = true;
 		return adRunning;
@@ -648,21 +650,6 @@ void WebDownloader::Stop()
 	debug("WebDownloader stopped successfully");
 }
 
-bool WebDownloader::Terminate()
-{
-	std::unique_ptr<Connection> connection = std::move(m_connection);
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		connection.reset();
-	}
-	return terminated;
-}
-
 void WebDownloader::FreeConnection()
 {
 	if (m_connection)

daemon/connect/WebDownloader.h

@@ -50,7 +50,6 @@ public:
 	virtual void Stop();
 	EStatus Download();
 	EStatus DownloadWithRedirects(int maxRedirects);
-	bool Terminate();
 	void SetInfoName(const char* infoName) { m_infoName = infoName; }
 	const char* GetInfoName() { return m_infoName; }
 	void SetUrl(const char* url);

daemon/extension/CommandScript.cpp

@@ -0,0 +1,131 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "CommandScript.h"
+#include "Options.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+
+static const int COMMANDPROCESS_SUCCESS = 93;
+static const int COMMANDPROCESS_ERROR = 94;
+
+bool CommandScriptController::StartScript(const char* scriptName, const char* command,
+	std::unique_ptr<Options::OptEntries> modifiedOptions)
+{
+	CommandScriptController* scriptController = new CommandScriptController();
+	scriptController->m_script = scriptName;
+	scriptController->m_command = command;
+	scriptController->m_logId = g_CommandScriptLog->Reset();
+	scriptController->m_modifiedOptions = std::move(modifiedOptions);
+
+	scriptController->SetAutoDestroy(true);
+
+	scriptController->Start();
+
+	for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
+	{
+		if (FileSystem::SameFilename(scriptName, script.GetName()))
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+void CommandScriptController::Run()
+{
+	ExecuteScriptList(m_script);
+}
+
+void CommandScriptController::ExecuteScript(ScriptConfig::Script* script)
+{
+	PrintMessage(Message::mkInfo, "Executing script %s with command %s", script->GetName(), *m_command);
+
+	SetArgs({script->GetLocation()});
+
+	BString<1024> infoName("script %s with command %s", script->GetName(), *m_command);
+	SetInfoName(infoName);
+
+	SetLogPrefix(script->GetDisplayName());
+	PrepareParams(script->GetName());
+
+	int exitCode = Execute();
+
+	infoName[0] = 'S'; // uppercase
+	SetLogPrefix(nullptr);
+
+	switch (exitCode)
+	{
+		case COMMANDPROCESS_SUCCESS:
+			PrintMessage(Message::mkInfo, "%s successful", *infoName);
+			break;
+
+		case COMMANDPROCESS_ERROR:
+		case -1: // Execute() returns -1 if the process could not be started (file not found or other problem)
+			PrintMessage(Message::mkError, "%s failed", *infoName);
+			break;
+
+		default:
+			PrintMessage(Message::mkError, "%s failed (terminated with unknown status)", *infoName);
+			break;
+	}
+}
+
+void CommandScriptController::PrepareParams(const char* scriptName)
+{
+	ResetEnv();
+
+	SetEnvVar("NZBCP_COMMAND", m_command);
+
+	PrepareEnvScript(nullptr, scriptName);
+}
+
+const char* CommandScriptController::GetOptValue(const char* name, const char* value)
+{
+	Options::OptEntry* entry = m_modifiedOptions->FindOption(name);
+	return entry ? entry->GetValue() : value;
+}
+
+void CommandScriptController::AddMessage(Message::EKind kind, const char * text)
+{
+	NzbScriptController::AddMessage(kind, text);
+	g_CommandScriptLog->AddMessage(m_logId, kind, text);
+}
+
+
+int CommandScriptLog::Reset()
+{
+	Guard guard(m_logMutex);
+	m_messages.clear();
+	return ++m_idScriptGen;
+}
+
+void CommandScriptLog::AddMessage(int scriptId, Message::EKind kind, const char * text)
+{
+	Guard guard(m_logMutex);
+
+	// save only messages from the last started script
+	if (scriptId == m_idScriptGen)
+	{
+		m_messages.emplace_back(++m_idMessageGen, kind, Util::CurrentTime(), text);
+	}
+}

daemon/extension/CommandScript.h

@@ -0,0 +1,63 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef COMMANDSCRIPT_H
+#define COMMANDSCRIPT_H
+
+#include "NzbScript.h"
+#include "Log.h"
+
+class CommandScriptController : public Thread, public NzbScriptController
+{
+public:
+	virtual void Run();
+	static bool StartScript(const char* scriptName, const char* command, std::unique_ptr<Options::OptEntries> modifiedOptions);
+
+protected:
+	virtual void ExecuteScript(ScriptConfig::Script* script);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+	virtual const char* GetOptValue(const char* name, const char* value);
+
+private:
+	CString m_script;
+	CString m_command;
+	int m_logId;
+	std::unique_ptr<Options::OptEntries> m_modifiedOptions;
+
+	void PrepareParams(const char* scriptName);
+};
+
+class CommandScriptLog
+{
+public:
+	GuardedMessageList GuardMessages() { return GuardedMessageList(&m_messages, &m_logMutex); }
+	int Reset();
+	void AddMessage(int scriptId, Message::EKind kind, const char* text);
+
+private:
+	MessageList m_messages;
+	Mutex m_logMutex;
+	int m_idMessageGen;
+	int m_idScriptGen;
+};
+
+extern CommandScriptLog* g_CommandScriptLog;
+
+#endif

daemon/extension/QueueScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -32,6 +32,7 @@ static const char* QUEUE_EVENT_NAMES[] = {
 	"URL_COMPLETED",
 	"NZB_MARKED",
 	"NZB_ADDED",
+	"NZB_NAMED",
 	"NZB_DOWNLOADED",
 	"NZB_DELETED" };
 
@@ -408,11 +409,10 @@ void QueueScriptCoordinator::CheckQueue()
 		return;
 	}
 
-	m_curItem.reset();
-
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 	Guard guard(m_queueMutex);
 
+	m_curItem.reset();
 	NzbInfo* curNzbInfo = nullptr;
 	Queue::iterator itCurItem;
 

daemon/extension/QueueScript.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,6 +33,7 @@ public:
 		qeUrlCompleted,
 		qeNzbMarked,
 		qeNzbAdded,
+		qeNzbNamed,
 		qeNzbDownloaded,
 		qeNzbDeleted // highest priority
 	};

daemon/feed/FeedCoordinator.cpp

@@ -87,7 +87,7 @@ void FeedCoordinator::Run()
 		usleep(20 * 1000);
 	}
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetReloadQueue())
+	if (g_Options->GetServerMode())
 	{
 		Guard guard(m_downloadsMutex);
 		g_DiskState->LoadFeeds(&m_feeds, &m_feedHistory);
@@ -118,8 +118,9 @@ void FeedCoordinator::Run()
 					for (FeedInfo* feedInfo : &m_feeds)
 					{
 						if (((feedInfo->GetInterval() > 0 &&
-							(current - feedInfo->GetLastUpdate() >= feedInfo->GetInterval() * 60 ||
-							 current < feedInfo->GetLastUpdate())) ||
+							 (feedInfo->GetNextUpdate() == 0 ||
+							  current >= feedInfo->GetNextUpdate() ||
+							  current < feedInfo->GetNextUpdate() - feedInfo->GetInterval() * 60)) ||
 							feedInfo->GetFetch()) &&
 							feedInfo->GetStatus() != FeedInfo::fsRunning)
 						{
@@ -182,8 +183,7 @@ void FeedCoordinator::Stop()
 
 void FeedCoordinator::ResetHangingDownloads()
 {
-	const int timeout = g_Options->GetTerminateTimeout();
-	if (timeout == 0)
+	if (g_Options->GetUrlTimeout() == 0)
 	{
 		return;
 	}
@@ -191,31 +191,15 @@ void FeedCoordinator::ResetHangingDownloads()
 	Guard guard(m_downloadsMutex);
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[timeout, tm](FeedDownloader* feedDownloader)
+	for (FeedDownloader* feedDownloader: m_activeDownloads)
+	{
+		if (tm - feedDownloader->GetLastUpdateTime() > g_Options->GetUrlTimeout() + 10 &&
+			feedDownloader->GetStatus() == FeedDownloader::adRunning)
 		{
-			if (tm - feedDownloader->GetLastUpdateTime() > timeout &&
-				feedDownloader->GetStatus() == FeedDownloader::adRunning)
-			{
-				debug("Terminating hanging download %s", feedDownloader->GetInfoName());
-				if (feedDownloader->Terminate())
-				{
-					error("Terminated hanging download %s", feedDownloader->GetInfoName());
-					feedDownloader->GetFeedInfo()->SetStatus(FeedInfo::fsUndefined);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s", feedDownloader->GetInfoName());
-				}
-
-				// it's not safe to destroy feedDownloader, because the state of object is unknown
-				delete feedDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging feed download %s", feedDownloader->GetInfoName());
+			feedDownloader->Stop();
+		}
+	}
 }
 
 void FeedCoordinator::LogDebugInfo()
@@ -291,6 +275,8 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), feedDownloader));
 	}
 
+	SchedulerNextUpdate(feedInfo, statusOK);
+
 	if (statusOK)
 	{
 		if (!feedInfo->GetPreview())
@@ -337,6 +323,30 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 	}
 }
 
+void FeedCoordinator::SchedulerNextUpdate(FeedInfo* feedInfo, bool success)
+{
+	time_t current = Util::CurrentTime();
+	int interval;
+
+	if (success)
+	{
+		interval = feedInfo->GetInterval() * 60;
+		feedInfo->SetLastInterval(0);
+	}
+	else
+	{
+		// On failure schedule next update sooner:
+		// starting with 1 minute and increasing, but not greater than FeedX.Interval
+		interval = feedInfo->GetLastInterval() * 2;
+		interval = std::max(interval, 60);
+		interval = std::min(interval, feedInfo->GetInterval() * 60);
+		feedInfo->SetLastInterval(interval);
+	}
+
+	detail("Scheduling update for feed %s in %i minute(s)", feedInfo->GetName(), interval / 60);
+	feedInfo->SetNextUpdate(current + interval);
+}
+
 void FeedCoordinator::FilterFeed(FeedInfo* feedInfo, FeedItemList* feedItems)
 {
 	debug("Filtering feed %s", feedInfo->GetName());
@@ -588,7 +598,7 @@ std::unique_ptr<FeedFile> FeedCoordinator::parseFeed(FeedInfo* feedInfo)
 		error("Feed file %s kept for troubleshooting (will be deleted on next successful feed fetch)", feedInfo->GetOutputFilename());
 		feedFile.reset();
 	}
-	return std::move(feedFile);
+	return feedFile;
 }
 
 void FeedCoordinator::DownloadQueueUpdate(Subject* caller, void* aspect)
@@ -620,11 +630,11 @@ bool FeedCoordinator::HasActiveDownloads()
 
 void FeedCoordinator::CheckSaveFeeds()
 {
-	debug("CheckSaveFeeds");
 	Guard guard(m_downloadsMutex);
 	if (m_save)
 	{
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		debug("CheckSaveFeeds: save");
+		if (g_Options->GetServerMode())
 		{
 			g_DiskState->SaveFeeds(&m_feeds, &m_feedHistory);
 		}

daemon/feed/FeedCoordinator.h

@@ -121,6 +121,7 @@ private:
 	void CleanupCache();
 	void CheckSaveFeeds();
 	std::unique_ptr<FeedFile> parseFeed(FeedInfo* feedInfo);
+	void SchedulerNextUpdate(FeedInfo* feedInfo, bool success);
 };
 
 extern FeedCoordinator* g_FeedCoordinator;

daemon/feed/FeedFile.cpp

@@ -360,6 +360,10 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 
 bool FeedFile::Parse()
 {
+#ifdef DISABLE_LIBXML2
+	error("Could not parse rss feed, program was compiled without libxml2 support");
+	return false;
+#else
 	xmlSAXHandler SAX_handler = {0};
 	SAX_handler.startElement = reinterpret_cast<startElementSAXFunc>(SAX_StartElement);
 	SAX_handler.endElement = reinterpret_cast<endElementSAXFunc>(SAX_EndElement);
@@ -378,6 +382,7 @@ bool FeedFile::Parse()
 	}
 
 	return true;
+#endif
 }
 
 void FeedFile::Parse_StartElement(const char *name, const char **atts)
@@ -566,7 +571,11 @@ void FeedFile::SAX_characters(FeedFile* file, const char * xmlstr, int len)
 
 void* FeedFile::SAX_getEntity(FeedFile* file, const char * name)
 {
+#ifdef DISABLE_LIBXML2
+	void* e = nullptr;
+#else
 	xmlEntityPtr e = xmlGetPredefinedEntity((xmlChar* )name);
+#endif
 	if (!e)
 	{
 		warn("entity not found");

daemon/feed/FeedFilter.cpp

@@ -897,7 +897,7 @@ void FeedFilter::Rule::ExpandRefValues(FeedItemInfo& feedItemInfo, CString* dest
 			break; // error
 		}
 
-		curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+		curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 	}
 
 	*destStr = std::move(curvalue);

daemon/feed/FeedInfo.h

@@ -51,6 +51,10 @@ public:
 	const char* GetExtensions() { return m_extensions; }
 	time_t GetLastUpdate() { return m_lastUpdate; }
 	void SetLastUpdate(time_t lastUpdate) { m_lastUpdate = lastUpdate; }
+	time_t GetNextUpdate() { return m_nextUpdate; }
+	void SetNextUpdate(time_t nextUpdate) { m_nextUpdate = nextUpdate; }
+	int GetLastInterval() { return m_lastInterval; }
+	void SetLastInterval(int lastInterval) { m_lastInterval = lastInterval; }
 	bool GetPreview() { return m_preview; }
 	void SetPreview(bool preview) { m_preview = preview; }
 	EStatus GetStatus() { return m_status; }
@@ -76,6 +80,8 @@ private:
 	CString m_extensions;
 	int m_priority;
 	time_t m_lastUpdate = 0;
+	time_t m_nextUpdate = 0;
+	int m_lastInterval = 0;
 	bool m_preview = false;
 	EStatus m_status = fsUndefined;
 	CString m_outputFilename;

daemon/frontend/NCursesFrontend.cpp

@@ -738,7 +738,7 @@ void NCursesFrontend::PrintFilename(FileInfo * fileInfo, int row, bool selected)
 	BString<1024> nzbNiceName;
 	if (m_showNzbname)
 	{
-		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR);
+		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR);
 	}
 
 	BString<1024> text("%s%i%s%s%s %s%s (%s%s)%s", Brace1, fileInfo->GetId(),

daemon/main/CommandLineParser.cpp

@@ -517,6 +517,10 @@ void CommandLineParser::InitCommandLine(int argc, const char* const_argv[])
 					{
 						m_editQueueAction = DownloadQueue::eaGroupParkDelete;
 					}
+					else if (!strcasecmp(optarg, "SF"))
+					{
+						m_editQueueAction = DownloadQueue::eaGroupSortFiles;
+					}
 					else if (!strcasecmp(optarg, "C") || !strcasecmp(optarg, "K") || !strcasecmp(optarg, "CP"))
 					{
 						// switch "K" is provided for compatibility with v. 0.8.0 and can be removed in future versions
@@ -741,11 +745,11 @@ void CommandLineParser::PrintUsage(const char* com)
 		"                 S          Print only server status\n"
 		"    <RegEx>                 Regular expression (only with options \"FR\", \"GR\")\n"
 		"                            using POSIX Extended Regular Expression Syntax\n"
-		"  -P, --pause   [D|O|S]  Pause server\n"
+		"  -P, --pause   [D|O|S]     Pause server\n"
 		"                 D          Pause download queue (default)\n"
 		"                 O          Pause post-processor queue\n"
 		"                 S          Pause scan of incoming nzb-directory\n"
-		"  -U, --unpause [D|O|S]  Unpause server\n"
+		"  -U, --unpause [D|O|S]     Unpause server\n"
 		"                 D          Unpause download queue (default)\n"
 		"                 O          Unpause post-processor queue\n"
 		"                 S          Unpause scan of incoming nzb-directory\n"
@@ -772,7 +776,6 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       T                    Move to top of queue\n"
 		"       B                    Move to bottom of queue\n"
 		"       D                    Delete\n"
-		"    - for files (F) and groups (G):\n"
 		"       P                    Pause\n"
 		"       U                    Resume (unpause)\n"
 		"    - for groups (G):\n"
@@ -784,6 +787,7 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       CP <name>            Set category and apply post-process parameters\n"
 		"       N <name>             Rename\n"
 		"       M                    Merge\n"
+		"       SF                   Sort inner files for optimal order\n"
 		"       S <name>             Split - create new group from selected files\n"
 		"       O <name>=<value>     Set post-process parameter\n"
 		"    - for post-jobs (O):\n"
@@ -890,7 +894,7 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			if (p)
 			{
 				BString<100> buf;
-				buf.Set(optarg, p - optarg);
+				buf.Set(optarg, (int)(p - optarg));
 				editQueueIdFrom = atoi(buf);
 				editQueueIdTo = atoi(p + 1);
 				if (editQueueIdFrom <= 0 || editQueueIdTo <= 0)

daemon/main/Maintenance.cpp

@@ -179,7 +179,7 @@ bool Maintenance::ReadPackageInfoStr(const char* key, CString& value)
 		return false;
 	}
 
-	int len = pend - p;
+	int len = (int)(pend - p);
 	if (len >= sizeof(fileName))
 	{
 		error("Could not parse file %s", *fileName);
@@ -373,7 +373,7 @@ bool Signature::ReadSignature()
 			{
 				hexSig[sigLen - 2] = '\0'; // trim trailing ",
 			}
-			for (; *hexSig && *(hexSig+1);)
+			while (*hexSig && *(hexSig+1) && output != m_signature + sizeof(m_signature))
 			{
 				uchar c1 = *hexSig++;
 				uchar c2 = *hexSig++;

daemon/main/Options.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -59,27 +59,28 @@ static const char* OPTION_RESTRICTEDUSERNAME	= "RestrictedUsername";
 static const char* OPTION_RESTRICTEDPASSWORD	= "RestrictedPassword";
 static const char* OPTION_ADDUSERNAME			= "AddUsername";
 static const char* OPTION_ADDPASSWORD			= "AddPassword";
+static const char* OPTION_FORMAUTH				= "FormAuth";
 static const char* OPTION_SECURECONTROL			= "SecureControl";
 static const char* OPTION_SECUREPORT			= "SecurePort";
 static const char* OPTION_SECURECERT			= "SecureCert";
 static const char* OPTION_SECUREKEY				= "SecureKey";
+static const char* OPTION_CERTSTORE				= "CertStore";
+static const char* OPTION_CERTCHECK				= "CertCheck";
 static const char* OPTION_AUTHORIZEDIP			= "AuthorizedIP";
 static const char* OPTION_ARTICLETIMEOUT		= "ArticleTimeout";
 static const char* OPTION_URLTIMEOUT			= "UrlTimeout";
-static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_REMOTETIMEOUT			= "RemoteTimeout";
 static const char* OPTION_FLUSHQUEUE			= "FlushQueue";
-static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
-static const char* OPTION_BROKENLOG				= "BrokenLog";
 static const char* OPTION_NZBLOG				= "NzbLog";
-static const char* OPTION_DECODE				= "Decode";
+static const char* OPTION_RAWARTICLE			= "RawArticle";
+static const char* OPTION_SKIPWRITE				= "SkipWrite";
 static const char* OPTION_ARTICLERETRIES		= "ArticleRetries";
 static const char* OPTION_ARTICLEINTERVAL		= "ArticleInterval";
 static const char* OPTION_URLRETRIES			= "UrlRetries";
 static const char* OPTION_URLINTERVAL			= "UrlInterval";
-static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
 static const char* OPTION_CONTINUEPARTIAL		= "ContinuePartial";
 static const char* OPTION_URLCONNECTIONS		= "UrlConnections";
-static const char* OPTION_LOGBUFFERSIZE			= "LogBufferSize";
+static const char* OPTION_LOGBUFFER				= "LogBuffer";
 static const char* OPTION_INFOTARGET			= "InfoTarget";
 static const char* OPTION_WARNINGTARGET			= "WarningTarget";
 static const char* OPTION_ERRORTARGET			= "ErrorTarget";
@@ -90,11 +91,13 @@ static const char* OPTION_PARREPAIR				= "ParRepair";
 static const char* OPTION_PARSCAN				= "ParScan";
 static const char* OPTION_PARQUICK				= "ParQuick";
 static const char* OPTION_POSTSTRATEGY			= "PostStrategy";
+static const char* OPTION_FILENAMING			= "FileNaming";
 static const char* OPTION_PARRENAME				= "ParRename";
 static const char* OPTION_PARBUFFER				= "ParBuffer";
 static const char* OPTION_PARTHREADS			= "ParThreads";
 static const char* OPTION_RARRENAME				= "RarRename";
 static const char* OPTION_HEALTHCHECK			= "HealthCheck";
+static const char* OPTION_DIRECTRENAME			= "DirectRename";
 static const char* OPTION_UMASK					= "UMask";
 static const char* OPTION_UPDATEINTERVAL		= "UpdateInterval";
 static const char* OPTION_CURSESNZBNAME			= "CursesNzbName";
@@ -106,14 +109,15 @@ static const char* OPTION_WRITEBUFFER			= "WriteBuffer";
 static const char* OPTION_NZBDIRINTERVAL		= "NzbDirInterval";
 static const char* OPTION_NZBDIRFILEAGE			= "NzbDirFileAge";
 static const char* OPTION_DISKSPACE				= "DiskSpace";
-static const char* OPTION_DUMPCORE				= "DumpCore";
+static const char* OPTION_CRASHTRACE			= "CrashTrace";
+static const char* OPTION_CRASHDUMP				= "CrashDump";
 static const char* OPTION_PARPAUSEQUEUE			= "ParPauseQueue";
 static const char* OPTION_SCRIPTPAUSEQUEUE		= "ScriptPauseQueue";
 static const char* OPTION_NZBCLEANUPDISK		= "NzbCleanupDisk";
 static const char* OPTION_PARTIMELIMIT			= "ParTimeLimit";
 static const char* OPTION_KEEPHISTORY			= "KeepHistory";
-static const char* OPTION_ACCURATERATE			= "AccurateRate";
 static const char* OPTION_UNPACK				= "Unpack";
+static const char* OPTION_DIRECTUNPACK			= "DirectUnpack";
 static const char* OPTION_UNPACKCLEANUPDISK		= "UnpackCleanupDisk";
 static const char* OPTION_UNRARCMD				= "UnrarCmd";
 static const char* OPTION_SEVENZIPCMD			= "SevenZipCmd";
@@ -134,6 +138,8 @@ static const char* OPTION_SHELLOVERRIDE			= "ShellOverride";
 static const char* OPTION_MONTHLYQUOTA			= "MonthlyQuota";
 static const char* OPTION_QUOTASTARTDAY			= "QuotaStartDay";
 static const char* OPTION_DAILYQUOTA			= "DailyQuota";
+static const char* OPTION_REORDERFILES			= "ReorderFiles";
+static const char* OPTION_UPDATECHECK			= "UpdateCheck";
 
 // obsolete options
 static const char* OPTION_POSTLOGKIND			= "PostLogKind";
@@ -160,6 +166,13 @@ static const char* OPTION_HISTORYCLEANUPDISK	= "HistoryCleanupDisk";
 static const char* OPTION_SCANSCRIPT			= "ScanScript";
 static const char* OPTION_QUEUESCRIPT			= "QueueScript";
 static const char* OPTION_FEEDSCRIPT			= "FeedScript";
+static const char* OPTION_DECODE				= "Decode";
+static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
+static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
+static const char* OPTION_ACCURATERATE			= "AccurateRate";
+static const char* OPTION_CREATEBROKENLOG		= "CreateBrokenLog";
+static const char* OPTION_BROKENLOG				= "BrokenLog";
 
 const char* BoolNames[] = { "yes", "no", "true", "false", "1", "0", "on", "off", "enable", "disable", "enabled", "disabled" };
 const int BoolValues[] = { 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
@@ -193,10 +206,13 @@ bool Options::OptEntry::Restricted()
 
 	bool restricted = !strcasecmp(m_name, OPTION_CONTROLIP) ||
 		!strcasecmp(m_name, OPTION_CONTROLPORT) ||
+		!strcasecmp(m_name, OPTION_FORMAUTH) ||
 		!strcasecmp(m_name, OPTION_SECURECONTROL) ||
 		!strcasecmp(m_name, OPTION_SECUREPORT) ||
 		!strcasecmp(m_name, OPTION_SECURECERT) ||
 		!strcasecmp(m_name, OPTION_SECUREKEY) ||
+		!strcasecmp(m_name, OPTION_CERTSTORE) ||
+		!strcasecmp(m_name, OPTION_CERTCHECK) ||
 		!strcasecmp(m_name, OPTION_AUTHORIZEDIP) ||
 		!strcasecmp(m_name, OPTION_DAEMONUSERNAME) ||
 		!strcasecmp(m_name, OPTION_UMASK) ||
@@ -405,7 +421,7 @@ void Options::InitDefaults()
 	SetOption(OPTION_QUEUEDIR, "${MainDir}/queue");
 	SetOption(OPTION_NZBDIR, "${MainDir}/nzb");
 	SetOption(OPTION_LOCKFILE, "${MainDir}/nzbget.lock");
-	SetOption(OPTION_LOGFILE, "${DestDir}/nzbget.log");
+	SetOption(OPTION_LOGFILE, "${MainDir}/nzbget.log");
 	SetOption(OPTION_SCRIPTDIR, "${MainDir}/scripts");
 	SetOption(OPTION_REQUIREDDIR, "");
 	SetOption(OPTION_WRITELOG, "append");
@@ -422,27 +438,28 @@ void Options::InitDefaults()
 	SetOption(OPTION_ADDUSERNAME, "");
 	SetOption(OPTION_ADDPASSWORD, "");
 	SetOption(OPTION_CONTROLPORT, "6789");
+	SetOption(OPTION_FORMAUTH, "no");
 	SetOption(OPTION_SECURECONTROL, "no");
 	SetOption(OPTION_SECUREPORT, "6791");
 	SetOption(OPTION_SECURECERT, "");
 	SetOption(OPTION_SECUREKEY, "");
+	SetOption(OPTION_CERTSTORE, "");
+	SetOption(OPTION_CERTCHECK, "no");
 	SetOption(OPTION_AUTHORIZEDIP, "");
 	SetOption(OPTION_ARTICLETIMEOUT, "60");
 	SetOption(OPTION_URLTIMEOUT, "60");
-	SetOption(OPTION_SAVEQUEUE, "yes");
+	SetOption(OPTION_REMOTETIMEOUT, "90");
 	SetOption(OPTION_FLUSHQUEUE, "yes");
-	SetOption(OPTION_RELOADQUEUE, "yes");
-	SetOption(OPTION_BROKENLOG, "yes");
 	SetOption(OPTION_NZBLOG, "yes");
-	SetOption(OPTION_DECODE, "yes");
+	SetOption(OPTION_RAWARTICLE, "no");
+	SetOption(OPTION_SKIPWRITE, "no");
 	SetOption(OPTION_ARTICLERETRIES, "3");
 	SetOption(OPTION_ARTICLEINTERVAL, "10");
 	SetOption(OPTION_URLRETRIES, "3");
 	SetOption(OPTION_URLINTERVAL, "10");
-	SetOption(OPTION_TERMINATETIMEOUT, "600");
 	SetOption(OPTION_CONTINUEPARTIAL, "no");
 	SetOption(OPTION_URLCONNECTIONS, "4");
-	SetOption(OPTION_LOGBUFFERSIZE, "1000");
+	SetOption(OPTION_LOGBUFFER, "1000");
 	SetOption(OPTION_INFOTARGET, "both");
 	SetOption(OPTION_WARNINGTARGET, "both");
 	SetOption(OPTION_ERRORTARGET, "both");
@@ -453,11 +470,13 @@ void Options::InitDefaults()
 	SetOption(OPTION_PARSCAN, "extended");
 	SetOption(OPTION_PARQUICK, "yes");
 	SetOption(OPTION_POSTSTRATEGY, "sequential");
+	SetOption(OPTION_FILENAMING, "article");
 	SetOption(OPTION_PARRENAME, "yes");
 	SetOption(OPTION_PARBUFFER, "16");
 	SetOption(OPTION_PARTHREADS, "1");
 	SetOption(OPTION_RARRENAME, "yes");
 	SetOption(OPTION_HEALTHCHECK, "none");
+	SetOption(OPTION_DIRECTRENAME, "no");
 	SetOption(OPTION_SCRIPTORDER, "");
 	SetOption(OPTION_EXTENSIONS, "");
 	SetOption(OPTION_DAEMONUSERNAME, "root");
@@ -472,14 +491,15 @@ void Options::InitDefaults()
 	SetOption(OPTION_NZBDIRINTERVAL, "5");
 	SetOption(OPTION_NZBDIRFILEAGE, "60");
 	SetOption(OPTION_DISKSPACE, "250");
-	SetOption(OPTION_DUMPCORE, "no");
+	SetOption(OPTION_CRASHTRACE, "no");
+	SetOption(OPTION_CRASHDUMP, "no");
 	SetOption(OPTION_PARPAUSEQUEUE, "no");
 	SetOption(OPTION_SCRIPTPAUSEQUEUE, "no");
 	SetOption(OPTION_NZBCLEANUPDISK, "no");
 	SetOption(OPTION_PARTIMELIMIT, "0");
 	SetOption(OPTION_KEEPHISTORY, "7");
-	SetOption(OPTION_ACCURATERATE, "no");
 	SetOption(OPTION_UNPACK, "no");
+	SetOption(OPTION_DIRECTUNPACK, "no");
 	SetOption(OPTION_UNPACKCLEANUPDISK, "no");
 #ifdef WIN32
 	SetOption(OPTION_UNRARCMD, "unrar.exe");
@@ -503,6 +523,8 @@ void Options::InitDefaults()
 	SetOption(OPTION_MONTHLYQUOTA, "0");
 	SetOption(OPTION_QUOTASTARTDAY, "1");
 	SetOption(OPTION_DAILYQUOTA, "0");
+	SetOption(OPTION_REORDERFILES, "no");
+	SetOption(OPTION_UPDATECHECK, "none");
 }
 
 void Options::InitOptFile()
@@ -659,6 +681,7 @@ void Options::InitOptions()
 	m_addPassword			= GetOption(OPTION_ADDPASSWORD);
 	m_secureCert			= GetOption(OPTION_SECURECERT);
 	m_secureKey				= GetOption(OPTION_SECUREKEY);
+	m_certStore				= GetOption(OPTION_CERTSTORE);
 	m_authorizedIp			= GetOption(OPTION_AUTHORIZEDIP);
 	m_lockFile				= GetOption(OPTION_LOCKFILE);
 	m_daemonUsername		= GetOption(OPTION_DAEMONUSERNAME);
@@ -674,7 +697,7 @@ void Options::InitOptions()
 	m_downloadRate			= ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024;
 	m_articleTimeout		= ParseIntValue(OPTION_ARTICLETIMEOUT, 10);
 	m_urlTimeout			= ParseIntValue(OPTION_URLTIMEOUT, 10);
-	m_terminateTimeout		= ParseIntValue(OPTION_TERMINATETIMEOUT, 10);
+	m_remoteTimeout			= ParseIntValue(OPTION_REMOTETIMEOUT, 10);
 	m_articleRetries		= ParseIntValue(OPTION_ARTICLERETRIES, 10);
 	m_articleInterval		= ParseIntValue(OPTION_ARTICLEINTERVAL, 10);
 	m_urlRetries			= ParseIntValue(OPTION_URLRETRIES, 10);
@@ -682,7 +705,7 @@ void Options::InitOptions()
 	m_controlPort			= ParseIntValue(OPTION_CONTROLPORT, 10);
 	m_securePort			= ParseIntValue(OPTION_SECUREPORT, 10);
 	m_urlConnections		= ParseIntValue(OPTION_URLCONNECTIONS, 10);
-	m_logBufferSize			= ParseIntValue(OPTION_LOGBUFFERSIZE, 10);
+	m_logBuffer				= ParseIntValue(OPTION_LOGBUFFER, 10);
 	m_rotateLog				= ParseIntValue(OPTION_ROTATELOG, 10);
 	m_umask					= ParseIntValue(OPTION_UMASK, 8);
 	m_updateInterval		= ParseIntValue(OPTION_UPDATEINTERVAL, 10);
@@ -708,34 +731,37 @@ void Options::InitOptions()
 	m_quotaStartDay			= ParseIntValue(OPTION_QUOTASTARTDAY, 10);
 	m_dailyQuota			= ParseIntValue(OPTION_DAILYQUOTA, 10);
 
-	m_brokenLog				= (bool)ParseEnumValue(OPTION_BROKENLOG, BoolCount, BoolNames, BoolValues);
 	m_nzbLog				= (bool)ParseEnumValue(OPTION_NZBLOG, BoolCount, BoolNames, BoolValues);
 	m_appendCategoryDir		= (bool)ParseEnumValue(OPTION_APPENDCATEGORYDIR, BoolCount, BoolNames, BoolValues);
 	m_continuePartial		= (bool)ParseEnumValue(OPTION_CONTINUEPARTIAL, BoolCount, BoolNames, BoolValues);
-	m_saveQueue				= (bool)ParseEnumValue(OPTION_SAVEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_flushQueue			= (bool)ParseEnumValue(OPTION_FLUSHQUEUE, BoolCount, BoolNames, BoolValues);
 	m_dupeCheck				= (bool)ParseEnumValue(OPTION_DUPECHECK, BoolCount, BoolNames, BoolValues);
 	m_parRepair				= (bool)ParseEnumValue(OPTION_PARREPAIR, BoolCount, BoolNames, BoolValues);
 	m_parQuick				= (bool)ParseEnumValue(OPTION_PARQUICK, BoolCount, BoolNames, BoolValues);
 	m_parRename				= (bool)ParseEnumValue(OPTION_PARRENAME, BoolCount, BoolNames, BoolValues);
 	m_rarRename				= (bool)ParseEnumValue(OPTION_RARRENAME, BoolCount, BoolNames, BoolValues);
-	m_reloadQueue			= (bool)ParseEnumValue(OPTION_RELOADQUEUE, BoolCount, BoolNames, BoolValues);
+	m_directRename			= (bool)ParseEnumValue(OPTION_DIRECTRENAME, BoolCount, BoolNames, BoolValues);
 	m_cursesNzbName			= (bool)ParseEnumValue(OPTION_CURSESNZBNAME, BoolCount, BoolNames, BoolValues);
 	m_cursesTime			= (bool)ParseEnumValue(OPTION_CURSESTIME, BoolCount, BoolNames, BoolValues);
 	m_cursesGroup			= (bool)ParseEnumValue(OPTION_CURSESGROUP, BoolCount, BoolNames, BoolValues);
 	m_crcCheck				= (bool)ParseEnumValue(OPTION_CRCCHECK, BoolCount, BoolNames, BoolValues);
 	m_directWrite			= (bool)ParseEnumValue(OPTION_DIRECTWRITE, BoolCount, BoolNames, BoolValues);
-	m_decode				= (bool)ParseEnumValue(OPTION_DECODE, BoolCount, BoolNames, BoolValues);
-	m_dumpCore				= (bool)ParseEnumValue(OPTION_DUMPCORE, BoolCount, BoolNames, BoolValues);
+	m_rawArticle			= (bool)ParseEnumValue(OPTION_RAWARTICLE, BoolCount, BoolNames, BoolValues);
+	m_skipWrite				= (bool)ParseEnumValue(OPTION_SKIPWRITE, BoolCount, BoolNames, BoolValues);
+	m_crashTrace			= (bool)ParseEnumValue(OPTION_CRASHTRACE, BoolCount, BoolNames, BoolValues);
+	m_crashDump				= (bool)ParseEnumValue(OPTION_CRASHDUMP, BoolCount, BoolNames, BoolValues);
 	m_parPauseQueue			= (bool)ParseEnumValue(OPTION_PARPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_scriptPauseQueue		= (bool)ParseEnumValue(OPTION_SCRIPTPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_nzbCleanupDisk		= (bool)ParseEnumValue(OPTION_NZBCLEANUPDISK, BoolCount, BoolNames, BoolValues);
-	m_accurateRate			= (bool)ParseEnumValue(OPTION_ACCURATERATE, BoolCount, BoolNames, BoolValues);
+	m_formAuth				= (bool)ParseEnumValue(OPTION_FORMAUTH, BoolCount, BoolNames, BoolValues);
 	m_secureControl			= (bool)ParseEnumValue(OPTION_SECURECONTROL, BoolCount, BoolNames, BoolValues);
 	m_unpack				= (bool)ParseEnumValue(OPTION_UNPACK, BoolCount, BoolNames, BoolValues);
+	m_directUnpack			= (bool)ParseEnumValue(OPTION_DIRECTUNPACK, BoolCount, BoolNames, BoolValues);
 	m_unpackCleanupDisk		= (bool)ParseEnumValue(OPTION_UNPACKCLEANUPDISK, BoolCount, BoolNames, BoolValues);
 	m_unpackPauseQueue		= (bool)ParseEnumValue(OPTION_UNPACKPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_urlForce				= (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues);
+	m_certCheck				= (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues);
+	m_reorderFiles			= (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues);
 
 	const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" };
 	const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses };
@@ -757,6 +783,11 @@ void Options::InitOptions()
 	const int PostStrategyCount = 4;
 	m_postStrategy = (EPostStrategy)ParseEnumValue(OPTION_POSTSTRATEGY, PostStrategyCount, PostStrategyNames, PostStrategyValues);
 
+	const char* FileNamingNames[] = { "auto", "article", "nzb" };
+	const int FileNamingValues[] = { nfAuto, nfArticle, nfNzb };
+	const int FileNamingCount = 4;
+	m_fileNaming = (EFileNaming)ParseEnumValue(OPTION_FILENAMING, FileNamingCount, FileNamingNames, FileNamingValues);
+
 	const char* HealthCheckNames[] = { "pause", "delete", "park", "none" };
 	const int HealthCheckValues[] = { hcPause, hcDelete, hcPark, hcNone };
 	const int HealthCheckCount = 4;
@@ -886,7 +917,7 @@ void Options::SetOption(const char* optname, const char* value)
 			const char* varvalue = GetOption(variable);
 			if (varvalue)
 			{
-				curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+				curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 			}
 			else
 			{
@@ -979,6 +1010,16 @@ void Options::InitServers()
 			m_tls |= tls;
 		}
 
+		const char* nipversion = GetOption(BString<100>("Server%i.IpVersion", n));
+		int ipversion = 0;
+		if (nipversion)
+		{
+			const char* IpVersionNames[] = {"auto", "ipv4", "ipv6"};
+			const int IpVersionValues[] = {0, 4, 6};
+			const int IpVersionCount = 3;
+			ipversion = ParseEnumValue(BString<100>("Server%i.IpVersion", n), IpVersionCount, IpVersionNames, IpVersionValues);
+		}
+
 		const char* ncipher = GetOption(BString<100>("Server%i.Cipher", n));
 		const char* nconnections = GetOption(BString<100>("Server%i.Connections", n));
 		const char* nretention = GetOption(BString<100>("Server%i.Retention", n));
@@ -999,6 +1040,7 @@ void Options::InitServers()
 				m_extender->AddNewsServer(n, active, nname,
 					nhost,
 					nport ? atoi(nport) : 119,
+					ipversion,
 					nusername, npassword,
 					joinGroup, tls, ncipher,
 					nconnections ? atoi(nconnections) : 1,
@@ -1460,7 +1502,7 @@ bool Options::SplitOptionString(const char* option, CString& optName, CString& o
 		return false;
 	}
 
-	optName.Set(option, eq - option);
+	optName.Set(option, (int)(eq - option));
 	optValue.Set(eq + 1);
 
 	ConvertOldOption(optName, optValue);
@@ -1495,7 +1537,8 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 			!strcasecmp(p, ".password") || !strcasecmp(p, ".joingroup") ||
 			!strcasecmp(p, ".encryption") || !strcasecmp(p, ".connections") ||
 			!strcasecmp(p, ".cipher") || !strcasecmp(p, ".group") ||
-			!strcasecmp(p, ".retention") || !strcasecmp(p, ".optional")))
+			!strcasecmp(p, ".retention") || !strcasecmp(p, ".optional") ||
+			!strcasecmp(p, ".notes") || !strcasecmp(p, ".ipversion")))
 		{
 			return true;
 		}
@@ -1558,7 +1601,13 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 		!strcasecmp(optname, OPTION_RELOADPOSTQUEUE) ||
 		!strcasecmp(optname, OPTION_PARCLEANUPQUEUE) ||
 		!strcasecmp(optname, OPTION_DELETECLEANUPDISK) ||
-		!strcasecmp(optname, OPTION_HISTORYCLEANUPDISK))
+		!strcasecmp(optname, OPTION_HISTORYCLEANUPDISK) ||
+		!strcasecmp(optname, OPTION_SAVEQUEUE) ||
+		!strcasecmp(optname, OPTION_RELOADQUEUE) ||
+		!strcasecmp(optname, OPTION_TERMINATETIMEOUT) ||
+		!strcasecmp(optname, OPTION_ACCURATERATE) ||
+		!strcasecmp(optname, OPTION_CREATEBROKENLOG) ||
+		!strcasecmp(optname, OPTION_BROKENLOG))
 	{
 		ConfigWarn("Option \"%s\" is obsolete, ignored", optname);
 		return true;
@@ -1678,9 +1727,20 @@ void Options::ConvertOldOption(CString& option, CString& value)
 		option = "ArticleInterval";
 	}
 
-	if (!strcasecmp(option, "CreateBrokenLog"))
+	if (!strcasecmp(option, "DumpCore"))
 	{
-		option = "BrokenLog";
+		option = OPTION_CRASHDUMP;
+	}
+
+	if (!strcasecmp(option, OPTION_DECODE))
+	{
+		option = OPTION_RAWARTICLE;
+		value = !strcasecmp(value, "no") ? "yes" : "no";
+	}
+
+	if (!strcasecmp(option, "LogBufferSize"))
+	{
+		option = OPTION_LOGBUFFER;
 	}
 }
 
@@ -1697,6 +1757,11 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_PARRENAME);
 		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_PARRENAME);
 	}
+	if (m_directRename)
+	{
+		LocateOptionSrcPos(OPTION_DIRECTRENAME);
+		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_DIRECTRENAME);
+	}
 #endif
 
 #ifdef DISABLE_CURSES
@@ -1713,13 +1778,53 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_SECURECONTROL);
 		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_SECURECONTROL);
 	}
+
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_CERTCHECK);
+	}
 #endif
 
-	if (!m_decode)
+#ifdef HAVE_OPENSSL
+#ifndef HAVE_X509_CHECK_HOST
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is limited because the program "
+			"was compiled with older OpenSSL version not supporting hostname validation (at least OpenSSL 1.0.2d is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER < 0x030104
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is disabled because the program "
+			"was compiled with older GnuTLS version not supporting certificate validation (at least GnuTLS 3.1.4 is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+	if (m_certCheck && m_certStore.Empty())
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Option \"%s\" requires proper configuration of option \"%s\"", OPTION_CERTCHECK, OPTION_CERTSTORE);
+		m_certCheck = false;
+	}
+
+	if (m_rawArticle)
 	{
 		m_directWrite = false;
 	}
 
+	if (m_skipWrite)
+	{
+		m_directRename = false;
+	}
+
 	// if option "ConfigTemplate" is not set, use "WebDir" as default location for template
 	// (for compatibility with versions 9 and 10).
 	if (m_configTemplate.Empty() && !m_noDiskAccess)

daemon/main/Options.h

@@ -92,6 +92,12 @@ public:
 		ppAggressive,
 		ppRocket
 	};
+	enum EFileNaming
+	{
+		nfAuto,
+		nfArticle,
+		nfNzb
+	};
 
 	class OptEntry
 	{
@@ -161,7 +167,7 @@ public:
 	{
 	public:
 		virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-			int port, const char* user, const char* pass, bool joinGroup,
+			int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 			bool tls, const char* cipher, int maxConnections, int retention,
 			int level, int group, bool optional) = 0;
 		virtual void AddFeed(int id, const char* name, const char* url, int interval,
@@ -197,7 +203,6 @@ public:
 	const char* GetConfigTemplate() { return m_configTemplate; }
 	const char* GetScriptDir() { return m_scriptDir; }
 	const char* GetRequiredDir() { return m_requiredDir; }
-	bool GetBrokenLog() const { return m_brokenLog; }
 	bool GetNzbLog() const { return m_nzbLog; }
 	EMessageTarget GetInfoTarget() const { return m_infoTarget; }
 	EMessageTarget GetWarningTarget() const { return m_warningTarget; }
@@ -206,15 +211,15 @@ public:
 	EMessageTarget GetDetailTarget() const { return m_detailTarget; }
 	int GetArticleTimeout() { return m_articleTimeout; }
 	int GetUrlTimeout() { return m_urlTimeout; }
-	int GetTerminateTimeout() { return m_terminateTimeout; }
-	bool GetDecode() { return m_decode; };
+	int GetRemoteTimeout() { return m_remoteTimeout; }
+	bool GetRawArticle() { return m_rawArticle; };
+	bool GetSkipWrite() { return m_skipWrite; };
 	bool GetAppendCategoryDir() { return m_appendCategoryDir; }
 	bool GetContinuePartial() { return m_continuePartial; }
 	int GetArticleRetries() { return m_articleRetries; }
 	int GetArticleInterval() { return m_articleInterval; }
 	int GetUrlRetries() { return m_urlRetries; }
 	int GetUrlInterval() { return m_urlInterval; }
-	bool GetSaveQueue() { return m_saveQueue; }
 	bool GetFlushQueue() { return m_flushQueue; }
 	bool GetDupeCheck() { return m_dupeCheck; }
 	const char* GetControlIp() { return m_controlIp; }
@@ -225,17 +230,19 @@ public:
 	const char* GetAddUsername() { return m_addUsername; }
 	const char* GetAddPassword() { return m_addPassword; }
 	int GetControlPort() { return m_controlPort; }
+	bool GetFormAuth() { return m_formAuth; }
 	bool GetSecureControl() { return m_secureControl; }
 	int GetSecurePort() { return m_securePort; }
 	const char* GetSecureCert() { return m_secureCert; }
 	const char* GetSecureKey() { return m_secureKey; }
+	const char* GetCertStore() { return m_certStore; }
+	bool GetCertCheck() { return m_certCheck; }
 	const char* GetAuthorizedIp() { return m_authorizedIp; }
 	const char* GetLockFile() { return m_lockFile; }
 	const char* GetDaemonUsername() { return m_daemonUsername; }
 	EOutputMode GetOutputMode() { return m_outputMode; }
-	bool GetReloadQueue() { return m_reloadQueue; }
 	int GetUrlConnections() { return m_urlConnections; }
-	int GetLogBufferSize() { return m_logBufferSize; }
+	int GetLogBuffer() { return m_logBuffer; }
 	EWriteLog GetWriteLog() { return m_writeLog; }
 	const char* GetLogFile() { return m_logFile; }
 	int GetRotateLog() { return m_rotateLog; }
@@ -263,14 +270,15 @@ public:
 	int GetNzbDirFileAge() { return m_nzbDirFileAge; }
 	int GetDiskSpace() { return m_diskSpace; }
 	bool GetTls() { return m_tls; }
-	bool GetDumpCore() { return m_dumpCore; }
+	bool GetCrashTrace() { return m_crashTrace; }
+	bool GetCrashDump() { return m_crashDump; }
 	bool GetParPauseQueue() { return m_parPauseQueue; }
 	bool GetScriptPauseQueue() { return m_scriptPauseQueue; }
 	bool GetNzbCleanupDisk() { return m_nzbCleanupDisk; }
 	int GetParTimeLimit() { return m_parTimeLimit; }
 	int GetKeepHistory() { return m_keepHistory; }
-	bool GetAccurateRate() { return m_accurateRate; }
 	bool GetUnpack() { return m_unpack; }
+	bool GetDirectUnpack() { return m_directUnpack; }
 	bool GetUnpackCleanupDisk() { return m_unpackCleanupDisk; }
 	const char* GetUnrarCmd() { return m_unrarCmd; }
 	const char* GetSevenZipCmd() { return m_sevenZipCmd; }
@@ -289,6 +297,9 @@ public:
 	int GetMonthlyQuota() { return m_monthlyQuota; }
 	int GetQuotaStartDay() { return m_quotaStartDay; }
 	int GetDailyQuota() { return m_dailyQuota; }
+	bool GetDirectRename() { return m_directRename; }
+	bool GetReorderFiles() { return m_reorderFiles; }
+	EFileNaming GetFileNaming() { return m_fileNaming; }
 
 	Categories* GetCategories() { return &m_categories; }
 	Category* FindCategory(const char* name, bool searchAliases) { return m_categories.FindCategory(name, searchAliases); }
@@ -347,19 +358,18 @@ private:
 	EMessageTarget m_errorTarget = mtScreen;
 	EMessageTarget m_debugTarget = mtNone;
 	EMessageTarget m_detailTarget = mtScreen;
-	bool m_decode = true;
-	bool m_brokenLog = false;
+	bool m_skipWrite = false;
+	bool m_rawArticle = false;
 	bool m_nzbLog = false;
 	int m_articleTimeout = 0;
 	int m_urlTimeout = 0;
-	int m_terminateTimeout = 0;
+	int m_remoteTimeout = 0;
 	bool m_appendCategoryDir = false;
 	bool m_continuePartial = false;
 	int m_articleRetries = 0;
 	int m_articleInterval = 0;
 	int m_urlRetries = 0;
 	int m_urlInterval = 0;
-	bool m_saveQueue = false;
 	bool m_flushQueue = false;
 	bool m_dupeCheck = false;
 	CString m_controlIp;
@@ -369,18 +379,20 @@ private:
 	CString m_restrictedPassword;
 	CString m_addUsername;
 	CString m_addPassword;
+	bool m_formAuth = false;
 	int m_controlPort = 0;
 	bool m_secureControl = false;
 	int m_securePort = 0;
 	CString m_secureCert;
 	CString m_secureKey;
+	CString m_certStore;
+	bool m_certCheck = false;
 	CString m_authorizedIp;
 	CString m_lockFile;
 	CString m_daemonUsername;
 	EOutputMode m_outputMode = omLoggable;
-	bool m_reloadQueue = false;
 	int m_urlConnections = 0;
-	int m_logBufferSize = 0;
+	int m_logBuffer = 0;
 	EWriteLog m_writeLog = wlAppend;
 	int m_rotateLog = 0;
 	CString m_logFile;
@@ -393,6 +405,7 @@ private:
 	int m_parBuffer = 0;
 	int m_parThreads = 0;
 	bool m_rarRename = false;
+	bool m_directRename = false;
 	EHealthCheck m_healthCheck = hcNone;
 	CString m_extensions;
 	CString m_scriptOrder;
@@ -408,14 +421,15 @@ private:
 	int m_nzbDirFileAge = 0;
 	int m_diskSpace = 0;
 	bool m_tls = false;
-	bool m_dumpCore = false;
+	bool m_crashTrace = false;
+	bool m_crashDump = false;
 	bool m_parPauseQueue = false;
 	bool m_scriptPauseQueue = false;
 	bool m_nzbCleanupDisk = false;
 	int m_parTimeLimit = 0;
 	int m_keepHistory = 0;
-	bool m_accurateRate = false;
 	bool m_unpack = false;
+	bool m_directUnpack = false;
 	bool m_unpackCleanupDisk = false;
 	CString m_unrarCmd;
 	CString m_sevenZipCmd;
@@ -434,6 +448,8 @@ private:
 	int m_monthlyQuota = 0;
 	int m_quotaStartDay = 0;
 	int m_dailyQuota = 0;
+	bool m_reorderFiles = false;
+	EFileNaming m_fileNaming = nfArticle;
 
 	// Current state
 	bool m_serverMode = false;

daemon/main/Scheduler.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -93,7 +93,10 @@ void Scheduler::CheckTasks()
 
 				for (Task* task : &m_taskList)
 				{
-					task->m_lastExecuted = 0;
+					if (task->m_hours != Task::STARTUP_TASK)
+					{
+						task->m_lastExecuted = 0;
+					}
 				}
 			}
 

daemon/main/StackTrace.cpp

@@ -147,17 +147,20 @@ LONG __stdcall ExceptionFilter(EXCEPTION_POINTERS* exPtrs)
 
 #ifdef DEBUG
 	PrintBacktrace(exPtrs->ContextRecord);
-#else
-	 info("Detailed exception information can be printed by debug version of NZBGet (available from download page)");
-#endif
-
-	ExitProcess(-1);
 	return EXCEPTION_CONTINUE_SEARCH;
+#else
+	info("Detailed crash information can be printed by debug version of NZBGet."
+		" For more info visit http://nzbget.net/crash-dump");
+	return EXCEPTION_EXECUTE_HANDLER;
+#endif
 }
 
 void InstallErrorHandler()
 {
-	SetUnhandledExceptionFilter(ExceptionFilter);
+	if (g_Options->GetCrashTrace())
+	{
+		SetUnhandledExceptionFilter(ExceptionFilter);
+	}
 }
 
 #else
@@ -171,7 +174,7 @@ std::vector<sighandler> SignalProcList;
 /**
 * activates the creation of core-files
 */
-void EnableDumpCore()
+void EnableCoreDump()
 {
 	rlimit rlim;
 	rlim.rlim_cur= RLIM_INFINITY;
@@ -248,9 +251,9 @@ void SignalProc(int signum)
 void InstallErrorHandler()
 {
 #ifdef HAVE_SYS_PRCTL_H
-	if (g_Options->GetDumpCore())
+	if (g_Options->GetCrashDump())
 	{
-		EnableDumpCore();
+		EnableCoreDump();
 	}
 #endif
 
@@ -258,7 +261,10 @@ void InstallErrorHandler()
 	signal(SIGTERM, SignalProc);
 	signal(SIGPIPE, SIG_IGN);
 #ifdef DEBUG
-	signal(SIGSEGV, SignalProc);
+	if (g_Options->GetCrashTrace())
+	{
+		signal(SIGSEGV, SignalProc);
+	}
 #endif
 #ifdef SIGCHLD_HANDLER
 	// it could be necessary on some systems to activate a handler for SIGCHLD

daemon/main/nzbget.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -39,7 +39,6 @@
 #include "PrePostProcessor.h"
 #include "HistoryCoordinator.h"
 #include "DupeCoordinator.h"
-#include "ParChecker.h"
 #include "Scheduler.h"
 #include "Scanner.h"
 #include "FeedCoordinator.h"
@@ -52,6 +51,8 @@
 #include "Util.h"
 #include "FileSystem.h"
 #include "StackTrace.h"
+#include "CommandScript.h"
+#include "YEncode.h"
 #ifdef WIN32
 #include "WinService.h"
 #include "WinConsole.h"
@@ -85,6 +86,7 @@ ArticleCache* g_ArticleCache;
 QueueScriptCoordinator* g_QueueScriptCoordinator;
 ServiceCoordinator* g_ServiceCoordinator;
 ScriptConfig* g_ScriptConfig;
+CommandScriptLog* g_CommandScriptLog; 
 #ifdef WIN32
 WinConsole* g_WinConsole;
 #endif
@@ -111,6 +113,7 @@ int main(int argc, char *argv[], char *argp[])
 #endif
 
 	Util::Init();
+	YEncode::init();
 
 	g_ArgumentCount = argc;
 	g_Arguments = (char*(*)[])argv;
@@ -144,7 +147,7 @@ int main(int argc, char *argv[], char *argp[])
 	InstallUninstallServiceCheck(argc, argv);
 #endif
 
-	srand(Util::CurrentTime());
+	srand((unsigned int)Util::CurrentTime());
 
 #ifdef WIN32
 	for (int i=0; i < argc; i++)
@@ -174,7 +177,7 @@ public:
 
 	// Options::Extender
 	virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-		int port, const char* user, const char* pass, bool joinGroup,
+		int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
 		int level, int group, bool optional);
 	virtual void AddFeed(int id, const char* name, const char* url, int interval,
@@ -205,6 +208,7 @@ private:
 	std::unique_ptr<QueueScriptCoordinator> m_queueScriptCoordinator;
 	std::unique_ptr<ServiceCoordinator> m_serviceCoordinator;
 	std::unique_ptr<ScriptConfig> m_scriptConfig;
+	std::unique_ptr<CommandScriptLog> m_commandScriptLog;
 #ifdef WIN32
 	std::unique_ptr<WinConsole> m_winConsole;
 #endif
@@ -280,6 +284,9 @@ void NZBGet::Init()
 
 	m_scanner->InitOptions();
 	m_queueScriptCoordinator->InitOptions();
+#ifndef DISABLE_TLS
+	TlsSocket::InitOptions(g_Options->GetCertCheck() ? g_Options->GetCertStore() : nullptr);
+#endif
 
 	if (m_commandLineParser->GetDaemonMode())
 	{
@@ -365,6 +372,9 @@ void NZBGet::CreateGlobals()
 	m_scriptConfig = std::make_unique<ScriptConfig>();
 	g_ScriptConfig = m_scriptConfig.get();
 
+	m_commandScriptLog = std::make_unique<CommandScriptLog>();
+	g_CommandScriptLog = m_commandScriptLog.get();
+
 	m_scheduler = std::make_unique<Scheduler>();
 
 	m_diskService = std::make_unique<DiskService>();
@@ -382,7 +392,7 @@ void NZBGet::BootConfig()
 	if (m_commandLineParser->GetPrintUsage() || m_commandLineParser->GetErrors() || g_ArgumentCount <= 1)
 	{
 		m_commandLineParser->PrintUsage(((const char**)(*g_Arguments))[0]);
-		exit(0);
+		exit(m_commandLineParser->GetPrintUsage() ? 0 : 1);
 	}
 
 	debug("Reading options");
@@ -437,6 +447,7 @@ void NZBGet::Cleanup()
 	g_QueueScriptCoordinator = nullptr;
 	g_Maintenance = nullptr;
 	g_StatMeter = nullptr;
+	g_CommandScriptLog = nullptr;
 #ifdef WIN32
 	g_WinConsole = nullptr;
 #endif
@@ -490,7 +501,11 @@ void NZBGet::StartRemoteServer()
 	m_remoteServer = std::make_unique<RemoteServer>(false);
 	m_remoteServer->Start();
 
-	if (m_options->GetSecureControl())
+	if (m_options->GetSecureControl()
+#ifndef WIN32
+		&& !(m_options->GetControlIp() && m_options->GetControlIp()[0] == '/')
+#endif
+)
 	{
 		m_remoteSecureServer = std::make_unique<RemoteServer>(true);
 		m_remoteSecureServer->Start();
@@ -503,37 +518,55 @@ void NZBGet::StopRemoteServer()
 	{
 		debug("stopping RemoteServer");
 		m_remoteServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteServer->IsRunning())
-		{
-			debug("Killing RemoteServer");
-			m_remoteServer->Kill();
-		}
-		debug("RemoteServer stopped");
 	}
 
 	if (m_remoteSecureServer)
 	{
 		debug("stopping RemoteSecureServer");
 		m_remoteSecureServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteSecureServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteSecureServer->IsRunning())
-		{
-			debug("Killing RemoteSecureServer");
-			m_remoteSecureServer->Kill();
-		}
-		debug("RemoteSecureServer stopped");
 	}
+
+	int maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		usleep(100 * 1000);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		m_remoteServer->ForceStop();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		m_remoteSecureServer->ForceStop();
+	}
+
+	maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		usleep(100 * 1000);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		debug("Killing RemoteServer");
+		m_remoteServer->Kill();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		debug("Killing RemoteSecureServer");
+		m_remoteSecureServer->Kill();
+	}
+
+	debug("RemoteServer stopped");
 }
 
 void NZBGet::StartFrontend()
@@ -694,106 +727,109 @@ void NZBGet::Run(bool reload)
 void NZBGet::ProcessClientRequest()
 {
 	RemoteClient Client;
+	bool ok = false;
 
 	switch (m_commandLineParser->GetClientOperation())
 	{
 		case CommandLineParser::opClientRequestListFiles:
-			Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListGroups:
-			Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListStatus:
-			Client.RequestServerList(false, false, nullptr);
+			ok = Client.RequestServerList(false, false, nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadPause:
-			Client.RequestServerPauseUnpause(true, rpDownload);
+			ok = Client.RequestServerPauseUnpause(true, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadUnpause:
-			Client.RequestServerPauseUnpause(false, rpDownload);
+			ok = Client.RequestServerPauseUnpause(false, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestSetRate:
-			Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
+			ok = Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
 			break;
 
 		case CommandLineParser::opClientRequestDumpDebug:
-			Client.RequestServerDumpDebug();
+			ok = Client.RequestServerDumpDebug();
 			break;
 
 		case CommandLineParser::opClientRequestEditQueue:
-			Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
+			ok = Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
 				m_commandLineParser->GetEditQueueOffset(), m_commandLineParser->GetEditQueueText(),
 				m_commandLineParser->GetEditQueueIdList(), m_commandLineParser->GetEditQueueNameList(),
 				(ERemoteMatchMode)m_commandLineParser->GetMatchMode());
 			break;
 
 		case CommandLineParser::opClientRequestLog:
-			Client.RequestServerLog(m_commandLineParser->GetLogLines());
+			ok = Client.RequestServerLog(m_commandLineParser->GetLogLines());
 			break;
 
 		case CommandLineParser::opClientRequestShutdown:
-			Client.RequestServerShutdown();
+			ok = Client.RequestServerShutdown();
 			break;
 
 		case CommandLineParser::opClientRequestReload:
-			Client.RequestServerReload();
+			ok = Client.RequestServerReload();
 			break;
 
 		case CommandLineParser::opClientRequestDownload:
-			Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
+			ok = Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
 				m_commandLineParser->GetAddCategory(), m_commandLineParser->GetAddTop(), m_commandLineParser->GetAddPaused(), m_commandLineParser->GetAddPriority(),
 				m_commandLineParser->GetAddDupeKey(), m_commandLineParser->GetAddDupeMode(), m_commandLineParser->GetAddDupeScore());
 			break;
 
 		case CommandLineParser::opClientRequestVersion:
-			Client.RequestServerVersion();
+			ok = Client.RequestServerVersion();
 			break;
 
 		case CommandLineParser::opClientRequestPostQueue:
-			Client.RequestPostQueue();
+			ok = Client.RequestPostQueue();
 			break;
 
 		case CommandLineParser::opClientRequestWriteLog:
-			Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
+			ok = Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
 			break;
 
 		case CommandLineParser::opClientRequestScanAsync:
-			Client.RequestScan(false);
+			ok = Client.RequestScan(false);
 			break;
 
 		case CommandLineParser::opClientRequestScanSync:
-			Client.RequestScan(true);
+			ok = Client.RequestScan(true);
 			break;
 
 		case CommandLineParser::opClientRequestPostPause:
-			Client.RequestServerPauseUnpause(true, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(true, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestPostUnpause:
-			Client.RequestServerPauseUnpause(false, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(false, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestScanPause:
-			Client.RequestServerPauseUnpause(true, rpScan);
+			ok = Client.RequestServerPauseUnpause(true, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestScanUnpause:
-			Client.RequestServerPauseUnpause(false, rpScan);
+			ok = Client.RequestServerPauseUnpause(false, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestHistory:
 		case CommandLineParser::opClientRequestHistoryAll:
-			Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
+			ok = Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
 			break;
 
 		case CommandLineParser::opClientNoOperation:
-			break;
+			return;
 	}
+
+	exit(ok ? 0 : 1);
 }
 
 void NZBGet::ProcessWebGet()
@@ -938,10 +974,10 @@ void NZBGet::Daemonize()
 #endif
 
 void NZBGet::AddNewsServer(int id, bool active, const char* name, const char* host,
-	int port, const char* user, const char* pass, bool joinGroup, bool tls,
+	int port, int ipVersion, const char* user, const char* pass, bool joinGroup, bool tls,
 	const char* cipher, int maxConnections, int retention, int level, int group, bool optional)
 {
-	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, user, pass, joinGroup,
+	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, ipVersion, user, pass, joinGroup,
 		tls, cipher, maxConnections, retention, level, group, optional));
 }
 

daemon/main/nzbget.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -40,6 +40,9 @@
 //#define HAVE_LIBGNUTLS
 #endif
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#define HAVE_X509_CHECK_HOST 1
+
 /* Define to the name of macro which returns the name of function being
 compiled */
 #define FUNCTION_MACRO_NAME __FUNCTION__
@@ -56,9 +59,6 @@ compiled */
 /* Define to 1 if variadic macros are supported */
 #define HAVE_VARIADIC_MACROS
 
-/* Define to 1 if libpar2 supports cancelling (needs a special patch) */
-#define HAVE_PAR2_CANCEL
-
 /* Define to 1 if function GetAddrInfo is supported */
 #define HAVE_GETADDRINFO
 
@@ -66,7 +66,16 @@ compiled */
 #define SOCKLEN_T socklen_t
 
 /* Define to 1 if you have the <regex.h> header file. */
+#ifndef DISABLE_REGEX
 #define HAVE_REGEX_H 1
+// Static linking to regex library
+#define REGEX_STATIC
+#endif
+
+#ifndef DISABLE_GZIP
+// Static linking to zlib library
+#define ZLIB_WINAPI
+#endif
 
 /* Suppress warnings */
 #define _CRT_SECURE_NO_DEPRECATE
@@ -74,13 +83,21 @@ compiled */
 /* Suppress warnings */
 #define _CRT_NONSTDC_NO_WARNINGS
 
+#ifndef _WIN64
 #define _USE_32BIT_TIME_T
+#endif
 
 #if _WIN32_WINNT < 0x0501
 #undef _WIN32_WINNT
 #define _WIN32_WINNT 0x0501
 #endif
 
+#ifdef _WIN64
+#define __amd64__
+#else
+#define __i686__
+#endif
+
 #ifdef _DEBUG
 // detection of memory leaks
 #define _CRTDBG_MAP_ALLOC
@@ -151,21 +168,28 @@ using namespace MSXML;
 #include <sys/resource.h>
 #include <sys/statvfs.h>
 #include <sys/wait.h>
+#include <sys/un.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <stdint.h>
 #include <pwd.h>
 #include <dirent.h>
 
+#ifndef DISABLE_LIBXML2
 #include <libxml/parser.h>
 #include <libxml/xmlreader.h>
 #include <libxml/xmlerror.h>
 #include <libxml/entities.h>
+#endif
 
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
 #endif
 
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#endif
+
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
 #endif
@@ -191,13 +215,19 @@ using namespace MSXML;
 #include <list>
 #include <set>
 #include <map>
+#include <unordered_map>
 #include <iterator>
 #include <algorithm>
-#include <iostream>
 #include <fstream>
 #include <memory>
 #include <functional>
 
+// NOTE: do not include <iostream> in "nzbget.h". <iostream> contains objects requiring
+// intialization, causing every unit in nzbget to have initialization routine. This in particular
+// is causing fatal problems in SIMD units which must not have static initialization because
+// they contain code with runtime CPU dispatching.
+//#include <iostream>
+
 #ifdef HAVE_LIBGNUTLS
 #ifdef WIN32
 #include <BaseTsd.h>
@@ -205,6 +235,7 @@ typedef SSIZE_T ssize_t;
 typedef int pid_t;
 #endif
 #include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
 #if GNUTLS_VERSION_NUMBER <= 0x020b00
 #define NEED_GCRYPT_LOCKING
 #endif
@@ -222,6 +253,8 @@ typedef int pid_t;
 #include <openssl/rsa.h>
 #include <openssl/sha.h>
 #include <openssl/pem.h>
+#include <openssl/x509v3.h>
+#include <openssl/comp.h>
 #endif /* HAVE_OPENSSL */
 
 #ifdef HAVE_REGEX_H
@@ -290,11 +323,22 @@ typedef int pid_t;
 #define FOPEN_WB "wbN"
 #define FOPEN_AB "abN"
 
+#define __SSE2__
+#define __SSSE3__
+#define __PCLMUL__
+
 #ifdef DEBUG
 // redefine "exit" to avoid printing memory leaks report when terminated because of wrong command line switches
 #define exit(code) ExitProcess(code)
 #endif
 
+#ifdef HAVE_OPENSSL
+FILE _iob[] = {*stdin, *stdout, *stderr};
+extern "C" FILE * __cdecl __iob_func(void) { return _iob; }
+// For static linking of OpenSSL libraries:
+#pragma comment (lib, "legacy_stdio_definitions.lib")
+#endif /* HAVE_OPENSSL */
+
 #else
 
 // POSIX
@@ -340,6 +384,13 @@ typedef unsigned long long uint64;
 
 typedef unsigned char uchar;
 
+// Assume little endian if byte order is not defined
+#ifndef __BYTE_ORDER
+#define __LITTLE_ENDIAN 1234
+#define __BIG_ENDIAN    4321
+#define __BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
 #ifdef __GNUC__
 #define PRINTF_SYNTAX(strindex) __attribute__ ((format (printf, strindex, strindex+1)))
 #define SCANF_SYNTAX(strindex) __attribute__ ((format (scanf, strindex, strindex+1)))

daemon/nntp/ArticleDownloader.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,7 +33,6 @@ ArticleDownloader::ArticleDownloader()
 {
 	debug("Creating ArticleDownloader");
 
-	m_articleWriter.SetOwner(this);
 	SetLastUpdateTimeNow();
 }
 
@@ -296,6 +295,11 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	m_writingStarted = false;
 	m_articleInfo->SetCrc(0);
 
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Reset();
+	}
+
 	if (m_connection->GetNewsServer()->GetJoinGroup())
 	{
 		// change group
@@ -316,14 +320,8 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	}
 
 	// retrieve article
-	for (int retry = 3; retry > 0; retry--)
-	{
-		response = m_connection->Request(BString<1024>("ARTICLE %s\r\n", m_articleInfo->GetMessageId()));
-		if ((response && !strncmp(response, "2", 1)) || m_connection->GetAuthError())
-		{
-			break;
-		}
-	}
+	response = m_connection->Request(BString<1024>("%s %s\r\n",
+		g_Options->GetRawArticle() ? "ARTICLE" : "BODY", m_articleInfo->GetMessageId()));
 
 	status = CheckResponse(response, "could not fetch article");
 	if (status != adFinished)
@@ -331,28 +329,16 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 		return status;
 	}
 
-	if (g_Options->GetDecode())
-	{
-		m_yDecoder.Clear();
-		m_yDecoder.SetCrcCheck(g_Options->GetCrcCheck());
-		m_uDecoder.Clear();
-	}
+	m_decoder.Clear();
+	m_decoder.SetCrcCheck(g_Options->GetCrcCheck());
+	m_decoder.SetRawMode(g_Options->GetRawArticle());
 
-	bool body = false;
-	bool end = false;
-	CharBuffer lineBuf(1024*10);
 	status = adRunning;
+	CharBuffer lineBuf(1024*4);
 
-	while (!IsStopped())
+	while (!IsStopped() && !m_decoder.GetEof())
 	{
-		time_t oldTime = m_lastUpdateTime;
-		SetLastUpdateTimeNow();
-		if (oldTime != m_lastUpdateTime)
-		{
-			AddServerData();
-		}
-
-		// Throttle the bandwidth
+		// throttle the bandwidth
 		while (!IsStopped() && (g_Options->GetDownloadRate() > 0.0f) &&
 			(g_StatMeter->CalcCurrentDownloadSpeed() > g_Options->GetDownloadRate() ||
 			g_StatMeter->CalcMomentaryDownloadSpeed() > g_Options->GetDownloadRate()))
@@ -361,17 +347,17 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 			usleep(10 * 1000);
 		}
 
-		int len = 0;
-		char* line = m_connection->ReadLine(lineBuf, lineBuf.Size(), &len);
-
-		g_StatMeter->AddSpeedReading(len);
-		if (g_Options->GetAccurateRate())
+		char* buffer;
+		int len;
+		m_connection->ReadBuffer(&buffer, &len);
+		if (len == 0)
 		{
-			AddServerData();
+			len = m_connection->TryRecv(lineBuf, lineBuf.Size());
+			buffer = lineBuf;
 		}
 
-		// Have we encountered a timeout?
-		if (!line)
+		// have we encountered a timeout?
+		if (len <= 0)
 		{
 			if (!IsStopped())
 			{
@@ -381,67 +367,25 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 			break;
 		}
 
-		//detect end of article
-		if (!strcmp(line, ".\r\n") || !strcmp(line, ".\n"))
+		g_StatMeter->AddSpeedReading(len);
+		time_t oldTime = m_lastUpdateTime;
+		SetLastUpdateTimeNow();
+		if (oldTime != m_lastUpdateTime)
 		{
-			end = true;
-			break;
+			AddServerData();
 		}
 
-		//detect lines starting with "." (marked as "..")
-		if (!strncmp(line, "..", 2))
-		{
-			line++;
-			len--;
-		}
-
-		if (!body)
-		{
-			// detect body of article
-			if (*line == '\r' || *line == '\n')
-			{
-				body = true;
-			}
-			// check id of returned article
-			else if (!strncmp(line, "Message-ID: ", 12))
-			{
-				char* p = line + 12;
-				if (strncmp(p, m_articleInfo->GetMessageId(), strlen(m_articleInfo->GetMessageId())))
-				{
-					if (char* e = strrchr(p, '\r')) *e = '\0'; // remove trailing CR-character
-					detail("Article %s @ %s failed: Wrong message-id, expected %s, returned %s", *m_infoName,
-						*m_connectionName, m_articleInfo->GetMessageId(), p);
-					status = adFailed;
-					break;
-				}
-			}
-		}
-
-		if (m_format == Decoder::efUnknown && g_Options->GetDecode())
-		{
-			m_format = Decoder::DetectFormat(line, len, body);
-			if (m_format != Decoder::efUnknown)
-			{
-				// sometimes news servers misbehave and send article body without new line separator between headers and body
-				// if we found decoder signature we know the body is already arrived
-				body = true;
-			}
-		}
+		// decode article data
+		len = m_decoder.DecodeBuffer(buffer, len);
 
 		// write to output file
-		if (((body && m_format != Decoder::efUnknown) || !g_Options->GetDecode()) && !Write(line, len))
+		if (len > 0 && !Write(buffer, len))
 		{
 			status = adFatalError;
 			break;
 		}
 	}
 
-	if (!end && status == adRunning && !IsStopped())
-	{
-		detail("Article %s @ %s failed: article incomplete", *m_infoName, *m_connectionName);
-		status = adFailed;
-	}
-
 	if (IsStopped())
 	{
 		status = adFailed;
@@ -500,89 +444,69 @@ ArticleDownloader::EStatus ArticleDownloader::CheckResponse(const char* response
 	}
 }
 
-bool ArticleDownloader::Write(char* line, int len)
+bool ArticleDownloader::Write(char* buffer, int len)
 {
 	const char* articleFilename = nullptr;
 	int64 articleFileSize = 0;
 	int64 articleOffset = 0;
 	int articleSize = 0;
 
-	if (g_Options->GetDecode())
+	if (!m_writingStarted)
 	{
-		if (m_format == Decoder::efYenc)
+		if (!g_Options->GetRawArticle())
 		{
-			len = m_yDecoder.DecodeBuffer(line, len);
-			articleFilename = m_yDecoder.GetArticleFilename();
-			articleFileSize = m_yDecoder.GetSize();
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			len = m_uDecoder.DecodeBuffer(line, len);
-			articleFilename = m_uDecoder.GetArticleFilename();
-		}
-		else
-		{
-			detail("Decoding %s failed: unsupported encoding", *m_infoName);
-			return false;
-		}
-
-		if (len > 0 && m_format == Decoder::efYenc)
-		{
-			if (m_yDecoder.GetBegin() == 0 || m_yDecoder.GetEnd() == 0)
+			articleFilename = m_decoder.GetArticleFilename();
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
-				return false;
+				if (m_decoder.GetBeginPos() == 0 || m_decoder.GetEndPos() == 0)
+				{
+					return false;
+				}
+				articleFileSize = m_decoder.GetSize();
+				articleOffset = m_decoder.GetBeginPos() - 1;
+				articleSize = (int)(m_decoder.GetEndPos() - m_decoder.GetBeginPos() + 1);
+				if (articleSize <= 0 || articleSize > 1024*1024*1024)
+				{
+					warn("Malformed article %s: size %i out of range", *m_infoName, articleSize);
+					return false;
+				}
 			}
-			articleOffset = m_yDecoder.GetBegin() - 1;
-			articleSize = (int)(m_yDecoder.GetEnd() - m_yDecoder.GetBegin() + 1);
 		}
-	}
 
-	if (!m_writingStarted && len > 0)
-	{
-		if (!m_articleWriter.Start(m_format, articleFilename, articleFileSize, articleOffset, articleSize))
+		if (!m_articleWriter.Start(m_decoder.GetFormat(), articleFilename, articleFileSize, articleOffset, articleSize))
 		{
 			return false;
 		}
 		m_writingStarted = true;
 	}
 
-	bool ok = len == 0 || m_articleWriter.Write(line, len);
+	bool ok = m_articleWriter.Write(buffer, len);
+
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Append(buffer, len);
+	}
 
 	return ok;
 }
 
 ArticleDownloader::EStatus ArticleDownloader::DecodeCheck()
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
-		Decoder* decoder = nullptr;
-		if (m_format == Decoder::efYenc)
-		{
-			decoder = &m_yDecoder;
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			decoder = &m_uDecoder;
-		}
-		else
-		{
-			detail("Decoding %s failed: no binary data or unsupported encoding format", *m_infoName);
-			return adFailed;
-		}
-
-		Decoder::EStatus status = decoder->Check();
+		Decoder::EStatus status = m_decoder.Check();
 
 		if (status == Decoder::dsFinished)
 		{
-			if (decoder->GetArticleFilename())
+			if (m_decoder.GetArticleFilename())
 			{
-				m_articleFilename = decoder->GetArticleFilename();
+				m_articleFilename = m_decoder.GetArticleFilename();
 			}
 
-			if (m_format == Decoder::efYenc)
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
 				m_articleInfo->SetCrc(g_Options->GetCrcCheck() ?
-					m_yDecoder.GetCalculatedCrc() : m_yDecoder.GetExpectedCrc());
+					m_decoder.GetCalculatedCrc() : m_decoder.GetExpectedCrc());
 			}
 
 			return adFinished;
@@ -643,22 +567,6 @@ void ArticleDownloader::Stop()
 	debug("ArticleDownloader stopped successfully");
 }
 
-bool ArticleDownloader::Terminate()
-{
-	NntpConnection* connection = m_connection;
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		g_StatMeter->AddServerData(connection->FetchTotalBytesRead(), connection->GetNewsServer()->GetId());
-		g_ServerPool->FreeConnection(connection, true);
-	}
-	return terminated;
-}
-
 void ArticleDownloader::FreeConnection(bool keepConnected)
 {
 	if (m_connection)

daemon/nntp/ArticleDownloader.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -29,6 +29,15 @@
 #include "NntpConnection.h"
 #include "Decoder.h"
 #include "ArticleWriter.h"
+#include "Util.h"
+
+class ArticleContentAnalyzer
+{
+public:
+	virtual ~ArticleContentAnalyzer() {};
+	virtual void Reset() = 0;
+	virtual void Append(const void* buffer, int len) = 0;
+};
 
 class ArticleDownloader : public Thread, public Subject
 {
@@ -47,16 +56,6 @@ public:
 		adFatalError
 	};
 
-	class ArticleWriterImpl : public ArticleWriter
-	{
-	public:
-		void SetOwner(ArticleDownloader* owner) { m_owner = owner; }
-	protected:
-		virtual void SetLastUpdateTimeNow() { m_owner->SetLastUpdateTimeNow(); }
-	private:
-		ArticleDownloader* m_owner;
-	};
-
 	ArticleDownloader();
 	virtual ~ArticleDownloader();
 	void SetFileInfo(FileInfo* fileInfo) { m_fileInfo = fileInfo; }
@@ -67,7 +66,6 @@ public:
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 	virtual void Run();
 	virtual void Stop();
-	bool Terminate();
 	time_t GetLastUpdateTime() { return m_lastUpdateTime; }
 	void SetLastUpdateTimeNow();
 	const char* GetArticleFilename() { return m_articleFilename; }
@@ -77,6 +75,8 @@ public:
 	void SetConnection(NntpConnection* connection) { m_connection = connection; }
 	void CompleteFileParts() { m_articleWriter.CompleteFileParts(); }
 	int GetDownloadedSize() { return m_downloadedSize; }
+	void SetContentAnalyzer(std::unique_ptr<ArticleContentAnalyzer> contentAnalyzer) { m_contentAnalyzer = std::move(contentAnalyzer); }
+	ArticleContentAnalyzer* GetContentAnalyzer() { return m_contentAnalyzer.get(); }
 
 	void LogDebugInfo();
 
@@ -90,20 +90,19 @@ private:
 	CString m_connectionName;
 	CString m_articleFilename;
 	time_t m_lastUpdateTime;
-	Decoder::EFormat m_format = Decoder::efUnknown;
-	YDecoder m_yDecoder;
-	UDecoder m_uDecoder;
-	ArticleWriterImpl m_articleWriter;
+	Decoder m_decoder;
+	ArticleWriter m_articleWriter;
 	ServerStatList m_serverStats;
 	bool m_writingStarted;
 	int m_downloadedSize = 0;
+	std::unique_ptr<ArticleContentAnalyzer> m_contentAnalyzer;
 
 	EStatus Download();
 	EStatus DecodeCheck();
 	void FreeConnection(bool keepConnected);
 	EStatus CheckResponse(const char* response, const char* comment);
 	void SetStatus(EStatus status) { m_status = status; }
-	bool Write(char* line, int len);
+	bool Write(char* buffer, int len);
 	void AddServerData();
 };
 

daemon/nntp/ArticleWriter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -84,7 +84,7 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 			}
 
 			if (!outputInitialized && filename &&
-				FileSystem::FileExists(m_fileInfo->GetNzbInfo()->GetDestDir(), filename))
+				FileSystem::FileExists(BString<1024>("%s%c%s", m_fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, filename)))
 			{
 				m_duplicate = true;
 				return false;
@@ -106,7 +106,7 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 	}
 
 	// allocate cache buffer
-	if (g_Options->GetArticleCache() > 0 && g_Options->GetDecode() &&
+	if (g_Options->GetArticleCache() > 0 && !g_Options->GetRawArticle() &&
 		(!g_Options->GetDirectWrite() || m_format == Decoder::efYenc))
 	{
 		m_articleData = g_ArticleCache->Alloc(m_articleSize);
@@ -147,22 +147,31 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 
 bool ArticleWriter::Write(char* buffer, int len)
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		m_articlePtr += len;
 	}
 
-	if (g_Options->GetDecode() && m_articleData.GetData())
+	if (m_articlePtr > m_articleSize)
+	{
+		// An attempt to write beyond article border is detected.
+		// That's an error condition (damaged article).
+		// We return 'false' since this isn't a fatal disk error and
+		// article size mismatch will be detected in decoder check anyway.
+		return true;
+	}
+
+	if (!g_Options->GetRawArticle() && m_articleData.GetData())
 	{
-		if (m_articlePtr > m_articleSize)
-		{
-			detail("Decoding %s failed: article size mismatch", *m_infoName);
-			return false;
-		}
 		memcpy(m_articleData.GetData() + m_articlePtr - len, buffer, len);
 		return true;
 	}
 
+	if (g_Options->GetSkipWrite())
+	{
+		return true;
+	}
+
 	return m_outFile.Write(buffer, len) > 0;
 }
 
@@ -179,7 +188,7 @@ void ArticleWriter::Finish(bool success)
 
 	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
 
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		if (!directWrite && !m_articleData.GetData())
 		{
@@ -189,10 +198,9 @@ void ArticleWriter::Finish(bool success)
 					"Could not rename file %s to %s: %s", *m_tempFilename, m_resultFilename,
 					*FileSystem::GetLastErrorMessage());
 			}
+			FileSystem::DeleteFile(m_tempFilename);
 		}
 
-		FileSystem::DeleteFile(m_tempFilename);
-
 		if (m_articleData.GetData())
 		{
 			if (m_articleSize != m_articlePtr)
@@ -224,19 +232,20 @@ void ArticleWriter::Finish(bool success)
 /* creates output file and subdirectores */
 bool ArticleWriter::CreateOutputFile(int64 size)
 {
-	if (g_Options->GetDirectWrite() && FileSystem::FileExists(m_outputFilename) &&
-		FileSystem::FileSize(m_outputFilename) == size)
+	if (FileSystem::FileExists(m_outputFilename))
 	{
-		// keep existing old file from previous program session
-		return true;
+		if (FileSystem::FileSize(m_outputFilename) == size)
+		{
+			// keep existing old file from previous program session
+			return true;
+		}
+		// delete existing old file from previous program session
+		FileSystem::DeleteFile(m_outputFilename);
 	}
 
-	// delete eventually existing old file from previous program session
-	FileSystem::DeleteFile(m_outputFilename);
-
 	// ensure the directory exist
 	BString<1024> destDir;
-	destDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+	destDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 	CString errmsg;
 
 	if (!FileSystem::ForceDirectories(destDir, errmsg))
@@ -275,7 +284,7 @@ void ArticleWriter::BuildOutputFilename()
 		else
 		{
 			filename.Format("%s%c%i.out.tmp", m_fileInfo->GetNzbInfo()->GetDestDir(),
-				(int)PATH_SEPARATOR, m_fileInfo->GetId());
+				PATH_SEPARATOR, m_fileInfo->GetId());
 			m_fileInfo->SetOutputFilename(filename);
 		}
 
@@ -292,18 +301,20 @@ void ArticleWriter::CompleteFileParts()
 
 	BString<1024> nzbName;
 	BString<1024> nzbDestDir;
+	BString<1024> filename;
 
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
 		nzbName = m_fileInfo->GetNzbInfo()->GetName();
 		nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir();
+		filename = m_fileInfo->GetFilename();
 	}
 
-	BString<1024> infoFilename("%s%c%s", *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
+	BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename);
 
 	bool cached = m_fileInfo->GetCachedArticles() > 0;
 
-	if (!g_Options->GetDecode())
+	if (g_Options->GetRawArticle())
 	{
 		detail("Moving articles for %s", *infoFilename);
 	}
@@ -332,17 +343,17 @@ void ArticleWriter::CompleteFileParts()
 	CString ofn;
 	if (m_fileInfo->GetForceDirectWrite())
 	{
-		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, m_fileInfo->GetFilename());
+		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename);
 	}
 	else
 	{
-		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename);
 	}
 
 	DiskFile outfile;
 	BString<1024> tmpdestfile("%s.tmp", *ofn);
 
-	if (g_Options->GetDecode() && !directWrite)
+	if (!g_Options->GetRawArticle() && !directWrite)
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!outfile.Open(tmpdestfile, DiskFile::omWrite))
@@ -362,7 +373,7 @@ void ArticleWriter::CompleteFileParts()
 		}
 		tmpdestfile = *m_outputFilename;
 	}
-	else if (!g_Options->GetDecode())
+	else if (g_Options->GetRawArticle())
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!FileSystem::CreateDirectory(ofn))
@@ -390,7 +401,7 @@ void ArticleWriter::CompleteFileParts()
 		CharBuffer buffer;
 		bool firstArticle = true;
 
-		if (g_Options->GetDecode() && !directWrite)
+		if (!g_Options->GetRawArticle() && !directWrite)
 		{
 			buffer.Reserve(1024 * 64);
 		}
@@ -402,22 +413,27 @@ void ArticleWriter::CompleteFileParts()
 				continue;
 			}
 
-			if (g_Options->GetDecode() && !directWrite && pa->GetSegmentOffset() > -1 &&
+			if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 &&
 				pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1)
 			{
 				memset(buffer, 0, buffer.Size());
-				while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
-					outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				if (!g_Options->GetSkipWrite())
+				{
+					while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
+						outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				}
 			}
 
 			if (pa->GetSegmentContent())
 			{
-				outfile.Seek(pa->GetSegmentOffset());
-				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				if (!g_Options->GetSkipWrite())
+				{
+					outfile.Seek(pa->GetSegmentOffset());
+					outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				}
 				pa->DiscardSegment();
-				SetLastUpdateTimeNow();
 			}
-			else if (g_Options->GetDecode() && !directWrite)
+			else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite())
 			{
 				DiskFile infile;
 				if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead))
@@ -427,7 +443,6 @@ void ArticleWriter::CompleteFileParts()
 					{
 						cnt = (int)infile.Read(buffer, buffer.Size());
 						outfile.Write(buffer, cnt);
-						SetLastUpdateTimeNow();
 					}
 					infile.Close();
 				}
@@ -436,14 +451,14 @@ void ArticleWriter::CompleteFileParts()
 					m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1);
 					m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1);
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-						"Could not find file %s for %s%c%s [%i/%i]",
-						pa->GetResultFilename(), *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename(),
-						pa->GetPartNumber(), (int)m_fileInfo->GetArticles()->size());
+						"Could not find file %s for %s [%i/%i]",
+						pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(),
+						(int)m_fileInfo->GetArticles()->size());
 				}
 			}
-			else if (!g_Options->GetDecode())
+			else if (g_Options->GetRawArticle())
 			{
-				BString<1024> dstFileName("%s%c%03i", *ofn, (int)PATH_SEPARATOR, pa->GetPartNumber());
+				BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber());
 				if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName))
 				{
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
@@ -454,7 +469,7 @@ void ArticleWriter::CompleteFileParts()
 
 			if (m_format == Decoder::efYenc)
 			{
-				crc = firstArticle ? pa->GetCrc() : Util::Crc32Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
+				crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
 				firstArticle = false;
 			}
 		}
@@ -490,7 +505,7 @@ void ArticleWriter::CompleteFileParts()
 		{
 			debug("Checking old dir for: %s", *m_outputFilename);
 			BString<1024> oldDestDir;
-			oldDestDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+			oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 			if (FileSystem::DirEmpty(oldDestDir))
 			{
 				debug("Deleting old dir: %s", *oldDestDir);
@@ -520,29 +535,31 @@ void ArticleWriter::CompleteFileParts()
 			"%i of %i article downloads failed for \"%s\"",
 			m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(),
 			m_fileInfo->GetTotalArticles(), *infoFilename);
-
-		if (g_Options->GetBrokenLog())
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", *nzbDestDir, (int)PATH_SEPARATOR);
-			DiskFile file;
-			if (file.Open(brokenLogName, DiskFile::omAppend))
-			{
-				file.Print("%s (%i/%i)%s", m_fileInfo->GetFilename(), m_fileInfo->GetSuccessArticles(),
-					m_fileInfo->GetTotalArticles(), LINE_ENDING);
-				file.Close();
-			}
-		}
 	}
 	else
 	{
 		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename);
 	}
 
-	m_fileInfo->SetCrc(crc);
-	m_fileInfo->SetOutputFilename(ofn);
-
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
+
+		m_fileInfo->SetCrc(crc);
+		m_fileInfo->SetOutputFilename(ofn);
+
+		if (strcmp(m_fileInfo->GetFilename(), filename))
+		{
+			// file was renamed during completion, need to move the file
+			ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+			if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn))
+			{
+				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
+					"Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(),
+					*ofn, *FileSystem::GetLastErrorMessage());
+			}
+			m_fileInfo->SetOutputFilename(ofn);
+		}
+
 		if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir))
 		{
 			// destination directory was changed during completion, need to move the file
@@ -565,10 +582,18 @@ void ArticleWriter::FlushCache()
 		ArticleCache::FlushGuard flushGuard = g_ArticleCache->GuardFlush();
 
 		std::vector<ArticleInfo*> cachedArticles;
-		cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
+
+			if (m_fileInfo->GetFlushLocked())
+			{
+				return;
+			}
+
+			m_fileInfo->SetFlushLocked(true);
+
+			cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 			for (ArticleInfo* pa : m_fileInfo->GetArticles())
 			{
 				if (pa->GetSegmentContent())
@@ -593,6 +618,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not open file %s: %s", m_fileInfo->GetOutputFilename(),
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -608,6 +636,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not create file %s: %s", *destFile,
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -624,7 +655,10 @@ void ArticleWriter::FlushCache()
 				outfile.Seek(pa->GetSegmentOffset());
 			}
 
-			outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			if (!g_Options->GetSkipWrite())
+			{
+				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			}
 
 			flushedSize += pa->GetSegmentSize();
 			flushedArticles++;
@@ -649,6 +683,7 @@ void ArticleWriter::FlushCache()
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
 			m_fileInfo->SetCachedArticles(m_fileInfo->GetCachedArticles() - flushedArticles);
+			m_fileInfo->SetFlushLocked(false);
 		}
 	}
 
@@ -674,14 +709,14 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 	// move already downloaded files to new destination
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		BString<1024> oldFileName("%s%c%s", oldDestDir, (int)PATH_SEPARATOR, completedFile.GetFileName());
-		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName());
+		BString<1024> oldFileName("%s%c%s", oldDestDir, PATH_SEPARATOR, completedFile.GetFilename());
+		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename());
 
 		// check if file was not moved already
 		if (strcmp(oldFileName, newFileName))
 		{
 			// prevent overwriting of existing files
-			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFileName());
+			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFilename());
 
 			detail("Moving file %s to %s", *oldFileName, *newFileName);
 			if (!FileSystem::MoveFile(oldFileName, newFileName))
@@ -692,57 +727,6 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 		}
 	}
 
-	// move brokenlog.txt
-	if (g_Options->GetBrokenLog())
-	{
-		BString<1024> oldBrokenLogName("%s%c_brokenlog.txt", oldDestDir, (int)PATH_SEPARATOR);
-		if (FileSystem::FileExists(oldBrokenLogName))
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR);
-
-			detail("Moving file %s to %s", *oldBrokenLogName, *brokenLogName);
-			if (FileSystem::FileExists(brokenLogName))
-			{
-				// copy content to existing new file, then delete old file
-				DiskFile outfile;
-				if (outfile.Open(brokenLogName, DiskFile::omAppend))
-				{
-					DiskFile infile;
-					if (infile.Open(oldBrokenLogName, DiskFile::omRead))
-					{
-						CharBuffer buffer(1024 * 50);
-						int cnt = buffer.Size();
-						while (cnt == buffer.Size())
-						{
-							cnt = (int)infile.Read(buffer, buffer.Size());
-							outfile.Write(buffer, cnt);
-						}
-						infile.Close();
-						FileSystem::DeleteFile(oldBrokenLogName);
-					}
-					else
-					{
-						nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *oldBrokenLogName);
-					}
-					outfile.Close();
-				}
-				else
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
-				}
-			}
-			else
-			{
-				// move to new destination
-				if (!FileSystem::MoveFile(oldBrokenLogName, brokenLogName))
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not move file %s to %s: %s",
-						*oldBrokenLogName, *brokenLogName, *FileSystem::GetLastErrorMessage());
-				}
-			}
-		}
-	}
-
 	// delete old directory (if empty)
 	if (FileSystem::DirEmpty(oldDestDir))
 	{
@@ -787,7 +771,7 @@ CachedSegmentData ArticleCache::Alloc(int size)
 		p = malloc(size);
 		if (p)
 		{
-			if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+			if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 			{
 				g_DiskState->WriteCacheFlag();
 			}
@@ -807,6 +791,7 @@ bool ArticleCache::Realloc(CachedSegmentData* segment, int newSize)
 	{
 		m_allocated += newSize - segment->m_size;
 		segment->m_size = newSize;
+		segment->m_data = (char*)p;
 	}
 
 	return p;
@@ -820,7 +805,7 @@ void ArticleCache::Free(CachedSegmentData* segment)
 
 		Guard guard(m_allocMutex);
 		m_allocated -= segment->m_size;
-		if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+		if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 		{
 			g_DiskState->DeleteCacheFlag();
 		}
@@ -871,7 +856,7 @@ bool ArticleCache::CheckFlush(bool flushEverything)
 				if (fileInfo->GetCachedArticles() > 0 && (fileInfo->GetActiveDownloads() == 0 || flushEverything))
 				{
 					m_fileInfo = fileInfo;
-					infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
+					infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR, m_fileInfo->GetFilename());
 					break;
 				}
 			}

daemon/nntp/ArticleWriter.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -60,9 +60,6 @@ public:
 	static bool MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir);
 	void FlushCache();
 
-protected:
-	virtual void SetLastUpdateTimeNow() {}
-
 private:
 	FileInfo* m_fileInfo;
 	ArticleInfo* m_articleInfo;

daemon/nntp/Decoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,22 +22,129 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Util.h"
+#include "YEncode.h"
 
-const char* Decoder::FormatNames[] = { "Unknown", "yEnc", "UU" };
+Decoder::Decoder()
+{
+	debug("%s", YEncode::decode_simd ? "SIMD yEnc decoder can be used" : "SIMD yEnc decoder isn't available for this CPU");
+	debug("%s", YEncode::crc_simd ? "SIMD Crc routine can be used" : "SIMD Crc routine isn't available for this CPU");
+
+	Clear();
+}
 
 void Decoder::Clear()
 {
 	m_articleFilename.Clear();
+	m_body = false;
+	m_begin = false;
+	m_part = false;
+	m_end = false;
+	m_crc = false;
+	m_eof = false;
+	m_expectedCRC = 0;
+	m_crc32.Reset();
+	m_beginPos = 0;
+	m_endPos = 0;
+	m_size = 0;
+	m_endSize = 0;
+	m_outSize = 0;
+	m_state = 0;
+	m_crcCheck = false;
+	m_lineBuf.Reserve(1024*8);
+	m_lineBuf.SetLength(0);
 }
 
-Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
+/* At the beginning of article the processing goes line by line to find '=ybegin'-marker.
+ * Once the yEnc-data is started switches to blockwise processing.
+ * At the end of yEnc-data switches back to line by line mode to
+ * process '=yend'-marker and EOF-marker.
+ * UU-encoded articles are processed completely in line by line mode.
+ */
+int Decoder::DecodeBuffer(char* buffer, int len)
+{
+	if (m_rawMode)
+	{
+		ProcessRaw(buffer, len);
+		return len;
+	}
+
+	int outlen = 0;
+
+	if (m_body && m_format == efYenc)
+	{
+		outlen = DecodeYenc(buffer, buffer, len);
+		if (m_body)
+		{
+			return outlen;
+		}
+	}
+	else
+	{
+		m_lineBuf.Append(buffer, len);
+	}
+
+	char* line = (char*)m_lineBuf;
+	while (char* end = strchr(line, '\n'))
+	{
+		int llen = (int)(end - line + 1);
+
+		if (line[0] == '.' && line[1] == '\r')
+		{
+			m_eof = true;
+			m_lineBuf.SetLength(0);
+			return outlen;
+		}
+
+		if (m_format == efUnknown)
+		{
+			m_format = DetectFormat(line, llen);
+		}
+
+		if (m_format == efYenc)
+		{
+			ProcessYenc(line, llen);
+			if (m_body)
+			{
+				outlen = DecodeYenc(end + 1, buffer, m_lineBuf.Length() - (int)(end + 1 - m_lineBuf));
+				if (m_body)
+				{
+					m_lineBuf.SetLength(0);
+					return outlen;
+				}
+				line = (char*)m_lineBuf;
+				continue;
+			}
+		}
+		else if (m_format == efUx)
+		{
+			outlen += DecodeUx(line, llen);
+		}
+
+		line = end + 1;
+	}
+
+	if (*line)
+	{
+		len = m_lineBuf.Length() - (int)(line - m_lineBuf);
+		memmove((char*)m_lineBuf, line, len);
+		m_lineBuf.SetLength(len);
+	}
+	else
+	{
+		m_lineBuf.SetLength(0);
+	}
+
+	return outlen;
+}
+
+Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len)
 {
 	if (!strncmp(buffer, "=ybegin ", 8))
 	{
 		return efYenc;
 	}
 
-	if (inBody && (len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
+	if ((len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
 	{
 		return efUx;
 	}
@@ -64,139 +171,122 @@ Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
 	return efUnknown;
 }
 
-/**
-  * YDecoder: fast implementation of yEnc-Decoder
-  */
-
-YDecoder::YDecoder()
+void Decoder::ProcessYenc(char* buffer, int len)
 {
-	Clear();
-}
-
-void YDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_begin = false;
-	m_part = false;
-	m_end = false;
-	m_crc = false;
-	m_expectedCRC = 0;
-	m_calculatedCRC = 0xFFFFFFFF;
-	m_beginPos = 0;
-	m_endPos = 0;
-	m_size = 0;
-	m_endSize = 0;
-	m_crcCheck = false;
-}
-
-int YDecoder::DecodeBuffer(char* buffer, int len)
-{
-	if (m_body && !m_end)
+	if (!strncmp(buffer, "=ybegin ", 8))
 	{
-		if (!strncmp(buffer, "=yend ", 6))
+		m_begin = true;
+		char* pb = strstr(buffer, " name=");
+		if (pb)
 		{
-			m_end = true;
-			char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
-			if (pb)
-			{
-				m_crc = true;
-				pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
-				m_expectedCRC = strtoul(pb, nullptr, 16);
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_endSize = (int64)atoll(pb);
-			}
-			return 0;
+			pb += 6; //=strlen(" name=")
+			char* pe;
+			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++);
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 		}
-
-		char* iptr = buffer;
-		char* optr = buffer;
-		while (true)
+		pb = strstr(buffer, " size=");
+		if (pb)
 		{
-			switch (*iptr)
-			{
-				case '=':	//escape-sequence
-					iptr++;
-					*optr = *iptr - 64 - 42;
-					optr++;
-					break;
-				case '\n':	// ignored char
-				case '\r':	// ignored char
-					break;
-				case '\0':
-					goto BreakLoop;
-				default:	// normal char
-					*optr = *iptr - 42;
-					optr++;
-					break;
-			}
-			iptr++;
+			pb += 6; //=strlen(" size=")
+			m_size = (int64)atoll(pb);
 		}
-BreakLoop:
-
-		if (m_crcCheck)
+		m_part = strstr(buffer, " part=");
+		if (!m_part)
 		{
-			m_calculatedCRC = Util::Crc32m(m_calculatedCRC, (uchar *)buffer, (uint32)(optr - buffer));
-		}
-		return optr - buffer;
-	}
-	else
-	{
-		if (!m_part && !strncmp(buffer, "=ybegin ", 8))
-		{
-			m_begin = true;
-			char* pb = strstr(buffer, " name=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" name=")
-				char* pe;
-				for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-				m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_size = (int64)atoll(pb);
-			}
-			m_part = strstr(buffer, " part=");
-			if (!m_part)
-			{
-				m_body = true;
-				m_beginPos = 1;
-				m_endPos = m_size;
-			}
-		}
-		else if (m_part && !strncmp(buffer, "=ypart ", 7))
-		{
-			m_part = true;
 			m_body = true;
-			char* pb = strstr(buffer, " begin=");
-			if (pb)
-			{
-				pb += 7; //=strlen(" begin=")
-				m_beginPos = (int64)atoll(pb);
-			}
-			pb = strstr(buffer, " end=");
-			if (pb)
-			{
-				pb += 5; //=strlen(" end=")
-				m_endPos = (int64)atoll(pb);
-			}
+			m_beginPos = 1;
+			m_endPos = m_size;
+		}
+	}
+	else if (!strncmp(buffer, "=ypart ", 7))
+	{
+		m_part = true;
+		m_body = true;
+		char* pb = strstr(buffer, " begin=");
+		if (pb)
+		{
+			pb += 7; //=strlen(" begin=")
+			m_beginPos = (int64)atoll(pb);
+		}
+		pb = strstr(buffer, " end=");
+		if (pb)
+		{
+			pb += 5; //=strlen(" end=")
+			m_endPos = (int64)atoll(pb);
+		}
+	}
+	else if (!strncmp(buffer, "=yend ", 6))
+	{
+		m_end = true;
+		char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
+		if (pb)
+		{
+			m_crc = true;
+			pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
+			m_expectedCRC = strtoul(pb, nullptr, 16);
+		}
+		pb = strstr(buffer, " size=");
+		if (pb)
+		{
+			pb += 6; //=strlen(" size=")
+			m_endSize = (int64)atoll(pb);
 		}
 	}
-
-	return 0;
 }
 
-Decoder::EStatus YDecoder::Check()
+int Decoder::DecodeYenc(char* buffer, char* outbuf, int len)
 {
-	m_calculatedCRC ^= 0xFFFFFFFF;
+	const unsigned char* src = (unsigned char*)buffer;
+	unsigned char* dst = (unsigned char*)outbuf;
+
+	int endseq = YEncode::decode(&src, &dst, len, (YEncode::YencDecoderState*)&m_state);
+	int outlen = (int)((char*)dst - outbuf);
+
+	// endseq:
+	//   0: no end sequence found
+	//   1: \r\n=y sequence found, src points to byte after 'y'
+	//   2: \r\n.\r\n sequence found, src points to byte after last '\n'
+	if (endseq != 0)
+	{
+		// switch back to line mode to process '=yend'- or eof- marker
+		m_lineBuf.SetLength(0);
+		m_lineBuf.Append(endseq == 1 ? "=y" : ".\r\n");
+		int rem = len - (int)((const char*)src - buffer);
+		if (rem > 0)
+		{
+			m_lineBuf.Append((const char*)src, rem);
+		}
+		m_body = false;
+	}
+
+	if (m_crcCheck)
+	{
+		m_crc32.Append((uchar*)outbuf, (uint32)outlen);
+	}
+
+	m_outSize += outlen;
+
+	return outlen;
+}
+
+Decoder::EStatus Decoder::Check()
+{
+	switch (m_format)
+	{
+		case efYenc:
+			return CheckYenc();
+			 
+		case efUx:
+			return CheckUx();
+
+		default:
+			return dsUnknownError;
+	}
+}
+
+Decoder::EStatus Decoder::CheckYenc()
+{
+	m_calculatedCRC = m_crc32.Finish();
 
 	debug("Expected crc32=%x", m_expectedCRC);
 	debug("Calculated crc32=%x", m_calculatedCRC);
@@ -209,7 +299,7 @@ Decoder::EStatus YDecoder::Check()
 	{
 		return dsArticleIncomplete;
 	}
-	else if (!m_part && m_size != m_endSize)
+	else if ((!m_part && m_size != m_endSize) || (m_endSize != m_outSize))
 	{
 		return dsInvalidSize;
 	}
@@ -222,24 +312,7 @@ Decoder::EStatus YDecoder::Check()
 }
 
 
-/**
-  * UDecoder: supports UU encoding formats
-  */
-
-UDecoder::UDecoder()
-{
-	Clear();
-}
-
-void UDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_end = false;
-}
-
-/* DecodeBuffer-function uses portions of code from tool UUDECODE by Clem Dye
+/* DecodeUx-function uses portions of code from tool UUDECODE by Clem Dye
  * UUDECODE.c (http://www.bastet.com/uue.zip)
  * Copyright (C) 1998 Clem Dye
  *
@@ -248,7 +321,7 @@ void UDecoder::Clear()
 
 #define UU_DECODE_CHAR(c) (c == '`' ? 0 : (((c) - ' ') & 077))
 
-int UDecoder::DecodeBuffer(char* buffer, int len)
+int Decoder::DecodeUx(char* buffer, int len)
 {
 	if (!m_body)
 	{
@@ -264,7 +337,7 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			// extracting filename
 			char* pe;
 			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 
 			m_body = true;
 			return 0;
@@ -312,13 +385,13 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			}
 		}
 
-		return optr - buffer;
+		return (int)(optr - buffer);
 	}
 
 	return 0;
 }
 
-Decoder::EStatus UDecoder::Check()
+Decoder::EStatus Decoder::CheckUx()
 {
 	if (!m_body)
 	{
@@ -327,3 +400,50 @@ Decoder::EStatus UDecoder::Check()
 
 	return dsFinished;
 }
+
+void Decoder::ProcessRaw(char* buffer, int len)
+{
+	switch (m_state)
+	{
+		case 1:
+			m_eof = len >= 4 && buffer[0] == '\n' &&
+				buffer[1] == '.' && buffer[2] == '\r' && buffer[3] == '\n';
+			break;
+
+		case 2:
+			m_eof = len >= 3 && buffer[0] == '.' && buffer[1] == '\r' && buffer[2] == '\n';
+			break;
+
+		case 3:
+			m_eof = len >= 2 && buffer[0] == '\r' && buffer[1] == '\n';
+			break;
+
+		case 4:
+			m_eof = len >= 1 && buffer[0] == '\n';
+			break;
+	}
+
+	m_eof |= len >= 5 && strstr(buffer, "\r\n.\r\n");
+
+	if (len >= 4 && buffer[len-4] == '\r' && buffer[len-3] == '\n' &&
+		buffer[len-2] == '.' && buffer[len-1] == '\r')
+	{
+		m_state = 4;
+	}
+	else if (len >= 3 && buffer[len-3] == '\r' && buffer[len-2] == '\n' && buffer[len-1] == '.')
+	{
+		m_state = 3;
+	}
+	else if (len >= 2 && buffer[len-2] == '\r' && buffer[len-1] == '\n')
+	{
+		m_state = 2;
+	}
+	else if (len >= 1 && buffer[len-1] == '\r')
+	{
+		m_state = 1;
+	}
+	else
+	{
+		m_state = 0;
+	}
+}

daemon/nntp/Decoder.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #define DECODER_H
 
 #include "NString.h"
+#include "Util.h"
 
 class Decoder
 {
@@ -43,37 +44,26 @@ public:
 		efUx,
 	};
 
-	static const char* FormatNames[];
-
-	virtual ~Decoder() {}
-	virtual EStatus Check() = 0;
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len) = 0;
-	const char* GetArticleFilename() { return m_articleFilename; }
-	static EFormat DetectFormat(const char* buffer, int len, bool inBody);
-
-protected:
-	CString m_articleFilename;
-};
-
-class YDecoder: public Decoder
-{
-public:
-	YDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
+	Decoder();
+	EStatus Check();
+	void Clear();
+	int DecodeBuffer(char* buffer, int len);
 	void SetCrcCheck(bool crcCheck) { m_crcCheck = crcCheck; }
-	int64 GetBegin() { return m_beginPos; }
-	int64 GetEnd() { return m_endPos; }
+	void SetRawMode(bool rawMode) { m_rawMode = rawMode; }
+	EFormat GetFormat() { return m_format; }
+	int64 GetBeginPos() { return m_beginPos; }
+	int64 GetEndPos() { return m_endPos; }
 	int64 GetSize() { return m_size; }
 	uint32 GetExpectedCrc() { return m_expectedCRC; }
 	uint32 GetCalculatedCrc() { return m_calculatedCRC; }
+	bool GetEof() { return m_eof; }
+	const char* GetArticleFilename() { return m_articleFilename; }
 
-private:
+private: 
+	EFormat m_format = efUnknown;
 	bool m_begin;
 	bool m_part;
-	bool m_body;
+	bool m_body; 
 	bool m_end;
 	bool m_crc;
 	uint32 m_expectedCRC;
@@ -82,20 +72,22 @@ private:
 	int64 m_endPos;
 	int64 m_size;
 	int64 m_endSize;
+	int64 m_outSize;
+	bool m_eof;
 	bool m_crcCheck;
-};
+	char m_state;
+	bool m_rawMode = false;
+	CString m_articleFilename;
+	StringBuilder m_lineBuf;
+	Crc32 m_crc32;
 
-class UDecoder: public Decoder
-{
-public:
-	UDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
-
-private:
-	bool m_body;
-	bool m_end;
+	EFormat DetectFormat(const char* buffer, int len);
+	void ProcessYenc(char* buffer, int len);
+	int DecodeYenc(char* buffer, char* outbuf, int len);
+	EStatus CheckYenc();
+	int DecodeUx(char* buffer, int len);
+	EStatus CheckUx();
+	void ProcessRaw(char* buffer, int len);
 };
 
 #endif

daemon/nntp/NewsServer.cpp

@@ -22,10 +22,10 @@
 #include "nzbget.h"
 #include "NewsServer.h"
 
-NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port,
+NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 	const char* user, const char* pass, bool joinGroup, bool tls, const char* cipher,
 	int maxConnections, int retention, int level, int group, bool optional) :
-		m_id(id), m_active(active), m_port(port), m_level(level), m_normLevel(level),
+		m_id(id), m_active(active), m_port(port), m_ipVersion(ipVersion), m_level(level), m_normLevel(level),
 		m_group(group), m_maxConnections(maxConnections), m_joinGroup(joinGroup), m_tls(tls),
 		m_name(name), m_host(host ? host : ""), m_user(user ? user : ""), m_password(pass ? pass : ""),
 		m_cipher(cipher ? cipher : ""), m_retention(retention), m_optional(optional)

daemon/nntp/NewsServer.h

@@ -32,7 +32,7 @@
 class NewsServer
 {
 public:
-	NewsServer(int id, bool active, const char* name, const char* host, int port,
+	NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 		const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
 		int level, int group, bool optional);
@@ -45,6 +45,7 @@ public:
 	int GetGroup() { return m_group; }
 	const char* GetHost() { return m_host; }
 	int GetPort() { return m_port; }
+	int GetIpVersion() { return m_ipVersion; }
 	const char* GetUser() { return m_user; }
 	const char* GetPassword() { return m_password; }
 	int GetMaxConnections() { return m_maxConnections; }
@@ -67,6 +68,7 @@ private:
 	int m_group;
 	CString m_host;
 	int m_port;
+	int m_ipVersion;
 	CString m_user;
 	CString m_password;
 	int m_maxConnections;

daemon/nntp/NntpConnection.cpp

@@ -27,10 +27,13 @@
 
 static const int CONNECTION_LINEBUFFER_SIZE = 1024*10;
 
-NntpConnection::NntpConnection(NewsServer* newsServer) : Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
+NntpConnection::NntpConnection(NewsServer* newsServer) :
+	Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
 {
 	m_lineBuf.Reserve(CONNECTION_LINEBUFFER_SIZE);
 	SetCipher(newsServer->GetCipher());
+	SetIPVersion(newsServer->GetIpVersion() == 4 ? Connection::ipV4 :
+		newsServer->GetIpVersion() == 6 ? Connection::ipV6 : Connection::ipAuto);
 }
 
 const char* NntpConnection::Request(const char* req)
@@ -224,10 +227,7 @@ bool NntpConnection::Disconnect()
 {
 	if (m_status == csConnected)
 	{
-		if (!m_broken)
-		{
-			Request("quit\r\n");
-		}
+		Request("quit\r\n");
 		m_activeGroup = nullptr;
 	}
 	return Connection::Disconnect();

daemon/nntp/StatMeter.cpp

@@ -308,8 +308,6 @@ void StatMeter::AddSpeedReading(int bytes)
 	time_t curTime = Util::CurrentTime();
 	int nowSlot = (int)curTime / SPEEDMETER_SLOTSIZE;
 
-	Guard guard(g_Options->GetAccurateRate() ? &m_speedMutex : nullptr);
-
 	if (curTime != m_curSecTime)
 	{
 		m_curSecTime =	curTime;
@@ -490,7 +488,7 @@ void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
 	ServerVolume totalVolume = m_serverVolumes[0];
 
 	time_t locTime = Util::CurrentTime() + g_Options->GetLocalTimeOffset();
-	int daySlot = locTime / 86400 - totalVolume.GetFirstDay();
+	int daySlot = (int)(locTime / 86400) - totalVolume.GetFirstDay();
 
 	dayBytes = 0;
 	if (daySlot < (int)totalVolume.BytesPerDays()->size())
@@ -534,7 +532,7 @@ int StatMeter::CalcMonthSlots(ServerVolume& volume)
 			dayparts.tm_mon++;
 			prevMonth = Util::Timegm(&dayparts);
 		}
-		elapsedDays = (locCurTime - prevMonth) / 60 / 60 / 24 + 1;
+		elapsedDays = (int)(locCurTime - prevMonth) / 60 / 60 / 24 + 1;
 	}
 	else
 	{

daemon/nntp/StatMeter.h

@@ -106,7 +106,6 @@ private:
 	int m_speedBytesIndex;
 	int m_curSecBytes;
 	time_t m_curSecTime;
-	Mutex m_speedMutex;
 
 	// time
 	int64 m_allBytes = 0;

daemon/nserv/NServMain.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -44,6 +44,10 @@ struct NServOpts
 	bool generateNzb;
 	int segmentSize;
 	bool quit;
+	int latency;
+	int speed;
+	bool memCache;
+	bool paramError;
 
 	NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts);
 };
@@ -59,7 +63,7 @@ int NServMain(int argc, char* argv[])
 	Options::CmdOptList cmdOpts;
 	NServOpts opts(argc, argv, cmdOpts);
 
-	if (opts.dataDir.Empty())
+	if (opts.dataDir.Empty() || opts.paramError)
 	{
 		NServPrintUsage(argv[0]);
 		return 1;
@@ -85,6 +89,10 @@ int NServMain(int argc, char* argv[])
 	TlsSocket::Init();
 #endif
 
+#ifndef WIN32
+	signal(SIGPIPE, SIG_IGN);
+#endif
+
 	NServFrontend frontend;
 	frontend.Start();
 
@@ -105,11 +113,13 @@ int NServMain(int argc, char* argv[])
 	}
 
 	std::vector<std::unique_ptr<NntpServer>> instances;
+	NntpCache cache;
 
 	for (int i = 0; i < opts.instances; i++)
 	{
 		instances.emplace_back(std::make_unique<NntpServer>(i + 1, opts.bindAddress,
-			opts.firstPort + i, opts.secureCert, opts.secureKey, opts.dataDir, opts.cacheDir));
+			opts.firstPort + i, opts.secureCert, opts.secureKey, opts.dataDir, opts.cacheDir,
+			opts.latency, opts.speed, opts.memCache ? &cache : nullptr));
 		instances.back()->Start();
 	}
 
@@ -143,12 +153,15 @@ void NServPrintUsage(const char* com)
 		"    -d <data-dir>   - directory whose files will be served\n"
 		"  Optional switches:\n"
 		"    -c <cache-dir>  - directory to store encoded articles\n"
+		"    -m              - in-memory cache (unlimited, use with care)\n"
 		"    -l <log-file>   - write into log-file (disabled by default)\n"
 		"    -i <instances>  - number of server instances (default is 1)\n"
 		"    -b <address>    - ip address to bind to (default is 0.0.0.0)\n"
 		"    -p <port>       - port number for the first instance (default is 6791)\n"
 		"    -s <cert> <key> - paths to SSL certificate and key files\n"
 		"    -v <verbose>    - verbosity level 0..3 (default is 2)\n"
+		"    -w <msec>       - response latency (in milliseconds)\n"
+		"    -r <KB/s>       - speed throttling (in kilobytes per second)\n"
 		"    -z <seg-size>   - generate nzbs for all files in data-dir (size in bytes)\n"
 		"    -q              - quit after generating nzbs (in combination with -z)\n"
 		, FileSystem::BaseFileName(com));
@@ -162,9 +175,13 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 	generateNzb = false;
 	segmentSize = 500000;
 	quit = false;
+	latency = 0;
+	memCache = false;
+	speed = 0;
+	paramError = false;
 	int verbosity = 2;
 
-	char short_options[] = "b:c:d:l:p:i:s:v:z:q";
+	char short_options[] = "b:c:d:l:p:i:ms:v:w:r:z:q";
 
 	optind = 2;
 	while (true)
@@ -181,6 +198,10 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 				cacheDir = optind > argc ? nullptr : argv[optind - 1];
 				break;
 
+			case 'm':
+				memCache = true;
+				break;
+
 			case 'l':
 				logFile = optind > argc ? nullptr : argv[optind - 1];
 				break;
@@ -207,6 +228,14 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 				verbosity = atoi(optind > argc ? "1" : argv[optind - 1]);
 				break;
 
+			case 'w':
+				latency = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
+			case 'r':
+				speed = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
 			case 'z':
 				generateNzb = true;
 				segmentSize = atoi(optind > argc ? "500000" : argv[optind - 1]);
@@ -218,6 +247,11 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 		}
 	}
 
+	if (optind < argc)
+	{
+		paramError = true;
+	}
+
 	if (logFile.Empty())
 	{
 		cmdOpts.push_back("WriteLog=none");

daemon/nserv/NntpServer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -28,9 +28,10 @@ class NntpProcessor : public Thread
 {
 public:
 	NntpProcessor(int id, int serverId, const char* dataDir, const char* cacheDir,
-		const char* secureCert, const char* secureKey) :
+		const char* secureCert, const char* secureKey, int latency, int speed, NntpCache* cache) :
 		m_id(id), m_serverId(serverId), m_dataDir(dataDir), m_cacheDir(cacheDir),
-		m_secureCert(secureCert), m_secureKey(secureKey) {}
+		m_secureCert(secureCert), m_secureKey(secureKey), m_latency(latency),
+		m_speed(speed), m_cache(cache) {}
 	~NntpProcessor() { m_connection->Disconnect(); }
 	virtual void Run();
 	void SetConnection(std::unique_ptr<Connection>&& connection) { m_connection = std::move(connection); }
@@ -39,6 +40,8 @@ private:
 	int m_id;
 	int m_serverId;
 	std::unique_ptr<Connection> m_connection;
+	int m_latency;
+	int m_speed;
 	const char* m_dataDir;
 	const char* m_cacheDir;
 	const char* m_secureCert;
@@ -49,18 +52,29 @@ private:
 	int64 m_offset;
 	int m_size;
 	bool m_sendHeaders;
+	int64 m_start;
+	NntpCache* m_cache;
 
 	void ServArticle();
 	void SendSegment();
 	bool ServerInList(const char* servList);
+	void SendData(const char* buffer, int size);
 };
 
+
 void NntpServer::Run()
 {
 	debug("Entering NntpServer-loop");
 
 	info("Listening on port %i", m_port);
 
+#ifdef WIN32
+	if (m_speed > 0)
+	{
+		timeBeginPeriod(1);
+	}
+#endif
+
 	int num = 1;
 
 	while (!IsStopped())
@@ -93,8 +107,8 @@ void NntpServer::Run()
 			continue;
 		}
 		
-		NntpProcessor* commandThread = new NntpProcessor(num++, m_id,
-			m_dataDir, m_cacheDir, m_secureCert, m_secureKey);
+		NntpProcessor* commandThread = new NntpProcessor(num++, m_id, m_dataDir,
+			m_cacheDir, m_secureCert, m_secureKey, m_latency, m_speed, m_cache);
 		commandThread->SetAutoDestroy(true);
 		commandThread->SetConnection(std::move(acceptedConnection));
 		commandThread->Start();
@@ -134,6 +148,7 @@ void NntpProcessor::Run()
 	}
 #endif
 
+	info("[%i] Incoming connection from: %s", m_id, m_connection->GetHost() );
 	m_connection->WriteLine("200 Welcome (NServ)\r\n");
 
 	CharBuffer buf(1024);
@@ -199,6 +214,11 @@ void NntpProcessor::ServArticle()
 {
 	detail("[%i] Serving: %s", m_id, m_messageid);
 
+	if (m_latency)
+	{
+		usleep(1000 * m_latency);
+	}
+
 	bool ok = false;
 
 	const char* from = strchr(m_messageid, '?');
@@ -209,7 +229,7 @@ void NntpProcessor::ServArticle()
 
 	if (from && off && to && end)
 	{
-		m_filename.Set(m_messageid + 1, from - m_messageid - 1);
+		m_filename.Set(m_messageid + 1, (int)(from - m_messageid - 1));
 		m_part = atoi(from + 1);
 		m_offset = atoll(off + 1);
 		m_size = atoi(to + 1);
@@ -250,14 +270,32 @@ void NntpProcessor::SendSegment()
 {
 	detail("[%i] Sending segment %s (%i=%lli:%i)", m_id, *m_filename, m_part, (long long)m_offset, m_size);
 
+	if (m_speed > 0)
+	{
+		m_start = Util::GetCurrentTicks();
+	}
+
 	BString<1024> fullFilename("%s/%s", m_dataDir, *m_filename);
 	BString<1024> cacheFileDir("%s/%s", m_cacheDir, *m_filename);
 	BString<1024> cacheFileName("%i=%lli-%i", m_part, (long long)m_offset, m_size);
 	BString<1024> cacheFullFilename("%s/%s", *cacheFileDir, *cacheFileName);
+	BString<1024> cacheKey("%s/%s", *m_filename, *cacheFileName);
+
+	const char* cachedData = nullptr;
+	int cachedSize;
+	if (m_cache)
+	{
+		m_cache->Find(cacheKey, cachedData, cachedSize);
+	}
 
 	DiskFile cacheFile;
-	bool readCache = m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
-	bool writeCache = m_cacheDir && !readCache;
+	bool readCache = !cachedData && m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
+	bool writeCache = !cachedData && m_cacheDir && !readCache;
+	StringBuilder cacheMem;
+	if (m_cache && !cachedData)
+	{
+		cacheMem.Reserve((int)(m_size * 1.1));
+	}
 
 	CString errmsg;
 	if (writeCache && !FileSystem::ForceDirectories(cacheFileDir, errmsg))
@@ -270,23 +308,27 @@ void NntpProcessor::SendSegment()
 		error("Could not create file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
 	}
 
-	if (!readCache && !FileSystem::FileExists(fullFilename))
+	if (!cachedData && !readCache && !FileSystem::FileExists(fullFilename))
 	{
 		m_connection->WriteLine(CString::FormatStr("430 Article not found\r\n"));
 		return;
 	}
 
 	YEncoder encoder(fullFilename, m_part, m_offset, m_size, 
-		[con = m_connection.get(), writeCache, &cacheFile](const char* buf, int size)
+		[proc = this, writeCache, &cacheFile, &cacheMem](const char* buf, int size)
 		{
+			if (proc->m_cache)
+			{
+				cacheMem.Append(buf);
+			}
 			if (writeCache)
 			{
 				cacheFile.Write(buf, size);
 			}
-			con->Send(buf, size);
+			proc->SendData(buf, size);
 		});
 
-	if (!readCache && !encoder.OpenFile(errmsg))
+	if (!cachedData && !readCache && !encoder.OpenFile(errmsg))
 	{
 		m_connection->WriteLine(CString::FormatStr("403 %s\r\n", *errmsg));
 		return;
@@ -300,7 +342,11 @@ void NntpProcessor::SendSegment()
 		m_connection->WriteLine("\r\n");
 	}
 
-	if (readCache)
+	if (cachedData)
+	{
+		SendData(cachedData, cachedSize);
+	}
+	else if (readCache)
 	{
 		cacheFile.Seek(0, DiskFile::soEnd);
 		int size = (int)cacheFile.Position();
@@ -310,12 +356,88 @@ void NntpProcessor::SendSegment()
 		{
 			error("Could not read file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
 		}
-		m_connection->Send(buf, size);
+		if (m_cache)
+		{
+			cacheMem.Append(buf, size);
+		}
+		SendData(buf, size);
 	}
 	else
 	{
 		encoder.WriteSegment();
 	}
 
+	if (!cachedData && cacheMem.Length() > 0)
+	{
+		m_cache->Append(cacheKey, cacheMem, cacheMem.Length());
+	}
+
 	m_connection->WriteLine(".\r\n");
 }
+
+void NntpProcessor::SendData(const char* buffer, int size)
+{
+	if (m_speed == 0)
+	{
+		m_connection->Send(buffer, size);
+		return;
+	}
+
+	int64 expectedTime = (int64)1000 * size / (m_speed * 1024) - (Util::GetCurrentTicks() - m_start) / 1000;
+
+	int chunkNum = 21;
+	int chunkSize = size;
+	int pause = 0;
+
+	while (pause < 1 && chunkNum > 1)
+	{
+		chunkNum--;
+		chunkSize = size / chunkNum;
+		pause = (int)(expectedTime / chunkNum);
+	}
+
+	int sent = 0;
+	for (int i = 0; i < chunkNum; i++)
+	{
+		int len = sent + chunkSize < size ? chunkSize : size - sent;
+
+		while (sent + len < size && *(buffer + sent + len) != '\r')
+		{
+			len++;
+		}
+		
+		m_connection->Send(buffer + sent, len);
+		int64 now = Util::GetCurrentTicks();
+		if (now + pause * 1000 < m_start + expectedTime * 1000)
+		{
+			usleep(pause * 1000);
+		}
+		sent += len;
+	}
+}
+
+
+void NntpCache::Append(const char* key, const char* data, int len)
+{
+	Guard guard(m_lock);
+	if (!len)
+	{
+		len = strlen(data);
+	}
+	m_items.emplace(key, std::make_unique<CacheItem>(key, data, len));
+}
+
+bool NntpCache::Find(const char* key, const char*& data, int& size)
+{
+	Guard guard(m_lock);
+
+	CacheMap::iterator pos = m_items.find(key);
+	if (pos != m_items.end())
+	{
+		data = (*pos).second->m_data;
+		size = (*pos).second->m_size;
+		return true;
+	}
+
+	return false;
+}

daemon/nserv/NntpServer.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,14 +23,41 @@
 
 #include "Thread.h"
 #include "Connection.h"
+#include "Util.h"
+
+class NntpCache
+{
+public:
+	void Append(const char* key, const char* data, int len = 0);
+	bool Find(const char* key, const char*& data, int& size);
+
+private:
+	class CacheItem
+	{
+	public:
+		CacheItem(const char* key, const char* data, int size) :
+			m_key(key), m_data(data), m_size(size) {}
+
+		CString m_key;
+		CString m_data;
+		int m_size = 0;
+	};
+
+	typedef std::unordered_map<std::string, std::unique_ptr<CacheItem>> CacheMap;
+
+	CacheMap m_items;
+	Mutex m_lock;
+};
 
 class NntpServer : public Thread
 {
 public:
 	NntpServer(int id, const char* host, int port, const char* secureCert,
-		const char* secureKey, const char* dataDir, const char* cacheDir) :
+		const char* secureKey, const char* dataDir, const char* cacheDir,
+		int latency, int speed, NntpCache* cache) :
 		m_id(id), m_host(host), m_port(port), m_secureCert(secureCert),
-		m_secureKey(secureKey), m_dataDir(dataDir), m_cacheDir(cacheDir) {}
+		m_secureKey(secureKey), m_dataDir(dataDir), m_cacheDir(cacheDir),
+		m_latency(latency), m_speed(speed), m_cache(cache) {}
 	virtual void Run();
 	virtual void Stop();
 
@@ -38,11 +65,14 @@ private:
 	int m_id;
 	CString m_host;
 	int m_port;
+	int m_latency;
+	int m_speed;
 	CString m_dataDir;
 	CString m_cacheDir;
 	CString m_secureCert;
 	CString m_secureKey;
 	std::unique_ptr<Connection> m_connection;
+	NntpCache* m_cache;
 };
 
 #endif

daemon/nserv/YEncoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -67,7 +67,7 @@ void YEncoder::WriteSegment()
 	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%lli name=%s\r\n", m_part, (long long)m_fileSize, FileSystem::BaseFileName(m_filename)));
 	outbuf.Append(CString::FormatStr("=ypart begin=%lli end=%lli\r\n", (long long)(m_offset + 1), (long long)(m_offset + m_size)));
 
-	uint32 crc = 0xFFFFFFFF;
+	Crc32 crc;
 	CharBuffer inbuf(std::min(m_size, 16 * 1024 * 1024));
 	int lnsz = 0;
 	char* out = (char*)outbuf + outbuf.Length();
@@ -82,7 +82,7 @@ void YEncoder::WriteSegment()
 			return; // error;
 		}
 
-		crc = Util::Crc32m(crc, (uchar*)(const char*)inbuf, (int)readBytes);
+		crc.Append((uchar*)(const char*)inbuf, (int)readBytes);
 
 		char* in = inbuf;
 		while (readBytes > 0)
@@ -122,10 +122,8 @@ void YEncoder::WriteSegment()
 			}
 		}
 	}
-	crc ^= 0xFFFFFFFF;
-
 	m_diskfile.Close();
 
-	outbuf.Append(CString::FormatStr("=yend size=%i part=0 pcrc32=%08x\r\n", m_size, (unsigned int)crc));
+	outbuf.Append(CString::FormatStr("=yend size=%i part=0 pcrc32=%08x\r\n", m_size, (unsigned int)crc.Finish()));
 	m_writeFunc(outbuf, outbuf.Length());
 }

daemon/postprocess/DirectUnpack.cpp

@@ -0,0 +1,558 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "DirectUnpack.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "Options.h"
+
+void DirectUnpack::StartJob(NzbInfo* nzbInfo)
+{
+	DirectUnpack* directUnpack = new DirectUnpack();
+	directUnpack->m_nzbId = nzbInfo->GetId();
+	directUnpack->SetAutoDestroy(true);
+
+	nzbInfo->SetUnpackThread(directUnpack);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsRunning);
+
+	directUnpack->Start();
+}
+
+void DirectUnpack::Run()
+{
+	debug("Entering DirectUnpack-loop for %i", m_nzbId);
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo)
+		{
+			debug("Could not find NzbInfo for %i", m_nzbId);
+			return;
+		}
+
+		m_name = nzbInfo->GetName();
+		m_destDir = nzbInfo->GetDestDir();
+		m_finalDir = nzbInfo->BuildFinalDirName();
+
+		NzbParameter* parameter = nzbInfo->GetParameters()->Find("*Unpack:Password");
+		if (parameter)
+		{
+			m_password = parameter->GetValue();
+		}
+	}
+
+	m_infoName.Format("direct unpack for %s", *m_name);
+	m_infoNameUp.Format("Direct unpack for %s", *m_name); // first letter in upper case
+
+	FindArchiveFiles();
+
+	SetInfoName(m_infoName);
+	SetWorkingDir(m_destDir);
+
+	while (!IsStopped())
+	{
+		CString archive;
+		{
+			Guard guard(m_volumeMutex);
+			if (!m_archives.empty())
+			{
+				archive = std::move(m_archives.front());
+				m_archives.pop_front();
+			}
+		}
+
+		if (archive)
+		{
+			if (!m_processed)
+			{
+				PrintMessage(Message::mkInfo, "Directly unpacking %s", *m_name);
+				Cleanup();
+				CreateUnpackDir();
+				m_processed = true;
+			}
+
+			ExecuteUnrar(archive);
+
+			if (!m_unpackOk)
+			{
+				break;
+			}
+		}
+		else
+		{
+			if (m_nzbCompleted)
+			{
+				break;
+			}
+			usleep(100 * 1000);
+		}
+	}
+
+	if (!m_unpackOk)
+	{
+		Cleanup();
+	}
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo || nzbInfo->GetUnpackThread() != this)
+		{
+			debug("Could not find NzbInfo for %s", *m_infoName);
+			return;
+		}
+
+		nzbInfo->SetUnpackThread(nullptr);
+		nzbInfo->SetDirectUnpackStatus(m_processed ? m_unpackOk && !IsStopped() ? NzbInfo::nsSuccess : NzbInfo::nsFailure : NzbInfo::nsNone);
+
+		if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			nzbInfo->AddMessage(Message::mkInfo, BString<1024>("%s successful", *m_infoNameUp));
+		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsFailure)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("%s failed", *m_infoNameUp));
+		}
+
+		if (nzbInfo->GetPostInfo() && nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			std::move(m_extractedArchives.begin(), m_extractedArchives.end(),
+				std::back_inserter(*nzbInfo->GetPostInfo()->GetExtractedArchives()));
+		}
+
+		AddExtraTime(nzbInfo);
+
+		if (nzbInfo->GetPostInfo())
+		{
+			nzbInfo->GetPostInfo()->SetWorking(false);
+		}
+	}
+
+	debug("Exiting DirectUnpack-loop for %i", m_nzbId);
+}
+
+void DirectUnpack::ExecuteUnrar(const char* archiveName)
+{
+	// Format:
+	//   unrar x -y -p- -o+ -vp <archive.part0001.rar> <dest-dir>/_unpack/
+
+	ParamList params;
+	if (!PrepareCmdParams(g_Options->GetUnrarCmd(), &params, "unrar"))
+	{
+		return;
+	}
+
+	if (!params.Exists("x") && !params.Exists("e"))
+	{
+		params.emplace_back("x");
+	}
+
+	params.emplace_back("-y");
+
+	if (!m_password.Empty())
+	{
+		params.push_back(CString::FormatStr("-p%s", *m_password));
+	}
+	else
+	{
+		params.emplace_back("-p-");
+	}
+
+	if (!params.Exists("-o+") && !params.Exists("-o-"))
+	{
+		params.emplace_back("-o+");
+	}
+
+	params.emplace_back("-vp");
+
+	params.emplace_back(archiveName);
+	m_unpackExtendedDir = FileSystem::MakeExtendedPath(m_unpackDir, true);
+	params.push_back(*BString<1024>("%s%c", *m_unpackExtendedDir, PATH_SEPARATOR));
+	SetArgs(std::move(params));
+	SetLogPrefix("Unrar");
+	ResetEnv();
+
+	m_allOkMessageReceived = false;
+
+	SetNeedWrite(true);
+	m_unpacking = true;
+	int exitCode = Execute();
+	m_unpacking = false;
+	SetLogPrefix(nullptr);
+
+	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
+
+	if (!m_unpackOk && exitCode > 0)
+	{
+		PrintMessage(Message::mkError, "Unrar error code: %i", exitCode);
+	}
+}
+
+bool DirectUnpack::PrepareCmdParams(const char* command, ParamList* params, const char* infoName)
+{
+	if (FileSystem::FileExists(command))
+	{
+		params->emplace_back(command);
+		return true;
+	}
+
+	std::vector<CString> cmdArgs = Util::SplitCommandLine(command);
+	if (cmdArgs.empty())
+	{
+		PrintMessage(Message::mkError, "Could not start %s, failed to parse command line: %s", infoName, command);
+		m_unpackOk = false;
+		return false;
+	}
+
+	std::move(cmdArgs.begin(), cmdArgs.end(), std::back_inserter(*params));
+
+	return true;
+}
+
+void DirectUnpack::CreateUnpackDir()
+{
+	bool useInterDir = !Util::EmptyStr(g_Options->GetInterDir()) &&
+		!strncmp(m_destDir, g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
+		m_destDir[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
+
+	m_finalDirCreated = useInterDir && !FileSystem::DirectoryExists(m_finalDir);
+
+	const char* destDir = useInterDir && !m_finalDir.Empty() ? *m_finalDir : *m_destDir;
+
+	m_unpackDir.Format("%s%c%s", destDir, PATH_SEPARATOR, "_unpack");
+
+	detail("Unpacking into %s", *m_unpackDir);
+
+	CString errmsg;
+	if (!FileSystem::ForceDirectories(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not create directory %s: %s", *m_unpackDir, *errmsg);
+	}
+}
+
+void DirectUnpack::FindArchiveFiles()
+{
+	Guard guard(m_volumeMutex);
+
+	DirBrowser dir(m_destDir);
+	while (const char* filename = dir.Next())
+	{
+		if (IsMainArchive(filename))
+		{
+			BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+			if (!FileSystem::DirectoryExists(fullFilename))
+			{
+				m_archives.emplace_back(filename);
+			}
+		}
+	}
+}
+
+bool DirectUnpack::IsMainArchive(const char* filename)
+{
+	RegEx regExRarPart(".*\\.part([0-9]+)\\.rar$");
+	bool mainPart = Util::EndsWith(filename, ".rar", false) &&
+		(!regExRarPart.Match(filename) || atoi(filename + regExRarPart.GetMatchStart(1)) == 1);
+	return mainPart;
+}
+
+/**
+ * Unrar prints "Insert disk"-message without new line terminator.
+ * In order to become the message we analyze the output after every char.
+ */
+bool DirectUnpack::ReadLine(char* buf, int bufSize, FILE* stream)
+{
+	int i = 0;
+	bool backspace = false;
+
+	for (; i < bufSize - 1; i++)
+	{
+		int ch = fgetc(stream);
+
+		if (ch == '\b')
+		{
+			backspace = true;
+		}
+		if (!backspace)
+		{
+			buf[i] = ch;
+			buf[i+1] = '\0';
+		}
+		if (ch == EOF)
+		{
+			break;
+		}
+		if (ch == '\n')
+		{
+			i++;
+			break;
+		}
+		if (i > 35 && ch == ' ' &&
+			!strncmp(buf, "Insert disk with", 16) && strstr(buf, " [C]ontinue, [Q]uit "))
+		{
+			i++;
+			break;
+		}
+	}
+
+	// skip unnecessary progress messages
+	if (!strncmp(buf, "...", 3))
+	{
+		buf[0] = '\0';
+	}
+
+	buf[i] = '\0';
+
+	return i > 0;
+}
+
+void DirectUnpack::AddMessage(Message::EKind kind, const char* text)
+{
+	debug("%s", text);
+
+	BString<1024> msgText = text;
+	int len = strlen(text);
+
+	// Modify unrar messages for better readability:
+	// remove the destination path part from message "Extracting file.xxx"
+	if (!strncmp(text, "Unrar: Extracting  ", 19) &&
+		!strncmp(text + 19, m_unpackExtendedDir, strlen(m_unpackExtendedDir)))
+	{
+		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackExtendedDir) + 1);
+	}
+
+	if (!strncmp(text, "Unrar: Insert disk with", 23) && strstr(text, " [C]ontinue, [Q]uit"))
+	{
+		BString<1024> filename;
+		filename.Set(text + 24, (int)(strstr(text, " [C]ontinue, [Q]uit") - text - 24));
+		WaitNextVolume(filename);
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (nzbInfo)
+	{
+		nzbInfo->AddMessage(kind, msgText);
+	}
+	else
+	{
+		ScriptController::AddMessage(kind, msgText);
+	}
+
+	if (!strncmp(msgText, "Unrar: Extracting ", 18) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, msgText + 7);
+	}
+
+	if (!strncmp(text, "Unrar: Extracting from ", 23) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, text + 7);
+		m_extractedArchives.emplace_back(text + 23);
+	}
+
+	if (!IsStopped() && (
+		!strncmp(text, "Unrar: Checksum error in the encrypted file", 42) ||
+		!strncmp(text, "Unrar: CRC failed in the encrypted file", 39) ||
+		!strncmp(text, "Unrar: The specified password is incorrect.", 43) ||
+		strstr(text, " : packed data CRC failed in volume") ||
+		strstr(text, " : packed data checksum error in volume") ||
+		(len > 13 && !strncmp(text + len - 13, " - CRC failed", 13)) ||
+		(len > 18 && !strncmp(text + len - 18, " - checksum failed", 18)) ||
+		!strncmp(text, "Unrar: WARNING: You need to start extraction from a previous volume", 67)))
+	{
+		Stop(downloadQueue, nzbInfo);
+	}
+
+	if (!strncmp(text, "Unrar: All OK", 13))
+	{
+		m_allOkMessageReceived = true;
+	}
+}
+
+void DirectUnpack::Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("Stopping direct unpack for %s", *m_infoName);
+	if (m_processed)
+	{
+		if (nzbInfo)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Cancelling %s", *m_infoName));
+		}
+		else
+		{
+			warn("Cancelling %s", *m_infoName);
+		}
+	}
+	AddExtraTime(nzbInfo);
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetWorking(false);
+	}
+	Thread::Stop();
+	if (m_unpacking)
+	{
+		Terminate();
+	}
+}
+
+void DirectUnpack::WaitNextVolume(const char* filename)
+{
+	debug("WaitNextVolume for %s", filename);
+
+	// Stop direct unpack if destination directory was changed during unpack
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (nzbInfo && (strcmp(m_destDir, nzbInfo->GetDestDir()) ||
+			strcmp(m_finalDir, nzbInfo->BuildFinalDirName())))
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Destination directory changed for %s", nzbInfo->GetName()));
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+
+	BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+	if (FileSystem::FileExists(fullFilename))
+	{
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+	else
+	{
+		Guard guard(m_volumeMutex);
+		m_waitingFile = filename;
+		if (m_nzbCompleted)
+		{
+			// nzb completed but unrar waits for another volume
+			PrintMessage(Message::mkWarning, "Could not find volume %s", filename);
+			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+			NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+}
+
+void DirectUnpack::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo)
+{
+	debug("FileDownloaded for %s/%s", fileInfo->GetNzbInfo()->GetName(), fileInfo->GetFilename());
+
+	if (fileInfo->GetNzbInfo()->GetFailedArticles() > 0)
+	{
+		Stop(downloadQueue, fileInfo->GetNzbInfo());
+		return;
+	}
+
+	Guard guard(m_volumeMutex);
+	if (m_waitingFile && !strcasecmp(fileInfo->GetFilename(), m_waitingFile))
+	{
+		m_waitingFile = nullptr;
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+
+	if (IsMainArchive(fileInfo->GetFilename()))
+	{
+		m_archives.emplace_back(fileInfo->GetFilename());
+	}
+}
+
+void DirectUnpack::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDownloaded for %s", nzbInfo->GetName());
+
+	Guard guard(m_volumeMutex);
+	m_nzbCompleted = true;
+	if (m_waitingFile)
+	{
+		// nzb completed but unrar waits for another volume
+		nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Unrar: Could not find volume %s", *m_waitingFile));
+		Stop(downloadQueue, nzbInfo);
+		return;
+	}
+
+	m_extraStartTime = Util::CurrentTime();
+
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(m_progressLabel);
+	}
+}
+
+void DirectUnpack::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDeleted for %s", nzbInfo->GetName());
+
+	nzbInfo->SetUnpackThread(nullptr);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsFailure);
+	Stop(downloadQueue, nzbInfo);
+}
+
+// Remove _unpack-dir
+void DirectUnpack::Cleanup()
+{
+	debug("Cleanup for %s", *m_infoName);
+
+	CString errmsg;
+	if (FileSystem::DirectoryExists(m_unpackDir) &&
+		!FileSystem::DeleteDirectoryWithContent(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not delete temporary directory %s: %s", *m_unpackDir, *errmsg);
+	}
+
+	if (m_finalDirCreated)
+	{
+		FileSystem::DeleteDirectory(m_finalDir);
+	}
+}
+
+void DirectUnpack::SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel)
+{
+	m_progressLabel = progressLabel;
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(progressLabel);
+	}
+}
+
+void DirectUnpack::AddExtraTime(NzbInfo* nzbInfo)
+{
+	if (m_extraStartTime)
+	{
+		int extraTime = (int)(Util::CurrentTime() - m_extraStartTime);
+		nzbInfo->SetUnpackSec(nzbInfo->GetUnpackSec() + extraTime);
+		nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() + extraTime);
+		m_extraStartTime = 0;
+	}
+}
+
+bool DirectUnpack::IsArchiveFilename(const char* filename)
+{
+	if (Util::EndsWith(filename, ".rar", false))
+	{
+		return true;
+	}
+
+	RegEx regExRarMultiSeq(".*\\.[r-z][0-9][0-9]$");
+	return regExRarMultiSeq.Match(filename);
+}

daemon/postprocess/DirectUnpack.h

@@ -0,0 +1,86 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef DIRECTUNPACK_H
+#define DIRECTUNPACK_H
+
+#include "Log.h"
+#include "Thread.h"
+#include "DownloadInfo.h"
+#include "Script.h"
+
+class DirectUnpack : public Thread, public ScriptController
+{
+public:
+	virtual void Run();
+	static void StartJob(NzbInfo* nzbInfo);
+	void Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo);
+	void NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	static bool IsArchiveFilename(const char* filename);
+
+protected:
+	virtual bool ReadLine(char* buf, int bufSize, FILE* stream);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+
+private:
+	class ParamList : public std::vector<CString>
+	{
+	public:
+		bool Exists(const char* param) { return std::find(begin(), end(), param) != end(); }
+	};
+
+	typedef std::deque<CString> ArchiveList;
+
+	int m_nzbId;
+	CString m_name;
+	CString m_infoName;
+	CString m_infoNameUp;
+	CString m_destDir;
+	CString m_finalDir;
+	CString m_unpackDir;
+	CString m_unpackExtendedDir;
+	CString m_password;
+	CString m_waitingFile;
+	CString m_progressLabel;
+	bool m_allOkMessageReceived = false;
+	bool m_unpackOk = false;
+	bool m_finalDirCreated = false;
+	bool m_nzbCompleted = false;
+	Mutex m_volumeMutex;
+	ArchiveList m_archives;
+	bool m_processed = false;
+	bool m_unpacking = false;
+	time_t m_extraStartTime = 0;
+	ArchiveList m_extractedArchives;
+
+	void CreateUnpackDir();
+	void FindArchiveFiles();
+	void ExecuteUnrar(const char* archiveName);
+	bool PrepareCmdParams(const char* command, ParamList* params, const char* infoName);
+	void WaitNextVolume(const char* filename);
+	void Cleanup();
+	bool IsMainArchive(const char* filename);
+	void SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel);
+	void AddExtraTime(NzbInfo* nzbInfo);
+};
+
+#endif

daemon/postprocess/ParChecker.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -106,6 +106,11 @@ private:
 	volatile bool m_working = false;
 };
 
+class RepairCreatorPacket : public Par2::CreatorPacket
+{
+	friend class ParChecker;
+};
+
 Par2::Result Repairer::PreProcess(const char *parFilename)
 {
 	BString<100> memParam("-m%i", g_Options->GetParBuffer());
@@ -368,6 +373,7 @@ int ParChecker::StreamBuf::overflow(int ch)
 
 void ParChecker::Cleanup()
 {
+	Guard guard(m_repairerMutex);
 	m_repairer.reset();
 	m_queuedParFiles.clear();
 	m_processedFiles.clear();
@@ -408,14 +414,14 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 
 		if (!IsStopped())
 		{
-			BString<1024> fullParFilename( "%s%c%s", *m_destDir, (int)PATH_SEPARATOR, *parFilename);
+			BString<1024> fullParFilename( "%s%c%s", *m_destDir, PATH_SEPARATOR, *parFilename);
 
 			int baseLen = 0;
 			ParParser::ParseParFilename(parFilename, true, &baseLen, nullptr);
 			BString<1024> infoName;
 			infoName.Set(parFilename, baseLen);
 
-			BString<1024> parInfoName("%s%c%s", *m_nzbName, (int)PATH_SEPARATOR, *infoName);
+			BString<1024> parInfoName("%s%c%s", *m_nzbName, PATH_SEPARATOR, *infoName);
 			SetInfoName(parInfoName);
 
 			EStatus status = RunParCheck(fullParFilename);
@@ -425,11 +431,6 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 			{
 				allStatus = status;
 			}
-
-			if (g_Options->GetBrokenLog())
-			{
-				WriteBrokenLog(status);
-			}
 		}
 	}
 
@@ -464,6 +465,9 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		return psFailed;
 	}
 
+	CString creator = GetPacketCreator();
+	info("Recovery files created by: %s", creator.Empty() ? "<unknown program>" : *creator);
+
 	m_stage = ptVerifyingSources;
 	res = GetRepairer()->Process(false);
 
@@ -580,7 +584,8 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		{
 			m_errMsg = Par2CmdLineErrStr[res];
 		}
-		PrintMessage(Message::mkError, "Repair failed for %s: %s", *m_infoName, *m_errMsg);
+		PrintMessage(Message::mkError, "Repair failed for %s: %s. Recovery files created by: %s",
+			*m_infoName, *m_errMsg, creator.Empty() ? "<unknown program>" : *creator);
 	}
 
 	Cleanup();
@@ -594,7 +599,10 @@ int ParChecker::PreProcessPar()
 	{
 		Cleanup();
 
-		m_repairer = std::make_unique<Repairer>(this);
+		{
+			Guard guard(m_repairerMutex);
+			m_repairer = std::make_unique<Repairer>(this);
+		}
 
 		res = GetRepairer()->PreProcess(m_parFilename);
 		debug("ParChecker: PreProcess-result=%i", res);
@@ -814,7 +822,7 @@ bool ParChecker::AddSplittedFragments()
 	DirBrowser dir(m_destDir);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") && !IsParredFile(filename) && !IsProcessedFile(filename))
+		if (IsParredFile(filename) && !IsProcessedFile(filename))
 		{
 			for (Par2::Par2RepairerSourceFile *sourcefile : GetRepairer()->sourcefiles)
 			{
@@ -942,8 +950,7 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d
 	DirBrowser dir(directory);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") &&
-			(externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename))))
+		if (externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename)))
 		{
 			BString<1024> fullfilename("%s%c%s", directory, PATH_SEPARATOR, filename);
 			extrafiles.emplace_back(*fullfilename, FileSystem::FileSize(fullfilename));
@@ -1186,49 +1193,14 @@ void ParChecker::CheckEmptyFiles()
 
 void ParChecker::Cancel()
 {
-	GetRepairer()->cancelled = true;
-	QueueChanged();
-}
-
-void ParChecker::WriteBrokenLog(EStatus status)
-{
-	BString<1024> brokenLogName("%s%c_brokenlog.txt", *m_destDir, (int)PATH_SEPARATOR);
-
-	if (status != psRepairNotNeeded || FileSystem::FileExists(brokenLogName))
 	{
-		DiskFile file;
-		if (file.Open(brokenLogName, DiskFile::omAppend))
+		Guard guard(m_repairerMutex);
+		if (m_repairer)
 		{
-			if (status == psFailed)
-			{
-				if (IsStopped())
-				{
-					file.Print("Repair cancelled for %s\n", *m_infoName);
-				}
-				else
-				{
-					file.Print("Repair failed for %s: %s\n", *m_infoName, *m_errMsg);
-				}
-			}
-			else if (status == psRepairPossible)
-			{
-				file.Print("Repair possible for %s\n", *m_infoName);
-			}
-			else if (status == psRepaired)
-			{
-				file.Print("Successfully repaired %s\n", *m_infoName);
-			}
-			else if (status == psRepairNotNeeded)
-			{
-				file.Print("Repair not needed for %s\n", *m_infoName);
-			}
-			file.Close();
-		}
-		else
-		{
-			PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
+			m_repairer->GetRepairer()->cancelled = true;
 		}
 	}
+	QueueChanged();
 }
 
 void ParChecker::SaveSourceList()
@@ -1399,7 +1371,7 @@ bool ParChecker::VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32
 	{
 		const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 		Par2::u32 blockCrc = entry->crc;
-		parCrc = i == 0 ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+		parCrc = i == 0 ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 	}
 	debug("Block-CRC: %x, filename: %s", parCrc, FileSystem::BaseFileName(sourceFile->GetTargetFile()->FileName().c_str()));
 
@@ -1477,7 +1449,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 			}
 			const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 			Par2::u32 blockCrc = entry->crc;
-			parCrc = blockStart == i ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+			parCrc = blockStart == i ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 		}
 		else
 		{
@@ -1537,7 +1509,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 
 		if (segment.GetOffset() >= start && segment.GetOffset() + segment.GetSize() <= end)
 		{
-			downloadCrc = !started ? segment.GetCrc() : Util::Crc32Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
+			downloadCrc = !started ? segment.GetCrc() : Crc32::Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
 			started = true;
 		}
 
@@ -1555,7 +1527,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 				return false;
 			}
 
-			downloadCrc = Util::Crc32Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
+			downloadCrc = Crc32::Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
 
 			break;
 		}
@@ -1576,21 +1548,37 @@ bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, ui
 	}
 
 	CharBuffer buffer(1024 * 64);
-	uint32 downloadCrc = 0xFFFFFFFF;
+	Crc32 downloadCrc;
 
 	int cnt = buffer.Size();
 	while (cnt == buffer.Size() && start < end)
 	{
 		int needBytes = end - start + 1 > buffer.Size() ? buffer.Size() : (int)(end - start + 1);
 		cnt = (int)file.Read(buffer, needBytes);
-		downloadCrc = Util::Crc32m(downloadCrc, (uchar*)(char*)buffer, cnt);
+		downloadCrc.Append((uchar*)(char*)buffer, cnt);
 		start += cnt;
 	}
 
-	downloadCrc ^= 0xFFFFFFFF;
-
-	*downloadCrcOut = downloadCrc;
+	*downloadCrcOut = downloadCrc.Finish();
 	return true;
 }
 
+CString ParChecker::GetPacketCreator()
+{
+	Par2::CREATORPACKET* creatorpacket;
+	if (GetRepairer()->creatorpacket &&
+		(creatorpacket = (Par2::CREATORPACKET*)(((RepairCreatorPacket*)GetRepairer()->creatorpacket)->packetdata)))
+	{
+		int len = (int)(creatorpacket->header.length - sizeof(Par2::PACKET_HEADER));
+		BString<1024> creator;
+		if (len > 0)
+		{
+			creator.Set((const char*)creatorpacket->client, len);
+		}
+		return *creator;
+	}
+
+	return nullptr;
+}
+
 #endif

daemon/postprocess/ParChecker.h

@@ -186,6 +186,7 @@ private:
 	StreamBuf m_parErrStream{this, Message::mkError};
 	std::ostream m_parCout{&m_parOutStream};
 	std::ostream m_parCerr{&m_parErrStream};
+	Mutex m_repairerMutex;
 
 	// "m_repairer" should be of type "Par2::Par2Repairer", however to prevent the
 	// including of libpar2-headers into this header-file we use an empty abstract class.
@@ -204,7 +205,6 @@ private:
 	bool AddDupeFiles();
 	bool AddExtraFiles(bool onlyMissing, bool externalDir, const char* directory);
 	bool IsProcessedFile(const char* filename);
-	void WriteBrokenLog(EStatus status);
 	void SaveSourceList();
 	void DeleteLeftovers();
 	void signal_filename(std::string str);
@@ -220,6 +220,7 @@ private:
 		uint32* downloadCrc);
 	bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrc);
 	void CheckEmptyFiles();
+	CString GetPacketCreator();
 
 	friend class Repairer;
 };

daemon/postprocess/ParParser.cpp

@@ -44,7 +44,7 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 			bool exists = false;
 			for (CString& filename2 : fileList)
 			{
-				exists = SameParCollection(filename, filename2);
+				exists = SameParCollection(filename, filename2, true);
 				if (exists)
 				{
 					break;
@@ -59,11 +59,11 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 	return fileList && !fileList->empty();
 }
 
-bool ParParser::SameParCollection(const char* filename1, const char* filename2)
+bool ParParser::SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames)
 {
 	int baseLen1 = 0, baseLen2 = 0;
-	return ParseParFilename(filename1, false, &baseLen1, nullptr) &&
-		ParseParFilename(filename2, false, &baseLen2, nullptr) &&
+	return ParseParFilename(filename1, confirmedFilenames, &baseLen1, nullptr) &&
+		ParseParFilename(filename2, confirmedFilenames, &baseLen2, nullptr) &&
 		baseLen1 == baseLen2 &&
 		!strncasecmp(filename1, filename2, baseLen1);
 }

daemon/postprocess/ParParser.h

@@ -31,7 +31,7 @@ public:
 
 	static bool FindMainPars(const char* path, ParFileList* fileList);
 	static bool ParseParFilename(const char* parFilename, bool confirmedFilename, int* baseNameLen, int* blocks);
-	static bool SameParCollection(const char* filename1, const char* filename2);
+	static bool SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames);
 };
 
 #endif

daemon/postprocess/PrePostProcessor.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -34,6 +34,7 @@
 #include "NzbFile.h"
 #include "QueueScript.h"
 #include "ParParser.h"
+#include "DirectUnpack.h"
 
 PrePostProcessor::PrePostProcessor()
 {
@@ -51,7 +52,7 @@ void PrePostProcessor::Run()
 		usleep(20 * 1000);
 	}
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetReloadQueue())
+	if (g_Options->GetServerMode())
 	{
 		SanitisePostQueue();
 	}
@@ -76,7 +77,7 @@ void PrePostProcessor::WaitJobs()
 {
 	debug("PrePostProcessor: waiting for jobs to complete");
 
-	// wait 5 seconds until all jobs gracefully finish
+	// wait 5 seconds until all post-processing jobs gracefully finish
 	time_t waitStart = Util::CurrentTime();
 	while (Util::CurrentTime() < waitStart + 5)
 	{
@@ -91,7 +92,7 @@ void PrePostProcessor::WaitJobs()
 		usleep(200 * 1000);
 	}
 
-	// kill remaining jobs; not safe but we can't wait any longer
+	// kill remaining post-processing jobs; not safe but we can't wait any longer
 	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 		for (NzbInfo* postJob : m_activeJobs)
@@ -107,13 +108,41 @@ void PrePostProcessor::WaitJobs()
 		}
 	}
 
+	// wait 5 seconds until direct unpack threads gracefully finish
+	waitStart = Util::CurrentTime();
+	while (Util::CurrentTime() < waitStart + 5)
+	{
+		{
+			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+			if (std::find_if(downloadQueue->GetQueue()->begin(),
+				downloadQueue->GetQueue()->end(),
+				[](const std::unique_ptr<NzbInfo>& nzbInfo)
+				{
+					return nzbInfo->GetUnpackThread() != nullptr;
+				}) == downloadQueue->GetQueue()->end())
+			{
+				break;
+			}
+		}
+		usleep(200 * 1000);
+	}
+
+	// disconnect remaining direct unpack jobs
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			nzbInfo->SetUnpackThread(nullptr);
+		}
+	}
+
 	debug("PrePostProcessor: Jobs are completed");
 }
 
 void PrePostProcessor::Stop()
 {
 	Thread::Stop();
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
 	for (NzbInfo* postJob : m_activeJobs)
 	{
@@ -122,6 +151,14 @@ void PrePostProcessor::Stop()
 			postJob->GetPostInfo()->GetPostThread()->Stop();
 		}
 	}
+
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		if (nzbInfo->GetUnpackThread())
+		{
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->Stop(downloadQueue, nzbInfo);
+		}
+	}
 }
 
 /**
@@ -168,6 +205,10 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 	{
 		NzbAdded(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
+	else if (queueAspect->action == DownloadQueue::eaNzbNamed)
+	{
+		g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeNzbNamed);
+	}
 	else if (queueAspect->action == DownloadQueue::eaNzbDeleted &&
 		queueAspect->nzbInfo->GetDeleting() &&
 		!queueAspect->nzbInfo->GetPostInfo() &&
@@ -182,9 +223,9 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 	else if ((queueAspect->action == DownloadQueue::eaFileCompleted ||
 		queueAspect->action == DownloadQueue::eaFileDeleted))
 	{
-		if (queueAspect->action == DownloadQueue::eaFileCompleted && !queueAspect->nzbInfo->GetPostInfo())
+		if (queueAspect->action == DownloadQueue::eaFileCompleted)
 		{
-			g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeFileDownloaded);
+			FileDownloaded(queueAspect->downloadQueue, queueAspect->nzbInfo, queueAspect->fileInfo);
 		}
 
 #ifndef DISABLE_PARCHECK
@@ -206,7 +247,7 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			 queueAspect->fileInfo->GetDupeDeleted()) &&
 			queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() != NzbInfo::dsHealth &&
 			!queueAspect->nzbInfo->GetPostInfo() &&
-			IsNzbFileCompleted(queueAspect->nzbInfo, true))
+			queueAspect->nzbInfo->IsDownloadCompleted(true))
 		{
 			queueAspect->nzbInfo->PrintMessage(Message::mkInfo,
 				"Collection %s completely downloaded", queueAspect->nzbInfo->GetName());
@@ -217,7 +258,7 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			(queueAspect->action == DownloadQueue::eaFileCompleted &&
 			 queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() > NzbInfo::dsNone)) &&
 			!queueAspect->nzbInfo->GetPostInfo() &&
-			IsNzbFileCompleted(queueAspect->nzbInfo, false))
+			queueAspect->nzbInfo->IsDownloadCompleted(false))
 		{
 			queueAspect->nzbInfo->PrintMessage(Message::mkInfo,
 				"Collection %s deleted from queue", queueAspect->nzbInfo->GetName());
@@ -238,8 +279,13 @@ void PrePostProcessor::NzbAdded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 {
 	if (g_Options->GetParCheck() != Options::pcForce)
 	{
-		downloadQueue->EditEntry(nzbInfo->GetId(),
-			DownloadQueue::eaGroupPauseExtraPars, nullptr);
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupPauseExtraPars, nullptr);
+	}
+
+	if (g_Options->GetReorderFiles() && nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Reordering files for %s", nzbInfo->GetName());
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr);
 	}
 
 	if (nzbInfo->GetDeleteStatus() == NzbInfo::dsDupe ||
@@ -263,7 +309,7 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 		g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeNzbDeleted);
 	}
 
-	if (!nzbInfo->GetPostInfo() && g_Options->GetDecode())
+	if (!nzbInfo->GetPostInfo() && !g_Options->GetRawArticle() && !g_Options->GetSkipWrite())
 	{
 		nzbInfo->PrintMessage(Message::mkInfo, "Queueing %s for post-processing", nzbInfo->GetName());
 
@@ -277,16 +323,33 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 			nzbInfo->SetParStatus(NzbInfo::psSkipped);
 		}
 
+		if (nzbInfo->GetUnpackThread())
+		{
+			nzbInfo->GetPostInfo()->SetWorking(true);
+			m_activeJobs.push_back(nzbInfo);
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDownloaded(downloadQueue, nzbInfo);
+		}
+
 		downloadQueue->Save();
 	}
 	else
 	{
+		if (nzbInfo->GetUnpackThread())
+		{
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDownloaded(downloadQueue, nzbInfo);
+		}
+
 		NzbCompleted(downloadQueue, nzbInfo, true);
 	}
 }
 
 void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 {
+	if (nzbInfo->GetUnpackThread())
+	{
+		((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDeleted(downloadQueue, nzbInfo);
+	}
+
 	if (nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
 	{
 		nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
@@ -355,20 +418,20 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo)
 		// download was cancelled, deleting already downloaded files from disk
 		for (CompletedFile& completedFile: nzbInfo->GetCompletedFiles())
 		{
-			BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName());
+			BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename());
 			if (FileSystem::FileExists(fullFileName))
 			{
-				detail("Deleting file %s", completedFile.GetFileName());
+				detail("Deleting file %s", completedFile.GetFilename());
 				FileSystem::DeleteFile(fullFileName);
 			}
 		}
 
-		// delete .out.tmp-files and _brokenlog.txt
+		// delete .out.tmp-files
 		DirBrowser dir(nzbInfo->GetDestDir());
 		while (const char* filename = dir.Next())
 		{
 			int len = strlen(filename);
-			if ((len > 8 && !strcmp(filename + len - 8, ".out.tmp")) || !strcmp(filename, "_brokenlog.txt"))
+			if (len > 8 && !strcmp(filename + len - 8, ".out.tmp"))
 			{
 				BString<1024> fullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, filename);
 				detail("Deleting file %s", filename);
@@ -377,10 +440,7 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo)
 		}
 
 		// delete old directory (if empty)
-		if (FileSystem::DirEmpty(nzbInfo->GetDestDir()))
-		{
-			FileSystem::RemoveDirectory(nzbInfo->GetDestDir());
-		}
+		FileSystem::DeleteDirectory(nzbInfo->GetDestDir());
 	}
 }
 
@@ -433,7 +493,8 @@ void PrePostProcessor::CleanupJobs(DownloadQueue* downloadQueue)
 		[processor = this, downloadQueue](NzbInfo* postJob)
 		{
 			PostInfo* postInfo = postJob->GetPostInfo();
-			if (!postInfo->GetWorking())
+			if (!postInfo->GetWorking() &&
+				!(postInfo->GetPostThread() && postInfo->GetPostThread()->IsRunning()))
 			{
 				delete postInfo->GetPostThread();
 				postInfo->SetPostThread(nullptr);
@@ -511,10 +572,12 @@ NzbInfo* PrePostProcessor::PickNextJob(DownloadQueue* downloadQueue, bool allowP
 	{
 		if (nzbInfo1->GetPostInfo() && !nzbInfo1->GetPostInfo()->GetWorking() &&
 			!g_QueueScriptCoordinator->HasJob(nzbInfo1->GetId(), nullptr) &&
+			nzbInfo1->GetDirectUnpackStatus() != NzbInfo::nsRunning &&
 			(!nzbInfo || nzbInfo1->GetPriority() > nzbInfo->GetPriority()) &&
 			(!g_Options->GetPausePostProcess() || nzbInfo1->GetForcePriority()) &&
 			(allowPar || !nzbInfo1->GetPostInfo()->GetNeedParCheck()) &&
-			IsNzbFileCompleted(nzbInfo1, true))
+			(std::find(m_activeJobs.begin(), m_activeJobs.end(), nzbInfo1) == m_activeJobs.end()) &&
+			nzbInfo1->IsDownloadCompleted(true))
 		{
 			nzbInfo = nzbInfo1;
 		}
@@ -541,8 +604,6 @@ void PrePostProcessor::CheckPostQueue()
 			break;
 		}
 
-		Util::SetStandByMode(false);
-
 		m_activeJobs.push_back(postJob);
 
 		PostInfo* postInfo = postJob->GetPostInfo();
@@ -561,6 +622,8 @@ void PrePostProcessor::CheckPostQueue()
 		downloadQueue->Save();
 		UpdatePauseState();
 	}
+
+	Util::SetStandByMode(m_activeJobs.empty());
 }
 
 void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo, bool allowPar)
@@ -643,7 +706,7 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 	}
 #endif
 
-	NzbParameter* unpackParameter = postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:", false);
+	NzbParameter* unpackParameter = postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:");
 	bool wantUnpack = !(unpackParameter && !strcasecmp(unpackParameter->GetValue(), "no"));
 	bool unpack = wantUnpack && postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone &&
 		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone;
@@ -730,7 +793,7 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post
 
 	nzbInfo->LeavePostProcess();
 
-	if (IsNzbFileCompleted(nzbInfo, true))
+	if (nzbInfo->IsDownloadCompleted(true))
 	{
 		NzbCompleted(downloadQueue, nzbInfo, false);
 	}
@@ -738,25 +801,6 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post
 	m_queuedJobs--;
 }
 
-bool PrePostProcessor::IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars)
-{
-	if (nzbInfo->GetActiveDownloads())
-	{
-		return false;
-	}
-
-	for (FileInfo* fileInfo : nzbInfo->GetFileList())
-	{
-		if ((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) &&
-			!fileInfo->GetDeleted())
-		{
-			return false;
-		}
-	}
-
-	return true;
-}
-
 void PrePostProcessor::UpdatePauseState()
 {
 	bool needPause = false;
@@ -837,6 +881,11 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 						postInfo->GetPostThread()->Stop();
 						ok = true;
 					}
+					else if (postInfo->GetNzbInfo()->GetUnpackThread())
+					{
+						((DirectUnpack*)postInfo->GetNzbInfo()->GetUnpackThread())->NzbDeleted(downloadQueue, postInfo->GetNzbInfo());
+						ok = true;
+					}
 					else
 					{
 						error("Internal error in PrePostProcessor::QueueDelete");
@@ -846,7 +895,16 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 				{
 					postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
 						"Deleting queued post-job %s", postInfo->GetNzbInfo()->GetName());
+
 					JobCompleted(downloadQueue, postInfo);
+
+					m_activeJobs.erase(std::remove_if(m_activeJobs.begin(), m_activeJobs.end(),
+						[postInfo](NzbInfo* postJob)
+						{
+							return postInfo == postJob->GetPostInfo();
+						}),
+						m_activeJobs.end());
+
 					ok = true;
 				}
 				break;
@@ -854,5 +912,39 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 		}
 	}
 
+	if (ok)
+	{
+		downloadQueue->Save();
+	}
+
 	return ok;
 }
+
+void PrePostProcessor::FileDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileInfo* fileInfo)
+{
+	if (!nzbInfo->GetPostInfo())
+	{
+		g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeFileDownloaded);
+	}
+
+	if (g_Options->GetDirectUnpack() && !g_Options->GetRawArticle() && !g_Options->GetSkipWrite())
+	{
+		bool allowPar;
+		if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsNone &&
+			nzbInfo->GetDirectRenameStatus() != NzbInfo::tsRunning &&
+			DirectUnpack::IsArchiveFilename(fileInfo->GetFilename()) &&
+			CanRunMoreJobs(&allowPar))
+		{
+			NzbParameter* unpackParameter = nzbInfo->GetParameters()->Find("*Unpack:");
+			bool wantUnpack = !(unpackParameter && !strcasecmp(unpackParameter->GetValue(), "no"));
+			if (wantUnpack && nzbInfo->GetFailedArticles() == 0)
+			{
+				DirectUnpack::StartJob(nzbInfo);
+			}
+		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsRunning)
+		{
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->FileDownloaded(downloadQueue, fileInfo);
+		}
+	}
+}

daemon/postprocess/PrePostProcessor.h

@@ -61,8 +61,8 @@ private:
 	bool PostQueueDelete(DownloadQueue* downloadQueue, IdList* idList);
 	void DownloadQueueUpdate(void* aspect);
 	void DeleteCleanup(NzbInfo* nzbInfo);
-	bool IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars);
 	void WaitJobs();
+	void FileDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileInfo* fileInfo);
 };
 
 extern PrePostProcessor* g_PrePostProcessor;

daemon/postprocess/RarReader.cpp

@@ -268,6 +268,12 @@ RarVolume::RarBlock RarVolume::ReadRar3Block(DiskFile& file)
 	uint16 size = ((uint16)buf[6] << 8) + buf[5];
 
 	uint32 blocksize = size;
+	if (m_encrypted)
+	{
+		// Align to 16 bytes
+		blocksize = (blocksize + ((~blocksize + 1) & (16 - 1)));
+	}
+
 	block.trailsize = blocksize - sizeof(buf);
 
 	uint8 addbuf[4];

daemon/postprocess/RarRenamer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -202,7 +202,8 @@ void RarRenamer::MakeSets()
 	// find first volumes and create initial incomplete sets
 	for (RarVolume& volume : m_volumes)
 	{
-		if (!volume.GetFiles()->empty() && volume.GetVolumeNo() == 0)
+		if (!volume.GetFiles()->empty() && volume.GetVolumeNo() == 0 &&
+			!volume.GetFiles()->front().GetSplitBefore())
 		{
 			m_sets.push_back({&volume});
 		}
@@ -217,6 +218,8 @@ void RarRenamer::MakeSets()
 			while (found)
 			{
 				found = false;
+				std::vector<RarVolume*> candidates;
+
 				RarVolume* lastVolume = set.back();
 				for (RarVolume& volume : *volumes)
 				{
@@ -224,20 +227,47 @@ void RarRenamer::MakeSets()
 						volume.GetVolumeNo() == lastVolume->GetVolumeNo() + 1 &&
 						volume.GetVersion() == lastVolume->GetVersion() &&
 						lastVolume->GetHasNextVolume() &&
-						((volume.GetFiles()->at(0).GetSplitBefore() &&
-						 lastVolume->GetFiles()->at(0).GetSplitAfter() &&
-						 !strcmp(volume.GetFiles()->at(0).GetFilename(), lastVolume->GetFiles()->at(0).GetFilename())) ||
-						 (!volume.GetFiles()->at(0).GetSplitBefore() && !lastVolume->GetFiles()->at(0).GetSplitAfter())))
+						((volume.GetFiles()->front().GetSplitBefore() &&
+						 lastVolume->GetFiles()->back().GetSplitAfter() &&
+						 !strcmp(volume.GetFiles()->front().GetFilename(), lastVolume->GetFiles()->back().GetFilename())) ||
+						 (!volume.GetFiles()->front().GetSplitBefore() && !lastVolume->GetFiles()->back().GetSplitAfter())))
 					{
-						debug("   adding %s", FileSystem::BaseFileName(volume.GetFilename()));
-						set.push_back(&volume);
-						found = true;
-						break;
+						debug("   found candidate %s", FileSystem::BaseFileName(volume.GetFilename()));
+						candidates.push_back(&volume);
 					}
 				}
+
+				RarVolume* nextVolume = nullptr;
+
+				if (candidates.size() > 1)
+				{
+					for (RarVolume* volume : candidates)
+					{
+						if (SameArchiveName(FileSystem::BaseFileName(set[0]->GetFilename()),
+							FileSystem::BaseFileName(volume->GetFilename()), set[0]->GetNewNaming()))
+						{
+							nextVolume = volume;
+							break;
+						}
+					}
+				}
+
+				if (!nextVolume && !candidates.empty())
+				{
+					nextVolume = candidates.front();
+				}
+
+				if (nextVolume)
+				{
+					debug("   adding %s", FileSystem::BaseFileName(nextVolume->GetFilename()));
+					set.push_back(nextVolume);
+					found = true;
+				}
 			}
 
-			bool completed = !set.back()->GetHasNextVolume();
+			RarVolume* lastVolume = set.back();
+			bool completed = !lastVolume->GetHasNextVolume() &&
+				(lastVolume->GetFiles()->empty() || !lastVolume->GetFiles()->back().GetSplitAfter());
 
 			return !completed;
 		}),
@@ -254,6 +284,42 @@ void RarRenamer::MakeSets()
 	}
 }
 
+bool RarRenamer::SameArchiveName(const char* filename1, const char* filename2, bool newNaming)
+{
+	if (strlen(filename1) != strlen(filename2))
+	{
+		return false;
+	}
+
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+
+	if (!(ext1 && ext2 && strlen(ext1) == strlen(ext2)))
+	{
+		return false;
+	}
+
+	if (newNaming)
+	{
+		if (ext1 == filename1 || ext2 == filename2)
+		{
+			return false;
+		}
+		BString<1024> name1, name2;
+		name1.Set(filename1, (int)(ext1 - filename1));
+		name2.Set(filename2, (int)(ext2 - filename2));
+		ext1 = strrchr(name1, '.');
+		ext2 = strrchr(name2, '.');
+		return ext1 && ext2 && strlen(ext1) == strlen(ext2) &&
+			!strncmp(ext1, ".part", 5) && !strncmp(ext2, ".part", 5) &&
+			!strncmp(name1, name2, ext1 - name1);
+	}
+	else
+	{
+		return !strncmp(filename1, filename2, ext1 - filename1);
+	}
+}
+
 bool RarRenamer::IsSetProperlyNamed(RarVolumeSet& set)
 {
 	RegEx regExPart(".*.part([0-9]+)\\.rar$");

daemon/postprocess/RarRenamer.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -76,6 +76,7 @@ private:
 	void MakeSets();
 	bool IsSetProperlyNamed(RarVolumeSet& set);
 	RarFile* FindMainFile(RarVolumeSet& set);
+	static bool SameArchiveName(const char* filename1, const char* filename2, bool newNaming);
 };
 
 #endif

daemon/postprocess/Rename.cpp

@@ -137,7 +137,7 @@ void RenameController::ExecRename(const char* destDir, const char* finalDir, con
 		m_rarRenamer.SetInfoName(nzbName);
 		m_rarRenamer.SetIgnoreExt(g_Options->GetUnpackIgnoreExt());
 
-		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password", false);
+		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password");
 		if (parameter)
 		{
 			m_rarRenamer.SetPassword(parameter->GetValue());
@@ -207,13 +207,13 @@ void RenameController::UpdateRarRenameProgress()
 /**
 *  Update file name in the CompletedFiles-list of NZBInfo
 */
-void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFileName)
+void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFilename)
 {
 	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
 	{
-		if (!strcasecmp(completedFile.GetFileName(), oldFilename))
+		if (!strcasecmp(completedFile.GetFilename(), oldFilename))
 		{
-			completedFile.SetFileName(newFileName);
+			completedFile.SetFilename(newFilename);
 			break;
 		}
 	}

daemon/postprocess/Rename.h

@@ -78,8 +78,8 @@ private:
 	protected:
 		virtual void UpdateProgress() { m_owner->UpdateRarRenameProgress(); }
 		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
-		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName)
-			{ m_owner->RegisterRenamedFile(oldFilename, newFileName); }
+		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFilename)
+			{ m_owner->RegisterRenamedFile(oldFilename, newFilename); }
 		virtual bool IsStopped() { return m_owner->IsStopped(); };
 	private:
 		RenameController* m_owner;
@@ -92,7 +92,7 @@ private:
 
 	void ExecRename(const char* destDir, const char* finalDir, const char* nzbName);
 	void RenameCompleted();
-	void RegisterRenamedFile(const char* oldFilename, const char* newFileName);
+	void RegisterRenamedFile(const char* oldFilename, const char* newFilename);
 };
 
 #endif

daemon/postprocess/Repair.cpp

@@ -74,7 +74,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 
 	for (CompletedFile& completedFile2 : m_postInfo->GetNzbInfo()->GetCompletedFiles())
 	{
-		if (!strcasecmp(completedFile2.GetFileName(), filename))
+		if (!strcasecmp(completedFile2.GetFilename(), filename))
 		{
 			completedFile = &completedFile2;
 			break;
@@ -85,7 +85,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 		return ParChecker::fsUnknown;
 	}
 
-	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFileName()), completedFile->GetCrc(), (int)completedFile->GetStatus());
+	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFilename()), completedFile->GetCrc(), (int)completedFile->GetStatus());
 
 	*crc = completedFile->GetCrc();
 
@@ -246,7 +246,7 @@ bool RepairController::AddPar(FileInfo* fileInfo, bool deleted)
 	bool sameCollection = fileInfo->GetNzbInfo() == m_parChecker.GetPostInfo()->GetNzbInfo();
 	if (sameCollection && !deleted)
 	{
-		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), (int)PATH_SEPARATOR, fileInfo->GetFilename());
+		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename());
 		m_parChecker.AddParFile(fullFilename);
 	}
 	else
@@ -320,7 +320,7 @@ bool RepairController::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename
 	}
 
 	std::sort(availableBlocks.begin(), availableBlocks.end(),
-		[](BlockInfo& block1, BlockInfo& block2)
+		[](const BlockInfo& block1, const BlockInfo& block2)
 		{
 			return block1.m_blockCount < block2.m_blockCount;
 		});
@@ -355,7 +355,7 @@ bool RepairController::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename
 		{
 			if (blockInfo.m_fileInfo->GetPaused())
 			{
-				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), (int)PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
+				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
 				blockInfo.m_fileInfo->SetPaused(false);
 				blockInfo.m_fileInfo->SetExtraPriority(true);
 			}
@@ -410,7 +410,8 @@ void RepairController::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo,
 
 			if (exactParName)
 			{
-				useFile = ParParser::SameParCollection(fileInfo->GetFilename(), FileSystem::BaseFileName(parFilename));
+				useFile = ParParser::SameParCollection(fileInfo->GetFilename(),
+					FileSystem::BaseFileName(parFilename), fileInfo->GetFilenameConfirmed());
 			}
 			else if (strictParName)
 			{

daemon/postprocess/Unpack.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -50,17 +50,7 @@ void UnpackController::StartJob(PostInfo* postInfo)
 void UnpackController::Run()
 {
 	time_t start = Util::CurrentTime();
-
-	m_cleanedUpDisk = false;
-	m_finalDirCreated = false;
 	m_unpackOk = true;
-	m_unpackStartError = false;
-	m_unpackSpaceError = false;
-	m_unpackDecryptError = false;
-	m_unpackPasswordError = false;
-	m_autoTerminated = false;
-	m_passListTried = false;
-
 	bool unpack;
 
 	{
@@ -70,10 +60,10 @@ void UnpackController::Run()
 		m_finalDir = m_postInfo->GetNzbInfo()->GetFinalDir();
 		m_name = m_postInfo->GetNzbInfo()->GetName();
 
-		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:", false);
+		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:");
 		unpack = !(parameter && !strcasecmp(parameter->GetValue(), "no"));
 
-		parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password", false);
+		parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password");
 		if (parameter)
 		{
 			m_password = parameter->GetValue();
@@ -103,7 +93,7 @@ void UnpackController::Run()
 			m_postInfo->GetLastUnpackStatus() == (int)NzbInfo::usPassword ?
 				 "%s failed: checksum error in the encrypted file. Corrupt file or wrong password." : "%s failed.",
 			*m_infoNameUp);
-		m_postInfo->GetNzbInfo()->SetUnpackStatus((NzbInfo::EUnpackStatus)m_postInfo->GetLastUnpackStatus());
+		m_postInfo->GetNzbInfo()->SetUnpackStatus((NzbInfo::EPostUnpackStatus)m_postInfo->GetLastUnpackStatus());
 	}
 	else if (unpack && hasFiles)
 	{
@@ -113,7 +103,29 @@ void UnpackController::Run()
 
 		if (m_hasRarFiles)
 		{
-			UnpackArchives(upUnrar, false);
+			if (m_hasNotUnpackedRarFiles || m_unpackDirCreated)
+			{
+				if (m_postInfo->GetNzbInfo()->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+				{
+					if (m_unpackDirCreated)
+					{
+						PrintMessage(Message::mkWarning, "Could not find files unpacked by direct unpack, unpacking all files again");
+					}
+					else
+					{
+						PrintMessage(Message::mkInfo, "Found archive files not processed by direct unpack, unpacking all files again");
+					}
+				}
+
+				// Discard info about extracted archives to prevent reusing on next unpack attempt
+				m_postInfo->GetExtractedArchives()->clear();
+
+				UnpackArchives(upUnrar, false);
+			}
+			else
+			{
+				PrintMessage(Message::mkInfo, "Using directly unpacked files");
+			}
 		}
 
 		if (m_hasSevenZipFiles && m_unpackOk)
@@ -179,7 +191,7 @@ void UnpackController::UnpackArchives(EUnpacker unpacker, bool multiVolumes)
 		if (!m_unpackOk && m_hasParFiles && !m_unpackPasswordError &&
 			m_postInfo->GetNzbInfo()->GetParStatus() <= NzbInfo::psSkipped)
 		{
-			// for rar4- or 7z-archives try par-check first, before trying password file
+			debug("For rar4- or 7z-archives try par-check first, before trying password file");
 			return;
 		}
 	}
@@ -206,6 +218,7 @@ void UnpackController::UnpackArchives(EUnpacker unpacker, bool multiVolumes)
 			(m_unpackDecryptError || m_unpackPasswordError) &&
 			infile.ReadLine(password, sizeof(password) - 1))
 		{
+			debug("Password line: %s", password);
 			// trim trailing <CR> and <LF>
 			char* end = password + strlen(password) - 1;
 			while (end >= password && (*end == '\n' || *end == '\r')) *end-- = '\0';
@@ -277,7 +290,8 @@ void UnpackController::ExecuteUnrar(const char* password)
 	}
 
 	params.emplace_back("*.rar");
-	params.push_back(FileSystem::MakeExtendedPath(BString<1024>("%s%c", *m_unpackDir, PATH_SEPARATOR), true));
+	m_unpackExtendedDir = FileSystem::MakeExtendedPath(m_unpackDir, true);
+	params.push_back(*BString<1024>("%s%c", *m_unpackExtendedDir, PATH_SEPARATOR));
 	SetArgs(std::move(params));
 	SetLogPrefix("Unrar");
 	ResetEnv();
@@ -291,7 +305,7 @@ void UnpackController::ExecuteUnrar(const char* password)
 	SetProgressLabel("");
 
 	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
-	m_unpackStartError = exitCode == -1;
+	m_unpackStartError = exitCode == -1 && !m_autoTerminated;
 	m_unpackSpaceError = exitCode == 5;
 	m_unpackPasswordError |= exitCode == 11; // only for rar5-archives
 
@@ -346,7 +360,7 @@ void UnpackController::ExecuteSevenZip(const char* password, bool multiVolumes)
 	SetProgressLabel("");
 
 	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
-	m_unpackStartError = exitCode == -1;
+	m_unpackStartError = exitCode == -1 && !m_autoTerminated;
 
 	if (!m_unpackOk && exitCode > 0)
 	{
@@ -604,6 +618,7 @@ void UnpackController::CreateUnpackDir()
 	const char* destDir = !m_finalDir.Empty() ? *m_finalDir : *m_destDir;
 
 	m_unpackDir.Format("%s%c%s", destDir, PATH_SEPARATOR, "_unpack");
+	m_unpackDirCreated = !FileSystem::DirectoryExists(m_unpackDir);
 
 	detail("Unpacking into %s", *m_unpackDir);
 
@@ -616,12 +631,6 @@ void UnpackController::CreateUnpackDir()
 
 void UnpackController::CheckArchiveFiles()
 {
-	m_hasRarFiles = false;
-	m_hasRenamedArchiveFiles = false;
-	m_hasSevenZipFiles = false;
-	m_hasSevenZipMultiFiles = false;
-	m_hasSplittedFiles = false;
-
 	RegEx regExRar(".*\\.rar$");
 	RegEx regExRarMultiSeq(".*\\.[r-z][0-9][0-9]$");
 	RegEx regExSevenZip(".*\\.7z$");
@@ -641,6 +650,10 @@ void UnpackController::CheckArchiveFiles()
 			if (regExRar.Match(filename))
 			{
 				m_hasRarFiles = true;
+				m_hasNotUnpackedRarFiles |= std::find(
+					m_postInfo->GetExtractedArchives()->begin(),
+					m_postInfo->GetExtractedArchives()->end(),
+					filename) == m_postInfo->GetExtractedArchives()->end();
 			}
 			else if (regExSevenZip.Match(filename))
 			{
@@ -731,7 +744,7 @@ bool UnpackController::Cleanup()
 
 	if (!m_unpackOk && m_finalDirCreated)
 	{
-		FileSystem::RemoveDirectory(m_finalDir);
+		FileSystem::DeleteDirectory(m_finalDir);
 	}
 
 	if (m_unpackOk && ok && g_Options->GetUnpackCleanupDisk())
@@ -839,9 +852,9 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 	// Modify unrar messages for better readability:
 	// remove the destination path part from message "Extracting file.xxx"
 	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: Extracting  ", 19) &&
-		!strncmp(text + 19, m_unpackDir, strlen(m_unpackDir)))
+		!strncmp(text + 19, m_unpackExtendedDir, strlen(m_unpackExtendedDir)))
 	{
-		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackDir) + 1);
+		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackExtendedDir) + 1);
 	}
 
 	m_postInfo->GetNzbInfo()->AddMessage(kind, msgText);

daemon/postprocess/Unpack.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -65,27 +65,30 @@ private:
 	CString m_destDir;
 	CString m_finalDir;
 	CString m_unpackDir;
+	CString m_unpackExtendedDir;
 	CString m_password;
-	bool m_interDir;
-	bool m_allOkMessageReceived;
-	bool m_noFilesMessageReceived;
-	bool m_hasParFiles;
-	bool m_hasRarFiles;
-	bool m_hasRenamedArchiveFiles;
-	bool m_hasSevenZipFiles;
-	bool m_hasSevenZipMultiFiles;
-	bool m_hasSplittedFiles;
-	bool m_unpackOk;
-	bool m_unpackStartError;
-	bool m_unpackSpaceError;
-	bool m_unpackDecryptError;
-	bool m_unpackPasswordError;
-	bool m_cleanedUpDisk;
-	bool m_autoTerminated;
 	EUnpacker m_unpacker;
-	bool m_finalDirCreated;
+	bool m_interDir = false;
+	bool m_allOkMessageReceived = false;
+	bool m_noFilesMessageReceived = false;
+	bool m_hasParFiles = false;
+	bool m_hasRarFiles = false;
+	bool m_hasNotUnpackedRarFiles = false;
+	bool m_hasRenamedArchiveFiles = false;
+	bool m_hasSevenZipFiles = false;
+	bool m_hasSevenZipMultiFiles = false;
+	bool m_hasSplittedFiles = false;
+	bool m_unpackOk = false;
+	bool m_unpackStartError = false;
+	bool m_unpackSpaceError = false;
+	bool m_unpackDecryptError = false;
+	bool m_unpackPasswordError = false;
+	bool m_cleanedUpDisk = false;
+	bool m_autoTerminated = false;
+	bool m_finalDirCreated = false;
+	bool m_unpackDirCreated = false;
+	bool m_passListTried = false;
 	FileList m_joinedFiles;
-	bool m_passListTried;
 
 	void ExecuteUnpack(EUnpacker unpacker, const char* password, bool multiVolumes);
 	void ExecuteUnrar(const char* password);

daemon/queue/DirectRenamer.cpp

@@ -0,0 +1,556 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "DirectRenamer.h"
+#include "Options.h"
+#include "FileSystem.h"
+#include "ParParser.h"
+
+#ifndef DISABLE_PARCHECK
+#include "par2cmdline.h"
+#include "par2fileformat.h"
+#include "md5.h"
+#endif
+
+class RenameContentAnalyzer : public ArticleContentAnalyzer
+{
+public:
+	virtual void Reset();
+	virtual void Append(const void* buffer, int len);
+	void Finish();
+	const char* GetHash16k() { return m_hash16k; }
+	bool GetParFile() { return m_parFile; }
+	const char* GetParSetId() { return m_parSetId; }
+
+private:
+#ifndef DISABLE_PARCHECK
+	Par2::MD5Context m_md5Context;
+	char m_signature[sizeof(Par2::PACKET_HEADER)];
+#endif
+	int m_dataSize = 0;
+	CString m_hash16k;
+	CString m_parSetId;
+	bool m_parFile = false;
+};
+
+#ifndef DISABLE_PARCHECK
+class DirectParRepairer : public Par2::Par2Repairer
+{
+public:
+	DirectParRepairer() : Par2::Par2Repairer(m_nout, m_nout) {};
+	friend class DirectParLoader;
+
+private:
+	class NullStreamBuf : public std::streambuf {};
+	NullStreamBuf m_nullbuf;
+	std::ostream m_nout{&m_nullbuf};
+};
+
+class DirectParLoader : public Thread
+{
+public:
+	static void StartLoader(DirectRenamer* owner, NzbInfo* nzbInfo);
+	virtual void Run();
+
+private:
+	typedef std::vector<CString> ParFiles;
+
+	DirectRenamer* m_owner;
+	ParFiles m_parFiles;
+	DirectRenamer::FileHashList m_parHashes;
+	int m_nzbId;
+
+	void LoadParFile(const char* parFile);
+};
+
+void DirectParLoader::StartLoader(DirectRenamer* owner, NzbInfo* nzbInfo)
+{
+	nzbInfo->PrintMessage(Message::mkInfo, "Directly checking renamed files for %s", nzbInfo->GetName());
+
+	DirectParLoader* directParLoader = new DirectParLoader();
+	directParLoader->m_owner = owner;
+	directParLoader->m_nzbId = nzbInfo->GetId();
+
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (completedFile.GetParFile())
+		{
+			directParLoader->m_parFiles.emplace_back(BString<1024>("%s%c%s",
+				nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename()));
+		}
+	}
+
+	directParLoader->SetAutoDestroy(true);
+	directParLoader->Start();
+}
+
+void DirectParLoader::Run()
+{
+	debug("Started DirectParLoader");
+
+	for (CString& parFile : m_parFiles)
+	{
+		LoadParFile(parFile);
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (nzbInfo)
+	{
+		// nzb is still in queue
+		m_owner->RenameFiles(downloadQueue, nzbInfo, &m_parHashes);
+	}
+}
+
+void DirectParLoader::LoadParFile(const char* parFile)
+{
+	DirectParRepairer repairer;
+
+	if (!repairer.LoadPacketsFromFile(parFile))
+	{
+		warn("Could not load par2-file %s", parFile);
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (!nzbInfo)
+	{
+		// nzb isn't in queue anymore
+		return;
+	}
+
+	nzbInfo->PrintMessage(Message::mkInfo, "Loaded par2-file %s for direct-rename", FileSystem::BaseFileName(parFile));
+
+	for (std::pair<const Par2::MD5Hash, Par2::Par2RepairerSourceFile*>& entry : repairer.sourcefilemap)
+	{
+		if (IsStopped())
+		{
+			break;
+		}
+
+		Par2::Par2RepairerSourceFile* sourceFile = entry.second;
+		if (!sourceFile || !sourceFile->GetDescriptionPacket())
+		{
+			nzbInfo->PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", FileSystem::BaseFileName(parFile));
+			return;
+		}
+		std::string filename = Par2::DiskFile::TranslateFilename(sourceFile->GetDescriptionPacket()->FileName());
+		std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print();
+
+		debug("file: %s, hash-16k: %s", filename.c_str(), hash.c_str());
+		m_parHashes.emplace_back(filename.c_str(), hash.c_str());
+	}
+}
+#endif
+
+std::unique_ptr<ArticleContentAnalyzer> DirectRenamer::MakeArticleContentAnalyzer()
+{
+	return std::make_unique<RenameContentAnalyzer>();
+}
+
+void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo,
+	ArticleInfo* articleInfo, ArticleContentAnalyzer* articleContentAnalyzer)
+{
+	debug("Applying analyzer data %s for ", fileInfo->GetFilename());
+
+	RenameContentAnalyzer* contentAnalyzer = (RenameContentAnalyzer*)articleContentAnalyzer;
+	contentAnalyzer->Finish();
+
+	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+
+	// we don't support analyzing of files split into articles smaller than 16KB
+	if (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1)
+	{
+		fileInfo->SetHash16k(contentAnalyzer->GetHash16k());
+		debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), fileInfo->GetHash16k());
+	}
+
+	fileInfo->GetNzbInfo()->PrintMessage(Message::mkDetail, "Detected %s %s", (contentAnalyzer->GetParFile() ? "par2-file" : "non-par2-file"), fileInfo->GetFilename());
+
+	if (fileInfo->GetParFile() != contentAnalyzer->GetParFile())
+	{
+		debug("Changing par2-flag for %s", fileInfo->GetFilename());
+		fileInfo->SetParFile(contentAnalyzer->GetParFile());
+
+		int delta = fileInfo->GetParFile() ? 1 : -1;
+
+		nzbInfo->SetParSize(nzbInfo->GetParSize() + fileInfo->GetSize() * delta);
+		nzbInfo->SetParCurrentSuccessSize(nzbInfo->GetParCurrentSuccessSize() + fileInfo->GetSuccessSize() * delta);
+		nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() +
+			fileInfo->GetFailedSize() * delta + fileInfo->GetMissedSize() * delta);
+		nzbInfo->SetParFailedSize(nzbInfo->GetParFailedSize() + fileInfo->GetMissedSize() * delta);
+		nzbInfo->SetRemainingParCount(nzbInfo->GetRemainingParCount() + 1 * delta);
+
+		if (!fileInfo->GetParFile() && fileInfo->GetPaused())
+		{
+			fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Resuming non-par2-file %s", fileInfo->GetFilename());
+			fileInfo->SetPaused(false);
+		}
+
+		downloadQueue->Save();
+	}
+
+	if (fileInfo->GetParFile())
+	{
+		fileInfo->SetParSetId(contentAnalyzer->GetParSetId());
+		debug("file: %s; setid: %s", fileInfo->GetFilename(), fileInfo->GetParSetId());
+	}
+
+	CheckState(downloadQueue, nzbInfo);
+}
+
+void DirectRenamer::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo)
+{
+	CheckState(downloadQueue, fileInfo->GetNzbInfo());
+}
+
+void DirectRenamer::CheckState(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+#ifndef DISABLE_PARCHECK
+	if (nzbInfo->GetDirectRenameStatus() > NzbInfo::tsRunning)
+	{
+		return;
+	}
+
+	// check if all first articles are successfully downloaded (1)
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (Util::EmptyStr(fileInfo->GetHash16k()) ||
+			(fileInfo->GetParFile() && Util::EmptyStr(fileInfo->GetParSetId())))
+		{
+			return;
+		}
+	}
+
+	// check if all first articles are successfully downloaded (2)
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (Util::EmptyStr(completedFile.GetHash16k()) ||
+			(completedFile.GetParFile() && Util::EmptyStr(completedFile.GetParSetId())))
+		{
+			return;
+		}
+	}
+
+	if (!nzbInfo->GetWaitingPar())
+	{
+		// all first articles downloaded
+		UnpausePars(nzbInfo);
+		nzbInfo->SetWaitingPar(true);
+		downloadQueue->Save();
+	}
+
+	if (nzbInfo->GetWaitingPar() && !nzbInfo->GetLoadingPar())
+	{
+		// check if all par2-files scheduled for downloading already completed
+		FileList::iterator pos = std::find_if(
+			nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+			[](std::unique_ptr<FileInfo>& fileInfo)
+		{
+			return fileInfo->GetExtraPriority();
+		});
+
+		if (pos == nzbInfo->GetFileList()->end())
+		{
+			// all wanted par2-files are downloaded
+			nzbInfo->SetLoadingPar(true);
+			DirectParLoader::StartLoader(this, nzbInfo);
+			return;
+		}
+	}
+#endif
+}
+
+// Unpause smallest par-files from each par-set
+void DirectRenamer::UnpausePars(NzbInfo* nzbInfo)
+{
+	ParFileList parFiles;
+	CollectPars(nzbInfo, &parFiles);
+
+	std::vector<CString> parsets;
+
+	// sort by size
+	std::sort(parFiles.begin(), parFiles.end(),
+		[nzbInfo](const ParFile& parFile1, const ParFile& parFile2)
+		{
+			FileInfo* fileInfo1 = nzbInfo->GetFileList()->Find(const_cast<ParFile&>(parFile1).GetId());
+			FileInfo* fileInfo2 = nzbInfo->GetFileList()->Find(const_cast<ParFile&>(parFile2).GetId());
+			return (!fileInfo1 && fileInfo2) ||
+				(fileInfo1 && fileInfo2 && fileInfo1->GetSize() < fileInfo2->GetSize());
+		});
+
+	// 1. count already downloaded files
+	for (ParFile& parFile : parFiles)
+	{
+		if (parFile.GetCompleted())
+		{
+			parsets.emplace_back(parFile.GetSetId());
+		}
+	}
+
+	// 2. find smallest par-file from each par-set from not yet completely downloaded files
+	for (ParFile& parFile : parFiles)
+	{
+		std::vector<CString>::iterator pos = std::find(parsets.begin(), parsets.end(), parFile.GetSetId());
+		if (pos == parsets.end())
+		{
+			// this par-set is not yet downloaded
+			parsets.emplace_back(parFile.GetSetId());
+
+			FileInfo* fileInfo = nzbInfo->GetFileList()->Find(parFile.GetId());
+			if (fileInfo)
+			{
+				nzbInfo->PrintMessage(Message::mkInfo, "Increasing priority for par2-file %s", fileInfo->GetFilename());
+				fileInfo->SetPaused(false);
+				fileInfo->SetExtraPriority(true);
+			}
+		}
+	}
+}
+
+void DirectRenamer::CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles)
+{
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (fileInfo->GetParFile())
+		{
+			parFiles->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(), fileInfo->GetParSetId(), false);
+		}
+	}
+
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (completedFile.GetParFile())
+		{
+			parFiles->emplace_back(completedFile.GetId(), completedFile.GetFilename(), completedFile.GetParSetId(), true);
+		}
+	}
+}
+
+void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes)
+{
+	int renamedCount = 0;
+
+	bool renamePars = NeedRenamePars(nzbInfo);
+	int vol = 1;
+
+	// rename in-progress files
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		CString newName;
+		if (fileInfo->GetParFile() && renamePars)
+		{
+			newName = BuildNewParName(fileInfo->GetFilename(), nzbInfo->GetDestDir(), fileInfo->GetParSetId(), vol);
+		}
+		else if (!fileInfo->GetParFile())
+		{
+			newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k());
+		}
+
+		if (newName)
+		{
+			bool written = fileInfo->GetOutputFilename() &&
+				!Util::EndsWith(fileInfo->GetOutputFilename(), ".out.tmp", true);
+			if (!written)
+			{
+				nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s",
+					fileInfo->GetFilename(), *newName);
+				fileInfo->SetFilename(newName);
+				fileInfo->SetFilenameConfirmed(true);
+				renamedCount++;
+			}
+			else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName))
+			{
+				fileInfo->SetFilename(newName);
+				fileInfo->SetFilenameConfirmed(true);
+				renamedCount++;
+			}
+		}
+	}
+
+	// rename completed files
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		CString newName;
+		if (completedFile.GetParFile() && renamePars)
+		{
+			newName = BuildNewParName(completedFile.GetFilename(), nzbInfo->GetDestDir(), completedFile.GetParSetId(), vol);
+		}
+		else if (!completedFile.GetParFile())
+		{
+			newName = BuildNewRegularName(completedFile.GetFilename(), parHashes, completedFile.GetHash16k());
+		}
+
+		if (newName && RenameCompletedFile(nzbInfo, completedFile.GetFilename(), newName))
+		{
+			completedFile.SetFilename(newName);
+			renamedCount++;
+		}
+	}
+
+	if (renamedCount > 0)
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Successfully renamed %i file(s) for %s", renamedCount, nzbInfo->GetName());
+	}
+	else
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "No renamed files found for %s", nzbInfo->GetName());
+	}
+
+	RenameCompleted(downloadQueue, nzbInfo);
+}
+
+CString DirectRenamer::BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k)
+{
+	if (Util::EmptyStr(hash16k))
+	{
+		return nullptr;
+	}
+
+	FileHashList::iterator pos = std::find_if(parHashes->begin(), parHashes->end(),
+		[hash16k](FileHash& parHash)
+	{
+		return !strcmp(parHash.GetHash(), hash16k);
+	});
+
+	if (pos != parHashes->end())
+	{
+		FileHash& parHash = *pos;
+		if (strcasecmp(oldName, parHash.GetFilename()))
+		{
+			return parHash.GetFilename();
+		}
+	}
+
+	return nullptr;
+}
+
+CString DirectRenamer::BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol)
+{
+	BString<1024> newName;
+	BString<1024> destFileName;
+
+	// trying to reuse file suffix
+	const char* suffix = strstr(oldName, ".vol");
+	const char* extension = suffix ? strrchr(suffix, '.') : nullptr;
+	if (suffix && extension && !strcasecmp(extension, ".par2"))
+	{
+		newName.Format("%s%s", setId, suffix);
+		destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName);
+	}
+
+	while (destFileName.Empty() || FileSystem::FileExists(destFileName))
+	{
+		newName.Format("%s.vol%03i+01.PAR2", setId, vol);
+		destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName);
+		vol++;
+	}
+
+	return *newName;
+}
+
+bool DirectRenamer::NeedRenamePars(NzbInfo* nzbInfo)
+{
+	// renaming is needed if par2-files from same par-set have different base names
+	// or if any par2-file has non .par2-extension
+	ParFileList parFiles;
+	CollectPars(nzbInfo, &parFiles);
+
+	for (ParFile& parFile : parFiles)
+	{
+		if (!Util::EndsWith(parFile.GetFilename(), ".par2", false))
+		{
+			return true;
+		}
+
+		for (ParFile& parFile2 : parFiles)
+		{
+			if (&parFile != &parFile2 && !strcmp(parFile.GetSetId(), parFile2.GetSetId()) &&
+				!ParParser::SameParCollection(parFile.GetFilename(), parFile2.GetFilename(), false))
+			{
+					return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+bool DirectRenamer::RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName)
+{
+	BString<1024> oldFullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, oldName);
+	BString<1024> newFullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, newName);
+	nzbInfo->PrintMessage(Message::mkInfo, "Renaming completed file %s to %s", oldName, newName);
+	if (!FileSystem::MoveFile(oldFullFilename, newFullFilename))
+	{
+		nzbInfo->PrintMessage(Message::mkError, "Could not rename completed file %s to %s: %s",
+			*oldFullFilename, *newFullFilename, *FileSystem::GetLastErrorMessage());
+		return false;
+	}
+	return true;
+}
+
+
+void RenameContentAnalyzer::Reset()
+{
+#ifndef DISABLE_PARCHECK
+	m_md5Context.Reset();
+#endif
+	m_dataSize = 0;
+}
+
+void RenameContentAnalyzer::Append(const void* buffer, int len)
+{
+#ifndef DISABLE_PARCHECK
+	if (m_dataSize < sizeof(m_signature))
+	{
+		memcpy(m_signature + m_dataSize, buffer, std::min((size_t)len, sizeof(m_signature) - m_dataSize));
+	}
+
+	if (m_dataSize >= sizeof(m_signature) && (*(Par2::MAGIC*)m_signature) == Par2::packet_magic)
+	{
+		m_parFile = true;
+		m_parSetId = ((Par2::PACKET_HEADER*)m_signature)->setid.print().c_str();
+	}
+
+	int rem16kSize = std::min(len, 16 * 1024 - m_dataSize);
+	if (rem16kSize > 0)
+	{
+		m_md5Context.Update(buffer, rem16kSize);
+	}
+
+	m_dataSize += len;
+#endif
+}
+
+// Must be called with locked DownloadQueue
+void RenameContentAnalyzer::Finish()
+{
+#ifndef DISABLE_PARCHECK
+	Par2::MD5Hash hash;
+	m_md5Context.Final(hash);
+
+	m_hash16k = hash.print().c_str();
+#endif
+}

daemon/queue/DirectRenamer.h

@@ -0,0 +1,84 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef DIRECTRENAMER_H
+#define DIRECTRENAMER_H
+
+#include "ArticleDownloader.h"
+
+class DirectRenamer
+{
+public:
+	class FileHash
+	{
+	public:
+		FileHash(const char* filename, const char* hash) :
+			m_filename(filename), m_hash(hash) {}
+		const char* GetFilename() { return m_filename; }
+		const char* GetHash() { return m_hash; }
+
+	private:
+		CString m_filename;
+		CString m_hash;
+	};
+
+	typedef std::deque<FileHash> FileHashList;
+
+	class ParFile
+	{
+	public:
+		ParFile(int id, const char* filename, const char* setId, bool completed) :
+			m_id(id), m_filename(filename), m_setId(setId), m_completed(completed) {}
+		int GetId() { return m_id; }
+		const char* GetFilename() { return m_filename; }
+		const char* GetSetId() { return m_setId; }
+		bool GetCompleted() { return m_completed; }
+
+	private:
+		int m_id;
+		CString m_filename;
+		CString m_setId;
+		bool m_completed = false;
+	};
+
+	typedef std::deque<ParFile> ParFileList;
+
+	std::unique_ptr<ArticleContentAnalyzer> MakeArticleContentAnalyzer();
+	void ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo,
+		ArticleInfo* articleInfo, ArticleContentAnalyzer* articleContentAnalyzer);
+	void FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo);
+
+protected:
+	virtual void RenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo) = 0;
+
+private:
+	void CheckState(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void UnpausePars(NzbInfo* nzbInfo);
+	void RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes);
+	bool RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName);
+	bool NeedRenamePars(NzbInfo* nzbInfo);
+	void CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles);
+	CString BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k);
+	CString BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol);
+
+	friend class DirectParLoader;
+};
+
+#endif

daemon/queue/DiskState.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,6 +27,10 @@
 #include "FileSystem.h"
 
 static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version ";
+const int DISKSTATE_QUEUE_VERSION = 60;
+const int DISKSTATE_FILE_VERSION = 5;
+const int DISKSTATE_STATS_VERSION = 3;
+const int DISKSTATE_FEEDS_VERSION = 3;
 
 class StateDiskFile : public DiskFile
 {
@@ -265,7 +269,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 	bool ok = true;
 
 	{
-		StateFile stateFile("queue", 59, true);
+		StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
 		if (!downloadQueue->GetQueue()->empty())
 		{
 			StateDiskFile* outfile = stateFile.BeginWrite();
@@ -288,7 +292,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 
 	if (saveHistory)
 	{
-		StateFile stateFile("history", 59, true);
+		StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
 		if (!downloadQueue->GetHistory()->empty())
 		{
 			StateDiskFile* outfile = stateFile.BeginWrite();
@@ -320,7 +324,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 	int formatVersion = 0;
 
 	{
-		StateFile stateFile("queue", 59, true);
+		StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
 		if (stateFile.FileExists())
 		{
 			StateDiskFile* infile = stateFile.BeginRead();
@@ -349,7 +353,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 
 	if (formatVersion == 0 || formatVersion >= 57)
 	{
-		StateFile stateFile("history", 59, true);
+		StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
 		if (stateFile.FileExists())
 		{
 			StateDiskFile* infile = stateFile.BeginRead();
@@ -361,6 +365,8 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 		}
 	}
 
+	LoadAllFileInfos(downloadQueue);
+
 	CleanupQueueDir(downloadQueue);
 
 	if (!LoadAllFileStates(downloadQueue, servers)) goto error;
@@ -428,9 +434,10 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i,%i,%i,%i,%i", (int)nzbInfo->GetPriority(),
 		nzbInfo->GetPostInfo() ? (int)nzbInfo->GetPostInfo()->GetStage() + 1 : 0,
 		(int)nzbInfo->GetDeletePaused(), (int)nzbInfo->GetManyDupeFiles(), nzbInfo->GetFeedId());
-	outfile.PrintLine("%i,%i,%i,%i,%i,%i,%i,%i", (int)nzbInfo->GetParStatus(), (int)nzbInfo->GetUnpackStatus(),
-		(int)nzbInfo->GetMoveStatus(), (int)nzbInfo->GetParRenameStatus(), (int)nzbInfo->GetRarRenameStatus(), 
-		(int)nzbInfo->GetDeleteStatus(), (int)nzbInfo->GetMarkStatus(), (int)nzbInfo->GetUrlStatus());
+	outfile.PrintLine("%i,%i,%i,%i,%i,%i,%i,%i,%i", (int)nzbInfo->GetParStatus(), (int)nzbInfo->GetUnpackStatus(),
+		(int)nzbInfo->GetMoveStatus(), (int)nzbInfo->GetParRenameStatus(), (int)nzbInfo->GetRarRenameStatus(),
+		(int)nzbInfo->GetDirectRenameStatus(), (int)nzbInfo->GetDeleteStatus(), (int)nzbInfo->GetMarkStatus(),
+		(int)nzbInfo->GetUrlStatus());
 	outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetUnpackCleanedUpDisk(), (int)nzbInfo->GetHealthPaused(),
 		(int)nzbInfo->GetAddUrlPaused());
 	outfile.PrintLine("%i,%i,%i", nzbInfo->GetFileCount(), nzbInfo->GetParkedFileCount(),
@@ -466,8 +473,11 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i", (int)nzbInfo->GetCompletedFiles()->size());
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		outfile.PrintLine("%i,%i,%u,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
-			completedFile.GetCrc(), completedFile.GetFileName());
+		outfile.PrintLine("%i,%i,%u,%i,%s,%s,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
+			completedFile.GetCrc(), (int)completedFile.GetParFile(),
+			completedFile.GetHash16k() ? completedFile.GetHash16k() : "",
+			completedFile.GetParSetId() ? completedFile.GetParSetId() : "",
+			completedFile.GetFilename());
 	}
 
 	outfile.PrintLine("%i", (int)nzbInfo->GetParameters()->size());
@@ -568,17 +578,33 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 	}
 	nzbInfo->SetFeedId(feedId);
 
-	int parStatus, unpackStatus, moveStatus, parRenameStatus, rarRenameStatus, deleteStatus, markStatus, urlStatus;
-	if (formatVersion < 58 && infile.ScanLine("%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
-		&parRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 7) goto error;
-	rarRenameStatus = 0;
-	if (formatVersion >= 58 && infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
-		&parRenameStatus, &rarRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 8) goto error;
+	int parStatus, unpackStatus, moveStatus, parRenameStatus, rarRenameStatus,
+		directRenameStatus, deleteStatus, markStatus, urlStatus;
+	if (formatVersion >= 60)
+	{
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i,%i", &parStatus,
+			&unpackStatus, &moveStatus, &parRenameStatus, &rarRenameStatus, &directRenameStatus,
+			&deleteStatus, &markStatus, &urlStatus) != 9) goto error;
+	}
+	else if (formatVersion >= 58)
+	{
+		directRenameStatus = 0;
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i", &parStatus,
+			&unpackStatus, &moveStatus, &parRenameStatus, &rarRenameStatus, &deleteStatus,
+			&markStatus, &urlStatus) != 8) goto error;
+	}
+	else
+	{
+		rarRenameStatus = directRenameStatus = 0;
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
+			&parRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 7) goto error;
+	}
 	nzbInfo->SetParStatus((NzbInfo::EParStatus)parStatus);
-	nzbInfo->SetUnpackStatus((NzbInfo::EUnpackStatus)unpackStatus);
+	nzbInfo->SetUnpackStatus((NzbInfo::EPostUnpackStatus)unpackStatus);
 	nzbInfo->SetMoveStatus((NzbInfo::EMoveStatus)moveStatus);
-	nzbInfo->SetParRenameStatus((NzbInfo::ERenameStatus)parRenameStatus);
-	nzbInfo->SetRarRenameStatus((NzbInfo::ERenameStatus)rarRenameStatus);
+	nzbInfo->SetParRenameStatus((NzbInfo::EPostRenameStatus)parRenameStatus);
+	nzbInfo->SetRarRenameStatus((NzbInfo::EPostRenameStatus)rarRenameStatus);
+	nzbInfo->SetDirectRenameStatus((NzbInfo::EDirectRenameStatus)directRenameStatus);
 	nzbInfo->SetDeleteStatus((NzbInfo::EDeleteStatus)deleteStatus);
 	nzbInfo->SetMarkStatus((NzbInfo::EMarkStatus)markStatus);
 	if (nzbInfo->GetKind() == NzbInfo::nkNzb ||
@@ -690,10 +716,31 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		char* fileName = buf;
 		int status = 0;
 		uint32 crc = 0;
+		int parFile = 0;
+		char* hash16k = nullptr;
+		char* parSetId = nullptr;
 
 		if (formatVersion >= 49)
 		{
-			if (formatVersion >= 50)
+			if (formatVersion >= 60)
+			{
+				if (sscanf(buf, "%i,%i,%u,%i", &id, &status, &crc, &parFile) != 4) goto error;
+				hash16k = strchr(buf, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k)
+				{
+					parSetId = strchr(++hash16k, ',');
+					if (parSetId)
+					{
+						*parSetId++ = '\0';
+						fileName = strchr(parSetId, ',');
+						if (fileName) *fileName = '\0';
+					}
+				}
+			}
+			else if (formatVersion >= 50)
 			{
 				if (sscanf(buf, "%i,%i,%u", &id, &status, &crc) != 3) goto error;
 				fileName = strchr(buf, ',');
@@ -711,7 +758,10 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 			}
 		}
 
-		nzbInfo->GetCompletedFiles()->emplace_back(id, fileName, (CompletedFile::EStatus)status, crc);
+		nzbInfo->GetCompletedFiles()->emplace_back(id, fileName,
+			(CompletedFile::EStatus)status, crc, (bool)parFile,
+			Util::EmptyStr(hash16k) ? nullptr : hash16k,
+			Util::EmptyStr(parSetId) ? nullptr : parSetId);
 	}
 
 	int parameterCount;
@@ -775,19 +825,14 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 
 		std::unique_ptr<FileInfo> fileInfo = std::make_unique<FileInfo>();
 		fileInfo->SetId(id);
-
-		bool res = LoadFile(fileInfo.get(), true, false);
-		if (res)
+		fileInfo->SetPaused(paused);
+		if (formatVersion < 56)
 		{
-			fileInfo->SetPaused(paused);
-			if (formatVersion < 56)
-			{
-				fileInfo->SetTime(time);
-			}
-			fileInfo->SetExtraPriority(extraPriority != 0);
-			fileInfo->SetNzbInfo(nzbInfo);
-			nzbInfo->GetFileList()->Add(std::move(fileInfo));
+			fileInfo->SetTime(time);
 		}
+		fileInfo->SetExtraPriority((bool)extraPriority);
+		fileInfo->SetNzbInfo(nzbInfo);
+		nzbInfo->GetFileList()->Add(std::move(fileInfo));
 	}
 
 	return true;
@@ -840,7 +885,7 @@ bool DiskState::SaveFile(FileInfo* fileInfo)
 	debug("Saving FileInfo %i to disk", fileInfo->GetId());
 
 	BString<100> filename("%i", fileInfo->GetId());
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* outfile = stateFile.BeginWrite();
 	if (!outfile)
@@ -848,15 +893,15 @@ bool DiskState::SaveFile(FileInfo* fileInfo)
 		return false;
 	}
 
-	return SaveFileInfo(fileInfo, *outfile) && stateFile.FinishWrite();
+	return SaveFileInfo(fileInfo, *outfile, true) && stateFile.FinishWrite();
 }
 
-bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
+bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles)
 {
 	outfile.PrintLine("%s", fileInfo->GetSubject());
 	outfile.PrintLine("%s", fileInfo->GetFilename());
 
-	outfile.PrintLine("%i", (int)fileInfo->GetTime());
+	outfile.PrintLine("%i,%i", (int)fileInfo->GetFilenameConfirmed(), (int)fileInfo->GetTime());
 
 	uint32 High, Low;
 	Util::SplitInt64(fileInfo->GetSize(), &High, &Low);
@@ -874,11 +919,14 @@ bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
 		outfile.PrintLine("%s", *group);
 	}
 
-	outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
-	for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+	if (articles)
 	{
-		outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
-		outfile.PrintLine("%s", articleInfo->GetMessageId());
+		outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
+		for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+		{
+			outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
+			outfile.PrintLine("%s", articleInfo->GetMessageId());
+		}
 	}
 
 	return true;
@@ -894,7 +942,7 @@ bool DiskState::LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles)
 	debug("Loading FileInfo %i from disk", fileInfo->GetId());
 
 	BString<100> filename("%i", fileInfo->GetId());
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* infile = stateFile.BeginRead();
 	if (!infile)
@@ -915,7 +963,14 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	if (fileSummary) fileInfo->SetFilename(buf);
 
-	if (formatVersion >= 4)
+	if (formatVersion >= 5)
+	{
+		int time, filenameConfirmed;
+		if (infile.ScanLine("%i,%i", &filenameConfirmed, &time) != 2) goto error;
+		if (fileSummary) fileInfo->SetFilenameConfirmed((bool)filenameConfirmed);
+		if (fileSummary) fileInfo->SetTime((time_t)time);
+	}
+	else if (formatVersion >= 4)
 	{
 		int time;
 		if (infile.ScanLine("%i", &time) != 1) goto error;
@@ -948,10 +1003,9 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
 		if (fileSummary) fileInfo->GetGroups()->push_back(buf);
 	}
 
-	if (infile.ScanLine("%i", &size) != 1) goto error;
-
 	if (articles)
 	{
+		if (infile.ScanLine("%i", &size) != 1) goto error;
 		for (int i = 0; i < size; i++)
 		{
 			int PartNumber, PartSize;
@@ -979,7 +1033,7 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, bool completed)
 	debug("Saving FileState %i to disk", fileInfo->GetId());
 
 	BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* outfile = stateFile.BeginWrite();
 	if (!outfile)
@@ -1001,6 +1055,8 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool c
 	outfile.PrintLine("%u,%u,%u,%u,%u,%u", High1, Low1, High2, Low2, High3, Low3);
 
 	outfile.PrintLine("%s", fileInfo->GetFilename());
+	outfile.PrintLine("%s", fileInfo->GetHash16k());
+	outfile.PrintLine("%i", (int)fileInfo->GetParFile());
 
 	SaveServerStats(fileInfo->GetServerStats(), outfile);
 
@@ -1020,7 +1076,7 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, bool complet
 	debug("Loading FileInfo %i from disk", fileInfo->GetId());
 
 	BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* infile = stateFile.BeginRead();
 	if (!infile)
@@ -1046,13 +1102,23 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil
 	fileInfo->SetSuccessSize(Util::JoinInt64(High2, Low2));
 	fileInfo->SetFailedSize(Util::JoinInt64(High3, Low3));
 
+	char buf[1024];
+
 	if (formatVersion >= 4)
 	{
-		char buf[1024];
 		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 		fileInfo->SetFilename(buf);
 	}
 
+	if (formatVersion >= 5)
+	{
+		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+		fileInfo->SetHash16k(*buf ? buf : nullptr);
+		int parFile = 0;
+		if (infile.ScanLine("%i", &parFile) != 1) goto error;
+		fileInfo->SetParFile((bool)parFile);
+	}
+
 	if (!LoadServerStats(fileInfo->GetServerStats(), servers, infile)) goto error;
 
 	int completedArticles;
@@ -1091,6 +1157,13 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil
 			status = ArticleInfo::aiUndefined;
 		}
 
+		if (status == ArticleInfo::aiFinished && !g_Options->GetDirectWrite() &&
+			!fileInfo->GetForceDirectWrite() && !pa->GetResultFilename())
+		{
+			pa->SetResultFilename(BString<1024>("%s%c%i.%03i", g_Options->GetTempDir(),
+				PATH_SEPARATOR, fileInfo->GetId(), pa->GetPartNumber()));
+		}
+
 		// don't allow all articles be completed or the file will stuck.
 		// such states should never be saved on disk but just in case.
 		if (completedArticles == size - 1 && !completed)
@@ -1330,9 +1403,23 @@ void DiskState::CleanupTempDir(DownloadQueue* downloadQueue)
 	DirBrowser dir(g_Options->GetTempDir());
 	while (const char* filename = dir.Next())
 	{
+		bool garbage = strstr(filename, ".tmp") || strstr(filename, ".dec");
+
 		int id, part;
-		if (strstr(filename, ".tmp") || strstr(filename, ".dec") ||
-			(sscanf(filename, "%i.%i", &id, &part) == 2))
+		if (!garbage && sscanf(filename, "%i.%i", &id, &part) == 2)
+		{
+			garbage = true;
+			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+			{
+				if (nzbInfo->GetFileList()->Find(id))
+				{
+					garbage = false;
+					break;
+				}
+			}
+		}
+
+		if (garbage)
 		{
 			BString<1024> fullFilename("%s%c%s", g_Options->GetTempDir(), PATH_SEPARATOR, filename);
 			FileSystem::DeleteFile(fullFilename);
@@ -1342,6 +1429,50 @@ void DiskState::CleanupTempDir(DownloadQueue* downloadQueue)
 
 void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 {
+	// Prepare sorted id lists for faster search
+
+	std::vector<int> nzbIdList;
+	std::vector<int> fileIdList;
+
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		nzbIdList.push_back(nzbInfo->GetId());
+
+		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		{
+			fileIdList.push_back(fileInfo->GetId());
+		}
+
+		for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+		{
+			fileIdList.push_back(completedFile.GetId());
+		}
+	}
+
+	for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
+	{
+		if (historyInfo->GetKind() == HistoryInfo::hkNzb)
+		{
+			NzbInfo* nzbInfo = historyInfo->GetNzbInfo();
+			nzbIdList.push_back(nzbInfo->GetId());
+
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
+			{
+				fileIdList.push_back(fileInfo->GetId());
+			}
+
+			for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+			{
+				fileIdList.push_back(completedFile.GetId());
+			}
+		}
+	}
+
+	std::sort(nzbIdList.begin(), nzbIdList.end());
+	std::sort(fileIdList.begin(), fileIdList.end());
+
+	// Do cleanup
+
 	int deletedFiles = 0;
 
 	DirBrowser dir(g_Options->GetQueueDir());
@@ -1354,74 +1485,12 @@ void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 		if ((sscanf(filename, "%i%c", &id, &suffix) == 2 && (suffix == 's' || suffix == 'c')) ||
 			(sscanf(filename, "%i", &id) == 1 && !strchr(filename, '.')))
 		{
-			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
-			{
-				for (FileInfo* fileInfo : nzbInfo->GetFileList())
-				{
-					if (fileInfo->GetId() == id)
-					{
-						goto next;
-					}
-				}
-
-				for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
-				{
-					if (completedFile.GetId() == id)
-					{
-						goto next;
-					}
-				}
-			}
-
-			for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
-			{
-				if (historyInfo->GetKind() == HistoryInfo::hkNzb)
-				{
-					NzbInfo* nzbInfo = historyInfo->GetNzbInfo();
-
-					for (FileInfo* fileInfo : nzbInfo->GetFileList())
-					{
-						if (fileInfo->GetId() == id)
-						{
-							goto next;
-						}
-					}
-
-					for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
-					{
-						if (completedFile.GetId() == id)
-						{
-							goto next;
-						}
-					}
-				}
-			}
-
-			del = true;
+			del = !std::binary_search(fileIdList.begin(), fileIdList.end(), id);
 		}
 
 		if (!del && sscanf(filename, "n%i.log", &id) == 1)
 		{
-			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
-			{
-				if (nzbInfo->GetId() == id)
-				{
-					goto next;
-				}
-			}
-
-			for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
-			{
-				if (historyInfo->GetKind() == HistoryInfo::hkNzb)
-				{
-					if (historyInfo->GetNzbInfo()->GetId() == id)
-					{
-						goto next;
-					}
-				}
-			}
-
-			del = true;
+			del = !std::binary_search(nzbIdList.begin(), nzbIdList.end(), id);
 		}
 
 		if (del)
@@ -1431,8 +1500,6 @@ void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 			FileSystem::DeleteFile(fullFilename);
 			deletedFiles++;
 		}
-
-		next:;
 	}
 
 	if (deletedFiles > 0)
@@ -1450,7 +1517,7 @@ bool DiskState::SaveFeeds(Feeds* feeds, FeedHistory* feedHistory)
 {
 	debug("Saving feeds state to disk");
 
-	StateFile stateFile("feeds", 3, true);
+	StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);
 
 	if (feeds->empty() && feedHistory->empty())
 	{
@@ -1478,7 +1545,7 @@ bool DiskState::LoadFeeds(Feeds* feeds, FeedHistory* feedHistory)
 {
 	debug("Loading feeds state from disk");
 
-	StateFile stateFile("feeds", 3, true);
+	StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);
 
 	if (!stateFile.FileExists())
 	{
@@ -1620,6 +1687,119 @@ void DiskState::CalcFileStats(DownloadQueue* downloadQueue, int formatVersion)
 	}
 }
 
+bool DiskState::SaveAllFileInfos(DownloadQueue* downloadQueue)
+{
+	bool ok = true;
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, true);
+	if (!downloadQueue->GetQueue()->empty())
+	{
+		StateDiskFile* outfile = stateFile.BeginWrite();
+		if (!outfile)
+		{
+			return false;
+		}
+
+		// save file-infos
+
+		int fileCount = 0;
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			fileCount += nzbInfo->GetFileList()->size();
+		}
+		outfile->PrintLine("%i", fileCount);
+
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
+			{
+				outfile->PrintLine("%i", fileInfo->GetId());
+				SaveFileInfo(fileInfo, *outfile, false);
+			}
+		}
+
+		// now rename to dest file name
+		ok = stateFile.FinishWrite();
+	}
+	else
+	{
+		stateFile.Discard();
+	}
+
+	return ok;
+}
+
+bool DiskState::LoadAllFileInfos(DownloadQueue* downloadQueue)
+{
+	if (downloadQueue->GetQueue()->empty())
+	{
+		return true;
+	}
+
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
+	StateDiskFile* infile = nullptr;
+	bool useHibernate = false;
+
+	if (stateFile.FileExists())
+	{
+		infile = stateFile.BeginRead();
+		useHibernate = infile != nullptr;
+		if (useHibernate)
+		{
+			int fileCount = 0;
+			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+			{
+				fileCount += nzbInfo->GetFileList()->size();
+			}
+			int size = 0;
+			useHibernate = infile->ScanLine("%i", &size) == 1 && size == fileCount;
+		}
+		if (!useHibernate)
+		{
+			stateFile.Discard();
+		}
+	}
+
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		RawFileList brokenFileInfos;
+
+		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		{
+			bool res = false;
+			if (useHibernate)
+			{
+				int id = 0;
+				infile->ScanLine("%i", &id);
+				if (id == fileInfo->GetId())
+				{
+					res = LoadFileInfo(fileInfo, *infile, stateFile.GetFileVersion(), true, false);
+				}
+			}
+			if (!res)
+			{
+				res = LoadFile(fileInfo, true, false);
+			}
+			if (!res)
+			{
+				brokenFileInfos.push_back(fileInfo);
+			}
+		}
+
+		for (FileInfo* fileInfo : brokenFileInfos)
+		{
+			nzbInfo->GetFileList()->Remove(fileInfo);
+		}
+	}
+
+	return true;
+}
+
+void DiskState::DiscardQuickFileInfos()
+{
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
+	stateFile.Discard();
+}
+
 bool DiskState::LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers)
 {
 	BString<1024> cacheFlagFilename("%s%c%s", g_Options->GetQueueDir(), PATH_SEPARATOR, "acache");
@@ -1667,7 +1847,7 @@ bool DiskState::SaveStats(Servers* servers, ServerVolumes* serverVolumes)
 {
 	debug("Saving stats to disk");
 
-	StateFile stateFile("stats", 3, true);
+	StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);
 
 	if (servers->empty())
 	{
@@ -1695,7 +1875,7 @@ bool DiskState::LoadStats(Servers* servers, ServerVolumes* serverVolumes, bool*
 {
 	debug("Loading stats from disk");
 
-	StateFile stateFile("stats", 3, true);
+	StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);
 
 	if (!stateFile.FileExists())
 	{

daemon/queue/DiskState.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -38,6 +38,8 @@ public:
 	bool LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers);
 	bool SaveFile(FileInfo* fileInfo);
 	bool LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles);
+	bool SaveAllFileInfos(DownloadQueue* downloadQueue);
+	void DiscardQuickFileInfos();
 	bool SaveFileState(FileInfo* fileInfo, bool completed);
 	bool LoadFileState(FileInfo* fileInfo, Servers* servers, bool completed);
 	bool LoadArticles(FileInfo* fileInfo);
@@ -55,7 +57,7 @@ public:
 	void LoadNzbMessages(int nzbId, MessageList* messages);
 
 private:
-	bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile);
+	bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles);
 	bool LoadFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, int formatVersion, bool fileSummary, bool articles);
 	bool SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool completed);
 	bool LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFile& infile, int formatVersion, bool completed);
@@ -76,6 +78,7 @@ private:
 	bool SaveVolumeStat(ServerVolumes* serverVolumes, StateDiskFile& outfile);
 	bool LoadVolumeStat(Servers* servers, ServerVolumes* serverVolumes, StateDiskFile& infile, int formatVersion);
 	void CalcFileStats(DownloadQueue* downloadQueue, int formatVersion);
+	bool LoadAllFileInfos(DownloadQueue* downloadQueue);
 	bool LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers);
 	void SaveServerStats(ServerStatList* serverStatList, StateDiskFile& outfile);
 	bool LoadServerStats(ServerStatList* serverStatList, Servers* servers, StateDiskFile& infile);

daemon/queue/DownloadInfo.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -40,7 +40,7 @@ void NzbParameterList::SetParameter(const char* name, const char* value)
 	iterator pos = std::find_if(begin(), end(),
 		[name](NzbParameter& parameter)
 		{
-			return !strcmp(parameter.GetName(), name);
+			return !strcasecmp(parameter.GetName(), name);
 		});
 
 	if (emptyVal && pos != end())
@@ -57,12 +57,11 @@ void NzbParameterList::SetParameter(const char* name, const char* value)
 	}
 }
 
-NzbParameter* NzbParameterList::Find(const char* name, bool caseSensitive)
+NzbParameter* NzbParameterList::Find(const char* name)
 {
 	for (NzbParameter& parameter : this)
 	{
-		if ((caseSensitive && !strcmp(parameter.GetName(), name)) ||
-			(!caseSensitive && !strcasecmp(parameter.GetName(), name)))
+		if (!strcasecmp(parameter.GetName(), name))
 		{
 			return &parameter;
 		}
@@ -376,13 +375,13 @@ void NzbInfo::AddMessage(Message::EKind kind, const char * text)
 
 	m_messages.emplace_back(++m_idMessageGen, kind, Util::CurrentTime(), text);
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetNzbLog())
+	if (g_Options->GetServerMode() && g_Options->GetNzbLog())
 	{
 		g_DiskState->AppendNzbMessage(m_id, kind, text);
 		m_messageCount++;
 	}
 
-	while (m_messages.size() > (uint32)g_Options->GetLogBufferSize())
+	while (m_messages.size() > (uint32)g_Options->GetLogBuffer())
 	{
 		m_messages.pop_front();
 	}
@@ -470,13 +469,20 @@ void NzbInfo::SetActiveDownloads(int activeDownloads)
 		if (activeDownloads > 0)
 		{
 			m_downloadStartTime = Util::CurrentTime();
+			m_downloadStartSec = m_downloadSec;
 		}
 		else
 		{
-			m_downloadSec += Util::CurrentTime() - m_downloadStartTime;
+			m_downloadSec = m_downloadStartSec + (int)(Util::CurrentTime() - m_downloadStartTime);
 			m_downloadStartTime = 0;
+			m_changed = true;
 		}
 	}
+	else if (activeDownloads > 0)
+	{
+		m_downloadSec = m_downloadStartSec + (int)(Util::CurrentTime() - m_downloadStartTime);
+		m_changed = true;
+	}
 	m_activeDownloads = activeDownloads;
 }
 
@@ -645,6 +651,7 @@ void NzbInfo::UpdateCurrentStats()
 	m_currentFailedSize = m_failedSize;
 	m_parCurrentSuccessSize = m_parSuccessSize;
 	m_parCurrentFailedSize = m_parFailedSize;
+	m_extraPriority = 0;
 
 	m_currentServerStats.ListOp(&m_serverStats, ServerStatList::soSet);
 
@@ -655,6 +662,7 @@ void NzbInfo::UpdateCurrentStats()
 		m_currentFailedArticles += fileInfo->GetFailedArticles();
 		m_currentSuccessSize += fileInfo->GetSuccessSize();
 		m_currentFailedSize += fileInfo->GetFailedSize();
+		m_extraPriority += fileInfo->GetExtraPriority() ? 1 : 0;
 
 		if (fileInfo->GetPaused())
 		{
@@ -678,6 +686,7 @@ void NzbInfo::UpdateCompletedStats(FileInfo* fileInfo)
 	m_failedSize += fileInfo->GetFailedSize();
 	m_failedArticles += fileInfo->GetFailedArticles();
 	m_successArticles += fileInfo->GetSuccessArticles();
+	m_extraPriority -= fileInfo->GetExtraPriority() ? 1 : 0;
 
 	if (fileInfo->GetParFile())
 	{
@@ -706,6 +715,7 @@ void NzbInfo::UpdateDeletedStats(FileInfo* fileInfo)
 	m_currentSuccessArticles -= fileInfo->GetSuccessArticles();
 	m_currentFailedArticles -= fileInfo->GetFailedArticles() + fileInfo->GetMissedArticles();
 	m_remainingSize -= fileInfo->GetRemainingSize();
+	m_extraPriority -= fileInfo->GetExtraPriority() ? 1 : 0;
 
 	if (fileInfo->GetParFile())
 	{
@@ -725,6 +735,24 @@ void NzbInfo::UpdateDeletedStats(FileInfo* fileInfo)
 	m_currentServerStats.ListOp(fileInfo->GetServerStats(), ServerStatList::soSubtract);
 }
 
+bool NzbInfo::IsDownloadCompleted(bool ignorePausedPars)
+{
+	if (m_activeDownloads)
+	{
+		return false;
+	}
+
+	for (FileInfo* fileInfo : &m_fileList)
+	{
+		if ((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) &&
+			!fileInfo->GetDeleted())
+		{
+			return false;
+		}
+	}
+
+	return true;
+}
 
 void ArticleInfo::AttachSegment(std::unique_ptr<SegmentData> content, int64 offset, int size)
 {
@@ -771,6 +799,15 @@ void FileInfo::SetPaused(bool paused)
 	m_paused = paused;
 }
 
+void FileInfo::SetExtraPriority(bool extraPriority)
+{
+	if (m_extraPriority != extraPriority && m_nzbInfo)
+	{
+		m_nzbInfo->SetExtraPriority(m_nzbInfo->GetExtraPriority() + (extraPriority ? 1 : -1));
+	}
+	m_extraPriority = extraPriority;
+}
+
 void FileInfo::MakeValidFilename()
 {
 	m_filename = FileSystem::MakeValidFilename(m_filename);
@@ -791,9 +828,10 @@ void FileInfo::SetActiveDownloads(int activeDownloads)
 }
 
 
-CompletedFile::CompletedFile(int id, const char* fileName, EStatus status, uint32 crc) :
-	m_id(id), m_fileName(fileName), m_status(status), m_crc(crc)
-
+CompletedFile::CompletedFile(int id, const char* filename, EStatus status, uint32 crc,
+	bool parFile, const char* hash16k, const char* parSetId) :
+	m_id(id), m_filename(filename), m_status(status), m_crc(crc), m_parFile(parFile),
+	m_hash16k(hash16k), m_parSetId(parSetId)
 {
 	if (FileInfo::m_idMax < m_id)
 	{

daemon/queue/DownloadInfo.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -177,7 +177,7 @@ public:
 	bool GetOutputInitialized() { return m_outputInitialized; }
 	void SetOutputInitialized(bool outputInitialized) { m_outputInitialized = outputInitialized; }
 	bool GetExtraPriority() { return m_extraPriority; }
-	void SetExtraPriority(bool extraPriority) { m_extraPriority = extraPriority; }
+	void SetExtraPriority(bool extraPriority);
 	int GetActiveDownloads() { return m_activeDownloads; }
 	void SetActiveDownloads(int activeDownloads);
 	bool GetDupeDeleted() { return m_dupeDeleted; }
@@ -192,6 +192,13 @@ public:
 	void SetPartialState(EPartialState partialState) { m_partialState = partialState; }
 	uint32 GetCrc() { return m_crc; }
 	void SetCrc(uint32 crc) { m_crc = crc; }
+	const char* GetHash16k() { return m_hash16k; }
+	void SetHash16k(const char* hash16k) { m_hash16k = hash16k; }
+	const char* GetParSetId() { return m_parSetId; }
+	void SetParSetId(const char* parSetId) { m_parSetId = parSetId; }
+	bool GetFlushLocked() { return m_flushLocked; }
+	void SetFlushLocked(bool flushLocked) { m_flushLocked = flushLocked; }
+
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 
 private:
@@ -228,6 +235,9 @@ private:
 	bool m_forceDirectWrite = false;
 	EPartialState m_partialState = psNone;
 	uint32 m_crc = 0;
+	CString m_hash16k;
+	CString m_parSetId;
+	bool m_flushLocked = false;
 
 	static int m_idGen;
 	static int m_idMax;
@@ -249,18 +259,27 @@ public:
 		cfFailure
 	};
 
-	CompletedFile(int id, const char* fileName, EStatus status, uint32 crc);
+	CompletedFile(int id, const char* filename, EStatus status, uint32 crc, 
+		bool parFile, const char* hash16k, const char* parSetId);
 	int GetId() { return m_id; }
-	void SetFileName(const char* fileName) { m_fileName = fileName; }
-	const char* GetFileName() { return m_fileName; }
+	void SetFilename(const char* filename) { m_filename = filename; }
+	const char* GetFilename() { return m_filename; }
+	bool GetParFile() { return m_parFile; }
 	EStatus GetStatus() { return m_status; }
 	uint32 GetCrc() { return m_crc; }
+	const char* GetHash16k() { return m_hash16k; }
+	void SetHash16k(const char* hash16k) { m_hash16k = hash16k; }
+	const char* GetParSetId() { return m_parSetId; }
+	void SetParSetId(const char* parSetId) { m_parSetId = parSetId; }
 
 private:
 	int m_id;
-	CString m_fileName;
+	CString m_filename;
 	EStatus m_status;
 	uint32 m_crc;
+	bool m_parFile;
+	CString m_hash16k;
+	CString m_parSetId;
 };
 
 typedef std::deque<CompletedFile> CompletedFileList;
@@ -288,7 +307,7 @@ class NzbParameterList : public NzbParameterListBase
 {
 public:
 	void SetParameter(const char* name, const char* value);
-	NzbParameter* Find(const char* name, bool caseSensitive);
+	NzbParameter* Find(const char* name);
 	void CopyFrom(NzbParameterList* sourceParameters);
 };
 
@@ -332,7 +351,15 @@ enum EDupeMode
 class NzbInfo
 {
 public:
-	enum ERenameStatus
+	enum EDirectRenameStatus
+	{
+		tsNone,
+		tsRunning,
+		tsFailure,
+		tsSuccess
+	};
+
+	enum EPostRenameStatus
 	{
 		rsNone,
 		rsSkipped,
@@ -350,7 +377,15 @@ public:
 		psManual
 	};
 
-	enum EUnpackStatus
+	enum EDirectUnpackStatus
+	{
+		nsNone,
+		nsRunning,
+		nsFailure,
+		nsSuccess
+	};
+
+	enum EPostUnpackStatus
 	{
 		usNone,
 		usSkipped,
@@ -477,6 +512,9 @@ public:
 	void SetCurrentFailedArticles(int currentFailedArticles) { m_currentFailedArticles = currentFailedArticles; }
 	int GetPriority() { return m_priority; }
 	void SetPriority(int priority) { m_priority = priority; }
+	int GetExtraPriority() { return m_extraPriority; }
+	void SetExtraPriority(int extraPriority) { m_extraPriority = extraPriority; }
+	bool HasExtraPriority() { return m_extraPriority > 0; }
 	bool GetForcePriority() { return m_priority >= FORCE_PRIORITY; }
 	time_t GetMinTime() { return m_minTime; }
 	void SetMinTime(time_t minTime) { m_minTime = minTime; }
@@ -485,14 +523,18 @@ public:
 	void BuildDestDirName();
 	CString BuildFinalDirName();
 	CompletedFileList* GetCompletedFiles() { return &m_completedFiles; }
-	ERenameStatus GetParRenameStatus() { return m_parRenameStatus; }
-	void SetParRenameStatus(ERenameStatus renameStatus) { m_parRenameStatus = renameStatus; }
-	ERenameStatus GetRarRenameStatus() { return m_rarRenameStatus; }
-	void SetRarRenameStatus(ERenameStatus renameStatus) { m_rarRenameStatus = renameStatus; }
+	void SetDirectRenameStatus(EDirectRenameStatus renameStatus) { m_directRenameStatus = renameStatus; }
+	EDirectRenameStatus GetDirectRenameStatus() { return m_directRenameStatus; }
+	EPostRenameStatus GetParRenameStatus() { return m_parRenameStatus; }
+	void SetParRenameStatus(EPostRenameStatus renameStatus) { m_parRenameStatus = renameStatus; }
+	EPostRenameStatus GetRarRenameStatus() { return m_rarRenameStatus; }
+	void SetRarRenameStatus(EPostRenameStatus renameStatus) { m_rarRenameStatus = renameStatus; }
 	EParStatus GetParStatus() { return m_parStatus; }
 	void SetParStatus(EParStatus parStatus) { m_parStatus = parStatus; }
-	EUnpackStatus GetUnpackStatus() { return m_unpackStatus; }
-	void SetUnpackStatus(EUnpackStatus unpackStatus) { m_unpackStatus = unpackStatus; }
+	EDirectUnpackStatus GetDirectUnpackStatus() { return m_directUnpackStatus; }
+	void SetDirectUnpackStatus(EDirectUnpackStatus directUnpackStatus) { m_directUnpackStatus = directUnpackStatus; }
+	EPostUnpackStatus GetUnpackStatus() { return m_unpackStatus; }
+	void SetUnpackStatus(EPostUnpackStatus unpackStatus) { m_unpackStatus = unpackStatus; }
 	ECleanupStatus GetCleanupStatus() { return m_cleanupStatus; }
 	void SetCleanupStatus(ECleanupStatus cleanupStatus) { m_cleanupStatus = cleanupStatus; }
 	EMoveStatus GetMoveStatus() { return m_moveStatus; }
@@ -556,6 +598,8 @@ public:
 	void SetUnpackSec(int unpackSec) { m_unpackSec = unpackSec; }
 	time_t GetDownloadStartTime() { return m_downloadStartTime; }
 	void SetDownloadStartTime(time_t downloadStartTime) { m_downloadStartTime = downloadStartTime; }
+	bool GetChanged() { return m_changed; }
+	void SetChanged(bool changed) { m_changed = changed; }
 	void SetReprocess(bool reprocess) { m_reprocess = reprocess; }
 	bool GetReprocess() { return m_reprocess; }
 	time_t GetQueueScriptTime() { return m_queueScriptTime; }
@@ -577,9 +621,18 @@ public:
 	void SetMessageCount(int messageCount) { m_messageCount = messageCount; }
 	int GetCachedMessageCount() { return m_cachedMessageCount; }
 	GuardedMessageList GuardCachedMessages() { return GuardedMessageList(&m_messages, &m_logMutex); }
+	bool GetAllFirst() { return m_allFirst; }
+	void SetAllFirst(bool allFirst) { m_allFirst = allFirst; }
+	bool GetWaitingPar() { return m_waitingPar; }
+	void SetWaitingPar(bool waitingPar) { m_waitingPar = waitingPar; }
+	bool GetLoadingPar() { return m_loadingPar; }
+	void SetLoadingPar(bool loadingPar) { m_loadingPar = loadingPar; }
+	Thread* GetUnpackThread() { return m_unpackThread; }
+	void SetUnpackThread(Thread* unpackThread) { m_unpackThread = unpackThread; }
 	void UpdateCurrentStats();
 	void UpdateCompletedStats(FileInfo* fileInfo);
 	void UpdateDeletedStats(FileInfo* fileInfo);
+	bool IsDownloadCompleted(bool ignorePausedPars);
 
 	static const int FORCE_PRIORITY = 900;
 
@@ -617,11 +670,14 @@ private:
 	time_t m_minTime = 0;
 	time_t m_maxTime = 0;
 	int m_priority = 0;
+	int m_extraPriority = 0;
 	CompletedFileList m_completedFiles;
-	ERenameStatus m_parRenameStatus = rsNone;
-	ERenameStatus m_rarRenameStatus = rsNone;
+	EDirectRenameStatus m_directRenameStatus = tsNone;
+	EPostRenameStatus m_parRenameStatus = rsNone;
+	EPostRenameStatus m_rarRenameStatus = rsNone;
 	EParStatus m_parStatus = psNone;
-	EUnpackStatus m_unpackStatus = usNone;
+	EDirectUnpackStatus m_directUnpackStatus = nsNone;
+	EPostUnpackStatus m_unpackStatus = usNone;
 	ECleanupStatus m_cleanupStatus = csNone;
 	EMoveStatus m_moveStatus = msNone;
 	EDeleteStatus m_deleteStatus = dsNone;
@@ -655,17 +711,23 @@ private:
 	std::unique_ptr<PostInfo> m_postInfo;
 	int64 m_downloadedSize = 0;
 	time_t m_downloadStartTime = 0;
+	int m_downloadStartSec = 0;
 	int m_downloadSec = 0;
 	int m_postTotalSec = 0;
 	int m_parSec = 0;
 	int m_repairSec = 0;
 	int m_unpackSec = 0;
 	bool m_reprocess = false;
+	bool m_changed = false;
 	time_t m_queueScriptTime = 0;
 	bool m_parFull = false;
 	int m_messageCount = 0;
 	int m_cachedMessageCount = 0;
 	int m_feedId = 0;
+	bool m_allFirst = false;
+	bool m_waitingPar = false;
+	bool m_loadingPar = false;
+	Thread* m_unpackThread = nullptr;
 
 	static int m_idGen;
 	static int m_idMax;
@@ -698,6 +760,7 @@ public:
 	};
 
 	typedef std::vector<CString> ParredFiles;
+	typedef std::vector<CString> ExtractedArchives;
 
 	NzbInfo* GetNzbInfo() { return m_nzbInfo; }
 	void SetNzbInfo(NzbInfo* nzbInfo) { m_nzbInfo = nzbInfo; }
@@ -736,6 +799,7 @@ public:
 	Thread* GetPostThread() { return m_postThread; }
 	void SetPostThread(Thread* postThread) { m_postThread = postThread; }
 	ParredFiles* GetParredFiles() { return &m_parredFiles; }
+	ExtractedArchives* GetExtractedArchives() { return &m_extractedArchives; }
 
 private:
 	NzbInfo* m_nzbInfo = nullptr;
@@ -757,6 +821,7 @@ private:
 	time_t m_stageTime = 0;
 	Thread* m_postThread = nullptr;
 	ParredFiles m_parredFiles;
+	ExtractedArchives m_extractedArchives;
 };
 
 typedef std::vector<int> IdList;
@@ -850,6 +915,7 @@ public:
 		eaNzbFound,
 		eaNzbAdded,
 		eaNzbDeleted,
+		eaNzbNamed,
 		eaFileCompleted,
 		eaFileDeleted,
 		eaUrlCompleted
@@ -872,7 +938,7 @@ public:
 		eaFileResume, // resume (unpause) files
 		eaFileDelete, // delete files
 		eaFilePauseAllPars, // pause only (all) pars (does not affect other files)
-		eaFilePauseExtraPars, // pause only (almost all) pars, except main par-file (does not affect other files)
+		eaFilePauseExtraPars, // pause (almost all) pars, except main par-file (does not affect other files)
 		eaFileReorder, // set file order
 		eaFileSplit, // split - create new group from selected files
 		eaGroupMoveOffset, // move group to offset relative to the current position in download-queue
@@ -898,6 +964,7 @@ public:
 		eaGroupSetDupeScore, // set duplicate score
 		eaGroupSetDupeMode, // set duplicate mode
 		eaGroupSort, // sort groups
+		eaGroupSortFiles, // sort files for optimal download order
 		eaPostDelete, // cancel post-processing
 		eaHistoryDelete, // hide history-item
 		eaHistoryFinalDelete, // delete history-item

daemon/queue/DupeCoordinator.cpp

@@ -31,8 +31,8 @@ bool DupeCoordinator::SameNameOrKey(const char* name1, const char* dupeKey1,
 	const char* name2, const char* dupeKey2)
 {
 	bool hasDupeKeys = !Util::EmptyStr(dupeKey1) && !Util::EmptyStr(dupeKey2);
-	return (hasDupeKeys && !strcmp(dupeKey1, dupeKey2)) ||
-		(!hasDupeKeys && !strcmp(name1, name2));
+	return (hasDupeKeys && !strcasecmp(dupeKey1, dupeKey2)) ||
+		(!hasDupeKeys && !strcasecmp(name1, name2));
 }
 
 /**

daemon/queue/HistoryCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -87,7 +87,7 @@ void HistoryCoordinator::ServiceWork()
 
 void HistoryCoordinator::DeleteDiskFiles(NzbInfo* nzbInfo)
 {
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		// delete parked files
 		g_DiskState->DiscardFiles(nzbInfo);
@@ -130,8 +130,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 	for (FileInfo* fileInfo : nzbInfo->GetFileList())
 	{
 		nzbInfo->UpdateCompletedStats(fileInfo);
-		nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(),
-			fileInfo->GetFilename(), CompletedFile::cfNone, 0);
+		nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(),
+			CompletedFile::cfNone, 0, fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId());
 	}
 
 	// Cleaning up parked files if par-check was successful or unpack was successful or
@@ -169,7 +169,7 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 			(completedFile.GetStatus() == CompletedFile::cfPartial &&
 			 &completedFile == &*nzbInfo->GetCompletedFiles()->rbegin()))
 		{
-			nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFileName());
+			nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFilename());
 			nzbInfo->SetParkedFileCount(nzbInfo->GetParkedFileCount() + 1);
 		}
 	}
@@ -178,6 +178,11 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 	nzbInfo->SetRemainingParCount(0);
 	nzbInfo->SetParking(false);
 
+	if (nzbInfo->GetDirectRenameStatus() == NzbInfo::tsRunning)
+	{
+		nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure);
+	}
+
 	nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName());
 }
 
@@ -379,6 +384,7 @@ void HistoryCoordinator::MoveToQueue(DownloadQueue* downloadQueue, HistoryList::
 	if (!nzbInfo->GetUnpackCleanedUpDisk())
 	{
 		nzbInfo->SetUnpackStatus(NzbInfo::usNone);
+		nzbInfo->SetDirectUnpackStatus(NzbInfo::nsNone);
 		nzbInfo->SetCleanupStatus(NzbInfo::csNone);
 		nzbInfo->SetParRenameStatus(NzbInfo::rsNone);
 		nzbInfo->SetRarRenameStatus(NzbInfo::rsNone);
@@ -493,6 +499,8 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	nzbInfo->SetParStatus(NzbInfo::psNone);
 	nzbInfo->SetParRenameStatus(NzbInfo::rsNone);
 	nzbInfo->SetRarRenameStatus(NzbInfo::rsNone);
+	nzbInfo->SetDirectRenameStatus(NzbInfo::tsNone);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsNone);
 	nzbInfo->SetDownloadedSize(0);
 	nzbInfo->SetDownloadSec(0);
 	nzbInfo->SetPostTotalSec(0);
@@ -500,6 +508,9 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	nzbInfo->SetRepairSec(0);
 	nzbInfo->SetUnpackSec(0);
 	nzbInfo->SetExtraParBlocks(0);
+	nzbInfo->SetAllFirst(false);
+	nzbInfo->SetWaitingPar(false);
+	nzbInfo->SetLoadingPar(false);
 	nzbInfo->GetCompletedFiles()->clear();
 	nzbInfo->GetServerStats()->clear();
 	nzbInfo->GetCurrentServerStats()->clear();
@@ -587,7 +598,7 @@ void HistoryCoordinator::HistoryRetry(DownloadQueue* downloadQueue, HistoryList:
 				  g_DiskState->LoadFileState(fileInfo.get(), g_ServerPool->GetServers(), true) &&
 				  (resetFailed || fileInfo->GetRemainingSize() > 0))))
 			{
-				fileInfo->SetFilename(completedFile.GetFileName());
+				fileInfo->SetFilename(completedFile.GetFilename());
 				fileInfo->SetNzbInfo(nzbInfo);
 
 				BString<1024> outputFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename());

daemon/queue/NzbFile.cpp

@@ -374,7 +374,7 @@ void NzbFile::ProcessFiles()
 
 	CalcHashes();
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		for (FileInfo* fileInfo : m_nzbInfo->GetFileList())
 		{
@@ -595,6 +595,10 @@ bool NzbFile::ParseNzb(IUnknown* nzb)
 
 bool NzbFile::Parse()
 {
+#ifdef DISABLE_LIBXML2
+	error("Could not parse rss feed, program was compiled without libxml2 support");
+	return false;
+#else
 	xmlSAXHandler SAX_handler = {0};
 	SAX_handler.startElement = reinterpret_cast<startElementSAXFunc>(SAX_StartElement);
 	SAX_handler.endElement = reinterpret_cast<endElementSAXFunc>(SAX_EndElement);
@@ -623,6 +627,7 @@ bool NzbFile::Parse()
 	ProcessFiles();
 
 	return true;
+#endif
 }
 
 void NzbFile::Parse_StartElement(const char *name, const char **atts)
@@ -805,7 +810,11 @@ void NzbFile::SAX_characters(NzbFile* file, const char * xmlstr, int len)
 
 void* NzbFile::SAX_getEntity(NzbFile* file, const char * name)
 {
+#ifdef DISABLE_LIBXML2
+	void* e = nullptr;
+#else
 	xmlEntityPtr e = xmlGetPredefinedEntity((xmlChar* )name);
+#endif
 	if (!e)
 	{
 		file->m_nzbInfo->AddMessage(Message::mkWarning, "entity not found");

daemon/queue/QueueCoordinator.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -58,9 +58,15 @@ void QueueCoordinator::CoordinatorDownloadQueue::Save()
 		return;
 	}
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		g_DiskState->SaveDownloadQueue(this, m_historyChanged);
+		m_stateChanged = true;
+	}
+
+	for (NzbInfo* nzbInfo : GetQueue())
+	{
+		nzbInfo->SetChanged(false);
 	}
 
 	m_wantSave = false;
@@ -71,7 +77,6 @@ QueueCoordinator::QueueCoordinator()
 {
 	debug("Creating QueueCoordinator");
 
-	m_downloadQueue.m_owner = this;
 	CoordinatorDownloadQueue::Init(&m_downloadQueue);
 }
 
@@ -98,11 +103,11 @@ void QueueCoordinator::Load()
 	bool perfectServerMatch = true;
 	bool queueLoaded = false;
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue())
+	if (g_Options->GetServerMode())
 	{
 		statLoaded = g_StatMeter->Load(&perfectServerMatch);
 
-		if (g_Options->GetReloadQueue() && g_DiskState->DownloadQueueExists())
+		if (g_DiskState->DownloadQueueExists())
 		{
 			queueLoaded = g_DiskState->LoadDownloadQueue(downloadQueue, g_ServerPool->GetServers());
 		}
@@ -129,7 +134,7 @@ void QueueCoordinator::Load()
 		downloadQueue->Save();
 
 		// re-save file states into diskstate to update server ids
-		if (g_Options->GetServerMode() && g_Options->GetSaveQueue())
+		if (g_Options->GetServerMode())
 		{
 			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 			{
@@ -225,7 +230,7 @@ void QueueCoordinator::Run()
 			wasStandBy = standBy;
 			if (standBy)
 			{
-				SavePartialState();
+				SaveAllPartialState();
 			}
 		}
 
@@ -248,16 +253,18 @@ void QueueCoordinator::Run()
 			ResetHangingDownloads();
 			if (!standBy)
 			{
-				SavePartialState();
+				SaveAllPartialState();
 			}
 			resetCounter = 0;
 			g_StatMeter->IntervalCheck();
+			g_Log->IntervalCheck();
 			AdjustDownloadsLimit();
 		}
 	}
 
 	WaitJobs();
-	SavePartialState();
+	SaveAllPartialState();
+	SaveAllFileState();
 
 	debug("Exiting QueueCoordinator-loop");
 }
@@ -327,7 +334,7 @@ NzbInfo* QueueCoordinator::AddNzbFileToQueue(std::unique_ptr<NzbInfo> nzbInfo, N
 		for (FileInfo* fileInfo: nzbInfo->GetFileList())
 		{
 			allPaused &= fileInfo->GetPaused();
-			if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+			if (g_Options->GetServerMode())
 			{
 				g_DiskState->DiscardFile(fileInfo->GetId(), true, false, false);
 			}
@@ -394,11 +401,6 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 {
 	debug("CheckDupeFileInfos");
 
-	if (!g_Options->GetDupeCheck() || nzbInfo->GetDupeMode() == dmForce)
-	{
-		return;
-	}
-
 	RawFileList dupeList;
 
 	int index1 = 0;
@@ -415,9 +417,19 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 				(fileInfo->GetSize() < fileInfo2->GetSize() ||
 				 (fileInfo->GetSize() == fileInfo2->GetSize() && index2 < index1)))
 			{
-				warn("File \"%s\" appears twice in collection, adding only the biggest file", fileInfo->GetFilename());
-				dupe = true;
-				break;
+				// If more than two files have same filename we don't filter them out since that
+				// naming might be intentional and correct filenames must be read from article bodies.
+				int dupeCount = std::count_if(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+					[fileInfo2](std::unique_ptr<FileInfo>& fileInfo3)
+					{
+						return !strcmp(fileInfo3->GetFilename(), fileInfo2->GetFilename());
+					});
+				if (dupeCount == 2)
+				{
+					warn("File \"%s\" appears twice in collection, adding only the biggest file", fileInfo->GetFilename());
+					dupe = true;
+					break;
+				}
 			}
 		}
 		if (dupe)
@@ -431,7 +443,7 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 	{
 		nzbInfo->UpdateDeletedStats(fileInfo);
 		nzbInfo->GetFileList()->Remove(fileInfo);
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		if (g_Options->GetServerMode())
 		{
 			g_DiskState->DiscardFile(fileInfo->GetId(), true, false, false);
 		}
@@ -460,8 +472,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 	// if the file doesn't have any articles left for download, we store that fact and search again,
 	// ignoring all files which were previously marked as not having any articles.
 
-	// special case: if the file has ExtraPriority-flag set, it has the highest priority and the
-	// Paused-flag is ignored.
+	// special case: if the file has ExtraPriority-flag set, it has the highest priority.
 
 	//debug("QueueCoordinator::GetNextArticle()");
 
@@ -476,20 +487,34 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 
 		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 		{
-			for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
+			bool nzbHigherPriority = fileInfo &&
+				((nzbInfo->HasExtraPriority() == fileInfo->GetNzbInfo()->HasExtraPriority() &&
+					nzbInfo->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
+					(nzbInfo->HasExtraPriority() > fileInfo->GetNzbInfo()->HasExtraPriority()));
+
+			bool nzbPaused = nzbInfo->GetFileList()->size() - nzbInfo->GetPausedFileCount() <= 0;
+
+			if ((!fileInfo || nzbHigherPriority) && !nzbPaused &&
+				(!(g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) || nzbInfo->GetForcePriority()))
 			{
-				if ((checkedFiles.empty() ||
-					 std::find(checkedFiles.begin(), checkedFiles.end(), fileInfo1) == checkedFiles.end()) &&
-					!fileInfo1->GetPaused() && !fileInfo1->GetDeleted() &&
-					(g_Options->GetPropagationDelay() == 0 ||
-					 (int)fileInfo1->GetTime() < (int)curDate - g_Options->GetPropagationDelay()) &&
-					(!(g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) || nzbInfo->GetForcePriority()) &&
-					(!fileInfo ||
-					 (fileInfo1->GetExtraPriority() == fileInfo->GetExtraPriority() &&
-					  fileInfo1->GetNzbInfo()->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
-					 (fileInfo1->GetExtraPriority() > fileInfo->GetExtraPriority())))
+				for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
 				{
-					fileInfo = fileInfo1;
+					bool alreadyChecked = !checkedFiles.empty() &&
+						std::find(checkedFiles.begin(), checkedFiles.end(), fileInfo1) != checkedFiles.end();
+
+					bool propagationWait = g_Options->GetPropagationDelay() > 0 &&
+						(int)fileInfo1->GetTime() + g_Options->GetPropagationDelay() >= (int)curDate;
+
+					bool higherPriority = fileInfo &&
+						((fileInfo1->GetExtraPriority() == fileInfo->GetExtraPriority() &&
+							fileInfo1->GetNzbInfo()->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
+							(fileInfo1->GetExtraPriority() > fileInfo->GetExtraPriority()));
+
+					if (!alreadyChecked && !propagationWait && !fileInfo1->GetPaused() && 
+						!fileInfo1->GetDeleted() && (!fileInfo || higherPriority))
+					{
+						fileInfo = fileInfo1;
+					}
 				}
 			}
 		}
@@ -500,7 +525,15 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 			break;
 		}
 
-		if (fileInfo->GetArticles()->empty() && g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		if (g_Options->GetDirectRename() &&
+			fileInfo->GetNzbInfo()->GetDirectRenameStatus() <= NzbInfo::tsRunning &&
+			!fileInfo->GetNzbInfo()->GetAllFirst() &&
+			GetNextFirstArticle(fileInfo->GetNzbInfo(), fileInfo, articleInfo))
+		{
+			return true;
+		}
+
+		if (fileInfo->GetArticles()->empty() && g_Options->GetServerMode())
 		{
 			g_DiskState->LoadArticles(fileInfo);
 			LoadPartialState(fileInfo);
@@ -512,8 +545,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 			if (article->GetStatus() == ArticleInfo::aiUndefined)
 			{
 				articleInfo = article;
-				ok = true;
-				break;
+				return true;
 			}
 		}
 
@@ -525,7 +557,39 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 		}
 	}
 
-	return ok;
+	return false;
+}
+
+bool QueueCoordinator::GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo, ArticleInfo* &articleInfo)
+{
+	// find a file not renamed yet
+	for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
+	{
+		if (!fileInfo1->GetFilenameConfirmed())
+		{
+			if (fileInfo1->GetArticles()->empty() && g_Options->GetServerMode())
+			{
+				g_DiskState->LoadArticles(fileInfo1);
+				LoadPartialState(fileInfo1);
+			}
+			if (!fileInfo1->GetArticles()->empty())
+			{
+				ArticleInfo* article = fileInfo1->GetArticles()->at(0).get();
+				if (article->GetStatus() == ArticleInfo::aiUndefined)
+				{
+					fileInfo = fileInfo1;
+					articleInfo = article;
+					nzbInfo->SetDirectRenameStatus(NzbInfo::tsRunning);
+					return true;
+				}
+			}
+		}
+	}
+
+	// no more files for renaming remained
+	nzbInfo->SetAllFirst(true);
+
+	return false;
 }
 
 void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* articleInfo, NntpConnection* connection)
@@ -539,7 +603,12 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art
 	articleDownloader->SetArticleInfo(articleInfo);
 	articleDownloader->SetConnection(connection);
 
-	BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size());
+	if (articleInfo->GetPartNumber() == 1 && g_Options->GetDirectRename() && !g_Options->GetRawArticle())
+	{
+		articleDownloader->SetContentAnalyzer(m_directRenamer.MakeArticleContentAnalyzer());
+	}
+
+	BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size());
 	articleDownloader->SetInfoName(infoName);
 
 	articleInfo->SetStatus(ArticleInfo::aiRunning);
@@ -568,12 +637,14 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 	debug("Article downloaded");
 
 	FileInfo* fileInfo = articleDownloader->GetFileInfo();
-	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
-	ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
-	bool retry = false;
-	bool fileCompleted = false;
+	bool completeFileParts = false;
 
 	{
+		NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+		ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
+		bool retry = false;
+		bool fileCompleted = false;
+
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
 		if (articleDownloader->GetStatus() == ArticleDownloader::adFinished)
@@ -598,6 +669,10 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 		{
 			articleInfo->SetStatus(ArticleInfo::aiUndefined);
 			retry = true;
+			if (articleInfo->GetPartNumber() == 1)
+			{
+				nzbInfo->SetAllFirst(false);
+			}
 		}
 
 		if (!retry)
@@ -619,13 +694,22 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 			articleDownloader->GetStatus() == ArticleDownloader::adFinished &&
 			articleDownloader->GetArticleFilename())
 		{
-			fileInfo->SetFilename(articleDownloader->GetArticleFilename());
-			fileInfo->MakeValidFilename();
+			// in "FileNaming=auto"-mode prefer filename from nzb-file to filename read from article
+			// if the name from article seems to be obfuscated
+			bool useFilenameFromArticle = g_Options->GetFileNaming() == Options::nfArticle ||
+				(g_Options->GetFileNaming() == Options::nfAuto &&
+				 !Util::AlphaNum(articleDownloader->GetArticleFilename()) &&
+				 !nzbInfo->GetManyDupeFiles());
+			if (useFilenameFromArticle)
+			{
+				fileInfo->SetFilename(articleDownloader->GetArticleFilename());
+				fileInfo->MakeValidFilename();
+			}
 			fileInfo->SetFilenameConfirmed(true);
 			if (g_Options->GetDupeCheck() &&
 				nzbInfo->GetDupeMode() != dmForce &&
 				!nzbInfo->GetManyDupeFiles() &&
-				FileSystem::FileExists(nzbInfo->GetDestDir(), fileInfo->GetFilename()))
+				FileSystem::FileExists(BString<1024>("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename())))
 			{
 				warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", fileInfo->GetFilename());
 				fileCompleted = false;
@@ -634,6 +718,11 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 			}
 		}
 
+		if (articleDownloader->GetContentAnalyzer() && articleDownloader->GetStatus() == ArticleDownloader::adFinished)
+		{
+			m_directRenamer.ArticleDownloaded(downloadQueue, fileInfo, articleInfo, articleDownloader->GetContentAnalyzer());
+		}
+
 		nzbInfo->SetDownloadedSize(nzbInfo->GetDownloadedSize() + articleDownloader->GetDownloadedSize());
 
 		CheckHealth(downloadQueue, fileInfo);
@@ -642,35 +731,44 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 		{
 			fileCompleted = true;
 		}
+
+		completeFileParts = fileCompleted && (!fileInfo->GetDeleted() || nzbInfo->GetParking());
+
+		if (!completeFileParts)
+		{
+			DeleteDownloader(downloadQueue, articleDownloader, false);
+		}
 	}
 
-	bool deleteFileObj = false;
-
-	if (fileCompleted && (!fileInfo->GetDeleted() || nzbInfo->GetParking()))
+	if (completeFileParts)
 	{
 		// all jobs done
 		articleDownloader->CompleteFileParts();
 		fileInfo->SetPartialChanged(false);
-		deleteFileObj = true;
-	}
 
-	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		DeleteDownloader(downloadQueue, articleDownloader, true);
+	}
+}
 
-		bool hasOtherDownloaders = fileInfo->GetActiveDownloads() > 1;
-		deleteFileObj |= fileInfo->GetDeleted() && !hasOtherDownloaders;
+void QueueCoordinator::DeleteDownloader(DownloadQueue* downloadQueue,
+	ArticleDownloader* articleDownloader, bool fileCompleted)
+{
+	FileInfo* fileInfo = articleDownloader->GetFileInfo();
+	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+	bool hasOtherDownloaders = fileInfo->GetActiveDownloads() > 1;
+	bool deleteFileObj = fileCompleted || (fileInfo->GetDeleted() && !hasOtherDownloaders);
 
-		// remove downloader from downloader list
-		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), articleDownloader));
+	// remove downloader from downloader list
+	m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), articleDownloader));
 
-		fileInfo->SetActiveDownloads(fileInfo->GetActiveDownloads() - 1);
-		nzbInfo->SetActiveDownloads(nzbInfo->GetActiveDownloads() - 1);
+	fileInfo->SetActiveDownloads(fileInfo->GetActiveDownloads() - 1);
+	nzbInfo->SetActiveDownloads(nzbInfo->GetActiveDownloads() - 1);
 
-		if (deleteFileObj)
-		{
-			DeleteFileInfo(downloadQueue, fileInfo, fileCompleted);
-			downloadQueue->Save();
-		}
+	if (deleteFileObj)
+	{
+		DeleteFileInfo(downloadQueue, fileInfo, fileCompleted);
+		downloadQueue->Save();
 	}
 }
 
@@ -702,7 +800,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 		fileInfo->GetSuccessArticles() > 0 || fileInfo->GetFailedArticles() > 0 ? CompletedFile::cfPartial :
 		CompletedFile::cfNone;
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		g_DiskState->DiscardFile(fileInfo->GetId(), fileStatus == CompletedFile::cfSuccess || (fileDeleted && !parking), true, false);
 		if (fileStatus == CompletedFile::cfPartial && (completed || parking))
@@ -723,7 +821,19 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 			completed && fileInfo->GetOutputFilename() ?
 			FileSystem::BaseFileName(fileInfo->GetOutputFilename()) : fileInfo->GetFilename(),
 			fileStatus,
-			fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0);
+			fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0,
+			fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId());
+	}
+
+	if (g_Options->GetDirectRename())
+	{
+		m_directRenamer.FileDownloaded(downloadQueue, fileInfo);
+	}
+
+	if (nzbInfo->GetDirectRenameStatus() == NzbInfo::tsRunning &&
+		!nzbInfo->GetDeleting() && nzbInfo->IsDownloadCompleted(true))
+	{
+		DiscardDirectRename(downloadQueue, nzbInfo);
 	}
 
 	std::unique_ptr<FileInfo> srcFileInfo = nzbInfo->GetFileList()->Remove(fileInfo);
@@ -756,30 +866,45 @@ void QueueCoordinator::DiscardTempFiles(FileInfo* fileInfo)
 	}
 }
 
-void QueueCoordinator::SavePartialState()
+void QueueCoordinator::SaveAllPartialState()
 {
-	if (!(g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetContinuePartial()))
+	if (!g_Options->GetServerMode())
 	{
 		return;
 	}
 
+	bool hasUnsavedData = false;
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 	{
-		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		if (g_Options->GetContinuePartial())
 		{
-			if (fileInfo->GetPartialChanged())
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
 			{
-				debug("Saving partial state for %s", fileInfo->GetFilename());
-				if (fileInfo->GetPartialState() == FileInfo::psCompleted)
-				{
-					g_DiskState->DiscardFile(fileInfo->GetId(), false, false, true);
-				}
-				g_DiskState->SaveFileState(fileInfo, false);
-				fileInfo->SetPartialChanged(false);
-				fileInfo->SetPartialState(FileInfo::psPartial);
+				SavePartialState(fileInfo);
 			}
 		}
+		hasUnsavedData |= nzbInfo->GetChanged();
+	}
+
+	if (hasUnsavedData)
+	{
+		downloadQueue->Save();
+	}
+}
+
+void QueueCoordinator::SavePartialState(FileInfo* fileInfo)
+{
+	if (fileInfo->GetPartialChanged())
+	{
+		debug("Saving partial state for %s", fileInfo->GetFilename());
+		if (fileInfo->GetPartialState() == FileInfo::psCompleted)
+		{
+			g_DiskState->DiscardFile(fileInfo->GetId(), false, false, true);
+		}
+		g_DiskState->SaveFileState(fileInfo, false);
+		fileInfo->SetPartialChanged(false);
+		fileInfo->SetPartialState(FileInfo::psPartial);
 	}
 }
 
@@ -802,6 +927,15 @@ void QueueCoordinator::LoadPartialState(FileInfo* fileInfo)
 	}
 }
 
+void QueueCoordinator::SaveAllFileState()
+{
+	if (g_Options->GetServerMode() && m_downloadQueue.m_stateChanged)
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		g_DiskState->SaveAllFileInfos(downloadQueue);
+	}
+}
+
 void QueueCoordinator::CheckHealth(DownloadQueue* downloadQueue, FileInfo* fileInfo)
 {
 	if (g_Options->GetHealthCheck() == Options::hcNone ||
@@ -858,7 +992,7 @@ void QueueCoordinator::LogDebugInfo()
 
 void QueueCoordinator::ResetHangingDownloads()
 {
-	if (g_Options->GetTerminateTimeout() == 0 && g_Options->GetArticleTimeout() == 0)
+	if (g_Options->GetArticleTimeout() == 0)
 	{
 		return;
 	}
@@ -866,46 +1000,16 @@ void QueueCoordinator::ResetHangingDownloads()
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[tm](ArticleDownloader* articleDownloader)
+	for (ArticleDownloader* articleDownloader : m_activeDownloads)
+	{
+		if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetArticleTimeout() + 1 &&
+			articleDownloader->GetStatus() == ArticleDownloader::adRunning)
 		{
-			if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetArticleTimeout() + 1 &&
-				articleDownloader->GetStatus() == ArticleDownloader::adRunning)
-			{
-				error("Cancelling hanging download %s @ %s", articleDownloader->GetInfoName(),
-					articleDownloader->GetConnectionName());
-				articleDownloader->Stop();
-			}
-
-			if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetTerminateTimeout() &&
-				articleDownloader->GetStatus() == ArticleDownloader::adRunning)
-			{
-				ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
-				debug("Terminating hanging download %s", articleDownloader->GetInfoName());
-				if (articleDownloader->Terminate())
-				{
-					error("Terminated hanging download %s @ %s", articleDownloader->GetInfoName(),
-						articleDownloader->GetConnectionName());
-					articleInfo->SetStatus(ArticleInfo::aiUndefined);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s @ %s", articleDownloader->GetInfoName(),
-						articleDownloader->GetConnectionName());
-				}
-
-				articleDownloader->GetFileInfo()->SetActiveDownloads(articleDownloader->GetFileInfo()->GetActiveDownloads() - 1);
-				articleDownloader->GetFileInfo()->GetNzbInfo()->SetActiveDownloads(articleDownloader->GetFileInfo()->GetNzbInfo()->GetActiveDownloads() - 1);
-				articleDownloader->GetFileInfo()->GetNzbInfo()->SetDownloadedSize(articleDownloader->GetFileInfo()->GetNzbInfo()->GetDownloadedSize() + articleDownloader->GetDownloadedSize());
-
-				// it's not safe to destroy pArticleDownloader, because the state of object is unknown
-				delete articleDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging download %s @ %s", articleDownloader->GetInfoName(),
+				articleDownloader->GetConnectionName());
+			articleDownloader->Stop();
+		}
+	}
 }
 
 /*
@@ -1183,3 +1287,143 @@ bool QueueCoordinator::SplitQueueEntries(DownloadQueue* downloadQueue, RawFileLi
 
 	return true;
 }
+
+void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (g_Options->GetServerMode() && !fileInfo->GetArticles()->empty())
+		{
+			// save new file name into disk state file
+			g_DiskState->SaveFile(fileInfo);
+		}
+	}
+
+	DiscardDirectRename(downloadQueue, nzbInfo);
+
+	nzbInfo->SetDirectRenameStatus(NzbInfo::tsSuccess);
+
+	if (g_Options->GetParCheck() != Options::pcForce)
+	{
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupResume, nullptr);
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupPauseAllPars, nullptr);
+	}
+
+	if (g_Options->GetReorderFiles())
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Reordering files for %s", nzbInfo->GetName());
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr);
+	}
+
+	downloadQueue->Save();
+
+	DownloadQueue::Aspect namedAspect = { DownloadQueue::eaNzbNamed, downloadQueue, nzbInfo, nullptr };
+	downloadQueue->Notify(&namedAspect);
+}
+
+void QueueCoordinator::DiscardDirectRename(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	int64 discardedSize = 0;
+	int discardedCount = 0;
+
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (fileInfo->GetParFile() && fileInfo->GetCompletedArticles() == 1 &&
+			fileInfo->GetActiveDownloads() == 0)
+		{
+			bool locked = false;
+			{
+				Guard contentGuard = g_ArticleCache->GuardContent();
+				locked = fileInfo->GetFlushLocked();
+				if (!locked)
+				{
+					fileInfo->SetFlushLocked(true);
+				}
+			}
+
+			if (!locked)
+			{
+				// discard downloaded articles from partially downloaded par-files
+				discardedSize += fileInfo->GetSuccessSize();
+				discardedCount++;
+
+				DiscardDownloadedArticles(nzbInfo, fileInfo);
+			}
+
+			if (!locked)
+			{
+				Guard contentGuard = g_ArticleCache->GuardContent();
+				fileInfo->SetFlushLocked(false);
+			}
+		}
+
+		if (g_Options->GetServerMode() &&
+			!fileInfo->GetArticles()->empty() && g_Options->GetContinuePartial() &&
+			fileInfo->GetActiveDownloads() == 0 && fileInfo->GetCachedArticles() == 0)
+		{
+			// discard article infos to free up memory if possible
+			debug("Discarding article infos for %s/%s", nzbInfo->GetName(), fileInfo->GetFilename());
+			fileInfo->SetPartialChanged(true);
+			SavePartialState(fileInfo);
+			fileInfo->GetArticles()->clear();
+		}
+	}
+
+	if (discardedSize > 0)
+	{
+		nzbInfo->PrintMessage(Message::mkDetail, "Discarded %s from %i files used for direct renaming",
+			*Util::FormatSize(discardedSize), discardedCount);
+	}
+}
+
+void QueueCoordinator::DiscardDownloadedArticles(NzbInfo* nzbInfo, FileInfo* fileInfo)
+{
+	nzbInfo->SetRemainingSize(nzbInfo->GetRemainingSize() + fileInfo->GetSuccessSize() + fileInfo->GetFailedSize());
+	if (fileInfo->GetPaused())
+	{
+		nzbInfo->SetPausedSize(nzbInfo->GetPausedSize() + fileInfo->GetSuccessSize() + fileInfo->GetFailedSize());
+	}
+	nzbInfo->GetCurrentServerStats()->ListOp(fileInfo->GetServerStats(), ServerStatList::soSubtract);
+	fileInfo->GetServerStats()->clear();
+
+	nzbInfo->SetCurrentSuccessArticles(nzbInfo->GetCurrentSuccessArticles() - fileInfo->GetSuccessArticles());
+	nzbInfo->SetCurrentSuccessSize(nzbInfo->GetCurrentSuccessSize() - fileInfo->GetSuccessSize());
+	nzbInfo->SetParCurrentSuccessSize(nzbInfo->GetParCurrentSuccessSize() - fileInfo->GetSuccessSize());
+	fileInfo->SetSuccessSize(0);
+	fileInfo->SetSuccessArticles(0);
+
+	nzbInfo->SetCurrentFailedArticles(nzbInfo->GetCurrentFailedArticles() - fileInfo->GetFailedArticles());
+	nzbInfo->SetCurrentFailedSize(nzbInfo->GetCurrentFailedSize() - fileInfo->GetFailedSize());
+	nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() - fileInfo->GetFailedSize());
+	fileInfo->SetFailedSize(0);
+	fileInfo->SetFailedArticles(0);
+
+	fileInfo->SetCompletedArticles(0);
+	fileInfo->SetRemainingSize(fileInfo->GetSize() - fileInfo->GetMissedSize());
+
+	// discard temporary files
+	DiscardTempFiles(fileInfo);
+	g_DiskState->DiscardFile(fileInfo->GetId(), false, true, false);
+
+	fileInfo->SetOutputFilename(nullptr);
+	fileInfo->SetOutputInitialized(false);
+	fileInfo->SetCachedArticles(0);
+	fileInfo->SetPartialChanged(false);
+	fileInfo->SetPartialState(FileInfo::psNone);
+
+	if (g_Options->GetServerMode())
+	{
+		// free up memory used by articles if possible
+		fileInfo->GetArticles()->clear();
+	}
+	else
+	{
+		// reset article states if discarding isn't possible
+		for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+		{
+			articleInfo->SetStatus(ArticleInfo::aiUndefined);
+			articleInfo->SetResultFilename(nullptr);
+			articleInfo->DiscardSegment();
+		}
+	}
+}

daemon/queue/QueueCoordinator.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -30,6 +30,7 @@
 #include "Observer.h"
 #include "QueueEditor.h"
 #include "NntpConnection.h"
+#include "DirectRenamer.h"
 
 class QueueCoordinator : public Thread, public Observer, public Debuggable
 {
@@ -60,6 +61,7 @@ private:
 	class CoordinatorDownloadQueue : public DownloadQueue
 	{
 	public:
+		CoordinatorDownloadQueue(QueueCoordinator* owner) : m_owner(owner) {}
 		virtual bool EditEntry(int ID, EEditAction action, const char* args);
 		virtual bool EditList(IdList* idList, NameList* nameList, EMatchMode matchMode,
 			EEditAction action, const char* args);
@@ -70,26 +72,46 @@ private:
 		bool m_massEdit = false;
 		bool m_wantSave = false;
 		bool m_historyChanged = false;
+		bool m_stateChanged = false;
 		friend class QueueCoordinator;
 	};
 
-	CoordinatorDownloadQueue m_downloadQueue;
+	class CoordinatorDirectRenamer : public DirectRenamer
+	{
+	public:
+		CoordinatorDirectRenamer(QueueCoordinator* owner) : m_owner(owner) {}
+	protected:
+		virtual void RenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+			{ m_owner->DirectRenameCompleted(downloadQueue, nzbInfo); }
+	private:
+		QueueCoordinator* m_owner;
+	};
+
+	CoordinatorDownloadQueue m_downloadQueue{this};
 	ActiveDownloads m_activeDownloads;
 	QueueEditor m_queueEditor;
+	CoordinatorDirectRenamer m_directRenamer{this};
 	bool m_hasMoreJobs = true;
 	int m_downloadsLimit;
 	int m_serverConfigGeneration = 0;
 
 	bool GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
+	bool GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
 	void StartArticleDownload(FileInfo* fileInfo, ArticleInfo* articleInfo, NntpConnection* connection);
 	void ArticleCompleted(ArticleDownloader* articleDownloader);
+	void DeleteDownloader(DownloadQueue* downloadQueue, ArticleDownloader* articleDownloader, bool fileCompleted);
 	void DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fileInfo, bool completed);
+	void DirectRenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void DiscardDirectRename(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void DiscardDownloadedArticles(NzbInfo* nzbInfo, FileInfo* fileInfo);
 	void CheckHealth(DownloadQueue* downloadQueue, FileInfo* fileInfo);
 	void ResetHangingDownloads();
 	void AdjustDownloadsLimit();
 	void Load();
-	void SavePartialState();
+	void SaveAllPartialState();
+	void SavePartialState(FileInfo* fileInfo);
 	void LoadPartialState(FileInfo* fileInfo);
+	void SaveAllFileState();
 	void WaitJobs();
 };
 

daemon/queue/QueueEditor.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -29,6 +29,7 @@
 #include "PrePostProcessor.h"
 #include "HistoryCoordinator.h"
 #include "UrlCoordinator.h"
+#include "ParParser.h"
 
 const int MAX_ID = 1000000000;
 
@@ -436,7 +437,11 @@ bool QueueEditor::InternEditList(ItemList* itemList,
 						{
 							EditGroup(item.m_nzbInfo, action, args);
 						}
+						break;
 
+					case DownloadQueue::eaGroupSortFiles:
+						SortGroupFiles(item.m_nzbInfo);
+						break;
 
 					default:
 						// suppress compiler warning "enumeration not handled in switch"
@@ -645,7 +650,7 @@ void QueueEditor::PrepareList(ItemList* itemList, IdList* idList,
 bool QueueEditor::BuildIdListFromNameList(IdList* idList, NameList* nameList, DownloadQueue::EMatchMode matchMode, DownloadQueue::EEditAction action)
 {
 #ifndef HAVE_REGEX_H
-	if (matchMode == mmRegEx)
+	if (matchMode == DownloadQueue::mmRegEx)
 	{
 		return false;
 	}
@@ -835,10 +840,7 @@ void QueueEditor::PausePars(RawFileList* fileList, bool extraParsOnly)
 
 	for (FileInfo* fileInfo : fileList)
 	{
-		BString<1024> loFileName = fileInfo->GetFilename();
-		for (char* p = loFileName; *p; p++) *p = tolower(*p); // convert string to lowercase
-
-		if (strstr(loFileName, ".par2"))
+		if (fileInfo->GetParFile())
 		{
 			if (!extraParsOnly)
 			{
@@ -846,6 +848,8 @@ void QueueEditor::PausePars(RawFileList* fileList, bool extraParsOnly)
 			}
 			else
 			{
+				BString<1024> loFileName = fileInfo->GetFilename();
+				for (char* p = loFileName; *p; p++) *p = tolower(*p); // convert string to lowercase
 				if (strstr(loFileName, ".vol"))
 				{
 					Vols.push_back(fileInfo);
@@ -1259,6 +1263,54 @@ void QueueEditor::SetNzbDupeParam(NzbInfo* nzbInfo, DownloadQueue::EEditAction a
 	}
 }
 
+void QueueEditor::SortGroupFiles(NzbInfo* nzbInfo)
+{
+	debug("QueueEditor: sorting inner files for '%s'", nzbInfo->GetName());
+
+	std::sort(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+		[](const std::unique_ptr<FileInfo>& fileInfo1, const std::unique_ptr<FileInfo>& fileInfo2)
+		{
+			if (!fileInfo1->GetParFile() && !fileInfo2->GetParFile())
+			{
+				// ".rar"-files are ordered before ".r01", etc. files
+				int len1 = strlen(fileInfo1->GetFilename());
+				int len2 = strlen(fileInfo2->GetFilename());
+				const char* ext1 = strrchr(fileInfo1->GetFilename(), '.');
+				const char* ext2 = strrchr(fileInfo2->GetFilename(), '.');
+				int ext1len = ext1 ? strlen(ext1) : 0;
+				int ext2len = ext2 ? strlen(ext2) : 0;
+				bool sameBaseName = len1 == len2 && ext1len == 4 && ext2len == 4 &&
+					!strncmp(fileInfo1->GetFilename(), fileInfo2->GetFilename(), len1 - 4);
+				if (sameBaseName && !strcmp(ext1, ".rar") && ext2[1] == 'r' &&
+					isdigit(ext2[2]) && isdigit(ext2[3]))
+				{
+					return true;
+				}
+				else if (sameBaseName && !strcmp(ext2, ".rar") && ext1[1] == 'r' &&
+					isdigit(ext1[2]) && isdigit(ext1[3]))
+				{
+					return false;
+				}
+			}
+			else if (fileInfo1->GetParFile() && fileInfo2->GetParFile() &&
+				ParParser::SameParCollection(fileInfo1->GetFilename(), fileInfo2->GetFilename(),
+					fileInfo1->GetFilenameConfirmed() && fileInfo2->GetFilenameConfirmed()))
+			{
+				return fileInfo1->GetSize() < fileInfo2->GetSize();
+			}
+			else if (!fileInfo1->GetParFile() && fileInfo2->GetParFile())
+			{
+				return true;
+			}
+			else if (fileInfo1->GetParFile() && !fileInfo2->GetParFile())
+			{
+				return false;
+			}
+
+			return strcmp(fileInfo1->GetFilename(), fileInfo2->GetFilename()) < 0;
+		});
+}
+
 bool QueueEditor::DeleteUrl(NzbInfo* nzbInfo, DownloadQueue::EEditAction action)
 {
 	return g_UrlCoordinator->DeleteQueueEntry(m_downloadQueue, nzbInfo, action == DownloadQueue::eaGroupFinalDelete);

daemon/queue/QueueEditor.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -68,6 +68,7 @@ private:
 	void DeleteEntry(FileInfo* fileInfo);
 	void MoveEntry(FileInfo* fileInfo, int offset);
 	void MoveGroup(NzbInfo* nzbInfo, int offset);
+	void SortGroupFiles(NzbInfo* nzbInfo);
 	bool ItemListContainsItem(ItemList* itemList, int id);
 
 	friend class GroupSorter;

daemon/queue/Scanner.cpp

@@ -28,6 +28,8 @@
 #include "Util.h"
 #include "FileSystem.h"
 
+int Scanner::m_idGen = 0;
+
 Scanner::QueueData::QueueData(const char* filename, const char* nzbName, const char* category,
 	int priority, const char* dupeKey, int dupeScore, EDupeMode dupeMode,
 	NzbParameterList* parameters, bool addTop, bool addPaused, NzbInfo* urlInfo,
@@ -369,7 +371,10 @@ void Scanner::InitPPParameters(const char* category, NzbParameterList* parameter
 		}
 	}
 
-	parameters->SetParameter("*Unpack:", unpack ? "yes" : "no");
+	if (!parameters->Find("*Unpack:"))
+	{
+		parameters->SetParameter("*Unpack:", unpack ? "yes" : "no");
+	}
 
 	if (!Util::EmptyStr(extensions))
 	{
@@ -379,10 +384,12 @@ void Scanner::InitPPParameters(const char* category, NzbParameterList* parameter
 		{
 			for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
 			{
+				BString<1024> paramName("%s:", scriptName);
 				if ((script.GetPostScript() || script.GetQueueScript()) &&
+					!parameters->Find(paramName) &&
 					FileSystem::SameFilename(scriptName, script.GetName()))
 				{
-					parameters->SetParameter(BString<1024>("%s:", scriptName), "yes");
+					parameters->SetParameter(paramName, "yes");
 				}
 			}
 		}
@@ -493,8 +500,8 @@ Scanner::EAddStatus Scanner::AddExternalFile(const char* nzbName, const char* ca
 	}
 	else
 	{
-		int num = 1;
-		while (num == 1 || FileSystem::FileExists(tempFileName))
+		int num = ++m_idGen;
+		while (tempFileName.Empty() || FileSystem::FileExists(tempFileName))
 		{
 			tempFileName.Format("%s%cnzb-%i.tmp", g_Options->GetTempDir(), PATH_SEPARATOR, num);
 			num++;

daemon/queue/Scanner.h

@@ -114,6 +114,7 @@ private:
 	QueueList m_queueList;
 	bool m_scanning = false;
 	Mutex m_scanMutex;
+	static int m_idGen;
 
 	void CheckIncomingNzbs(const char* directory, const char* category, bool checkStat);
 	bool AddFileToQueue(const char* filename, const char* nzbName, const char* category,

daemon/queue/UrlCoordinator.cpp

@@ -158,8 +158,7 @@ void UrlCoordinator::Stop()
 
 void UrlCoordinator::ResetHangingDownloads()
 {
-	const int timeout = g_Options->GetTerminateTimeout();
-	if (timeout == 0)
+	if (g_Options->GetUrlTimeout() == 0)
 	{
 		return;
 	}
@@ -167,32 +166,15 @@ void UrlCoordinator::ResetHangingDownloads()
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[timeout, tm](UrlDownloader* urlDownloader)
+	for (UrlDownloader* urlDownloader: m_activeDownloads)
+	{
+		if (tm - urlDownloader->GetLastUpdateTime() > g_Options->GetUrlTimeout() + 10 &&
+			urlDownloader->GetStatus() == UrlDownloader::adRunning)
 		{
-			if (tm - urlDownloader->GetLastUpdateTime() > timeout &&
-				urlDownloader->GetStatus() == UrlDownloader::adRunning)
-			{
-				NzbInfo* nzbInfo = urlDownloader->GetNzbInfo();
-				debug("Terminating hanging download %s", urlDownloader->GetInfoName());
-				if (urlDownloader->Terminate())
-				{
-					error("Terminated hanging download %s", urlDownloader->GetInfoName());
-					nzbInfo->SetUrlStatus(NzbInfo::lsNone);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s", urlDownloader->GetInfoName());
-				}
-
-				// it's not safe to destroy urlDownloader, because the state of object is unknown
-				delete urlDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging url download %s", urlDownloader->GetInfoName());
+			urlDownloader->Stop();
+		}
+	}
 }
 
 void UrlCoordinator::LogDebugInfo()

daemon/remote/BinRpc.cpp

@@ -871,7 +871,7 @@ void EditQueueBinCommand::Execute()
 	else
 	{
 #ifndef HAVE_REGEX_H
-		if ((QueueEditor::EMatchMode)matchMode == QueueEditor::mmRegEx)
+		if ((DownloadQueue::EMatchMode)matchMode == DownloadQueue::mmRegEx)
 		{
 			SendBoolResponse(false, "Edit-Command failed: the program was compiled without RegEx-support");
 			return;

daemon/remote/MessageBase.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,7 +22,7 @@
 #ifndef MESSAGEBASE_H
 #define MESSAGEBASE_H
 
-static const int32 NZBMESSAGE_SIGNATURE = 0x6E7A6230; // = "nzb-XX" (protocol version)
+static const int32 NZBMESSAGE_SIGNATURE = 0x6E7A6231; // = "nzb-XX" (protocol version)
 static const int NZBREQUESTFILENAMESIZE = 512;
 static const int NZBREQUESTPASSWORDSIZE = 32;
 

daemon/remote/RemoteServer.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -60,7 +60,7 @@ void RemoteServer::Run()
 			m_connection = std::make_unique<Connection>(g_Options->GetControlIp(),
 				m_tls ? g_Options->GetSecurePort() : g_Options->GetControlPort(),
 				m_tls);
-			m_connection->SetTimeout(g_Options->GetUrlTimeout());
+			m_connection->SetTimeout(g_Options->GetRemoteTimeout());
 			m_connection->SetSuppressErrors(false);
 			bind = m_connection->Bind();
 		}
@@ -83,13 +83,19 @@ void RemoteServer::Run()
 			continue;
 		}
 
-		RequestProcessor* commandThread = new RequestProcessor();
-		commandThread->SetAutoDestroy(true);
-		commandThread->SetConnection(std::move(acceptedConnection));
+		if (!IsStopped())
+		{
+			RequestProcessor* commandThread = new RequestProcessor();
+			commandThread->SetAutoDestroy(true);
+			commandThread->SetConnection(std::move(acceptedConnection));
 #ifndef DISABLE_TLS
-		commandThread->SetTls(m_tls);
+			commandThread->SetTls(m_tls);
 #endif
-		commandThread->Start();
+			Guard guard(m_processorsMutex);
+			m_activeProcessors.push_back(commandThread);
+			commandThread->Attach(this);
+			commandThread->Start();
+		}
 	}
 
 	if (m_connection)
@@ -97,6 +103,19 @@ void RemoteServer::Run()
 		m_connection->Disconnect();
 	}
 
+	// waiting for request processors
+	debug("RemoteServer: waiting for request processor to complete");
+	bool completed = false;
+	while (!completed)
+	{
+		{
+			Guard guard(m_processorsMutex);
+			completed = m_activeProcessors.size() == 0;
+		}
+		usleep(100 * 1000);
+	}
+	debug("RemoteServer: request processor are completed");
+
 	debug("Exiting RemoteServer-loop");
 }
 
@@ -106,11 +125,39 @@ void RemoteServer::Stop()
 	if (m_connection)
 	{
 		m_connection->SetSuppressErrors(true);
+		m_connection->SetForceClose(true);
 		m_connection->Cancel();
-#ifdef WIN32
-		m_connection->Disconnect();
-#endif
+
+		debug("Stopping RequestProcessors");
+		Guard guard(m_processorsMutex);
+		for (RequestProcessor* requestProcessor : m_activeProcessors)
+		{
+			requestProcessor->Stop();
+		}
+		debug("RequestProcessors are notified");
 	}
+	debug("RemoteServer stop end");
+}
+
+void RemoteServer::ForceStop()
+{
+	debug("Killing RequestProcessors");
+	Guard guard(m_processorsMutex);
+	for (RequestProcessor* requestProcessor : m_activeProcessors)
+	{
+		requestProcessor->Kill();
+	}
+	m_activeProcessors.clear();
+	debug("RequestProcessors are killed");
+}
+
+void RemoteServer::Update(Subject* caller, void* aspect)
+{
+	debug("Notification from RequestProcessor received");
+
+	RequestProcessor* requestProcessor = (RequestProcessor*)caller;
+	Guard guard(m_processorsMutex);
+	m_activeProcessors.erase(std::find(m_activeProcessors.begin(), m_activeProcessors.end(), requestProcessor));
 }
 
 //*****************************************************************
@@ -123,8 +170,22 @@ RequestProcessor::~RequestProcessor()
 
 void RequestProcessor::Run()
 {
-	bool ok = false;
+	Execute();
+	Notify(nullptr);
+}
 
+void RequestProcessor::Stop()
+{
+	Thread::Stop();
+#ifdef WIN32
+	m_connection->SetForceClose(true);
+#endif
+	m_connection->Cancel();
+}
+
+void RequestProcessor::Execute()
+{
+	bool ok = false;
 	m_connection->SetSuppressErrors(true);
 
 #ifndef DISABLE_TLS
@@ -136,7 +197,7 @@ void RequestProcessor::Run()
 #endif
 
 	// Read the first 4 bytes to determine request type
-	int signature = 0;
+	uint32 signature = 0;
 	if (!m_connection->Recv((char*)&signature, 4))
 	{
 		debug("Could not read request signature");
@@ -156,39 +217,18 @@ void RequestProcessor::Run()
 		!strncmp((char*)&signature, "OPTI", 4))
 	{
 		// HTTP request received
-		char buffer[1024];
-		if (m_connection->ReadLine(buffer, sizeof(buffer), nullptr))
+		ok = true;
+		while (ServWebRequest((char*)&signature))
 		{
-			WebProcessor::EHttpMethod httpMethod = WebProcessor::hmGet;
-			char* url = buffer;
-			if (!strncmp((char*)&signature, "POST", 4))
+			if (!m_connection->Recv((char*)&signature, 4))
 			{
-				httpMethod = WebProcessor::hmPost;
-				url++;
+				debug("Could not read request signature");
+				break;
 			}
-			if (!strncmp((char*)&signature, "OPTI", 4) && strlen(url) > 4)
-			{
-				httpMethod = WebProcessor::hmOptions;
-				url += 4;
-			}
-			if (char* p = strchr(url, ' '))
-			{
-				*p = '\0';
-			}
-
-			debug("url: %s", url);
-
-			WebProcessor processor;
-			processor.SetConnection(m_connection.get());
-			processor.SetUrl(url);
-			processor.SetHttpMethod(httpMethod);
-			processor.Execute();
-
-			m_connection->SetGracefull(true);
-			m_connection->Disconnect();
-
-			ok = true;
 		}
+
+		m_connection->SetGracefull(true);
+		m_connection->Disconnect();
 	}
 
 	if (!ok)
@@ -196,3 +236,45 @@ void RequestProcessor::Run()
 		warn("Non-nzbget request received on port %i from %s", m_tls ? g_Options->GetSecurePort() : g_Options->GetControlPort(), m_connection->GetRemoteAddr());
 	}
 }
+
+bool RequestProcessor::ServWebRequest(const char* signature)
+{
+	// HTTP request received
+	char buffer[1024];
+	if (!m_connection->ReadLine(buffer, sizeof(buffer), nullptr))
+	{
+		return false;
+	}
+
+	WebProcessor::EHttpMethod httpMethod = WebProcessor::hmGet;
+	char* url = buffer;
+	if (!strncmp(signature, "POST", 4))
+	{
+		httpMethod = WebProcessor::hmPost;
+		url++;
+	}
+	else if (!strncmp(signature, "OPTI", 4) && strlen(url) > 4)
+	{
+		httpMethod = WebProcessor::hmOptions;
+		url += 4;
+	}
+	else if (!(!strncmp(signature, "GET ", 4)))
+	{
+		return false;
+	}
+
+	if (char* p = strchr(url, ' '))
+	{
+		*p = '\0';
+	}
+
+	debug("url: %s", url);
+
+	WebProcessor processor;
+	processor.SetConnection(m_connection.get());
+	processor.SetUrl(url);
+	processor.SetHttpMethod(httpMethod);
+	processor.Execute();
+
+	return processor.GetKeepAlive();
+}

daemon/remote/RemoteServer.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -24,30 +24,43 @@
 
 #include "Thread.h"
 #include "Connection.h"
+#include "Observer.h"
 
-class RemoteServer : public Thread
+class RequestProcessor;
+
+class RemoteServer : public Thread, public Observer
 {
 public:
 	RemoteServer(bool tls) : m_tls(tls) {}
 	virtual void Run();
 	virtual void Stop();
+	void ForceStop();
+	void Update(Subject* caller, void* aspect);
 
 private:
+	typedef std::deque<RequestProcessor*> RequestProcessors;
+
 	bool m_tls;
 	std::unique_ptr<Connection> m_connection;
+	RequestProcessors m_activeProcessors;
+	Mutex m_processorsMutex;
 };
 
-class RequestProcessor : public Thread
+class RequestProcessor : public Thread, public Subject
 {
 public:
 	~RequestProcessor();
 	virtual void Run();
+	virtual void Stop();
 	void SetTls(bool tls) { m_tls = tls; }
 	void SetConnection(std::unique_ptr<Connection>&& connection) { m_connection = std::move(connection); }
 
 private:
 	bool m_tls;
 	std::unique_ptr<Connection> m_connection;
+
+	bool ServWebRequest(const char* signature);
+	void Execute();
 };
 
 #endif

daemon/remote/WebServer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -26,6 +26,13 @@
 #include "Util.h"
 #include "FileSystem.h"
 
+#ifndef DISABLE_PARCHECK
+#include "par2cmdline.h"
+#include "md5.h"
+#endif
+
+static const char* ERR_HTTP_OK = "200 OK";
+static const char* ERR_HTTP_NOT_MODIFIED = "304 Not Modified";
 static const char* ERR_HTTP_BAD_REQUEST = "400 Bad Request";
 static const char* ERR_HTTP_NOT_FOUND = "404 Not Found";
 static const char* ERR_HTTP_SERVICE_UNAVAILABLE = "503 Service Unavailable";
@@ -90,7 +97,10 @@ void WebProcessor::Execute()
 
 	ParseUrl();
 
-	if (!CheckCredentials())
+	m_rpcRequest = XmlRpcProcessor::IsRpcRequest(m_url);
+	m_authorized = CheckCredentials();
+
+	if ((!g_Options->GetFormAuth() || m_rpcRequest) && !m_authorized)
 	{
 		SendAuthResponse();
 		return;
@@ -124,11 +134,12 @@ void WebProcessor::ParseHeaders()
 	{
 		if (char* pe = strrchr(p, '\r')) *pe = '\0';
 		debug("header=%s", p);
+
 		if (!strncasecmp(p, "Content-Length: ", 16))
 		{
 			m_contentLen = atoi(p + 16);
 		}
-		if (!strncasecmp(p, "Authorization: Basic ", 21))
+		else if (!strncasecmp(p, "Authorization: Basic ", 21) && Util::EmptyStr(m_authInfo))
 		{
 			char* authInfo64 = p + 21;
 			if (strlen(authInfo64) > sizeof(m_authInfo))
@@ -138,20 +149,47 @@ void WebProcessor::ParseHeaders()
 			}
 			m_authInfo[WebUtil::DecodeBase64(authInfo64, 0, m_authInfo)] = '\0';
 		}
-		if (!strncasecmp(p, "Accept-Encoding: ", 17))
+		else if (!strncasecmp(p, "X-Authorization: Basic ", 23))
+		{
+			char* authInfo64 = p + 23;
+			if (strlen(authInfo64) > sizeof(m_authInfo))
+			{
+				error("Invalid-request: auth-info too big");
+				return;
+			}
+			m_authInfo[WebUtil::DecodeBase64(authInfo64, 0, m_authInfo)] = '\0';
+		}
+		else if (!strncasecmp(p, "Accept-Encoding: ", 17))
 		{
 			m_gzip = strstr(p, "gzip");
 		}
-		if (!strncasecmp(p, "Origin: ", 8))
+		else if (!strncasecmp(p, "Origin: ", 8))
 		{
 			m_origin = p + 8;
 		}
-		if (!strncasecmp(p, "X-Auth-Token: ", 14))
+		else if (!strncasecmp(p, "Cookie: ", 8))
 		{
-			strncpy(m_authToken, p + 14, sizeof(m_authToken)-1);
-			m_authToken[sizeof(m_authToken)-1] = '\0';
+			debug("%s", p);
+			const char* tok = strstr(p, "Auth-Token=");
+			if (tok && tok[11] != ';' && tok[11] != '\0')
+			{
+				strncpy(m_authToken, tok + 11, sizeof(m_authToken)-1);
+				m_authToken[sizeof(m_authToken)-1] = '\0';
+			}
 		}
-		if (*p == '\0')
+		else if (!strncasecmp(p, "X-Forwarded-For: ", 17))
+		{
+			m_forwardedFor = p + 17;
+		}
+		else if (!strncasecmp(p, "If-None-Match: ", 15))
+		{
+			m_oldETag = p + 15;
+		}
+		else if (!strncasecmp(p, "Connection: keep-alive", 22))
+		{
+			m_keepAlive = true;
+		}
+		else if (*p == '\0')
 		{
 			break;
 		}
@@ -159,7 +197,7 @@ void WebProcessor::ParseHeaders()
 
 	debug("URL=%s", *m_url);
 	debug("Authorization=%s", m_authInfo);
-	debug("X-Auth-Token=%s", m_authToken);
+	debug("Auth-Token=%s", m_authToken);
 }
 
 void WebProcessor::ParseUrl()
@@ -245,8 +283,10 @@ bool WebProcessor::CheckCredentials()
 		}
 		else
 		{
-			warn("Request received on port %i from %s, but username or password invalid (%s:%s)",
-				g_Options->GetControlPort(), m_connection->GetRemoteAddr(), m_authInfo, pw);
+			warn("Request received on port %i from %s%s, but username (%s) or password invalid",
+				g_Options->GetControlPort(), m_connection->GetRemoteAddr(),
+				!m_forwardedFor.Empty() ? (char*)BString<1024>(" (forwarded for: %s)", *m_forwardedFor) : "",
+				m_authInfo);
 			return false;
 		}
 	}
@@ -281,7 +321,7 @@ void WebProcessor::Dispatch()
 		return;
 	}
 
-	if (XmlRpcProcessor::IsRpcRequest(m_url))
+	if (m_rpcRequest)
 	{
 		XmlRpcProcessor processor;
 		processor.SetRequest(m_request);
@@ -289,7 +329,7 @@ void WebProcessor::Dispatch()
 		processor.SetUserAccess((XmlRpcProcessor::EUserAccess)m_userAccess);
 		processor.SetUrl(m_url);
 		processor.Execute();
-		SendBodyResponse(processor.GetResponse(), strlen(processor.GetResponse()), processor.GetContentType());
+		SendBodyResponse(processor.GetResponse(), strlen(processor.GetResponse()), processor.GetContentType(), processor.IsSafeMethod());
 		return;
 	}
 
@@ -305,40 +345,42 @@ void WebProcessor::Dispatch()
 		return;
 	}
 
-	// for security reasons we allow only characters "0..9 A..Z a..z . - _ /" in the URLs
+	// for security reasons we allow only characters "0..9 A..Z a..z . - + _ / ?" in the URLs
 	// we also don't allow ".." in the URLs
 	for (char *p = m_url; *p; p++)
 	{
 		if (!((*p >= '0' && *p <= '9') || (*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z') ||
-			*p == '.' || *p == '-' || *p == '_' || *p == '/') || (*p == '.' && p[1] == '.'))
+			*p == '.' || *p == '-' || *p == '+' || *p == '?' || *p == '_' || *p == '/') || (*p == '.' && p[1] == '.'))
 		{
 			SendErrorResponse(ERR_HTTP_NOT_FOUND, true);
 			return;
 		}
 	}
 
-	const char *defRes = "";
-	if (m_url[strlen(m_url)-1] == '/')
+	if (!strncmp(m_url, "/combined.", 10) && strchr(m_url, '?'))
 	{
-		// default file in directory (if not specified) is "index.html"
-		defRes = "index.html";
+		SendMultiFileResponse();
+	}
+	else
+	{
+		SendSingleFileResponse();
 	}
-
-	BString<1024> disk_filename("%s%s%s", g_Options->GetWebDir(), *m_url, defRes);
-
-	SendFileResponse(disk_filename);
 }
 
 void WebProcessor::SendAuthResponse()
 {
 	const char* AUTH_RESPONSE_HEADER =
-		"HTTP/1.0 401 Unauthorized\r\n"
-		"WWW-Authenticate: Basic realm=\"NZBGet\"\r\n"
-		"Connection: close\r\n"
+		"HTTP/1.1 401 Unauthorized\r\n"
+		"%s"
+		"Connection: %s\r\n"
 		"Content-Type: text/plain\r\n"
+		"Content-Length: 0\r\n"
 		"Server: nzbget-%s\r\n"
 		"\r\n";
-	BString<1024> responseHeader(AUTH_RESPONSE_HEADER, Util::VersionRevision());
+
+	BString<1024> responseHeader(AUTH_RESPONSE_HEADER,
+		g_Options->GetFormAuth() ? "" : "WWW-Authenticate: Basic realm=\"NZBGet\"\r\n",
+		m_keepAlive ? "keep-alive" : "close", Util::VersionRevision());
 
 	// Send the response answer
 	debug("ResponseHeader=%s", *responseHeader);
@@ -349,8 +391,9 @@ void WebProcessor::SendOptionsResponse()
 {
 	const char* OPTIONS_RESPONSE_HEADER =
 		"HTTP/1.1 200 OK\r\n"
-		"Connection: close\r\n"
+		"Connection: %s\r\n"
 		//"Content-Type: plain/text\r\n"
+		"Content-Length: 0\r\n"
 		"Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n"
 		"Access-Control-Allow-Origin: %s\r\n"
 		"Access-Control-Allow-Credentials: true\r\n"
@@ -359,6 +402,7 @@ void WebProcessor::SendOptionsResponse()
 		"Server: nzbget-%s\r\n"
 		"\r\n";
 	BString<1024> responseHeader(OPTIONS_RESPONSE_HEADER,
+		m_keepAlive ? "keep-alive" : "close", 
 		m_origin.Str(), Util::VersionRevision());
 
 	// Send the response answer
@@ -369,8 +413,8 @@ void WebProcessor::SendOptionsResponse()
 void WebProcessor::SendErrorResponse(const char* errCode, bool printWarning)
 {
 	const char* RESPONSE_HEADER =
-		"HTTP/1.0 %s\r\n"
-		"Connection: close\r\n"
+		"HTTP/1.1 %s\r\n"
+		"Connection: %s\r\n"
 		"Content-Length: %i\r\n"
 		"Content-Type: text/html\r\n"
 		"Server: nzbget-%s\r\n"
@@ -385,7 +429,9 @@ void WebProcessor::SendErrorResponse(const char* errCode, bool printWarning)
 		errCode, errCode);
 	int pageContentLen = responseBody.Length();
 
-	BString<1024> responseHeader(RESPONSE_HEADER, errCode, pageContentLen, Util::VersionRevision());
+	BString<1024> responseHeader(RESPONSE_HEADER, errCode,
+		m_keepAlive ? "keep-alive" : "close",
+		pageContentLen, Util::VersionRevision());
 
 	// Send the response answer
 	m_connection->Send(responseHeader, responseHeader.Length());
@@ -395,35 +441,66 @@ void WebProcessor::SendErrorResponse(const char* errCode, bool printWarning)
 void WebProcessor::SendRedirectResponse(const char* url)
 {
 	const char* REDIRECT_RESPONSE_HEADER =
-		"HTTP/1.0 301 Moved Permanently\r\n"
+		"HTTP/1.1 301 Moved Permanently\r\n"
 		"Location: %s\r\n"
-		"Connection: close\r\n"
+		"Connection: %s\r\n"
+		"Content-Length: 0\r\n"
 		"Server: nzbget-%s\r\n"
 		"\r\n";
-	BString<1024> responseHeader(REDIRECT_RESPONSE_HEADER, url, Util::VersionRevision());
+	BString<1024> responseHeader(REDIRECT_RESPONSE_HEADER, url,
+		m_keepAlive ? "keep-alive" : "close", Util::VersionRevision());
 
 	// Send the response answer
 	debug("ResponseHeader=%s", *responseHeader);
 	m_connection->Send(responseHeader, responseHeader.Length());
 }
 
-void WebProcessor::SendBodyResponse(const char* body, int bodyLen, const char* contentType)
+void WebProcessor::SendBodyResponse(const char* body, int bodyLen, const char* contentType, bool cachable)
 {
 	const char* RESPONSE_HEADER =
-		"HTTP/1.1 200 OK\r\n"
-		"Connection: close\r\n"
+		"HTTP/1.1 %s\r\n"
+		"Connection: %s\r\n"
 		"Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n"
 		"Access-Control-Allow-Origin: %s\r\n"
 		"Access-Control-Allow-Credentials: true\r\n"
 		"Access-Control-Max-Age: 86400\r\n"
 		"Access-Control-Allow-Headers: Content-Type, Authorization\r\n"
-		"X-Auth-Token: %s\r\n"
+		"Set-Cookie: Auth-Type=%s\r\n"
+		"Set-Cookie: Auth-Token=%s; HttpOnly\r\n"
 		"Content-Length: %i\r\n"
 		"%s"					// Content-Type: xxx
 		"%s"					// Content-Encoding: gzip
+		"%s"					// ETag
 		"Server: nzbget-%s\r\n"
 		"\r\n";
 
+	BString<1024> eTagHeader;
+	bool unchanged = false;
+
+	if (cachable)
+	{
+		BString<1024> newETag;
+
+#ifndef DISABLE_PARCHECK
+		Par2::MD5Hash hash;
+		Par2::MD5Context md5;
+		md5.Update(body, bodyLen);
+		md5.Final(hash);
+		newETag.Format("\"%s\"", hash.print().c_str());
+#else
+		uint32 hash = Util::HashBJ96(body, bodyLen, 0);
+		newETag.Format("\"%x\"", hash);
+#endif
+
+		unchanged = m_oldETag && !strcmp(newETag, m_oldETag);
+		if (unchanged)
+		{
+			body = "";
+			bodyLen = 0;
+		}
+		eTagHeader.Format("ETag: %s\r\n", *newETag);
+	}
+
 #ifndef DISABLE_GZIP
 	CharBuffer gbuf;
 	bool gzip = m_gzip && bodyLen > MAX_UNCOMPRESSED_SIZE;
@@ -453,22 +530,39 @@ void WebProcessor::SendBodyResponse(const char* body, int bodyLen, const char* c
 	}
 
 	BString<1024> responseHeader(RESPONSE_HEADER,
+		unchanged ? ERR_HTTP_NOT_MODIFIED : ERR_HTTP_OK,
+		m_keepAlive ? "keep-alive" : "close",
 		m_origin.Str(),
-		m_serverAuthToken[m_userAccess], bodyLen, *contentTypeHeader,
+		g_Options->GetFormAuth() ? "form" : "http",
+		m_authorized ? m_serverAuthToken[m_userAccess] : "",
+		bodyLen,
+		*contentTypeHeader,
 		gzip ? "Content-Encoding: gzip\r\n" : "",
+		cachable ? *eTagHeader : "",
 		Util::VersionRevision());
 
+	debug("[%s] (%s) %s", *m_url, *m_oldETag, *responseHeader);
+
 	// Send the request answer
 	m_connection->Send(responseHeader, responseHeader.Length());
 	m_connection->Send(body, bodyLen);
 }
 
-void WebProcessor::SendFileResponse(const char* filename)
+void WebProcessor::SendSingleFileResponse()
 {
-	debug("serving file: %s", filename);
+	const char *defRes = "";
+	if (m_url[strlen(m_url)-1] == '/')
+	{
+		// default file in directory (if not specified) is "index.html"
+		defRes = "index.html";
+	}
+
+	BString<1024> filename("%s%s%s", g_Options->GetWebDir(), *m_url, defRes);
+
+	debug("serving file: %s", *filename);
 
 	CharBuffer body;
-	if (!FileSystem::LoadFileIntoBuffer(filename, body, false))
+	if (!FileSystem::LoadFileIntoBuffer(filename, body, true))
 	{
 		// do not print warnings "404 not found" for certain files
 		bool ignorable = !strcmp(filename, "package-info.json") ||
@@ -479,7 +573,48 @@ void WebProcessor::SendFileResponse(const char* filename)
 		return;
 	}
 
-	SendBodyResponse(body, body.Size(), DetectContentType(filename));
+	const char* contentType = DetectContentType(filename);
+	int len = body.Size() - 1;
+
+#ifdef DEBUG
+	if (contentType && !strcmp(contentType, "text/html"))
+	{
+		Util::ReduceStr(body, "<!-- %if-debug%", "");
+		Util::ReduceStr(body, "<!-- %if-not-debug% -->", "<!--");
+		Util::ReduceStr(body, "<!-- %end% -->", "-->");
+		Util::ReduceStr(body, "%end% -->", "");
+		len = strlen(body);
+	}
+#endif
+
+	SendBodyResponse(body, len, contentType, true);
+}
+
+void WebProcessor::SendMultiFileResponse()
+{
+	debug("serving multiple files: %s", *m_url);
+
+	StringBuilder response;
+	char* filelist = strchr(m_url, '?');
+	*filelist++ = '\0';
+
+	Tokenizer tok(filelist, "+");
+	while (const char* filename = tok.Next())
+	{
+		BString<1024> diskFilename("%s%c%s", g_Options->GetWebDir(), PATH_SEPARATOR, filename);
+
+		CharBuffer body;
+		if (!FileSystem::LoadFileIntoBuffer(diskFilename, body, true))
+		{
+			warn("Web-Server: %s, Resource: /%s", ERR_HTTP_NOT_FOUND, filename);
+			SendErrorResponse(ERR_HTTP_NOT_FOUND, false);
+			return;
+		}
+
+		response.Append(body);
+	}
+
+	SendBodyResponse(response, response.Length(), DetectContentType(m_url), true);
 }
 
 const char* WebProcessor::DetectContentType(const char* filename)

daemon/remote/WebServer.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -39,6 +39,7 @@ public:
 	void SetConnection(Connection* connection) { m_connection = connection; }
 	void SetUrl(const char* url) { m_url = url; }
 	void SetHttpMethod(EHttpMethod httpMethod) { m_httpMethod = httpMethod; }
+	bool GetKeepAlive() { return m_keepAlive; }
 
 private:
 	enum EUserAccess
@@ -53,19 +54,25 @@ private:
 	CString m_url;
 	EHttpMethod m_httpMethod;
 	EUserAccess m_userAccess;
+	bool m_rpcRequest;
+	bool m_authorized;
 	bool m_gzip;
 	CString m_origin;
 	int m_contentLen;
 	char m_authInfo[256+1];
 	char m_authToken[48+1];
 	static char m_serverAuthToken[3][48+1];
+	CString m_forwardedFor;
+	CString m_oldETag;
+	bool m_keepAlive = false;
 
 	void Dispatch();
 	void SendAuthResponse();
 	void SendOptionsResponse();
 	void SendErrorResponse(const char* errCode, bool printWarning);
-	void SendFileResponse(const char* filename);
-	void SendBodyResponse(const char* body, int bodyLen, const char* contentType);
+	void SendSingleFileResponse();
+	void SendMultiFileResponse();
+	void SendBodyResponse(const char* body, int bodyLen, const char* contentType, bool cachable);
 	void SendRedirectResponse(const char* url);
 	const char* DetectContentType(const char* filename);
 	bool IsAuthorizedIp(const char* remoteAddr);

daemon/remote/XmlRpc.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,16 +33,24 @@
 #include "DiskState.h"
 #include "ScriptConfig.h"
 #include "QueueScript.h"
+#include "CommandScript.h"
 
 extern void ExitProc();
 extern void Reload();
 
+class SafeXmlCommand: public XmlCommand
+{
+public:
+	virtual bool IsSafeMethod() { return true; };
+};
+
 class ErrorXmlCommand: public XmlCommand
 {
 public:
 	ErrorXmlCommand(int errCode, const char* errText) :
 		m_errCode(errCode), m_errText(errText) {}
 	virtual void Execute();
+	virtual bool IsError() { return true; };
 
 private:
 	int m_errCode;
@@ -86,13 +94,13 @@ public:
 	virtual void Execute();
 };
 
-class VersionXmlCommand: public XmlCommand
+class VersionXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
 };
 
-class DumpDebugXmlCommand: public XmlCommand
+class DumpDebugXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -104,13 +112,13 @@ public:
 	virtual void Execute();
 };
 
-class StatusXmlCommand: public XmlCommand
+class StatusXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
 };
 
-class LogXmlCommand: public XmlCommand
+class LogXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -121,14 +129,14 @@ protected:
 	virtual GuardedMessageList GuardMessages();
 };
 
-class NzbInfoXmlCommand: public XmlCommand
+class NzbInfoXmlCommand: public SafeXmlCommand
 {
 protected:
 	void AppendNzbInfoFields(NzbInfo* nzbInfo);
 	void AppendPostInfoFields(PostInfo* postInfo, int logEntries, bool postQueue);
 };
 
-class ListFilesXmlCommand: public XmlCommand
+class ListFilesXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -186,19 +194,19 @@ private:
 	const char* DetectStatus(HistoryInfo* historyInfo);
 };
 
-class UrlQueueXmlCommand: public XmlCommand
+class UrlQueueXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
 };
 
-class ConfigXmlCommand: public XmlCommand
+class ConfigXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
 };
 
-class LoadConfigXmlCommand: public XmlCommand
+class LoadConfigXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -210,7 +218,7 @@ public:
 	virtual void Execute();
 };
 
-class ConfigTemplatesXmlCommand: public XmlCommand
+class ConfigTemplatesXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -237,7 +245,7 @@ public:
 	virtual void Execute();
 };
 
-class ReadUrlXmlCommand: public XmlCommand
+class ReadUrlXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -261,7 +269,7 @@ protected:
 	virtual GuardedMessageList GuardMessages();
 };
 
-class ServerVolumesXmlCommand: public XmlCommand
+class ServerVolumesXmlCommand: public SafeXmlCommand
 {
 public:
 	virtual void Execute();
@@ -306,6 +314,18 @@ private:
 	void PrintError(const char* errMsg);
 };
 
+class StartScriptXmlCommand : public XmlCommand
+{
+public:
+	virtual void Execute();
+};
+
+class LogScriptXmlCommand : public LogXmlCommand
+{
+protected:
+	virtual GuardedMessageList GuardMessages();
+};
+
 
 //*****************************************************************
 // XmlRpcProcessor
@@ -407,8 +427,17 @@ void XmlRpcProcessor::Dispatch()
 		command->SetHttpMethod(m_httpMethod);
 		command->SetUserAccess(m_userAccess);
 		command->PrepareParams();
-		command->Execute();
-		BuildResponse(command->GetResponse(), command->GetCallbackFunc(), command->GetFault(), requestId);
+		m_safeMethod = command->IsSafeMethod();
+		bool safeToExecute = m_safeMethod || m_httpMethod == XmlRpcProcessor::hmPost || m_protocol == XmlRpcProcessor::rpJsonPRpc;
+		if (safeToExecute || command->IsError())
+		{
+			command->Execute();
+			BuildResponse(command->GetResponse(), command->GetCallbackFunc(), command->GetFault(), requestId);
+		}
+		else
+		{
+			BuildErrorResponse(4, "Not safe procedure for HTTP-Method GET. Use Method POST instead");
+		}
 	}
 }
 
@@ -439,6 +468,7 @@ void XmlRpcProcessor::MutliCall()
 
 		std::unique_ptr<XmlCommand> command = CreateCommand(methodName);
 		command->SetRequest(requestPtr);
+		m_safeMethod |= command->IsSafeMethod();
 		command->Execute();
 
 		debug("MutliCall, Response=%s", command->GetResponse());
@@ -463,12 +493,7 @@ void XmlRpcProcessor::MutliCall()
 
 	if (error)
 	{
-		ErrorXmlCommand command(4, "Parse error");
-		command.SetRequest(m_request);
-		command.SetProtocol(rpXmlRpc);
-		command.PrepareParams();
-		command.Execute();
-		BuildResponse(command.GetResponse(), "", command.GetFault(), nullptr);
+		BuildErrorResponse(4, "Parse error");
 	}
 	else
 	{
@@ -531,6 +556,16 @@ void XmlRpcProcessor::BuildResponse(const char* response, const char* callbackFu
 	m_contentType = xmlRpc ? "text/xml" : "application/json";
 }
 
+void XmlRpcProcessor::BuildErrorResponse(int errCode, const char* errText)
+{
+	ErrorXmlCommand command(errCode, errText);
+	command.SetRequest(m_request);
+	command.SetProtocol(m_protocol);
+	command.PrepareParams();
+	command.Execute();
+	BuildResponse(command.GetResponse(), "", command.GetFault(), nullptr);
+}
+
 std::unique_ptr<XmlCommand> XmlRpcProcessor::CreateCommand(const char* methodName)
 {
 	std::unique_ptr<XmlCommand> command;
@@ -709,6 +744,14 @@ std::unique_ptr<XmlCommand> XmlRpcProcessor::CreateCommand(const char* methodNam
 	{
 		command = std::make_unique<TestServerXmlCommand>();
 	}
+	else if (!strcasecmp(methodName, "startscript"))
+	{
+		command = std::make_unique<StartScriptXmlCommand>();
+	}
+	else if (!strcasecmp(methodName, "logscript"))
+	{
+		command = std::make_unique<LogScriptXmlCommand>();
+	}
 	else
 	{
 		command = std::make_unique<ErrorXmlCommand>(1, "Invalid procedure");
@@ -1046,16 +1089,6 @@ void XmlCommand::DecodeStr(char* str)
 	}
 }
 
-bool XmlCommand::CheckSafeMethod()
-{
-	bool safe = m_httpMethod == XmlRpcProcessor::hmPost || m_protocol == XmlRpcProcessor::rpJsonPRpc;
-	if (!safe)
-	{
-		BuildErrorResponse(4, "Not safe procedure for HTTP-Method GET. Use Method POST instead");
-	}
-	return safe;
-}
-
 //*****************************************************************
 // Commands
 
@@ -1066,11 +1099,6 @@ void ErrorXmlCommand::Execute()
 
 void PauseUnpauseXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	bool ok = true;
 
 	g_Options->SetResumeTime(0);
@@ -1099,11 +1127,6 @@ void PauseUnpauseXmlCommand::Execute()
 // bool scheduleresume(int Seconds)
 void ScheduleResumeXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	int seconds = 0;
 	if (!NextParamAsInt(&seconds) || seconds < 0)
 	{
@@ -1120,22 +1143,12 @@ void ScheduleResumeXmlCommand::Execute()
 
 void ShutdownXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	BuildBoolResponse(true);
 	ExitProc();
 }
 
 void ReloadXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	BuildBoolResponse(true);
 	Reload();
 }
@@ -1158,11 +1171,6 @@ void DumpDebugXmlCommand::Execute()
 
 void SetDownloadRateXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	int rate = 0;
 	if (!NextParamAsInt(&rate) || rate < 0)
 	{
@@ -1344,8 +1352,8 @@ void StatusXmlCommand::Execute()
 	Util::SplitInt64(freeDiskSpace, &freeDiskSpaceHi, &freeDiskSpaceLo);
 	int freeDiskSpaceMB = (int)(freeDiskSpace / 1024 / 1024);
 
-	int serverTime = Util::CurrentTime();
-	int resumeTime = g_Options->GetResumeTime();
+	int serverTime = (int)Util::CurrentTime();
+	int resumeTime = (int)g_Options->GetResumeTime();
 	bool feedActive = g_FeedCoordinator->HasActiveDownloads();
 	int queuedScripts = g_QueueScriptCoordinator->GetQueueSize();
 
@@ -1978,6 +1986,10 @@ const char* ListGroupsXmlCommand::DetectStatus(NzbInfo* nzbInfo)
 		{
 			status = queueScriptActive ? "QS_EXECUTING" : "QS_QUEUED";
 		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsRunning)
+		{
+			status = "UNPACKING";
+		}
 		else
 		{
 			status = postStageName[nzbInfo->GetPostInfo()->GetStage()];
@@ -2039,6 +2051,7 @@ EditCommandEntry EditCommandNameMap[] = {
 	{ DownloadQueue::eaGroupSetDupeScore, "GroupSetDupeScore" },
 	{ DownloadQueue::eaGroupSetDupeMode, "GroupSetDupeMode" },
 	{ DownloadQueue::eaGroupSort, "GroupSort" },
+	{ DownloadQueue::eaGroupSortFiles, "GroupSortFiles" },
 	{ DownloadQueue::eaPostDelete, "PostDelete" },
 	{ DownloadQueue::eaHistoryDelete, "HistoryDelete" },
 	{ DownloadQueue::eaHistoryFinalDelete, "HistoryFinalDelete" },
@@ -2065,11 +2078,6 @@ EditCommandEntry EditCommandNameMap[] = {
 //    bool editqueue(string Command, int Offset, string Args, int[] IDs)
 void EditQueueXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	char* editCommand;
 	if (!NextParamAsStr(&editCommand))
 	{
@@ -2135,11 +2143,6 @@ void EditQueueXmlCommand::Execute()
 //   bool append(string NZBFilename, string Category, int Priority, bool AddToTop, string Content, bool AddPaused, string DupeKey, int DupeScore, string DupeMode)
 void DownloadXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	bool v13 = true;
 
 	char* nzbFilename;
@@ -2352,11 +2355,6 @@ void PostQueueXmlCommand::Execute()
 
 void WriteLogXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	char* kind;
 	char* text;
 	if (!NextParamAsStr(&kind) || !NextParamAsStr(&text))
@@ -2400,11 +2398,6 @@ void WriteLogXmlCommand::Execute()
 
 void ClearLogXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	g_Log->Clear();
 
 	BuildBoolResponse(true);
@@ -2412,11 +2405,6 @@ void ClearLogXmlCommand::Execute()
 
 void ScanXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	bool syncMode = false;
 	// optional parameter "SyncMode"
 	NextParamAsBool(&syncMode);
@@ -2922,11 +2910,6 @@ void ViewFeedXmlCommand::Execute()
 // bool fetchfeed(int ID)
 void FetchFeedXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	int id;
 	if (!NextParamAsInt(&id))
 	{
@@ -2942,11 +2925,6 @@ void FetchFeedXmlCommand::Execute()
 // bool editserver(int ID, bool Active)
 void EditServerXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	bool ok = false;
 	int first = true;
 
@@ -3066,11 +3044,6 @@ void CheckUpdatesXmlCommand::Execute()
 // bool startupdate(string branch)
 void StartUpdateXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	char* branchName;
 	if (!NextParamAsStr(&branchName))
 	{
@@ -3109,7 +3082,7 @@ GuardedMessageList LogUpdateXmlCommand::GuardMessages()
 	return g_Maintenance->GuardMessages();
 }
 
-// struct[] servervolumes()
+// struct[] servervolumes(bool BytesPerSeconds, bool BytesPerMinutes, bool BytesPerHours, bool BytesPerDays)
 void ServerVolumesXmlCommand::Execute()
 {
 	const char* XML_VOLUME_ITEM_START =
@@ -3180,6 +3153,10 @@ void ServerVolumesXmlCommand::Execute()
 
 	AppendResponse(IsJson() ? "[\n" : "<array><data>\n");
 
+	bool BytesPer[] = {true, true, true, true};
+	NextParamAsBool(&BytesPer[0]) && NextParamAsBool(&BytesPer[1]) &&
+		NextParamAsBool(&BytesPer[2]) && NextParamAsBool(&BytesPer[3]);
+
 	int index = 0;
 
 	for (ServerVolume& serverVolume : g_StatMeter->GuardServerVolumes())
@@ -3205,25 +3182,28 @@ void ServerVolumesXmlCommand::Execute()
 
 		for (int i=0; i<4; i++)
 		{
-			ServerVolume::VolumeArray* volumeArray = VolumeArrays[i];
-			const char* arrayName = VolumeNames[i];
-
-			AppendFmtResponse(IsJson() ? JSON_BYTES_ARRAY_START : XML_BYTES_ARRAY_START, arrayName);
-
-			int index2 = 0;
-			for (int64 bytes : *volumeArray)
+			if (BytesPer[i])
 			{
-				uint32 sizeHi, sizeLo, sizeMB;
-				Util::SplitInt64(bytes, &sizeHi, &sizeLo);
-				sizeMB = (int)(bytes / 1024 / 1024);
+				ServerVolume::VolumeArray* volumeArray = VolumeArrays[i];
+				const char* arrayName = VolumeNames[i];
 
-				AppendCondResponse(",\n", IsJson() && index2++ > 0);
-				AppendFmtResponse(IsJson() ? JSON_BYTES_ARRAY_ITEM : XML_BYTES_ARRAY_ITEM,
-						 sizeLo, sizeHi, sizeMB);
+				AppendFmtResponse(IsJson() ? JSON_BYTES_ARRAY_START : XML_BYTES_ARRAY_START, arrayName);
+
+				int index2 = 0;
+				for (int64 bytes : *volumeArray)
+				{
+					uint32 sizeHi, sizeLo, sizeMB;
+					Util::SplitInt64(bytes, &sizeHi, &sizeLo);
+					sizeMB = (int)(bytes / 1024 / 1024);
+
+					AppendCondResponse(",\n", IsJson() && index2++ > 0);
+					AppendFmtResponse(IsJson() ? JSON_BYTES_ARRAY_ITEM : XML_BYTES_ARRAY_ITEM,
+							 sizeLo, sizeHi, sizeMB);
+				}
+
+				AppendResponse(IsJson() ? JSON_BYTES_ARRAY_END : XML_BYTES_ARRAY_END);
+				AppendCondResponse(",\n", IsJson() && i < 3);
 			}
-
-			AppendResponse(IsJson() ? JSON_BYTES_ARRAY_END : XML_BYTES_ARRAY_END);
-			AppendCondResponse(",\n", IsJson() && i < 3);
 		}
 		AppendResponse(IsJson() ? JSON_VOLUME_ITEM_END : XML_VOLUME_ITEM_END);
 		index++;
@@ -3235,11 +3215,6 @@ void ServerVolumesXmlCommand::Execute()
 // bool resetservervolume(int serverid, string counter);
 void ResetServerVolumeXmlCommand::Execute()
 {
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	int serverId;
 	char* counter;
 	if (!NextParamAsInt(&serverId) || !NextParamAsStr(&counter))
@@ -3318,11 +3293,6 @@ void TestServerXmlCommand::Execute()
 	const char* XML_RESPONSE_STR_BODY = "<string>%s</string>";
 	const char* JSON_RESPONSE_STR_BODY = "\"%s\"";
 
-	if (!CheckSafeMethod())
-	{
-		return;
-	}
-
 	char* host;
 	int port;
 	char* username;
@@ -3339,12 +3309,23 @@ void TestServerXmlCommand::Execute()
 		return;
 	}
 
-	NewsServer server(0, true, "test server", host, port, username, password, false, encryption, cipher, 1, 0, 0, 0, false);
+	NewsServer server(0, true, "test server", host, port, 0, username, password, false, encryption, cipher, 1, 0, 0, 0, false);
 	TestConnection connection(&server, this);
 	connection.SetTimeout(timeout == 0 ? g_Options->GetArticleTimeout() : timeout);
 	connection.SetSuppressErrors(false);
 
 	bool ok = connection.Connect();
+	if (ok)
+	{
+		// generate a unique non-existent message-id since we don't want a real article to be returned
+		BString<1024> id;
+		while (id.Length() < 30)
+		{
+			id.AppendFmt("%i", rand());
+		}
+		const char* response = connection.Request(BString<1024>("ARTICLE <%s@nzbget.net>\r\n", *id));
+		ok = response && (*response == '4' || *response == '2');
+	}
 
 	BString<1024> content(IsJson() ? JSON_RESPONSE_STR_BODY : XML_RESPONSE_STR_BODY,
 		ok ? "" : m_errText.Empty() ? "Unknown error" : *m_errText);
@@ -3359,3 +3340,40 @@ void TestServerXmlCommand::PrintError(const char* errMsg)
 		m_errText = EncodeStr(errMsg);
 	}
 }
+
+// bool startscript(string script, string command, string context, struct[] options);
+void StartScriptXmlCommand::Execute()
+{
+	char* script;
+	char* command;
+	char* context;
+	if (!NextParamAsStr(&script) || !NextParamAsStr(&command) || !NextParamAsStr(&context))
+	{
+		BuildErrorResponse(2, "Invalid parameter");
+		return;
+	}
+
+	std::unique_ptr<Options::OptEntries> optEntries = std::make_unique<Options::OptEntries>();
+
+	char* name;
+	char* value;
+	char* dummy;
+	while ((IsJson() && NextParamAsStr(&dummy) && NextParamAsStr(&name) &&
+		NextParamAsStr(&dummy) && NextParamAsStr(&value)) ||
+		(!IsJson() && NextParamAsStr(&name) && NextParamAsStr(&value)))
+	{
+		DecodeStr(name);
+		DecodeStr(value);
+		optEntries->emplace_back(name, value);
+	}
+
+	bool ok = CommandScriptController::StartScript(script, command, std::move(optEntries));
+
+	BuildBoolResponse(ok);
+}
+
+// struct[] logscript(idfrom, entries)
+GuardedMessageList LogScriptXmlCommand::GuardMessages()
+{
+	return g_CommandScriptLog->GuardMessages();
+}

daemon/remote/XmlRpc.h

@@ -59,6 +59,7 @@ public:
 	const char* GetResponse() { return m_response; }
 	const char* GetContentType() { return m_contentType; }
 	static bool IsRpcRequest(const char* url);
+	bool IsSafeMethod() { return m_safeMethod; };
 
 private:
 	char* m_request = nullptr;
@@ -68,11 +69,13 @@ private:
 	EUserAccess m_userAccess;
 	CString m_url;
 	StringBuilder m_response;
+	bool m_safeMethod = false;
 
 	void Dispatch();
 	std::unique_ptr<XmlCommand> CreateCommand(const char* methodName);
 	void MutliCall();
 	void BuildResponse(const char* response, const char* callbackFunc, bool fault, const char* requestId);
+	void BuildErrorResponse(int errCode, const char* errText);
 };
 
 class XmlCommand
@@ -89,6 +92,8 @@ public:
 	const char* GetResponse() { return m_response; }
 	const char* GetCallbackFunc() { return m_callbackFunc; }
 	bool GetFault() { return m_fault; }
+	virtual bool IsSafeMethod() { return false; };
+	virtual bool IsError() { return false; };
 
 protected:
 	char* m_request = nullptr;
@@ -107,7 +112,6 @@ protected:
 	void AppendFmtResponse(const char* format, ...);
 	void AppendCondResponse(const char* part, bool cond);
 	bool IsJson();
-	bool CheckSafeMethod();
 	bool NextParamAsInt(int* value);
 	bool NextParamAsBool(bool* value);
 	bool NextParamAsStr(char** valueBuf);

daemon/util/FileSystem.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -417,7 +417,7 @@ CString FileSystem::MakeValidFilename(const char* filename, bool allowSlashes)
 CString FileSystem::MakeUniqueFilename(const char* destDir, const char* basename)
 {
 	CString result;
-	result.Format("%s%c%s", destDir, (int)PATH_SEPARATOR, basename);
+	result.Format("%s%c%s", destDir, PATH_SEPARATOR, basename);
 
 	int dupeNumber = 0;
 	while (FileExists(result))
@@ -428,7 +428,7 @@ CString FileSystem::MakeUniqueFilename(const char* destDir, const char* basename
 		if (extension && extension != basename)
 		{
 			BString<1024> filenameWithoutExt = basename;
-			int end = extension - basename;
+			int end = (int)(extension - basename);
 			filenameWithoutExt[end < 1024 ? end : 1024-1] = '\0';
 
 			if (!strcasecmp(extension, ".par2"))
@@ -442,12 +442,12 @@ CString FileSystem::MakeUniqueFilename(const char* destDir, const char* basename
 				}
 			}
 
-			result.Format("%s%c%s.duplicate%d%s", destDir, (int)PATH_SEPARATOR,
+			result.Format("%s%c%s.duplicate%d%s", destDir, PATH_SEPARATOR,
 				*filenameWithoutExt, dupeNumber, extension);
 		}
 		else
 		{
-			result.Format("%s%c%s.duplicate%d", destDir, (int)PATH_SEPARATOR,
+			result.Format("%s%c%s.duplicate%d", destDir, PATH_SEPARATOR,
 				basename, dupeNumber);
 		}
 	}
@@ -527,13 +527,6 @@ bool FileSystem::FileExists(const char* filename)
 #endif
 }
 
-bool FileSystem::FileExists(const char* path, const char* filenameWithoutPath)
-{
-	BString<1024> fullFilename("%s%c%s", path, (int)PATH_SEPARATOR, filenameWithoutPath);
-	bool exists = FileExists(fullFilename);
-	return exists;
-}
-
 bool FileSystem::DirectoryExists(const char* dirFilename)
 {
 #ifdef WIN32
@@ -580,6 +573,37 @@ bool FileSystem::RemoveDirectory(const char* dirFilename)
 #endif
 }
 
+/* Delete directory which is empty or contains only hidden files or directories (whose names start with dot) */
+bool FileSystem::DeleteDirectory(const char* dirFilename)
+{
+	if (RemoveDirectory(dirFilename))
+	{
+		return true;
+	}
+
+	// check if directory contains only hidden files (whose names start with dot)
+	{
+		DirBrowser dir(dirFilename);
+		while (const char* filename = dir.Next())
+		{
+			if (*filename != '.')
+			{
+				// calling RemoveDirectory to set correct errno
+				return RemoveDirectory(dirFilename);
+			}
+		}
+	} // make sure "DirBrowser dir" is destroyed (and has closed its handle) before we trying to delete the directory
+
+	CString errmsg;
+	if (!DeleteDirectoryWithContent(dirFilename, errmsg))
+	{
+		// calling RemoveDirectory to set correct errno
+		return RemoveDirectory(dirFilename);
+	}
+
+	return true;
+}
+
 bool FileSystem::DeleteDirectoryWithContent(const char* dirFilename, CString& errmsg)
 {
 	errmsg.Clear();
@@ -999,7 +1023,7 @@ CString FileSystem::WidePathToUtfPath(const wchar_t* wpath)
 #ifdef WIN32
 DirBrowser::DirBrowser(const char* path)
 {
-	BString<1024> mask("%s%c*.*", path, (int)PATH_SEPARATOR);
+	BString<1024> mask("%s%c*.*", path, PATH_SEPARATOR);
 	m_file = FindFirstFileW(FileSystem::UtfPathToWidePath(mask), &m_findData);
 	m_first = true;
 }

daemon/util/FileSystem.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -41,10 +41,15 @@ public:
 	static bool CopyFile(const char* srcFilename, const char* dstFilename);
 	static bool DeleteFile(const char* filename);
 	static bool FileExists(const char* filename);
-	static bool FileExists(const char* path, const char* filenameWithoutPath);
 	static bool DirectoryExists(const char* dirFilename);
 	static bool CreateDirectory(const char* dirFilename);
+
+	/* Delete empty directory */
 	static bool RemoveDirectory(const char* dirFilename);
+
+	/* Delete directory which is empty or contains only hidden files or directories */
+	static bool DeleteDirectory(const char* dirFilename);
+
 	static bool DeleteDirectoryWithContent(const char* dirFilename, CString& errmsg);
 	static bool ForceDirectories(const char* path, CString& errmsg);
 	static CString GetCurrentDirectory();

daemon/util/Log.cpp

@@ -75,31 +75,55 @@ void Log::Filelog(const char* msg, ...)
 
 	if ((int)rawtime/86400 != (int)m_lastWritten/86400 && g_Options->GetWriteLog() == Options::wlRotate)
 	{
+		if (m_logFile)
+		{
+			m_logFile.reset();
+		}
 		RotateLog();
 	}
 
 	m_lastWritten = rawtime;
 
-	DiskFile file;
-	if (file.Open(m_logFilename, DiskFile::omAppend))
+	if (!m_logFile)
 	{
+		m_logFile = std::make_unique<DiskFile>();
+		if (!m_logFile->Open(m_logFilename, DiskFile::omAppend))
+		{
+			perror(m_logFilename);
+			m_logFile.reset();
+			return;
+		}
+	}
+
+	m_logFile->Seek(0, DiskFile::soEnd);
+
 #ifdef WIN32
-		uint64 processId = GetCurrentProcessId();
-		uint64 threadId = GetCurrentThreadId();
+	uint64 processId = GetCurrentProcessId();
+	uint64 threadId = GetCurrentThreadId();
 #else
-		uint64 processId = (uint64)getpid();
-		uint64 threadId = (uint64)pthread_self();
+	uint64 processId = (uint64)getpid();
+	uint64 threadId = (uint64)pthread_self();
 #endif
 #ifdef DEBUG
-		file.Print("%s\t%llu\t%llu\t%s%s", time, processId, threadId, tmp2, LINE_ENDING);
+	m_logFile->Print("%s\t%llu\t%llu\t%s%s", time, processId, threadId, tmp2, LINE_ENDING);
 #else
-		file.Print("%s\t%s%s", time, tmp2, LINE_ENDING);
+	m_logFile->Print("%s\t%s%s", time, tmp2, LINE_ENDING);
 #endif
-		file.Close();
-	}
-	else
+
+	m_logFile->Flush();
+}
+
+void Log::IntervalCheck()
+{
+	// Close log-file on idle (if last write into log was more than a second ago)
+	if (m_logFile)
 	{
-		perror(m_logFilename);
+		time_t curTime = Util::CurrentTime() + g_Options->GetTimeCorrection();
+		if (std::abs(curTime - m_lastWritten) > 1)
+		{
+			Guard guard(m_logMutex);
+			m_logFile.reset();
+		}
 	}
 }
 
@@ -111,6 +135,11 @@ void debug(const char* filename, const char* funcname, int lineNr, const char* m
 void debug(const char* msg, ...)
 #endif
 {
+	if (!g_Log)
+	{
+		return;
+	}
+
 	char tmp1[1024];
 
 	va_list ap;
@@ -257,7 +286,7 @@ void Log::AddMessage(Message::EKind kind, const char * text)
 
 	if (m_optInit && g_Options)
 	{
-		while (m_messages.size() > (uint32)g_Options->GetLogBufferSize())
+		while (m_messages.size() > (uint32)g_Options->GetLogBuffer())
 		{
 			m_messages.pop_front();
 		}

daemon/util/Log.h

@@ -70,6 +70,7 @@ typedef std::deque<Message> MessageList;
 typedef GuardedPtr<MessageList> GuardedMessageList;
 
 class Debuggable;
+class DiskFile;
 
 class Log
 {
@@ -83,6 +84,7 @@ public:
 	void RegisterDebuggable(Debuggable* debuggable);
 	void UnregisterDebuggable(Debuggable* debuggable);
 	void LogDebugInfo();
+	void IntervalCheck();
 
 private:
 	typedef std::list<Debuggable*> Debuggables;
@@ -92,6 +94,7 @@ private:
 	Debuggables m_debuggables;
 	Mutex m_debugMutex;
 	CString m_logFilename;
+	std::unique_ptr<DiskFile> m_logFile;
 	uint32 m_idGen = 0;
 	time_t m_lastWritten = 0;
 	bool m_optInit = false;

daemon/util/Script.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -198,14 +198,15 @@ void ScriptController::PrepareEnvOptions(const char* stripPrefix)
 
 	for (Options::OptEntry& optEntry : g_Options->GuardOptEntries())
 	{
+		const char* value = GetOptValue(optEntry.GetName(), optEntry.GetValue());
 		if (stripPrefix && !strncmp(optEntry.GetName(), stripPrefix, prefixLen) &&
 			(int)strlen(optEntry.GetName()) > prefixLen)
 		{
-			SetEnvVarSpecial("NZBPO", optEntry.GetName() + prefixLen, optEntry.GetValue());
+			SetEnvVarSpecial("NZBPO", optEntry.GetName() + prefixLen, value);
 		}
 		else if (!stripPrefix)
 		{
-			SetEnvVarSpecial("NZBOP", optEntry.GetName(), optEntry.GetValue());
+			SetEnvVarSpecial("NZBOP", optEntry.GetName(), value);
 		}
 	}
 }
@@ -287,6 +288,7 @@ void ScriptController::PrepareArgs()
 					if (FormatMessage(FORMAT_MESSAGE_FROM_STRING | FORMAT_MESSAGE_ARGUMENT_ARRAY, command, 0, 0,
 						m_cmdLine, sizeof(m_cmdLine), (va_list*)args))
 					{
+						Util::TrimRight(Util::ReduceStr(m_cmdLine, "*", ""));
 						debug("CmdLine: %s", m_cmdLine);
 						return;
 					}
@@ -313,7 +315,8 @@ int ScriptController::Execute()
 	{
 #endif
 
-	int pipein = StartProcess();
+	int pipein = -1, pipeout = -1;
+	StartProcess(&pipein, &pipeout);
 	if (pipein == -1)
 	{
 		m_completed = true;
@@ -324,12 +327,28 @@ int ScriptController::Execute()
 	m_readpipe = fdopen(pipein, "r");
 	if (!m_readpipe)
 	{
-		PrintMessage(Message::mkError, "Could not open pipe to %s", *m_infoName);
+		PrintMessage(Message::mkError, "Could not open read pipe to %s", *m_infoName);
 		close(pipein);
+		close(pipeout);
 		m_completed = true;
 		return -1;
 	}
 
+	m_writepipe = 0;
+	if (m_needWrite)
+	{
+		// open the write end
+		m_writepipe = fdopen(pipeout, "w");
+		if (!m_writepipe)
+		{
+			PrintMessage(Message::mkError, "Could not open write pipe to %s", *m_infoName);
+			close(pipein);
+			close(pipeout);
+			m_completed = true;
+			return -1;
+		}
+	}
+
 #ifdef CHILD_WATCHDOG
 	debug("Creating child watchdog");
 	ChildWatchDog watchDog;
@@ -384,6 +403,11 @@ int ScriptController::Execute()
 		fclose(m_readpipe);
 	}
 
+	if (m_writepipe)
+	{
+		fclose(m_writepipe);
+	}
+
 	if (m_terminated && m_infoName)
 	{
 		warn("Interrupted %s", *m_infoName);
@@ -391,7 +415,7 @@ int ScriptController::Execute()
 
 	exitCode = 0;
 
-	if (!m_terminated && !m_detached)
+	if (!m_detached)
 	{
 		exitCode = WaitProcess();
 #ifndef WIN32
@@ -430,7 +454,7 @@ void ScriptController::BuildCommandLine(char* cmdLineBuf, int bufSize)
 /*
 * Returns file descriptor of the read-pipe or -1 on error.
 */
-int ScriptController::StartProcess()
+void ScriptController::StartProcess(int* pipein, int* pipeout)
 {
 	CString workingDir = m_workingDir;
 	if (workingDir.Empty())
@@ -449,6 +473,8 @@ int ScriptController::StartProcess()
 		cmdLine = cmdLineBuf;
 	}
 
+	debug("Starting process: %s", cmdLine);
+
 	WString wideWorkingDir = FileSystem::UtfPathToWidePath(workingDir);
 	if (strlen(workingDir) > 260 - 14)
 	{
@@ -456,21 +482,27 @@ int ScriptController::StartProcess()
 	}
 
 	// create pipes to write and read data
-	HANDLE readPipe, writePipe;
+	HANDLE readPipe, readProcPipe;
+	HANDLE writePipe = 0, writeProcPipe = 0;
 	SECURITY_ATTRIBUTES securityAttributes = { 0 };
 	securityAttributes.nLength = sizeof(securityAttributes);
 	securityAttributes.bInheritHandle = TRUE;
 
-	CreatePipe(&readPipe, &writePipe, &securityAttributes, 0);
-
+	CreatePipe(&readPipe, &readProcPipe, &securityAttributes, 0);
 	SetHandleInformation(readPipe, HANDLE_FLAG_INHERIT, 0);
 
+	if (m_needWrite)
+	{
+		CreatePipe(&writeProcPipe, &writePipe, &securityAttributes, 0);
+		SetHandleInformation(writePipe, HANDLE_FLAG_INHERIT, 0);
+	}
+
 	STARTUPINFOW startupInfo = { 0 };
 	startupInfo.cb = sizeof(startupInfo);
 	startupInfo.dwFlags = STARTF_USESTDHANDLES;
-	startupInfo.hStdInput = 0;
-	startupInfo.hStdOutput = writePipe;
-	startupInfo.hStdError = writePipe;
+	startupInfo.hStdInput = writeProcPipe;
+	startupInfo.hStdOutput = readProcPipe;
+	startupInfo.hStdError = readProcPipe;
 
 	PROCESS_INFORMATION processInfo = { 0 };
 
@@ -501,8 +533,10 @@ int ScriptController::StartProcess()
 			PrintMessage(Message::mkError, "Could not build short path for %s", workingDir);
 		}
 		CloseHandle(readPipe);
+		CloseHandle(readProcPipe);
 		CloseHandle(writePipe);
-		return -1;
+		CloseHandle(writeProcPipe);
+		return;
 	}
 
 	debug("Child Process-ID: %i", (int)processInfo.dwProcessId);
@@ -510,24 +544,37 @@ int ScriptController::StartProcess()
 	m_processId = processInfo.hProcess;
 	m_dwProcessId = processInfo.dwProcessId;
 
-	// close unused "write" end
-	CloseHandle(writePipe);
+	// close unused pipe ends
+	CloseHandle(readProcPipe);
+	CloseHandle(writeProcPipe);
 
-	int pipein = _open_osfhandle((intptr_t)readPipe, _O_RDONLY);
-	return pipein;
+	*pipein = _open_osfhandle((intptr_t)readPipe, _O_RDONLY);
+	if (m_needWrite)
+	{
+		*pipeout = _open_osfhandle((intptr_t)writePipe, _O_WRONLY);
+	}
 
 #else
 
-	int p[2];
-	int pipein;
-	int pipeout;
+	int pin[] = {0, 0};
+	int pout[] = {0, 0};
 
-	// create the pipe
-	if (pipe(p))
+	// create the pipes
+	if (pipe(pin))
 	{
-		PrintMessage(Message::mkError, "Could not open pipe: errno %i", errno);
-		return -1;
+		PrintMessage(Message::mkError, "Could not open read pipe: errno %i", errno);
+		return;
 	}
+	if (m_needWrite && pipe(pout))
+	{
+		PrintMessage(Message::mkError, "Could not open write pipe: errno %i", errno);
+		close(pin[0]);
+		close(pin[1]);
+		return;
+	}
+
+	*pipein = pin[0];
+	*pipeout = pout[1];
 
 	std::vector<char*> environmentStrings = m_environmentStrings.GetStrings();
 	char** envdata = environmentStrings.data();
@@ -537,8 +584,11 @@ int ScriptController::StartProcess()
 	args.emplace_back(nullptr);
 	char* const* argdata = (char* const*)args.data();
 
-	pipein = p[0];
-	pipeout = p[1];
+	debug("Starting  process: %s", script);
+	for (const char* arg : m_args)
+	{
+		debug("arg: %s", arg);
+	}
 
 	debug("forking");
 	pid_t pid = fork();
@@ -546,9 +596,14 @@ int ScriptController::StartProcess()
 	if (pid == -1)
 	{
 		PrintMessage(Message::mkError, "Could not start %s: errno %i", *m_infoName, errno);
-		close(pipein);
-		close(pipeout);
-		return -1;
+		close(pin[0]);
+		close(pin[1]);
+		if (m_needWrite)
+		{
+			close(pout[0]);
+			close(pout[1]);
+		}
+		return;
 	}
 	else if (pid == 0)
 	{
@@ -561,14 +616,19 @@ int ScriptController::StartProcess()
 		// create new process group (see Terminate() where it is used)
 		setsid();
 
-		// close up the "read" end
-		close(pipein);
-
 		// make the pipeout to be the same as stdout and stderr
-		dup2(pipeout, 1);
-		dup2(pipeout, 2);
+		dup2(pin[1], 1);
+		dup2(pin[1], 2);
+		close(pin[0]);
+		close(pin[1]);
 
-		close(pipeout);
+		if (m_needWrite)
+		{
+			// make the pipein to be the same as stdin
+			dup2(pout[0], 0);
+			close(pout[0]);
+			close(pout[1]);
+		}
 
 #ifdef CHILD_WATCHDOG
 		write(1, "\n", 1);
@@ -608,17 +668,20 @@ int ScriptController::StartProcess()
 
 	m_processId = pid;
 
-	// close unused "write" end
-	close(pipeout);
+	// close unused pipe ends
+	close(pin[1]);
+	if (m_needWrite)
+	{
+		close(pout[0]);
+	}
 #endif
-
-	return pipein;
 }
 
 int ScriptController::WaitProcess()
 {
 #ifdef WIN32
-	WaitForSingleObject(m_processId, INFINITE);
+	// wait max 60 seconds for terminated processes
+	WaitForSingleObject(m_processId, m_terminated ? 60 * 1000 : INFINITE);
 	DWORD exitCode = 0;
 	GetExitCodeProcess(m_processId, &exitCode);
 	return exitCode;
@@ -641,17 +704,6 @@ void ScriptController::Terminate()
 
 #ifdef WIN32
 	BOOL ok = TerminateProcess(m_processId, -1) || m_completed;
-	if (ok)
-	{
-		// wait 60 seconds for process to terminate
-		WaitForSingleObject(m_processId, 60 * 1000);
-	}
-	else
-	{
-		DWORD exitCode = 0;
-		GetExitCodeProcess(m_processId, &exitCode);
-		ok = exitCode != STILL_ACTIVE;
-	}
 #else
 	pid_t killId = m_processId;
 	if (getpgid(killId) == killId)
@@ -822,3 +874,9 @@ void ScriptController::PrintMessage(Message::EKind kind, const char* format, ...
 		AddMessage(kind, tmp2);
 	}
 }
+
+void ScriptController::Write(const char* str)
+{
+	fwrite(str, 1, strlen(str), m_writepipe);
+	fflush(m_writepipe);
+}

daemon/util/Script.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -78,8 +78,11 @@ protected:
 	void ResetEnv();
 	void PrepareEnvOptions(const char* stripPrefix);
 	void PrepareArgs();
-	int StartProcess();
+	virtual const char* GetOptValue(const char* name, const char* value) { return value; }
+	void StartProcess(int* pipein, int* pipeout);
 	int WaitProcess();
+	void SetNeedWrite(bool needWrite) { m_needWrite = needWrite; }
+	void Write(const char* str);
 #ifdef WIN32
 	void BuildCommandLine(char* cmdLineBuf, int bufSize);
 #endif
@@ -94,7 +97,9 @@ private:
 	bool m_terminated = false;
 	bool m_completed = false;
 	bool m_detached = false;
-	FILE* m_readpipe;
+	bool m_needWrite = false;
+	FILE* m_readpipe = 0;
+	FILE* m_writepipe = 0;
 #ifdef WIN32
 	HANDLE m_processId = 0;
 	DWORD m_dwProcessId = 0;

daemon/util/Thread.cpp

@@ -169,13 +169,15 @@ void* Thread::thread_handler(void* object)
 
 	debug("Thread-func exited");
 
-	thread->m_running = false;
-
 	if (thread->m_autoDestroy)
 	{
 		debug("Autodestroying Thread-object");
 		delete thread;
 	}
+	else
+	{
+		thread->m_running = false;
+	}
 
 	{
 		Guard guard(m_threadMutex);

daemon/util/Thread.h

@@ -95,7 +95,6 @@ public:
 
 	bool IsStopped() { return m_stopped; };
 	bool IsRunning() const { return m_running; }
-	void SetRunning(bool onOff) { m_running = onOff; }
 	bool GetAutoDestroy() { return m_autoDestroy; }
 	void SetAutoDestroy(bool autoDestroy) { m_autoDestroy = autoDestroy; }
 	static int GetThreadCount();

daemon/util/Util.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -20,6 +20,7 @@
 
 #include "nzbget.h"
 #include "Util.h"			
+#include "YEncode.h"
 
 #ifndef WIN32
 // function "code_revision" is automatically generated in file "code_revision.cpp" on each build
@@ -429,6 +430,49 @@ std::vector<CString> Util::SplitStr(const char* str, const char* separators)
 	return result;
 }
 
+bool Util::EndsWith(const char* str, const char* suffix, bool caseSensitive)
+{
+	if (!str)
+	{
+		return false;
+	}
+
+	if (EmptyStr(suffix))
+	{
+		return true;
+	}
+
+	int lenStr = strlen(str);
+	int lenSuf = strlen(suffix);
+
+	if (lenSuf > lenStr)
+	{
+		return false;
+	}
+
+	if (caseSensitive)
+	{
+		return !strcmp(str + lenStr - lenSuf, suffix);
+	}
+	else
+	{
+		return !strcasecmp(str + lenStr - lenSuf, suffix);
+	}
+}
+
+bool Util::AlphaNum(const char* str)
+{
+	for (const char* p = str; *p; p++)
+	{
+		char ch = *p;
+		if (!((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')))
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
 /* Calculate Hash using Bob Jenkins (1996) algorithm
  * http://burtleburtle.net/bob/c/lookup2.c
  */
@@ -607,158 +651,6 @@ void Util::SetStandByMode(bool standBy)
 #endif
 }
 
-static uint32 crc32_tab[] = {
-	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
-	0xe963a535, 0x9e6495a3,	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
-	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
-	0xf3b97148, 0x84be41de,	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
-	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,	0x14015c4f, 0x63066cd9,
-	0xfa0f3d63, 0x8d080df5,	0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
-	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,	0x35b5a8fa, 0x42b2986c,
-	0xdbbbc9d6, 0xacbcf940,	0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
-	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
-	0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
-	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,	0x76dc4190, 0x01db7106,
-	0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
-	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
-	0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
-	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
-	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
-	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
-	0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
-	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
-	0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
-	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
-	0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
-	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
-	0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
-	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
-	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
-	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
-	0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
-	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
-	0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
-	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
-	0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
-	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
-	0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
-	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
-	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
-	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
-	0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
-	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
-	0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
-	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
-	0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
-	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
-};
-
-/* This is a modified version of chksum_crc() from
- * crc32.c (http://www.koders.com/c/fid699AFE0A656F0022C9D6B9D1743E697B69CE5815.aspx)
- * (c) 1999,2000 Krzysztof Dabrowski
- * (c) 1999,2000 ElysiuM deeZine
- *
- * chksum_crc() -- to a given block, this one calculates the
- *				crc32-checksum until the length is
- *				reached. the crc32-checksum will be
- *				the result.
- */
-uint32 Util::Crc32m(uint32 startCrc, uchar *block, uint32 length)
-{
-	uint32 crc = startCrc;
-	for (uint32 i = 0; i < length; i++)
-	{
-		crc = ((crc >> 8) & 0x00FFFFFF) ^ crc32_tab[(crc ^ *block++) & 0xFF];
-	}
-	return crc;
-}
-
-uint32 Util::Crc32(uchar *block, uint32 length)
-{
-	return Util::Crc32m(0xFFFFFFFF, block, length) ^ 0xFFFFFFFF;
-}
-
-/* From zlib/crc32.c (http://www.zlib.net/)
- * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler
- */
-
-#define GF2_DIM 32      /* dimension of GF(2) vectors (length of CRC) */
-
-uint32 gf2_matrix_times(uint32 *mat, uint32 vec)
-{
-	uint32 sum;
-
-	sum = 0;
-	while (vec) {
-		if (vec & 1)
-			sum ^= *mat;
-		vec >>= 1;
-		mat++;
-	}
-	return sum;
-}
-
-void gf2_matrix_square(uint32 *square, uint32 *mat)
-{
-	int n;
-
-	for (n = 0; n < GF2_DIM; n++)
-		square[n] = gf2_matrix_times(mat, mat[n]);
-}
-
-uint32 Util::Crc32Combine(uint32 crc1, uint32 crc2, uint32 len2)
-{
-	int n;
-	uint32 row;
-	uint32 even[GF2_DIM];    /* even-power-of-two zeros operator */
-	uint32 odd[GF2_DIM];     /* odd-power-of-two zeros operator */
-
-	/* degenerate case (also disallow negative lengths) */
-	if (len2 <= 0)
-		return crc1;
-
-	/* put operator for one zero bit in odd */
-	odd[0] = 0xedb88320UL;          /* CRC-32 polynomial */
-	row = 1;
-	for (n = 1; n < GF2_DIM; n++) {
-		odd[n] = row;
-		row <<= 1;
-	}
-
-	/* put operator for two zero bits in even */
-	gf2_matrix_square(even, odd);
-
-	/* put operator for four zero bits in odd */
-	gf2_matrix_square(odd, even);
-
-	/* apply len2 zeros to crc1 (first square will put the operator for one
-	   zero byte, eight zero bits, in even) */
-	do {
-		/* apply zeros operator for this bit of len2 */
-		gf2_matrix_square(even, odd);
-		if (len2 & 1)
-			crc1 = gf2_matrix_times(even, crc1);
-		len2 >>= 1;
-
-		/* if no more bits set, then done */
-		if (len2 == 0)
-			break;
-
-		/* another iteration of the loop with odd and even swapped */
-		gf2_matrix_square(odd, even);
-		if (len2 & 1)
-			crc1 = gf2_matrix_times(odd, crc1);
-		len2 >>= 1;
-
-		/* if no more bits set, then done */
-	} while (len2 != 0);
-
-	/* return combined crc */
-	crc1 ^= crc2;
-
-	return crc1;
-}
-
 int Util::NumberOfCpuCores()
 {
 #ifdef WIN32
@@ -1530,7 +1422,7 @@ void URL::ParseUrl()
 		return;
 	}
 
-	m_protocol.Set(m_address, protEnd - m_address);
+	m_protocol.Set(m_address, (int)(protEnd - m_address));
 
 	char* hostStart = protEnd + 3;
 	char* slash = strchr(hostStart, '/');
@@ -1564,7 +1456,7 @@ void URL::ParseUrl()
 	if (slash)
 	{
 		char* resEnd = m_address + strlen(m_address);
-		m_resource.Set(slash, resEnd - slash + 1);
+		m_resource.Set(slash, (int)(resEnd - slash + 1));
 
 		hostEnd = slash - 1;
 	}
@@ -1582,7 +1474,7 @@ void URL::ParseUrl()
 		m_port = atoi(colon + 1);
 	}
 
-	m_host.Set(hostStart, hostEnd - hostStart + 1);
+	m_host.Set(hostStart, (int)(hostEnd - hostStart + 1));
 
 	m_valid = true;
 }
@@ -1644,7 +1536,7 @@ int RegEx::GetMatchStart(int index)
 #ifdef HAVE_REGEX_H
 	return m_matches[index].rm_so;
 #else
-	return nullptr;
+	return 0;
 #endif
 }
 
@@ -1689,14 +1581,14 @@ bool WildMask::Match(const char* text)
 			if (!qmark)
 			{
 				ExpandArray();
-				m_wildStart[m_wildCount-1] = str - text;
+				m_wildStart[m_wildCount-1] = (int)(str - text);
 				m_wildLen[m_wildCount-1] = 0;
 				qmark = true;
 			}
 		}
 		else if (m_wantsPositions && qmark)
 		{
-			m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+			m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 			qmark = false;
 		}
 
@@ -1711,7 +1603,7 @@ bool WildMask::Match(const char* text)
 
 	if (m_wantsPositions && qmark)
 	{
-		m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+		m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 		qmark = false;
 	}
 
@@ -1721,13 +1613,13 @@ bool WildMask::Match(const char* text)
 		{
 			if (m_wantsPositions && qmark)
 			{
-				m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+				m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 				qmark = false;
 			}
 			if (m_wantsPositions && !star)
 			{
 				ExpandArray();
-				m_wildStart[m_wildCount-1] = str - text;
+				m_wildStart[m_wildCount-1] = (int)(str - text);
 				m_wildLen[m_wildCount-1] = 0;
 				star = true;
 			}
@@ -1749,7 +1641,7 @@ bool WildMask::Match(const char* text)
 			if (m_wantsPositions && !qmark)
 			{
 				ExpandArray();
-				m_wildStart[m_wildCount-1] = str - text;
+				m_wildStart[m_wildCount-1] = (int)(str - text);
 				m_wildLen[m_wildCount-1] = 0;
 				qmark = true;
 			}
@@ -1761,12 +1653,12 @@ bool WildMask::Match(const char* text)
 		{
 			if (m_wantsPositions && qmark)
 			{
-				m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+				m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 				qmark = false;
 			}
 			else if (m_wantsPositions && star)
 			{
-				m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+				m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 				star = false;
 			}
 
@@ -1789,13 +1681,13 @@ bool WildMask::Match(const char* text)
 
 	if (m_wantsPositions && qmark)
 	{
-		m_wildLen[m_wildCount-1] = str - (text + m_wildStart[m_wildCount-1]);
+		m_wildLen[m_wildCount-1] = (int)(str - (text + m_wildStart[m_wildCount-1]));
 	}
 
 	if (*pat == '*' && m_wantsPositions && !star)
 	{
 		ExpandArray();
-		m_wildStart[m_wildCount-1] = str - text;
+		m_wildStart[m_wildCount-1] = (int)(str - text);
 		m_wildLen[m_wildCount-1] = strlen(str);
 	}
 
@@ -1944,3 +1836,101 @@ char* Tokenizer::Next()
 	}
 	return token;
 }
+
+void Crc32::Reset()
+{
+	static_assert(sizeof(m_state) >= sizeof(YEncode::crc_state), "m_state has invalid size");
+
+	YEncode::crc_init((YEncode::crc_state*)State());
+}
+
+void Crc32::Append(uchar* block, uint32 length)
+{
+	YEncode::crc_incr((YEncode::crc_state*)State(), block, length);
+}
+
+uint32 Crc32::Finish()
+{
+	return YEncode::crc_finish((YEncode::crc_state*)State());
+}
+
+/* From zlib/crc32.c (http://www.zlib.net/)
+ * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler
+ */
+
+#define GF2_DIM 32      /* dimension of GF(2) vectors (length of CRC) */
+
+uint32 gf2_matrix_times(uint32 *mat, uint32 vec)
+{
+	uint32 sum;
+
+	sum = 0;
+	while (vec) {
+		if (vec & 1)
+			sum ^= *mat;
+		vec >>= 1;
+		mat++;
+	}
+	return sum;
+}
+
+void gf2_matrix_square(uint32 *square, uint32 *mat)
+{
+	int n;
+
+	for (n = 0; n < GF2_DIM; n++)
+		square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+uint32 Crc32::Combine(uint32 crc1, uint32 crc2, uint32 len2)
+{
+	int n;
+	uint32 row;
+	uint32 even[GF2_DIM];    /* even-power-of-two zeros operator */
+	uint32 odd[GF2_DIM];     /* odd-power-of-two zeros operator */
+
+	/* degenerate case (also disallow negative lengths) */
+	if (len2 <= 0)
+		return crc1;
+
+	/* put operator for one zero bit in odd */
+	odd[0] = 0xedb88320UL;          /* CRC-32 polynomial */
+	row = 1;
+	for (n = 1; n < GF2_DIM; n++) {
+		odd[n] = row;
+		row <<= 1;
+	}
+
+	/* put operator for two zero bits in even */
+	gf2_matrix_square(even, odd);
+
+	/* put operator for four zero bits in odd */
+	gf2_matrix_square(odd, even);
+
+	/* apply len2 zeros to crc1 (first square will put the operator for one
+	   zero byte, eight zero bits, in even) */
+	do {
+		/* apply zeros operator for this bit of len2 */
+		gf2_matrix_square(even, odd);
+		if (len2 & 1)
+			crc1 = gf2_matrix_times(even, crc1);
+		len2 >>= 1;
+
+		/* if no more bits set, then done */
+		if (len2 == 0)
+			break;
+
+		/* another iteration of the loop with odd and even swapped */
+		gf2_matrix_square(odd, even);
+		if (len2 & 1)
+			crc1 = gf2_matrix_times(odd, crc1);
+		len2 >>= 1;
+
+		/* if no more bits set, then done */
+	} while (len2 != 0);
+
+	/* return combined crc */
+	crc1 ^= crc2;
+
+	return crc1;
+}

daemon/util/Util.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -49,6 +49,8 @@ public:
 	static char* Trim(char* str);
 	static bool EmptyStr(const char* str) { return !str || !*str; }
 	static std::vector<CString> SplitStr(const char* str, const char* separators);
+	static bool EndsWith(const char* str, const char* suffix, bool caseSensitive);
+	static bool AlphaNum(const char* str);
 
 	/* replace all occurences of szFrom to szTo in string szStr with a limitation that szTo must be shorter than szFrom */
 	static char* ReduceStr(char* str, const char* from, const char* to);
@@ -84,10 +86,6 @@ public:
 
 	static void Init();
 
-	static uint32 Crc32(uchar *block, uint32 length);
-	static uint32 Crc32m(uint32 startCrc, uchar *block, uint32 length);
-	static uint32 Crc32Combine(uint32 crc1, uint32 crc2, uint32 len2);
-
 	/*
 	* Returns number of available CPU cores or -1 if it could not be determined
 	*/
@@ -315,4 +313,24 @@ private:
 	bool m_working = false;
 };
 
+class Crc32
+{
+public:
+	Crc32() { Reset(); }
+	void Reset();
+	void Append(uchar* block, uint32 length);
+	uint32 Finish();
+	static uint32 Combine(uint32 crc1, uint32 crc2, uint32 len2);
+
+private:
+#if defined(WIN32) && !defined(_WIN64)
+	// VC++ in 32 bit mode can not "alignas(16)" dynamically allocated objects
+	alignas(8) uint32_t m_state[4 * 5 + 8]; // = YEncode::crc_state
+	void* State() { void* p = &m_state; size_t s = sizeof(m_state); return std::align(16, 4 * 5, p, s); }
+#else
+	alignas(16) uint32_t m_state[4 * 5]; // = YEncode::crc_state
+	void* State() { return &m_state; }
+#endif
+};
+
 #endif

daemon/windows/WinConsole.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -71,29 +71,11 @@ BOOL WINAPI WinConsole::ConsoleCtrlHandler(DWORD dwCtrlType)
 	}
 }
 
-WinConsole::WinConsole()
-{
-	m_initialArguments = nullptr;
-	m_initialArgumentCount = 0;
-	m_defaultArguments = nullptr;
-	m_iconData = nullptr;
-	m_modal = false;
-	m_autostart = false;
-	m_showTrayIcon = true;
-	m_showConsole = false;
-	m_showWebUI = true;
-	m_autoParam = false;
-	m_doubleClick = false;
-	m_trayWindow = 0;
-	m_running = false;
-	m_runningService = false;
-}
-
 WinConsole::~WinConsole()
 {
 	if (m_initialArguments)
 	{
-		g_Arguments =	(char*(*)[])m_initialArguments;
+		g_Arguments = (char*(*)[])m_initialArguments;
 		g_ArgumentCount = m_initialArgumentCount;
 	}
 	free(m_defaultArguments);
@@ -107,7 +89,6 @@ void WinConsole::InitAppMode()
 	m_instance = (HINSTANCE)GetModuleHandle(0);
 	DWORD processId;
 	GetWindowThreadProcessId(GetConsoleWindow(), &processId);
-	m_appMode = false;
 
 	if (GetCurrentProcessId() == processId && g_ArgumentCount == 1)
 	{
@@ -131,24 +112,27 @@ void WinConsole::InitAppMode()
 			{
 				break;
 			}
-			if (!strcmp((*g_Arguments)[i], "-app"))
+			else if (!strcmp((*g_Arguments)[i], "-app"))
 			{
 				m_appMode = true;
 			}
-
-			if (!strcmp((*g_Arguments)[i], "-auto"))
+			else if (!strcmp((*g_Arguments)[i], "-auto"))
 			{
 				m_autoParam = true;
 			}
+			else if (!strcmp((*g_Arguments)[i], "-A"))
+			{
+				m_addParam = true;
+			}
 		}
 
-		if (m_appMode)
+		if (m_appMode || m_autoParam)
 		{
 			m_initialArguments = (char**)g_Arguments;
 			m_initialArgumentCount = g_ArgumentCount;
 
 			// remove "-app" from command line
-			int argc = g_ArgumentCount - 1 - (m_autoParam ? 1 : 0);
+			int argc = g_ArgumentCount - (m_appMode ? 1 : 0) - (m_autoParam ? 1 : 0);
 			m_defaultArguments = (char**)malloc(sizeof(char*) * (argc + 2));
 
 			int p = 0;
@@ -166,7 +150,13 @@ void WinConsole::InitAppMode()
 		}
 	}
 
-	// m_bAppMode indicates whether the program was started as a standalone app
+	if (m_addParam)
+	{
+		RunAnotherInstance();
+		return;
+	}
+
+	// m_appMode indicates whether the program was started as a standalone app
 	// (not from a dos box window). In that case we hide the console window,
 	// show the tray icon and start in server mode
 
@@ -179,7 +169,7 @@ void WinConsole::InitAppMode()
 
 void WinConsole::Run()
 {
-	if (!m_appMode)
+	if (!m_appMode || m_addParam)
 	{
 		return;
 	}
@@ -317,6 +307,7 @@ LRESULT WinConsole::TrayWndProc(HWND hwndWin, UINT uMsg, WPARAM wParam, LPARAM l
 			return 0;
 
 		case UM_QUIT:
+		case WM_ENDSESSION:
 			ExitProc();
 			return 0;
 
@@ -466,12 +457,12 @@ void WinConsole::ShowAboutBox()
 	m_modal = false;
 }
 
-BOOL CALLBACK WinConsole::AboutDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR CALLBACK WinConsole::AboutDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	return g_WinConsole->AboutDialogProc(hwndDlg, uMsg, wParam, lParam);
 }
 
-BOOL WinConsole::AboutDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR WinConsole::AboutDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	switch(uMsg)
 	{
@@ -517,7 +508,7 @@ BOOL WinConsole::AboutDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM
 				(HWND)wParam == GetDlgItem(hwndDlg, IDC_ABOUT_GPL))
 			{
 				SetCursor(m_handCursor);
-				SetWindowLong(hwndDlg, DWL_MSGRESULT, TRUE);
+				SetWindowLongPtr(hwndDlg, DWLP_MSGRESULT, TRUE);
 				return TRUE;
 			}
 			return FALSE;
@@ -539,12 +530,12 @@ void WinConsole::ShowPrefsDialog()
 	m_modal = false;
 }
 
-BOOL CALLBACK WinConsole::PrefsDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR CALLBACK WinConsole::PrefsDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	return g_WinConsole->PrefsDialogProc(hwndDlg, uMsg, wParam, lParam);
 }
 
-BOOL WinConsole::PrefsDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR WinConsole::PrefsDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	switch(uMsg)
 	{
@@ -701,7 +692,7 @@ void WinConsole::ShowRunningDialog()
 	HWND hTrayWindow = FindWindow("NZBGet tray window", nullptr);
 	m_running = true;
 
-	int result = DialogBox(m_instance, MAKEINTRESOURCE(IDD_RUNNINGDIALOG), m_trayWindow, RunningDialogProcStat);
+	INT_PTR result = DialogBox(m_instance, MAKEINTRESOURCE(IDD_RUNNINGDIALOG), m_trayWindow, RunningDialogProcStat);
 
 	switch (result)
 	{
@@ -720,12 +711,12 @@ void WinConsole::ShowRunningDialog()
 	}
 }
 
-BOOL CALLBACK WinConsole::RunningDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR CALLBACK WinConsole::RunningDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	return g_WinConsole->RunningDialogProc(hwndDlg, uMsg, wParam, lParam);
 }
 
-BOOL WinConsole::RunningDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR WinConsole::RunningDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	switch(uMsg)
 	{
@@ -923,7 +914,7 @@ void WinConsole::ShowFactoryResetDialog()
 	HWND hTrayWindow = FindWindow("NZBGet tray window", nullptr);
 	m_running = true;
 
-	int result = DialogBox(m_instance, MAKEINTRESOURCE(IDD_FACTORYRESETDIALOG), m_trayWindow, FactoryResetDialogProcStat);
+	INT_PTR result = DialogBox(m_instance, MAKEINTRESOURCE(IDD_FACTORYRESETDIALOG), m_trayWindow, FactoryResetDialogProcStat);
 
 	switch (result)
 	{
@@ -933,12 +924,12 @@ void WinConsole::ShowFactoryResetDialog()
 	}
 }
 
-BOOL CALLBACK WinConsole::FactoryResetDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR CALLBACK WinConsole::FactoryResetDialogProcStat(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	return g_WinConsole->FactoryResetDialogProc(hwndDlg, uMsg, wParam, lParam);
 }
 
-BOOL WinConsole::FactoryResetDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+INT_PTR WinConsole::FactoryResetDialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
 	switch(uMsg)
 	{
@@ -1043,3 +1034,12 @@ void WinConsole::ResetFactoryDefaults()
 	mayStartBrowser = true;
 	Reload();
 }
+
+void WinConsole::RunAnotherInstance()
+{
+	ShowWindow(GetConsoleWindow(), SW_HIDE);
+	if (!(FindWindow("NZBGet tray window", nullptr) || IsServiceRunning()))
+	{
+		ShellExecute(0, "open", (*g_Arguments)[0], nullptr, nullptr, SW_SHOWNORMAL);
+	}
+}