#784: removed expired root certificate

from ca root certificate store: certificate “DST Root CA X3” used by
Lets Encrypt

.gitignore

@@ -26,11 +26,14 @@
 # GNU Autotools
 .deps/
 config.h
+config.h.in~
 config.log
 config.status
 Makefile
 stamp-h1
 autom4te.cache/
+.dirstamp
+*.o-*
 
 # Visual Studio User-specific files
 *.suo
@@ -57,6 +60,10 @@ ipch/
 *.svclog
 *.scc
 *.sln
+.vscode/
+
+# macOS
+.DS_Store
 
 # NZBGet specific
 nzbget
@@ -64,3 +71,4 @@ code_revision.cpp
 *.temp
 *.pyc
 pytest.ini
+.cache

.lgtm.yml

@@ -0,0 +1,17 @@
+# Configuration file for integration with http://lgtm.com
+
+path_classifiers:
+  library:
+    # exclude these directories from default alerts report:
+    - lib
+    - webui/lib
+
+extraction:
+  cpp:
+    configure:
+      command:
+        # compile with tests to activate scanning of C++ sources for tests
+        - ./configure --enable-tests
+
+queries:
+  - exclude: js/incomplete-sanitization  # this one gives false positives only and nothing useful

.travis.yml

@@ -4,6 +4,7 @@ language: cpp
 
 matrix:
   include:
+
     - compiler: gcc
       addons:
         apt:
@@ -11,7 +12,12 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             - g++-5
-      env: COMPILER=g++-5
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-5
+
     - compiler: gcc
       addons:
         apt:
@@ -19,7 +25,24 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             - g++-4.9
-      env: COMPILER=g++-4.9
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.9
+
+    - compiler: gcc
+      addons:
+        apt:
+          packages:
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.8
+        - CXXFLAGS="-std=c++11 -O2 -s"
+        - CONFIGUREOPTS="--disable-cpp-check"
+
     - compiler: clang
       addons:
         apt:
@@ -28,9 +51,17 @@ matrix:
             - llvm-toolchain-precise-3.6
           packages:
             - clang-3.6
-      env: COMPILER=clang++-3.6
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=clang++-3.6
+
+install:
+  - sudo pip install -U pytest
 
 script:
   - $COMPILER --version
-  - CXX=$COMPILER ./configure --enable-tests && make
-  - ./nzbget --tests exclude:[DupeMatcher]
+  - CXX=$COMPILER ./configure $CONFIGUREOPTS --enable-tests && make
+  - ./nzbget --tests
+  - cd tests/functional && pytest -v

COPYING

@@ -1,12 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -15,7 +15,7 @@ software--to make sure the software is free for all its users.  This
 General Public License applies to most of the Free Software
 Foundation's software and to any other program whose authors commit to
 using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
+the GNU Lesser General Public License instead.)  You can apply it to
 your programs, too.
 
   When we speak of free software, we are referring to freedom, not
@@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
+
+                    GNU GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License applies to any program or other work which contains
@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
     License.  (Exception: if the Program itself is interactive but
     does not normally print such an announcement, your work based on
     the Program is not required to print an announcement.)
-
+
 These requirements apply to the modified work as a whole.  If
 identifiable sections of that work are not derived from the Program,
 and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
 access to copy the source code from the same place counts as
 distribution of the source code, even though third parties are not
 compelled to copy the source along with the object code.
-
+
   4. You may not copy, modify, sublicense, or distribute the Program
 except as expressly provided under this License.  Any attempt
 otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
 
 This section is intended to make thoroughly clear what is believed to
 be a consequence of the rest of this License.
-
+
   8. If the distribution and/or use of the Program is restricted in
 certain countries either by patents or by copyrighted interfaces, the
 original copyright holder who places the Program under this License
@@ -255,7 +255,7 @@ make exceptions for this.  Our decision will be guided by the two goals
 of preserving the free status of all derivatives of our free software and
 of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
 
   If you develop a new program, and you want it to be of the greatest
 possible use to the public, the best way to achieve this is to make it
@@ -303,10 +303,9 @@ the "copyright" line and a pointer to where the full notice is found.
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -336,5 +335,5 @@ necessary.  Here is a sample; alter the names:
 This General Public License does not permit incorporating your program into
 proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

ChangeLog

@@ -1,6 +1,410 @@
+nzbget-21.2-testing:
+  - please see repository change log at
+    https://github.com/nzbget/nzbget/commits/develop
+
+nzbget-21.1:
+  - fixed crash on systems with 64-bit time;
+  - corrected icon in Windows "uninstall program" list;
+  - allow special characters in URL for username and password;
+  - improved reporting for binding errors on Windows;
+  - fixed unicode space characters in javascript files, which could cause issues
+    with nginx proxy;
+  - fixed negative values for "FileSizeLo" in json-rpc;
+  - corrected url detection in rpc-method "append";
+  - added support for new error messages in unrar 5.80;
+  - now always using snapshots when reading directory contents:
+    - in previous versions snapshots were used on macOS only;
+    - now they are used on all OSes;
+    - this solves issue with leftovers during directory cleanup, which could
+      happen on certain OSes when working with network drives;
+  - fixed file allocating on file systems where sparse files are not supported:
+    - the issue could happen when InterDir was located on a network drive;
+  - fixed crash caused by malformed nzb files;
+  - fixed GROUP command in nserv;
+  - updated url of the global certificate storage file in the build scripts;
+  - fixed: file selector in WebKit based browsers doesn't allow to choose the
+    same file again;
+  - removed outdated links from web interface;
+  - fixed PC sleep mode not working (Windows only);
+  - set "SameSite" attribute for cookies;
+  - corrected typo in about dialog of web interface;
+  - updated license text: changed address of Free Software Foundation and minor
+    formatting changes.
+
+nzbget-21.0:
+  - reworked duplicate handling to support URLs, especially when using RSS
+    feeds:
+      - queue items added via URLs (to be fetched by nzbget) are no longer
+        immediately fetched;
+      - instead url-items are handled by duplicate check similar to nzb-items
+        and may be placed into history as duplicate backups;
+      - if an url-item needs to be downloaded as backup for a failed other item
+        the nzb-file is fetched via provided URL;
+      - this greatly reduces the number of nzbs fetched from indexers when using
+        RSS feeds and duplicate handling;
+  - improved support for Android devices:
+    - now providing a separate installer package for Android;
+    - the package contains binaries built using Android NDK;
+    - this improves compatibility with Android, in particular with Android 8,
+      where general Linux installer version of NZBGet didn't work anymore due
+      to security changes in Android;
+    - android installer app is updated to use the new android installer package
+      instead of general Linux package;
+  - thoroughly optimised the program to reduce power consumption in idle state:
+    - number of CPU wake ups in idle state has been reduced from hundreds times
+      per second to about only one per second;
+  - optimisations for large queues with thousands of items:
+    - speed up saving of queue state and reduced number of queue state savings;
+    - improved queue state format to reduce amount of state data saved during
+      downloading;
+    - in tests download speed for very large queue (16000 items) has been
+      increased from 45 MB/s to 300 MB/s (comparing to 400 MB/s with small
+      queue); 
+  - added native support for aarch64 architecture (ARM 64 Bit CPU) in Linux and
+    Android installers;
+  - force par-check for nzbs without archives;
+  - added functional tests for unpack CRC error;
+  - click on nzbget logo in web-interface now switches to downloads tab instead
+    of showing "About dialog" which has been moved into settings;
+  - improved handling of files splitted via par2;
+  - added python 3 compatibility to EMail.py script;
+  - added python 3 compatibility to Logger.py script;
+  - proper UTF-8 encoding of email content in EMail.py script;
+  - improved error reporting for queue disk state corruption;
+  - updated unrar to 5.7 and 7-zip to 19.0;
+  - Windows installer now includes unrar in 32 bit and 64 bit variants;
+  - allowing wildcards in option AuthorizedIP;
+  - removed suggestion of RC4 cipher;
+  - better description of option UMask;
+  - integrated LGTM code analyser tool into project;
+  - fixed: failed downloads not having any par2- or archive- files were moved to
+    DestDir instead of remaining in InterDir;
+  - fixed crash when using FIPS version of OpenSSL;
+  - fixed compatibility issue with OpenSSL compiled without compression support;
+  - fixed deprecated OpenSSL calls;
+  - fixed potential crash in built-in web-server;
+  - fixed: statistics for session download time and speed may be way off on high
+    load;
+  - fixed many compilation warnings in GCC;
+  - fixed: macOS menubar widget could not connect if password contained special
+    characters;
+  - fixed: remote clients not displaying current download speed;
+  - fixed: remote server could crash when feed with invalid api request;
+  - fixed trimming of relative paths in config.
+
+nzbget-20.0:
+  - massive performance optimisations in downloader:
+        - improved yEnc decoder;
+        - improved CRC32 calculation;
+        - processing data in one pass;
+        - SIMD implementation of decoder and CRC functions on x86 and ARM CPUs;
+          SIMD code relies on node-yencode library by Anime Tosho
+          (https://github.com/animetosho/node-yencode);
+        - overall performance improvement up to +500% on x86 and +250% on ARM
+          (better speed or less CPU usage);
+  - using glibc instead of uClibc in universal installer builds for Linux:
+        - this significantly improves performance;
+        - compatibility with Android and other systems is hopefully also improved;
+        - in universal installer glibc is used on x86 and ARM;
+        - uClibc is still used on MIPS and PPC;
+  - performance optimisations in web-interface:
+        - reduced number of requests when loading webui by combining all
+          javascript-files into one and all css-files into one;
+        - reduced load time by calling multiple API-methods simultaneously;
+        - extensive use of browser caching for static files significantly
+          reduces the amount of data transferred on webui initialisation;
+        - extensive use of browser caching for API requests reduces the amount
+          of data transferred during webui status updates, especially when
+          nzbget is in idle state and there are no changes in download queue or
+          history;
+        - avoid work in browser on status updates if there are no changes in
+          download queue or history;
+        - support for keep alive connections significantly reduces overhead for
+          establishing connections on webui status updates, especially when
+          connecting to nzbget via TLS/SSL (avoiding TLS handshakes);
+  - a number of performance optimisations for large download queue with
+    thousands of items:
+        - much faster loading of queue from disk greatly improves program start
+          up time;
+        - improved queue management for faster download speed;
+  - now offering 64 bit binaries for Windows:
+        - installer includes 32 and 64 bit nzbget binaries;
+        - when updating from older versions the 64 bit binary is installed
+          automatically, although into the old location to keep all your
+          shortcuts intact;
+        - using word "windows" instead of "win32" in the setup file name;
+  - automatic update check:
+        - new option "UpdateCheck" to check for stable or testing versions (or
+          disable);
+        - when a new version is found a notification is shown;
+        - the update check is enabled by default for stable versions;
+  - significantly improved logging performance, especially in debug builds;
+  - par-check prints par2 creator application to help identify creator app
+    issues;
+  - added support for Unix domain sockets (POSIX only);
+  - better error handling when fetching rss feeds;
+  - updated POSIX build files to newer autotools version:
+        - compatibility with newer autotools;
+        - compatibility with newer platforms such as aarch64;
+  - better username/password validation when testing connection on settings
+    page;
+  - improved rar-renamer to better handle certain cases;
+  - new option "SkipWrite" for easier speed tests;
+  - support for redirect codes 303, 307 and 308 in web-client for fetching of
+    rss feeds and nzb-files;
+  - installer for FreeBSD is now built using Clang instead of GCC; this fixes
+    incompatibility with FreeBSD 11;
+  - universal Linux installer can now be used on FreeBSD (via Linux
+    compatibility mode);
+  - updated unrar to v5.50;
+  - more robust news server connection test;
+  - enhancements in NServ:
+        - memory cache to reduce disk load during speed tests - new command line
+          switch "-m";
+        - speed control - new command line switches "-w" and "-r";
+        - show IP address of incoming connection;
+  - changed default location of log-file;
+  - better handling of broken connections;
+  - removed obsolete or less useful options "SaveQueue", "ReloadQueue",
+    "TerminateTimeout", "AccurateRate", "BrokenLog";
+  - renamed option "LogBufferSize" to "LogBuffer";
+  - passwords of failed login attempts are no longer printed to log to improve
+    security;
+  - cookies in web interface are now saved with "httpOnly" attribute to improve
+    security;
+  - titles and duplicate keys in duplicate check are now case insensitive;
+  - added LibreSSL support;
+  - web interface now has a better icon for favorites or home screen of mobile
+    devices;
+  - improved duplicate detection for obfuscated downloads having files with
+    same subjects;
+  - direct rename and direct unpack are now active by default on new
+    installations, except for slow Linux systems;
+  - added advice for letsencrypt in option descriptions;
+  - fixed incorrect renaming in rar-renamer which could cause some downloads to
+    fail;
+  - fixed race condition in queue script coordinator which could cause crashes;
+  - fixed: post-processing parameters were sometimes case sensitive causing
+    issues;
+  - fixed DNS resolving issues on Android;
+  - fixed: backup servers not used on certain article decoding errors;
+  - fixed: when direct rename was active certain downloads with damaged
+    par2-files become paused at near completion and required manual resuming;
+  - fixed: crash when flushing article cache after direct rename;
+  - fixed: deleting active par-job may crash the program;
+  - fixed: functional tests may fail on Windows due to locked files;
+  - fixed: unpack using password file doesn't work on Windows;
+  - fixed: compiler error when building using GnuTLS;
+  - fixed: Linux installer failure on android emulator;
+  - fixed: options formatted as password fields when they shouldn't;
+  - fixed: slightly off article statistics after direct rename;
+  - fixed: NServ terminated if client interrupted connection;
+  - fixed: example pp-scripts may not work properly if nzbget password or
+    username contained special characters;
+  - fix in functional tests to not rely on sizes of externally generated files;
+  - fixed: option AuthorizedIP did not work with IPv6 addresses;
+  - fixed crash on certain malformed articles;
+  - fixed crash which could happen on Windows when reloading or terminating the
+    program;
+  - fixed logging of IPv6 addresses.
+
+nzbget-19.1:
+  - proper handling of changing category (and destination path) during direct
+    unpack; direct unpack now gracefully aborts with cleanup; the files will
+    be unpacked during post-processing stage;
+  - fixed: password protected downloads of certain kind may sometimes end up
+    with no files if direct unpack was active;
+  - fixed: rar-renamer mistakenly renamed some encrypted rar3 files causing
+    unnecessary repair;
+  - fixed: rar-renamer could not process some encrypted rar3-archives.
+
+nzbget-19.0:
+  - unpack during downloading:
+        - downloaded files can now be unpacked as soon as every archive part is
+          downloaded;
+        - new option "DirectUnpack" to activate direct unpacking;
+        - direct unpack works even with obfuscated downloads; option
+         "DirectRename" (see below) must be active for that;
+        - option "ReorderFiles" (see below) should be also active for optimal
+          file download order;
+        - direct unpack works for rar-archives; 7-zip archives and simply
+          splitted files are processed by default unpack module;
+        - direct unpack obviously works only for healthy download; if download
+          is damaged the direct unpack cancels and the download is unpacked
+          during post-processing stage after files are repaired;
+        - direct unpack reduces the time needed to complete download and
+      post-processing;
+        - it also allows to start watching of video files during download
+          (requires compatible video player software);
+  - renaming of obfuscated file names during downloading:
+        - correct file names for obfuscated downloads are now determined during
+          download stage (instead of post-processing stage);
+        - downloaded files are saved into disk directly with correct names;
+        - direct renaming uses par2-files to restore correct file names;
+        - new option "DirectRename" to activate direct renaming;
+        - new queue-event NZB_NAMED, sent after the inner files are renamed;
+  - automatic reordering of files:
+        - inner files within nzb reordered to ensure download of files in
+          archive parts order;
+        - the files are reordered when nzb is added to queue;
+        - if direct renaming is active (option "DirectRename") the files are
+      reordered again after the correct names becomes known;
+        - new option "ReorderFiles";
+        - new command "GroupSortFiles" in api-method "editqueue";
+        - new subcommand "SF" of remote command "-E/--edit";
+  - new option "FileNaming" to control how to name obfuscated files (before they
+    get renamed by par-rename, rar-rename or direct-rename);
+  - TLS certificate verification:
+        - when connecting to a news server (for downloading) or a web server
+         (for fetching of rss feeds and nzb-files) the authenticity of the
+          server is validated using server security certificate. If the check
+          fails that means the connection cannot be trusted and must be closed
+          with an error message explaining the security issue;
+        - new options "CertCheck" and "CertStore";
+        - official NZBGet packages come with activated certificate check;
+        - when updating from an older NZBGet version the option CertCheck will
+          be automatically activated when the settings is saved (switch to
+          Settings page in web-interface and click "Save all changed");
+  - authentication via form in web-interface as alternative to HTTP
+    authentication:
+        - that must help with password tools having issues with HTTP
+          authentication dialog;
+        - new option "FormAuth";
+  - drop-downs (context menus) for priority, category and status columns:
+        - quicker changing of priority and category;
+        - easier access to actions via drop-down (context menu) in status
+          column;
+  - extensions scripts can now be executed from settings page:
+        - script authors define custom buttons;
+        - when clicked the script is executed in a special mode and obtain extra
+          parameters;
+        - example script "Email.py" extended with button "Send test e-mail";
+  - on Windows NZBGet can now associate itself with nzb-files:
+        - use option in Windows installer to register NZBGet for nzb-files;
+  - unrar shipped within Linux package is now compiled with "fallocate" option
+    to improve compatibility with media players when watching videos during
+    downloading and unpacking;
+  - support for HTTP-header "X-Forwarded-For" in IP-logging;
+  - improvements in RSS feed view in phone mode;
+  - set name, password and dupe info when adding via URL by click on a button
+    near URL field in web-interface;
+  - backup-badge for items in history similar to downloads tab;
+  - show backup icon in history in phone theme;
+  - added support for ECC certificates in built-in web-server;
+  - save changes before performing actions in history dialog;
+  - proper exit code on client command success or failure.
+  - added host name to all error messages regarding connection issues;
+  - new button "Volume Statistics" in section "News Servers" of settings page;
+    shows the same volume data as in global statistics dialog;
+  - new option "ServerX.Notes" for user comments on news servers;
+  - new parameters for api-method "servervolumes" as a performance optimization
+    measure to reduce amount of transferred data;
+  - new option to force connection to news servers via ipv4 or ipv6;
+  - removed unnecessary requests to news servers;
+  - updated unrar to v5.40;
+  - clear script execution log before executing script;
+  - added support for crash dumps on Windows:
+        - renamed option "DumpCore" to "CrashDump";
+        - new option "CrashTrace" to make it possible to disable default
+          printing off call stack in order to produce more relevant crash dumps;
+  - fixed: startup scheduler tasks can be executed again;
+  - fixed: "fatal" messages when compiling from sources.
+  - fixed: per-nzb download statistics could be wrong if the program was
+    reloaded during downloading.
+  - fixed crash which may happen between post-processing steps;
+  - fixed: asterix (*) was sometimes passed as parameter to extension scripts
+    (Windows only);
+  - fixed potential crash during update via web-interface.
+
+nzbget-18.1:
+  - fixed: crash during download caused by a race condition;
+  - fixed: sleep mode did not work on Windows;
+  - fixed: queue was not saved after deleting of queued post-jobs;
+  - fixed: possible crash at the end of post-processing;
+  - fixed: "undefined" in reorder extension scripts.
+
 nzbget-18.0:
-  -  please see repository change log at
-     https://github.com/nzbget/nzbget/commits/develop
+  - automatic deobfuscation of rar-archives without par-files:
+        - obfuscated downloads not having par-files can now be successfully
+          unpacked;
+        - also helps with downloads where rar-files were obfuscated before
+          creating par-files;
+        - new options "RarRename" and "UnpackIgnoreExt";
+  - multi post-processing:
+        - in addition to classic post-processing strategy where items are
+          processed one after another it is now possible to post-process
+          multiple items at the same time;
+        - new option "PostStrategy" to choose from four: sequential, balanced,
+          aggressive, rocket;
+        - in "balanced" strategy downloads needing repair do not block other
+          items which are processed sequentially but simultaneously with
+          repairing item;
+        - in "aggressive" mode up to three items are post-processed at the same
+          time and in "rocket" mode up to six items (including up to two repair
+          tasks);
+  - unified extension scripts settings:
+        - options "PostScript", "QueueScript", "ScanScript" and "FeedScript"
+          were replaced with one option "Extensions";
+        - users don't need to know the technical details os extension scripts as
+          all scripts are now can be selected at one place;
+        - easier activation of complex extension scripts which previously needed
+          to be selected in multiple options for their proper work;
+  - reordering download queue with drag and drop in web-interface:
+        - new actions "GroupMoveBefore" and "GroupMoveAfter" in API-method
+          "editqueue";
+  - priorities are now displayed as a column instead of badge; that makes it
+    possible to manually sort on priority;
+  - removed vertical lines in tables; looks better in combination with new
+    priority column;
+  - keyboard shortcuts in web-interface;
+  - improved UI to prevent accidental deletion of many items:
+        - visual indication of records selected on other pages;
+        - extra warning when deleting many records from history;
+  - additional options in "custom pause dialog";
+  - better handing of damaged par2-files in par-renamer:
+        - if par-renamer can't load a (damaged) par2-file then another par2-file
+          is downloaded and par-renamer tries again;
+        - reverted non-strict par2-filename matching to handle article subjects
+          with non-parseable filenames;
+  - better handling of obfuscated par-files;
+  - splitted option "Retries" into "ArticleRetries" and "UrlRetries"; option
+    "RetryInterval" into "ArticleInterval" and "UrlInterval";
+  - scheduler tasks can be started at program launch:
+        - use asterisk as TaskX.Time;
+  - graceful termination of scheduler scripts:
+        - scripts receive signal SIGINT (CTRL+BREAK on Windows) before
+          termination;
+  - added support for nZEDb attributes in rss feeds;
+  - better cleanup handling: if parameter "unpack" is disabled for an nzb-file
+    the cleanup isn't performed for it;
+  - fields containing passwords are now displayed as protected fields;
+  - showing password-badge for nzbs with passwords;
+  - allow control of what tab is shown when opening web-interface:
+    add "#downloads", "#history", "#messages" or "#settings" to the URL,
+    for example "http://localhost:6789/#history" or
+    "http://localhost:6789/index.html#history";
+  - functional testing to ensure program quality:
+        - implemented built-in simple nntp server to be used for functional
+          testing;
+        - created a number of tests;
+        - new features come with additional tests;
+  - improved API-method "append" in combination with duplicate check; method
+    returns nzb-id also for items added straight to history;
+  - removed parameter "offset" from api-method "editqueue":
+        - when needed the "offset" is now passed within parameter "Args" as
+          string;
+        - old method signature is supported for compatibility;
+  - improved error reporting on feed parse errors;
+  - highlighting selected rows with alternative colors;
+  - improved selecting of par2-file for repair;
+  - splitted config section "Download Queue" and moved many options into new
+    section "Connection";
+  - disabled SSLv3 in built-in web-server;
+  - multiple recipients in the example pp-script "EMail.py";
+  - added compatibility with openssl 1.1.0;
+  - fixed TLS handshake error when using GnuTLS;
+  - fixed: sorting of selected items may give wrong results;
+  - fixed: search box filter in feed view were not reset.
 
 nzbget-17.1:
   - adjustments and fixes for "Retry failed articles" function, better handling

Makefile.am

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,6 +28,8 @@ nzbget_SOURCES = \
 	daemon/connect/WebDownloader.h \
 	daemon/extension/FeedScript.cpp \
 	daemon/extension/FeedScript.h \
+	daemon/extension/CommandScript.cpp \
+	daemon/extension/CommandScript.h \
 	daemon/extension/NzbScript.cpp \
 	daemon/extension/NzbScript.h \
 	daemon/extension/PostScript.cpp \
@@ -56,8 +58,8 @@ nzbget_SOURCES = \
 	daemon/frontend/LoggableFrontend.h \
 	daemon/frontend/NCursesFrontend.cpp \
 	daemon/frontend/NCursesFrontend.h \
-    daemon/main/CommandLineParser.cpp \
-    daemon/main/CommandLineParser.h \
+	daemon/main/CommandLineParser.cpp \
+	daemon/main/CommandLineParser.h \
 	daemon/main/DiskService.cpp \
 	daemon/main/DiskService.h \
 	daemon/main/Maintenance.cpp \
@@ -66,6 +68,8 @@ nzbget_SOURCES = \
 	daemon/main/nzbget.h \
 	daemon/main/Options.cpp \
 	daemon/main/Options.h \
+	daemon/main/WorkState.cpp \
+	daemon/main/WorkState.h \
 	daemon/main/Scheduler.cpp \
 	daemon/main/Scheduler.h \
 	daemon/main/StackTrace.cpp \
@@ -106,6 +110,10 @@ nzbget_SOURCES = \
 	daemon/postprocess/Repair.h \
 	daemon/postprocess/Unpack.cpp \
 	daemon/postprocess/Unpack.h \
+	daemon/postprocess/DirectUnpack.cpp \
+	daemon/postprocess/DirectUnpack.h \
+	daemon/queue/DirectRenamer.cpp \
+	daemon/queue/DirectRenamer.h \
 	daemon/queue/DiskState.cpp \
 	daemon/queue/DiskState.h \
 	daemon/queue/DownloadInfo.cpp \
@@ -208,6 +216,25 @@ nzbget_SOURCES += \
 	lib/par2/verificationpacket.h
 endif
 
+# Simd decoder and Crc32
+nzbget_SOURCES += \
+	lib/yencode/YEncode.h \
+	lib/yencode/SimdInit.cpp \
+	lib/yencode/SimdDecoder.cpp \
+	lib/yencode/ScalarDecoder.cpp \
+	lib/yencode/Sse2Decoder.cpp \
+	lib/yencode/Ssse3Decoder.cpp \
+	lib/yencode/PclmulCrc.cpp \
+	lib/yencode/NeonDecoder.cpp \
+	lib/yencode/AcleCrc.cpp \
+	lib/yencode/SliceCrc.cpp
+
+lib/yencode/Sse2Decoder.$(OBJEXT) : CXXFLAGS+=$(SSE2_CXXFLAGS)
+lib/yencode/Ssse3Decoder.$(OBJEXT) : CXXFLAGS+=$(SSSE3_CXXFLAGS)
+lib/yencode/PclmulCrc.$(OBJEXT) : CXXFLAGS+=$(PCLMUL_CXXFLAGS)
+lib/yencode/NeonDecoder.$(OBJEXT) : CXXFLAGS+=$(NEON_CXXFLAGS)
+lib/yencode/AcleCrc.$(OBJEXT) : CXXFLAGS+=$(ACLECRC_CXXFLAGS)
+
 AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/connect \
 	-I$(srcdir)/daemon/extension \
@@ -220,7 +247,8 @@ AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/remote \
 	-I$(srcdir)/daemon/util \
 	-I$(srcdir)/daemon/nserv \
-	-I$(srcdir)/lib/par2
+	-I$(srcdir)/lib/par2 \
+	-I$(srcdir)/lib/yencode
 
 if WITH_TESTS
 nzbget_SOURCES += \
@@ -235,6 +263,7 @@ nzbget_SOURCES += \
 	tests/postprocess/DupeMatcherTest.cpp \
 	tests/postprocess/RarRenamerTest.cpp \
 	tests/postprocess/RarReaderTest.cpp \
+	tests/postprocess/DirectUnpackTest.cpp \
 	tests/queue/NzbFileTest.cpp \
 	tests/nntp/ServerPoolTest.cpp \
 	tests/util/FileSystemTest.cpp \
@@ -276,9 +305,9 @@ windows_FILES = \
 	windows/resources/trayicon_idle.ico \
 	windows/resources/trayicon_paused.ico \
 	windows/resources/trayicon_working.ico \
-	windows/setup/nzbget-setup.nsi \
-	windows/setup/install.bmp \
-	windows/setup/uninstall.bmp
+	windows/resources/install.bmp \
+	windows/resources/uninstall.bmp \
+	windows/nzbget-setup.nsi
 
 osx_FILES = \
 	osx/App_Prefix.pch \
@@ -318,6 +347,7 @@ linux_FILES = \
 	linux/build-info.txt \
 	linux/build-nzbget \
 	linux/build-unpack \
+	linux/build-toolchain-android \
 	linux/build-toolchain-freebsd
 
 doc_FILES = \
@@ -362,7 +392,9 @@ webui_FILES = \
 	webui/img/favicon.ico \
 	webui/img/download-anim-green-2x.png \
 	webui/img/download-anim-orange-2x.png \
-	webui/img/transmit-reload-2x.gif
+	webui/img/transmit-reload-2x.gif \
+	webui/img/favicon-256x256-opaque.png \
+	webui/img/favicon-256x256.png
 
 scripts_FILES = \
 	scripts/EMail.py \
@@ -384,6 +416,14 @@ testdata_FILES = \
 	tests/testdata/parchecker/testfile.vol00+1.PAR2 \
 	tests/testdata/parchecker/testfile.vol01+2.PAR2 \
 	tests/testdata/parchecker/testfile.vol03+3.PAR2 \
+	tests/testdata/parchecker2/crc.txt \
+	tests/testdata/parchecker2/testfile.7z.001 \
+	tests/testdata/parchecker2/testfile.7z.002 \
+	tests/testdata/parchecker2/testfile.7z.003 \
+	tests/testdata/parchecker2/testfile.7z.par2 \
+	tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \
 	tests/testdata/rarrenamer/testfile3.part01.rar \
 	tests/testdata/rarrenamer/testfile3.part02.rar \
 	tests/testdata/rarrenamer/testfile3.part03.rar \
@@ -455,21 +495,21 @@ uninstall-conf:
 #    we create new file "code_revision.c" with empty revision number.
 code_revision.cpp: FORCE
 	@ if test -d ./.git ; then \
-		B="$(shell git branch | sed -n -e 's/^\* \(.*\)/\1/p')"; \
-		M="$(shell git status --porcelain)" ; \
+		B=`git branch | sed -n -e 's/^\* \(.*\)/\1/p'`; \
+		M=`git status --porcelain` ; \
 		if test "$$M" != "" ; then \
 			M="M" ; \
 		fi ; \
 		if test "$$B" = "master" ; then \
 			V="$$M" ; \
 		elif test "$$B" = "develop" ; then \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M" ; \
 		else \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M ($$B)" ; \
 		fi ; \
-		H="$(shell test -f ./code_revision.cpp && head -n 1 code_revision.cpp)"; \
+		H=`test -f ./code_revision.cpp && head -n 1 code_revision.cpp`; \
 		if test "/* $$V */" != "$$H" ; then \
 			( \
 			echo "/* $$V */" ;\

README.md

@@ -1,18 +1,19 @@
 # NZBGet #
 [![License](https://img.shields.io/badge/license-GPL-blue.svg)](http://www.gnu.org/licenses/)
 [![Build Status](https://img.shields.io/travis/nzbget/nzbget/develop.svg)](https://travis-ci.org/nzbget/nzbget)
+[![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/nzbget/nzbget.svg?label=code%20quality:%20c%2b%2b)](https://lgtm.com/projects/g/nzbget/nzbget/context:cpp)
+[![Code Quality: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/nzbget/nzbget.svg?label=code%20quality:%20js)](https://lgtm.com/projects/g/nzbget/nzbget/context:javascript)
+[![Total Alerts](https://img.shields.io/lgtm/alerts/g/nzbget/nzbget.svg)](https://lgtm.com/projects/g/nzbget/nzbget/alerts)
+
+[![Total downloads](https://img.shields.io/github/downloads/nzbget/nzbget/total.svg)](https://github.com/nzbget/nzbget/releases)
+[![Downloads (latest release)](https://img.shields.io/github/downloads/nzbget/nzbget/latest/total.svg?label=latest%20release)](https://github.com/nzbget/nzbget/releases/latest)
 
 NZBGet is a binary downloader, which downloads files from Usenet
 based on information given in nzb-files. 
 
-NZBGet is written in C++ and is known for its extraordinary performance and efficiency.
+NZBGet is written in C++ and is known for its performance and efficiency.
 
-NZBGet can be run at almost every platform - classic PCs, NAS, media players, SAT-receivers, WLAN-routers, etc.
-The download area provides precompiled binaries
-for Windows, Mac OS X and Linux (compatible with many CPUs and platform variants). For other platforms
-the program can be compiled from sources.
-
-- [Home page (nzbget.net)](http://nzbget.net) - for first time visitors, learn more about NZBGet;
-- [Downloads](http://nzbget.net/download) - get the binaries and sources;
-- [Documentation](https://github.com/nzbget/nzbget/wiki) - installation manuals, HOW-TOs, API;
-- [Forum](http://forum.nzbget.net) - get support, share your ideas, scripts, add-ons.
+NZBGet can run on almost any device - classic PC, NAS, media player, SAT-receiver, WLAN-router, etc.
+The download area provides precompiled binaries for Windows, macOS, Linux (compatible with
+many CPUs and platform variants), FreeBSD and Android. For other platforms
+the program can be compiled from sources.

config.h.in

@@ -3,16 +3,15 @@
 /* Define to 1 to include debug-code */
 #undef DEBUG
 
-/* Define to 1 if deleting of files during reading of directory is not
-   properly supported by OS */
-#undef DIRBROWSER_SNAPSHOT
-
 /* Define to 1 to not use curses */
 #undef DISABLE_CURSES
 
 /* Define to 1 to disable gzip-support */
 #undef DISABLE_GZIP
 
+/* Define to 1 to not use libxml2, only for development purposes */
+#undef DISABLE_LIBXML2
+
 /* Define to 1 to disable par-verification and repair */
 #undef DISABLE_PARCHECK
 
@@ -83,6 +82,9 @@
 /* Define to 1 to use GnuTLS library for TLS/SSL-support. */
 #undef HAVE_LIBGNUTLS
 
+/* Define to 1 if lockf is supported */
+#undef HAVE_LOCKF
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
@@ -98,6 +100,9 @@
 /* Define to 1 to use OpenSSL library for TLS/SSL-support and decryption. */
 #undef HAVE_OPENSSL
 
+/* Define to 1 if pthread_cancel is supported */
+#undef HAVE_PTHREAD_CANCEL
+
 /* Define to 1 if you have the <regex.h> header file. */
 #undef HAVE_REGEX_H
 
@@ -134,6 +139,9 @@
 /* Define to 1 if variadic macros are supported */
 #undef HAVE_VARIADIC_MACROS
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#undef HAVE_X509_CHECK_HOST
+
 /* Define to 1 to exclude debug-code */
 #undef NDEBUG
 
@@ -152,6 +160,9 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME
 
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
 
@@ -167,9 +178,10 @@
 /* Version number of package */
 #undef VERSION
 
-/* Define to 1 if your processor stores words with the most significant byte
-   first (like Motorola and SPARC, unlike Intel and VAX). */
-#undef WORDS_BIGENDIAN
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
 
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS

configure.ac

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2021 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,11 +20,11 @@
 #                                               -*- Autoconf -*-
 # Process this file with autoconf to produce a configure script.
 
-AC_PREREQ(2.59)
-AC_INIT(nzbget, 18.0-testing, hugbug@users.sourceforge.net)
+AC_PREREQ(2.65)
+AC_INIT(nzbget, 21.2-testing, hugbug@users.sourceforge.net)
 AC_CONFIG_AUX_DIR(posix)
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE([foreign])
+AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_CONFIG_SRCDIR([daemon/main/nzbget.cpp])
 AC_CONFIG_HEADERS([config.h])
 AM_MAINTAINER_MODE
@@ -65,8 +65,7 @@ fi
 dnl
 dnl Checks for header files.
 dnl
-AC_CHECK_HEADERS(sys/prctl.h)
-AC_CHECK_HEADERS(regex.h)
+AC_CHECK_HEADERS(sys/prctl.h regex.h endian.h getopt.h)
 
 
 dnl
@@ -78,6 +77,19 @@ AC_SEARCH_LIBS([inet_addr], [nsl])
 AC_SEARCH_LIBS([hstrerror], [resolv])
 
 
+dnl
+dnl Android NDK restrictions
+dnl
+AC_CHECK_FUNC(lockf,
+	[AC_CHECK_DECL(lockf,
+		[AC_DEFINE([HAVE_LOCKF], 1, [Define to 1 if lockf is supported])],,
+		[#include <unistd.h>])])
+AC_CHECK_FUNC(pthread_cancel,
+	[AC_CHECK_DECL(pthread_cancel,
+		[AC_DEFINE([HAVE_PTHREAD_CANCEL], 1, [Define to 1 if pthread_cancel is supported])],,
+		[#include <pthread.h>])])
+
+
 dnl
 dnl Getopt
 dnl
@@ -146,7 +158,7 @@ if test "$FOUND" = "no"; then
 		[	char* szHost; struct hostent hinfobuf; char* strbuf; int h_errnop;
 			struct hostent* hinfo = gethostbyname_r(szHost, &hinfobuf, strbuf, 1024, &h_errnop); ],
 		AC_MSG_RESULT([[yes, and it takes 5 arguments]])
-		FOUND="yes"                      
+		FOUND="yes"
 		AC_DEFINE([HAVE_GETHOSTBYNAME_R_5], 1, [Define to 1 if gethostbyname_r takes 5 arguments]),
 		FOUND="no")
 	
@@ -196,32 +208,20 @@ AC_TRY_COMPILE([
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (size_t*)NULL)],[
-    AC_MSG_RESULT(size_t)
-    SOCKLEN_T=size_t],[
-    AC_TRY_COMPILE([
+	AC_MSG_RESULT(size_t)
+	SOCKLEN_T=size_t],[
+	AC_TRY_COMPILE([
 #include <stddef.h>
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (int*)NULL)],[
-      AC_MSG_RESULT(int)
-      SOCKLEN_T=int],[
-      AC_MSG_WARN(could not determine)
-      SOCKLEN_T=int])])])
+	AC_MSG_RESULT(int)
+	SOCKLEN_T=int],[
+	AC_MSG_WARN(could not determine)
+	SOCKLEN_T=int])])])
 AC_DEFINE_UNQUOTED(SOCKLEN_T, $SOCKLEN_T, [Determine what socket length (socklen_t) data type is])
 
 
-dnl
-dnl Dir-browser's snapshot
-dnl
-AC_MSG_CHECKING(whether dir-browser snapshot workaround is needed)
-if test "$target_vendor" == "apple"; then
-	AC_MSG_RESULT([[yes]])
-	AC_DEFINE([DIRBROWSER_SNAPSHOT], 1, [Define to 1 if deleting of files during reading of directory is not properly supported by OS])
-else
-	AC_MSG_RESULT([[no]])
-fi
-
-
 dnl
 dnl check cpu cores via sysconf
 dnl
@@ -238,26 +238,36 @@ AC_TRY_COMPILE(
 dnl 
 dnl checks for libxml2 includes and libraries.
 dnl
-AC_ARG_WITH(libxml2_includes,
-	[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
-	[CPPFLAGS="${CPPFLAGS} -I${withval}"]
-	[INCVAL="yes"],
-	[INCVAL="no"])
-AC_ARG_WITH(libxml2_libraries,
-	[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
-	[LDFLAGS="${LDFLAGS} -L${withval}"]
-	[LIBVAL="yes"],
-	[LIBVAL="no"])
-if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
-	PKG_CHECK_MODULES(libxml2, libxml-2.0,
-		[LIBS="${LIBS} $libxml2_LIBS"]
-		[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+AC_MSG_CHECKING(whether to use libxml2)
+AC_ARG_ENABLE(libxml2,
+	[AS_HELP_STRING([--disable-libxml2], [do not use libxml2 (removes dependency from libxml2-library, only for development purposes)])],
+	[USELIBXML2=$enableval],
+	[USELIBXML2=yes] )
+AC_MSG_RESULT($USELIBXML2)
+if test "$USELIBXML2" = "yes"; then
+	AC_ARG_WITH(libxml2_includes,
+		[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
+		[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+		[INCVAL="yes"],
+		[INCVAL="no"])
+	AC_ARG_WITH(libxml2_libraries,
+		[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
+		[LDFLAGS="${LDFLAGS} -L${withval}"]
+		[LIBVAL="yes"],
+		[LIBVAL="no"])
+	if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+		PKG_CHECK_MODULES(libxml2, libxml-2.0,
+			[LIBS="${LIBS} $libxml2_LIBS"]
+			[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+			AC_MSG_ERROR("libxml2 library not found"))
+	fi
+	AC_CHECK_HEADER(libxml/tree.h,,
+		AC_MSG_ERROR("libxml2 header files not found"))
+	AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
 		AC_MSG_ERROR("libxml2 library not found"))
+else
+	AC_DEFINE([DISABLE_LIBXML2],1,[Define to 1 to not use libxml2, only for development purposes])
 fi
-AC_CHECK_HEADER(libxml/tree.h,,
-	AC_MSG_ERROR("libxml2 header files not found"))
-AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
-	AC_MSG_ERROR("libxml2 library not found"))
 
 
 dnl
@@ -327,11 +337,8 @@ AC_MSG_RESULT($ENABLEPARCHECK)
 if test "$ENABLEPARCHECK" = "yes"; then
 	dnl PAR2 checks. 
 	dnl 
-	dnl Checks for header files.
-	AC_CHECK_HEADERS([endian.h] [getopt.h])
 	dnl Checks for typedefs, structures, and compiler characteristics.
 	AC_TYPE_SIZE_T
-	AC_C_BIGENDIAN
 	AC_FUNC_FSEEKO
 	dnl Checks for library functions.
 	AC_CHECK_FUNCS([stricmp])
@@ -396,6 +403,8 @@ if test "$USETLS" = "yes"; then
 			if test "$FOUND" = "yes"; then
 				TLSLIB="OpenSSL"
 				AC_DEFINE([HAVE_OPENSSL],1,[Define to 1 to use OpenSSL library for TLS/SSL-support and decryption.])
+				AC_SEARCH_LIBS([X509_check_host], [crypto],
+					AC_DEFINE([HAVE_X509_CHECK_HOST],1,[Define to 1 if OpenSSL supports function "X509_check_host".]))
 			fi
 		fi
 	fi
@@ -539,6 +548,36 @@ else
 fi
 
 
+dnl
+dnl Determine if CPU supports SIMD instructions
+dnl
+AC_MSG_CHECKING(whether to use SIMD-optimized routines)
+USE_SIMD=no
+case $host_cpu in
+	i?86|x86_64)
+		SSE2_CXXFLAGS="-msse2"
+		SSSE3_CXXFLAGS="-mssse3"
+		PCLMUL_CXXFLAGS="-msse4.1 -mpclmul"
+		USE_SIMD=yes
+		;;
+	arm*)
+		NEON_CXXFLAGS="-mfpu=neon"
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+	aarch64)
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+esac
+AC_MSG_RESULT($USE_SIMD)
+AC_SUBST([SSE2_CXXFLAGS])
+AC_SUBST([SSSE3_CXXFLAGS])
+AC_SUBST([PCLMUL_CXXFLAGS])
+AC_SUBST([NEON_CXXFLAGS])
+AC_SUBST([ACLECRC_CXXFLAGS])
+
+
 dnl 
 dnl Some Linux systems require an empty signal handler for SIGCHLD
 dnl in order for exit codes to be correctly delivered to parent process.
@@ -597,11 +636,10 @@ dnl
 dnl variadic macros
 dnl 
 AC_MSG_CHECKING(for variadic macros)
-AC_COMPILE_IFELSE([
-	#define macro(...)   macrofunc(__VA_ARGS__)
-	int macrofunc(int a, int b) { return a + b; }
-	int test() { return macro(1, 2); }
-	],
+AC_TRY_COMPILE(
+	[ #define macro(...)   macrofunc(__VA_ARGS__) ]
+	[ int macrofunc(int a, int b) { return a + b; } ],
+	[ int a=macro(1, 2); ],
 	AC_MSG_RESULT([yes])
 	AC_DEFINE([HAVE_VARIADIC_MACROS], 1, Define to 1 if variadic macros are supported),
 	AC_MSG_RESULT([no]))

daemon/connect/Connection.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -41,13 +41,22 @@ public:
 		csConnected,
 		csDisconnected,
 		csListening,
-		csCancelled
+		csCancelled,
+		csBroken
+	};
+
+	enum EIPVersion
+	{
+		ipAuto,
+		ipV4,
+		ipV6
 	};
 
 	Connection(const char* host, int port, bool tls);
 	Connection(SOCKET socket, bool tls);
 	virtual ~Connection();
 	static void Init();
+	static void Final();
 	virtual bool Connect();
 	virtual bool Disconnect();
 	bool Bind();
@@ -65,12 +74,14 @@ public:
 	const char* GetCipher() { return m_cipher; }
 	void SetCipher(const char* cipher) { m_cipher = cipher; }
 	void SetTimeout(int timeout) { m_timeout = timeout; }
+	void SetIPVersion(EIPVersion ipVersion) { m_ipVersion = ipVersion; }
 	EStatus GetStatus() { return m_status; }
 	void SetSuppressErrors(bool suppressErrors);
 	bool GetSuppressErrors() { return m_suppressErrors; }
 	const char* GetRemoteAddr();
 	bool GetGracefull() { return m_gracefull; }
 	void SetGracefull(bool gracefull) { m_gracefull = gracefull; }
+	void SetForceClose(bool forceClose) { m_forceClose = forceClose; }
 #ifndef DISABLE_TLS
 	bool StartTls(bool isClient, const char* certFile, const char* keyFile);
 #endif
@@ -80,6 +91,7 @@ protected:
 	CString m_host;
 	int m_port;
 	bool m_tls;
+	EIPVersion m_ipVersion = ipAuto;
 	SOCKET m_socket = INVALID_SOCKET;
 	CString m_cipher;
 	CharBuffer m_readBuf;
@@ -90,8 +102,8 @@ protected:
 	bool m_suppressErrors = true;
 	BString<100> m_remoteAddr;
 	int m_totalBytesRead = 0;
-	bool m_broken = false;
 	bool m_gracefull = false;
+	bool m_forceClose = false;
 
 	struct SockAddr
 	{
@@ -124,12 +136,13 @@ protected:
 #endif
 #endif
 
-	void ReportError(const char* msgPrefix, const char* msgArg, bool PrintErrCode, int herrno = 0,
-		const char* herrMsg = nullptr);
+	void ReportError(const char* msgPrefix, const char* msgArg, bool printErrCode, int errCode = 0,
+		const char* errMsg = nullptr);
 	virtual void PrintError(const char* errMsg);
+	int GetLastNetworkError();
 	bool DoConnect();
 	bool DoDisconnect();
-	bool InitSocketOpts();
+	bool InitSocketOpts(SOCKET socket);
 	bool ConnectWithTimeout(void* address, int address_len);
 #ifndef HAVE_GETADDRINFO
 	in_addr_t ResolveHostAddr(const char* host);
@@ -139,10 +152,6 @@ protected:
 	int send(SOCKET s, const char* buf, int len, int flags);
 	void CloseTls();
 #endif
-
-private:
-	static void Final();
-	friend class ConnectionFinalizer;
 };
 
 #endif

daemon/connect/TlsSocket.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,17 +25,10 @@
 #include "TlsSocket.h"
 #include "Thread.h"
 #include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
 
-class TlsSocketFinalizer
-{
-public:
-	~TlsSocketFinalizer()
-	{
-		TlsSocket::Final();
-	}
-};
-
-std::unique_ptr<TlsSocketFinalizer> m_tlsSocketFinalizer;
+CString TlsSocket::m_certStore;
 
 #ifdef HAVE_LIBGNUTLS
 #ifdef NEED_GCRYPT_LOCKING
@@ -186,8 +179,6 @@ void TlsSocket::Init()
 	OpenSSL_add_all_algorithms();
 
 #endif /* HAVE_OPENSSL */
-
-	m_tlsSocketFinalizer = std::make_unique<TlsSocketFinalizer>();
 }
 
 void TlsSocket::Final()
@@ -195,18 +186,49 @@ void TlsSocket::Final()
 #ifdef HAVE_LIBGNUTLS
 	gnutls_global_deinit();
 #endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+#ifndef LIBRESSL_VERSION_NUMBER
+	FIPS_mode_set(0);
+#endif
+#ifdef NEED_CRYPTO_LOCKING
+	CRYPTO_set_locking_callback(nullptr);
+	CRYPTO_set_id_callback(nullptr);
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	ERR_remove_state(0);
+#endif
+#if	OPENSSL_VERSION_NUMBER >= 0x10002000L && ! defined (LIBRESSL_VERSION_NUMBER)
+	SSL_COMP_free_compression_methods();
+#endif
+	//ENGINE_cleanup();
+	CONF_modules_free();
+	CONF_modules_unload(1);
+#ifndef OPENSSL_NO_COMP
+	COMP_zlib_cleanup();
+#endif
+	ERR_free_strings();
+	EVP_cleanup();
+	CRYPTO_cleanup_all_ex_data();
+#endif /* HAVE_OPENSSL */
 }
 
 TlsSocket::~TlsSocket()
 {
 	Close();
+
+#ifdef HAVE_OPENSSL
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	ERR_remove_state(0);
+#endif
+#endif
 }
 
-void TlsSocket::ReportError(const char* errMsg)
+void TlsSocket::ReportError(const char* errMsg, bool suppressable)
 {
 #ifdef HAVE_LIBGNUTLS
 	const char* errstr = gnutls_strerror(m_retCode);
-	if (m_suppressErrors)
+	if (suppressable && m_suppressErrors)
 	{
 		debug("%s: %s", errMsg, errstr);
 	}
@@ -217,16 +239,14 @@ void TlsSocket::ReportError(const char* errMsg)
 #endif /* HAVE_LIBGNUTLS */
 
 #ifdef HAVE_OPENSSL
-	int errcode;
+	int errcode = ERR_get_error();
 	do
 	{
-		errcode = ERR_get_error();
-
 		char errstr[1024];
 		ERR_error_string_n(errcode, errstr, sizeof(errstr));
 		errstr[1024-1] = '\0';
 
-		if (m_suppressErrors)
+		if (suppressable && m_suppressErrors)
 		{
 			debug("%s: %s", errMsg, errstr);
 		}
@@ -238,6 +258,8 @@ void TlsSocket::ReportError(const char* errMsg)
 		{
 			PrintError(errMsg);
 		}
+
+		errcode = ERR_get_error();
 	} while (errcode);
 #endif /* HAVE_OPENSSL */
 }
@@ -254,7 +276,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_certificate_allocate_credentials(&cred);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -266,7 +288,7 @@ bool TlsSocket::Start()
 			m_certFile, m_keyFile, GNUTLS_X509_FMT_PEM);
 		if (m_retCode != 0)
 		{
-			ReportError("Could not load certificate or key file");
+			ReportError("Could not load certificate or key file", false);
 			Close();
 			return false;
 		}
@@ -276,7 +298,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_init(&sess, m_isClient ? GNUTLS_CLIENT : GNUTLS_SERVER);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
@@ -291,7 +313,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_priority_set_direct((gnutls_session_t)m_session, priority, nullptr);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not select cipher for TLS");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
@@ -301,7 +323,7 @@ bool TlsSocket::Start()
 		m_retCode = gnutls_server_name_set((gnutls_session_t)m_session, GNUTLS_NAME_DNS, m_host, m_host.Length());
 		if (m_retCode != 0)
 		{
-			ReportError("Could not set host name for TLS");
+			ReportError("Could not set hostname for TLS");
 			Close();
 			return false;
 		}
@@ -311,7 +333,7 @@ bool TlsSocket::Start()
 		(gnutls_certificate_credentials_t*)m_context);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not initialize TLS session");
+		ReportError("Could not initialize TLS session", false);
 		Close();
 		return false;
 	}
@@ -321,7 +343,13 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_handshake((gnutls_session_t)m_session);
 	if (m_retCode != 0)
 	{
-		ReportError("TLS handshake failed");
+		ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -335,7 +363,7 @@ bool TlsSocket::Start()
 
 	if (!m_context)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -343,47 +371,77 @@ bool TlsSocket::Start()
 	{
 		if (SSL_CTX_use_certificate_chain_file((SSL_CTX*)m_context, m_certFile) != 1)
 		{
-			ReportError("Could not load certificate file");
+			ReportError("Could not load certificate file", false);
 			Close();
 			return false;
 		}
 		if (SSL_CTX_use_PrivateKey_file((SSL_CTX*)m_context, m_keyFile, SSL_FILETYPE_PEM) != 1)
 		{
-			ReportError("Could not load key file");
+			ReportError("Could not load key file", false);
 			Close();
 			return false;
 		}
 		if (!SSL_CTX_set_options((SSL_CTX*)m_context, SSL_OP_NO_SSLv3))
 		{
-			ReportError("Could not select minimum protocol version for TLS");
+			ReportError("Could not select minimum protocol version for TLS", false);
 			Close();
 			return false;
 		}
+
+		// For ECC certificates
+		EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
+		if (!ecdh)
+		{
+			ReportError("Could not generate ecdh parameters for TLS", false);
+			Close();
+			return false;
+		}
+		if (!SSL_CTX_set_tmp_ecdh((SSL_CTX*)m_context, ecdh))
+		{
+			ReportError("Could not set ecdh parameters for TLS", false);
+			EC_KEY_free(ecdh);
+			Close();
+			return false;
+		}
+		EC_KEY_free(ecdh);
+	}
+
+	if (m_isClient && !m_certStore.Empty())
+	{
+		// Enable certificate validation
+		if (SSL_CTX_load_verify_locations((SSL_CTX*)m_context, m_certStore, nullptr) != 1)
+		{
+			ReportError("Could not set certificate store location", false);
+			Close();
+			return false;
+		}
+
+		SSL_CTX_set_verify((SSL_CTX*)m_context, SSL_VERIFY_PEER, nullptr);
 	}
 
 	m_session = SSL_new((SSL_CTX*)m_context);
 	if (!m_session)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
 
 	if (!m_cipher.Empty() && !SSL_set_cipher_list((SSL*)m_session, m_cipher))
 	{
-		ReportError("Could not select cipher for TLS");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
 
-	if (m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host))
+	if (m_isClient && m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host))
 	{
 		ReportError("Could not set host name for TLS");
 		Close();
 		return false;
 	}
 
-	if (!SSL_set_fd((SSL*)m_session, m_socket))
+	if (!SSL_set_fd((SSL*)m_session, (int)m_socket))
 	{
 		ReportError("Could not set the file descriptor for TLS");
 		Close();
@@ -393,7 +451,23 @@ bool TlsSocket::Start()
 	int error_code = m_isClient ? SSL_connect((SSL*)m_session) : SSL_accept((SSL*)m_session);
 	if (error_code < 1)
 	{
-		ReportError("TLS handshake failed");
+		long verifyRes = SSL_get_verify_result((SSL*)m_session);
+		if (verifyRes != X509_V_OK)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification",
+				*m_host, X509_verify_cert_error_string(verifyRes)));
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		}
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -403,6 +477,128 @@ bool TlsSocket::Start()
 #endif /* HAVE_OPENSSL */
 }
 
+bool TlsSocket::ValidateCert()
+{
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER >= 0x030104
+#if	GNUTLS_VERSION_NUMBER >= 0x030306
+	if (FileSystem::DirectoryExists(m_certStore))
+	{
+		if (gnutls_certificate_set_x509_trust_dir((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+	else
+#endif
+	{
+		if (gnutls_certificate_set_x509_trust_file((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+
+	unsigned int status = 0;
+	if (gnutls_certificate_verify_peers3((gnutls_session_t)m_session, m_host, &status) != 0 ||
+		gnutls_certificate_type_get((gnutls_session_t)m_session) != GNUTLS_CRT_X509)
+	{
+		ReportError("Could not verify TLS certificate");
+		return false;
+	}
+
+	if (status != 0)
+	{
+		if (status & GNUTLS_CERT_UNEXPECTED_OWNER)
+		{
+			// Extracting hostname from the certificate
+			unsigned int cert_list_size = 0;
+			const gnutls_datum_t* cert_list = gnutls_certificate_get_peers((gnutls_session_t)m_session, &cert_list_size);
+			if (cert_list_size > 0)
+			{
+				gnutls_x509_crt_t cert;
+				gnutls_x509_crt_init(&cert);
+				gnutls_x509_crt_import(cert, &cert_list[0], GNUTLS_X509_FMT_DER);
+				char dn[256];
+				size_t size = sizeof(dn);
+				if (gnutls_x509_crt_get_dn_by_oid(cert, GNUTLS_OID_X520_COMMON_NAME, 0, 0, dn, &size) == 0)
+				{
+					PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+						" For more info visit http://nzbget.net/certificate-verification", *m_host, dn));
+					gnutls_x509_crt_deinit(cert);
+					return false;
+				}
+				gnutls_x509_crt_deinit(cert);
+			}
+		}
+
+		gnutls_datum_t msgdata;
+		if (gnutls_certificate_verification_status_print(status, GNUTLS_CRT_X509, &msgdata, 0) == 0)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host, msgdata.data));
+			gnutls_free(&msgdata);
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS certificate verification failed for %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		}
+		return false;
+	}
+#endif
+	return true;
+#endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+	// verify a server certificate was presented during the negotiation
+	X509* cert = SSL_get_peer_certificate((SSL*)m_session);
+	if (!cert)
+	{
+		PrintError(BString<1024>("TLS certificate verification failed for %s: no certificate provided by server."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		return false;
+	}
+
+#ifdef HAVE_X509_CHECK_HOST
+	// hostname verification
+	if (!m_host.Empty() && X509_check_host(cert, m_host, m_host.Length(), 0, nullptr) != 1)
+	{
+		const unsigned char* certHost = nullptr;
+        // Find the position of the CN field in the Subject field of the certificate
+        int common_name_loc = X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1);
+        if (common_name_loc >= 0)
+		{
+			// Extract the CN field
+			X509_NAME_ENTRY* common_name_entry = X509_NAME_get_entry(X509_get_subject_name(cert), common_name_loc);
+			if (common_name_entry != nullptr)
+			{
+				// Convert the CN field to a C string
+				ASN1_STRING* common_name_asn1 = X509_NAME_ENTRY_get_data(common_name_entry);
+				if (common_name_asn1 != nullptr)
+				{
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+					certHost = ASN1_STRING_get0_data(common_name_asn1);
+#else
+					certHost = ASN1_STRING_data(common_name_asn1);
+#endif
+				}
+			}
+        }
+
+		PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host, certHost));
+		X509_free(cert);
+		return false;
+	}
+#endif
+
+	X509_free(cert);
+	return true;
+#endif /* HAVE_OPENSSL */
+}
+
 void TlsSocket::Close()
 {
 	if (m_session)

daemon/connect/TlsSocket.h

@@ -33,6 +33,8 @@ public:
 		m_certFile(certFile), m_keyFile(keyFile), m_cipher(cipher) {}
 	virtual ~TlsSocket();
 	static void Init();
+	static void InitOptions(const char* certStore) { m_certStore = certStore; }
+	static void Final();
 	bool Start();
 	void Close();
 	int Send(const char* buffer, int size);
@@ -41,27 +43,26 @@ public:
 
 protected:
 	virtual void PrintError(const char* errMsg);
-	
+
 private:
+	SOCKET m_socket;
 	bool m_isClient;
 	CString m_host;
 	CString m_certFile;
 	CString m_keyFile;
 	CString m_cipher;
-	SOCKET m_socket;
 	bool m_suppressErrors = false;
 	bool m_initialized = false;
 	bool m_connected = false;
 	int m_retCode;
+	static CString m_certStore;
 
 	// using "void*" to prevent the including of GnuTLS/OpenSSL header files into TlsSocket.h
 	void* m_context = nullptr;
 	void* m_session = nullptr;
 
-	void ReportError(const char* errMsg);
-
-	static void Final();
-	friend class TlsSocketFinalizer;
+	void ReportError(const char* errMsg, bool suppressable = true);
+	bool ValidateCert();
 };
 
 #endif

daemon/connect/WebDownloader.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "WebDownloader.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Util.h"
 #include "FileSystem.h"
 
@@ -72,19 +73,19 @@ void WebDownloader::Run()
 
 		if ((((Status == adFailed) && (remainedDownloadRetries > 1)) ||
 			((Status == adConnectError) && (remainedConnectRetries > 1)))
-			&& !IsStopped() && !(!m_force && g_Options->GetPauseDownload()))
+			&& !IsStopped() && !(!m_force && g_WorkState->GetPauseDownload()))
 		{
 			detail("Waiting %i sec to retry", g_Options->GetUrlInterval());
 			int msec = 0;
 			while (!IsStopped() && (msec < g_Options->GetUrlInterval() * 1000) &&
-				!(!m_force && g_Options->GetPauseDownload()))
+				!(!m_force && g_WorkState->GetPauseDownload()))
 			{
-				usleep(100 * 1000);
+				Util::Sleep(100);
 				msec += 100;
 			}
 		}
 
-		if (IsStopped() || (!m_force && g_Options->GetPauseDownload()))
+		if (IsStopped() || (!m_force && g_WorkState->GetPauseDownload()))
 		{
 			Status = adRetry;
 			break;
@@ -441,7 +442,9 @@ WebDownloader::EStatus WebDownloader::CheckResponse(const char* response)
 		warn("URL %s failed: %s", *m_infoName, hTTPResponse);
 		return adNotFound;
 	}
-	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3))
+	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3) ||
+		 !strncmp(hTTPResponse, "303", 3) || !strncmp(hTTPResponse, "307", 3) ||
+		 !strncmp(hTTPResponse, "308", 3))
 	{
 		m_redirecting = true;
 		return adRunning;
@@ -648,21 +651,6 @@ void WebDownloader::Stop()
 	debug("WebDownloader stopped successfully");
 }
 
-bool WebDownloader::Terminate()
-{
-	std::unique_ptr<Connection> connection = std::move(m_connection);
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		connection.reset();
-	}
-	return terminated;
-}
-
 void WebDownloader::FreeConnection()
 {
 	if (m_connection)

daemon/connect/WebDownloader.h

@@ -50,7 +50,6 @@ public:
 	virtual void Stop();
 	EStatus Download();
 	EStatus DownloadWithRedirects(int maxRedirects);
-	bool Terminate();
 	void SetInfoName(const char* infoName) { m_infoName = infoName; }
 	const char* GetInfoName() { return m_infoName; }
 	void SetUrl(const char* url);

daemon/extension/CommandScript.cpp

@@ -0,0 +1,131 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "CommandScript.h"
+#include "Options.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+
+static const int COMMANDPROCESS_SUCCESS = 93;
+static const int COMMANDPROCESS_ERROR = 94;
+
+bool CommandScriptController::StartScript(const char* scriptName, const char* command,
+	std::unique_ptr<Options::OptEntries> modifiedOptions)
+{
+	CommandScriptController* scriptController = new CommandScriptController();
+	scriptController->m_script = scriptName;
+	scriptController->m_command = command;
+	scriptController->m_logId = g_CommandScriptLog->Reset();
+	scriptController->m_modifiedOptions = std::move(modifiedOptions);
+
+	scriptController->SetAutoDestroy(true);
+
+	scriptController->Start();
+
+	for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
+	{
+		if (FileSystem::SameFilename(scriptName, script.GetName()))
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+void CommandScriptController::Run()
+{
+	ExecuteScriptList(m_script);
+}
+
+void CommandScriptController::ExecuteScript(ScriptConfig::Script* script)
+{
+	PrintMessage(Message::mkInfo, "Executing script %s with command %s", script->GetName(), *m_command);
+
+	SetArgs({script->GetLocation()});
+
+	BString<1024> infoName("script %s with command %s", script->GetName(), *m_command);
+	SetInfoName(infoName);
+
+	SetLogPrefix(script->GetDisplayName());
+	PrepareParams(script->GetName());
+
+	int exitCode = Execute();
+
+	infoName[0] = 'S'; // uppercase
+	SetLogPrefix(nullptr);
+
+	switch (exitCode)
+	{
+		case COMMANDPROCESS_SUCCESS:
+			PrintMessage(Message::mkInfo, "%s successful", *infoName);
+			break;
+
+		case COMMANDPROCESS_ERROR:
+		case -1: // Execute() returns -1 if the process could not be started (file not found or other problem)
+			PrintMessage(Message::mkError, "%s failed", *infoName);
+			break;
+
+		default:
+			PrintMessage(Message::mkError, "%s failed (terminated with unknown status)", *infoName);
+			break;
+	}
+}
+
+void CommandScriptController::PrepareParams(const char* scriptName)
+{
+	ResetEnv();
+
+	SetEnvVar("NZBCP_COMMAND", m_command);
+
+	PrepareEnvScript(nullptr, scriptName);
+}
+
+const char* CommandScriptController::GetOptValue(const char* name, const char* value)
+{
+	Options::OptEntry* entry = m_modifiedOptions->FindOption(name);
+	return entry ? entry->GetValue() : value;
+}
+
+void CommandScriptController::AddMessage(Message::EKind kind, const char * text)
+{
+	NzbScriptController::AddMessage(kind, text);
+	g_CommandScriptLog->AddMessage(m_logId, kind, text);
+}
+
+
+int CommandScriptLog::Reset()
+{
+	Guard guard(m_logMutex);
+	m_messages.clear();
+	return ++m_idScriptGen;
+}
+
+void CommandScriptLog::AddMessage(int scriptId, Message::EKind kind, const char * text)
+{
+	Guard guard(m_logMutex);
+
+	// save only messages from the last started script
+	if (scriptId == m_idScriptGen)
+	{
+		m_messages.emplace_back(++m_idMessageGen, kind, Util::CurrentTime(), text);
+	}
+}

daemon/extension/CommandScript.h

@@ -0,0 +1,63 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef COMMANDSCRIPT_H
+#define COMMANDSCRIPT_H
+
+#include "NzbScript.h"
+#include "Log.h"
+
+class CommandScriptController : public Thread, public NzbScriptController
+{
+public:
+	virtual void Run();
+	static bool StartScript(const char* scriptName, const char* command, std::unique_ptr<Options::OptEntries> modifiedOptions);
+
+protected:
+	virtual void ExecuteScript(ScriptConfig::Script* script);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+	virtual const char* GetOptValue(const char* name, const char* value);
+
+private:
+	CString m_script;
+	CString m_command;
+	int m_logId;
+	std::unique_ptr<Options::OptEntries> m_modifiedOptions;
+
+	void PrepareParams(const char* scriptName);
+};
+
+class CommandScriptLog
+{
+public:
+	GuardedMessageList GuardMessages() { return GuardedMessageList(&m_messages, &m_logMutex); }
+	int Reset();
+	void AddMessage(int scriptId, Message::EKind kind, const char* text);
+
+private:
+	MessageList m_messages;
+	Mutex m_logMutex;
+	int m_idMessageGen;
+	int m_idScriptGen;
+};
+
+extern CommandScriptLog* g_CommandScriptLog;
+
+#endif

daemon/extension/PostScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@
 #include "Log.h"
 #include "Util.h"
 #include "Options.h"
+#include "WorkState.h"
 
 static const int POSTPROCESS_PARCHECK = 92;
 static const int POSTPROCESS_SUCCESS = 93;
@@ -272,16 +273,16 @@ void PostScriptController::AddMessage(Message::EKind kind, const char* text)
 		m_postInfo->SetProgressLabel(text);
 	}
 
-	if (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = m_postInfo->GetStageTime();
 		time_t startTime = m_postInfo->GetStartTime();
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
+		while (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 
 			// update time stamps
 

daemon/extension/QueueScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -32,6 +32,7 @@ static const char* QUEUE_EVENT_NAMES[] = {
 	"URL_COMPLETED",
 	"NZB_MARKED",
 	"NZB_ADDED",
+	"NZB_NAMED",
 	"NZB_DOWNLOADED",
 	"NZB_DELETED" };
 
@@ -408,11 +409,10 @@ void QueueScriptCoordinator::CheckQueue()
 		return;
 	}
 
-	m_curItem.reset();
-
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 	Guard guard(m_queueMutex);
 
+	m_curItem.reset();
 	NzbInfo* curNzbInfo = nullptr;
 	Queue::iterator itCurItem;
 

daemon/extension/QueueScript.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,6 +33,7 @@ public:
 		qeUrlCompleted,
 		qeNzbMarked,
 		qeNzbAdded,
+		qeNzbNamed,
 		qeNzbDownloaded,
 		qeNzbDeleted // highest priority
 	};

daemon/feed/FeedCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "FeedCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "WebDownloader.h"
 #include "Util.h"
 #include "FileSystem.h"
@@ -29,6 +30,7 @@
 #include "FeedScript.h"
 #include "DiskState.h"
 #include "DupeCoordinator.h"
+#include "UrlCoordinator.h"
 
 std::unique_ptr<RegEx>& FeedCoordinator::FilterHelper::GetRegEx(int id)
 {
@@ -65,6 +67,9 @@ FeedCoordinator::FeedCoordinator()
 
 	m_downloadQueueObserver.m_owner = this;
 	DownloadQueue::Guard()->Attach(&m_downloadQueueObserver);
+
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
 }
 
 FeedCoordinator::~FeedCoordinator()
@@ -84,68 +89,77 @@ void FeedCoordinator::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetReloadQueue())
+	if (g_Options->GetServerMode())
 	{
 		Guard guard(m_downloadsMutex);
 		g_DiskState->LoadFeeds(&m_feeds, &m_feedHistory);
 	}
 
-	int sleepInterval = 100;
-	int updateCounter = 0;
-	int cleanupCounter = 60000;
-
+	time_t lastCleanup = 0;
 	while (!IsStopped())
 	{
-		usleep(sleepInterval * 1000);
-
-		updateCounter += sleepInterval;
-		if (updateCounter >= 1000)
+		// this code should not be called too often, once per second is OK
+		if (!g_WorkState->GetPauseDownload() || m_force || g_Options->GetUrlForce())
 		{
-			// this code should not be called too often, once per second is OK
+			Guard guard(m_downloadsMutex);
 
-			if (!g_Options->GetPauseDownload() || m_force || g_Options->GetUrlForce())
+			time_t current = Util::CurrentTime();
+			if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
 			{
-				Guard guard(m_downloadsMutex);
-
-				time_t current = Util::CurrentTime();
-				if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
+				m_force = false;
+				// check feed list and update feeds
+				for (FeedInfo* feedInfo : &m_feeds)
 				{
-					m_force = false;
-					// check feed list and update feeds
-					for (FeedInfo* feedInfo : &m_feeds)
+					if (((feedInfo->GetInterval() > 0 &&
+						 (feedInfo->GetNextUpdate() == 0 ||
+						  current >= feedInfo->GetNextUpdate() ||
+						  current < feedInfo->GetNextUpdate() - feedInfo->GetInterval() * 60)) ||
+						feedInfo->GetFetch()) &&
+						feedInfo->GetStatus() != FeedInfo::fsRunning)
 					{
-						if (((feedInfo->GetInterval() > 0 &&
-							(current - feedInfo->GetLastUpdate() >= feedInfo->GetInterval() * 60 ||
-							 current < feedInfo->GetLastUpdate())) ||
-							feedInfo->GetFetch()) &&
-							feedInfo->GetStatus() != FeedInfo::fsRunning)
-						{
-							StartFeedDownload(feedInfo, feedInfo->GetFetch());
-						}
-						else if (feedInfo->GetFetch())
-						{
-							m_force = true;
-						}
+						StartFeedDownload(feedInfo, feedInfo->GetFetch());
+					}
+					else if (feedInfo->GetFetch())
+					{
+						m_force = true;
 					}
 				}
 			}
-
-			CheckSaveFeeds();
-			ResetHangingDownloads();
-			updateCounter = 0;
 		}
 
-		cleanupCounter += sleepInterval;
-		if (cleanupCounter >= 60000)
+		CheckSaveFeeds();
+		ResetHangingDownloads();
+
+		if (std::abs(Util::CurrentTime() - lastCleanup) >= 60)
 		{
 			// clean up feed history once a minute
 			CleanupHistory();
 			CleanupCache();
 			CheckSaveFeeds();
-			cleanupCounter = 0;
+			lastCleanup = Util::CurrentTime();
+		}
+
+		Guard guard(m_downloadsMutex);
+		if (m_force)
+		{
+			// don't sleep too long if there active feeds scheduled for redownload
+			m_waitCond.WaitFor(m_downloadsMutex, 1000, [&]{ return IsStopped(); });
+		}
+		else
+		{
+			// no active jobs, we can sleep longer:
+			//  - if option "UrlForce" is active or if the feed list is empty we need to wake up
+			//    only when a new feed preview is requested. We could wait indefinitely for that
+			//    but we need to do some job every now and then and therefore we sleep only 60 seconds.
+			//  - if option "UrlForce" is disabled we need also to wake up when state "DownloadPaused"
+			//    is changed. We detect this via notification from 'WorkState'. However such
+			//    notifications are not 100% reliable due to possible race conditions. Therefore
+			//    we sleep for max. 5 seconds.
+			int waitInterval = g_Options->GetUrlForce() || m_feeds.empty() ? 60000 : 5000;
+			m_waitCond.WaitFor(m_downloadsMutex, waitInterval, [&]{ return m_force || IsStopped(); });
 		}
 	}
 
@@ -159,7 +173,7 @@ void FeedCoordinator::Run()
 			completed = m_activeDownloads.size() == 0;
 		}
 		CheckSaveFeeds();
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 	debug("FeedCoordinator: Downloads are completed");
@@ -178,12 +192,20 @@ void FeedCoordinator::Stop()
 		feedDownloader->Stop();
 	}
 	debug("UrlDownloads are notified");
+
+	// Resume Run() to exit it
+	m_waitCond.NotifyAll();
+}
+
+void FeedCoordinator::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_force = true;
+	m_waitCond.NotifyAll();
 }
 
 void FeedCoordinator::ResetHangingDownloads()
 {
-	const int timeout = g_Options->GetTerminateTimeout();
-	if (timeout == 0)
+	if (g_Options->GetUrlTimeout() == 0)
 	{
 		return;
 	}
@@ -191,31 +213,15 @@ void FeedCoordinator::ResetHangingDownloads()
 	Guard guard(m_downloadsMutex);
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[timeout, tm](FeedDownloader* feedDownloader)
+	for (FeedDownloader* feedDownloader: m_activeDownloads)
+	{
+		if (tm - feedDownloader->GetLastUpdateTime() > g_Options->GetUrlTimeout() + 10 &&
+			feedDownloader->GetStatus() == FeedDownloader::adRunning)
 		{
-			if (tm - feedDownloader->GetLastUpdateTime() > timeout &&
-				feedDownloader->GetStatus() == FeedDownloader::adRunning)
-			{
-				debug("Terminating hanging download %s", feedDownloader->GetInfoName());
-				if (feedDownloader->Terminate())
-				{
-					error("Terminated hanging download %s", feedDownloader->GetInfoName());
-					feedDownloader->GetFeedInfo()->SetStatus(FeedInfo::fsUndefined);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s", feedDownloader->GetInfoName());
-				}
-
-				// it's not safe to destroy feedDownloader, because the state of object is unknown
-				delete feedDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging feed download %s", feedDownloader->GetInfoName());
+			feedDownloader->Stop();
+		}
+	}
 }
 
 void FeedCoordinator::LogDebugInfo()
@@ -291,6 +297,8 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), feedDownloader));
 	}
 
+	SchedulerNextUpdate(feedInfo, statusOK);
+
 	if (statusOK)
 	{
 		if (!feedInfo->GetPreview())
@@ -322,12 +330,10 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 				m_save = true;
 			}
 
-			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 			for (std::unique_ptr<NzbInfo>& nzbInfo : addedNzbs)
 			{
-				downloadQueue->GetQueue()->Add(std::move(nzbInfo));
+				g_UrlCoordinator->AddUrlToQueue(std::move(nzbInfo), false);
 			}
-			downloadQueue->Save();
 		}
 		feedInfo->SetStatus(FeedInfo::fsFinished);
 	}
@@ -337,6 +343,30 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 	}
 }
 
+void FeedCoordinator::SchedulerNextUpdate(FeedInfo* feedInfo, bool success)
+{
+	time_t current = Util::CurrentTime();
+	int interval;
+
+	if (success)
+	{
+		interval = feedInfo->GetInterval() * 60;
+		feedInfo->SetLastInterval(0);
+	}
+	else
+	{
+		// On failure schedule next update sooner:
+		// starting with 1 minute and increasing, but not greater than FeedX.Interval
+		interval = feedInfo->GetLastInterval() * 2;
+		interval = std::max(interval, 60);
+		interval = std::min(interval, feedInfo->GetInterval() * 60);
+		feedInfo->SetLastInterval(interval);
+	}
+
+	detail("Scheduling update for feed %s in %i minute(s)", feedInfo->GetName(), interval / 60);
+	feedInfo->SetNextUpdate(current + interval);
+}
+
 void FeedCoordinator::FilterFeed(FeedInfo* feedInfo, FeedItemList* feedItems)
 {
 	debug("Filtering feed %s", feedInfo->GetName());
@@ -445,6 +475,9 @@ std::unique_ptr<NzbInfo> FeedCoordinator::CreateNzbInfo(FeedInfo* feedInfo, Feed
 	nzbInfo->SetDupeKey(feedItemInfo.GetDupeKey());
 	nzbInfo->SetDupeScore(feedItemInfo.GetDupeScore());
 	nzbInfo->SetDupeMode(feedItemInfo.GetDupeMode());
+	nzbInfo->SetSize(feedItemInfo.GetSize());
+	nzbInfo->SetMinTime(feedItemInfo.GetTime());
+	nzbInfo->SetMaxTime(feedItemInfo.GetTime());
 
 	return nzbInfo;
 }
@@ -510,12 +543,15 @@ std::shared_ptr<FeedItemList> FeedCoordinator::PreviewFeed(int id,
 			}
 
 			StartFeedDownload(feedInfo.get(), true);
+
+			m_force = true;
+			m_waitCond.NotifyAll();
 		}
 
 		// wait until the download in a separate thread completes
 		while (feedInfo->GetStatus() == FeedInfo::fsRunning)
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 		}
 
 		// now can process the feed
@@ -574,6 +610,8 @@ void FeedCoordinator::FetchFeed(int id)
 			m_force = true;
 		}
 	}
+
+	m_waitCond.NotifyAll();
 }
 
 std::unique_ptr<FeedFile> FeedCoordinator::parseFeed(FeedInfo* feedInfo)
@@ -588,7 +626,7 @@ std::unique_ptr<FeedFile> FeedCoordinator::parseFeed(FeedInfo* feedInfo)
 		error("Feed file %s kept for troubleshooting (will be deleted on next successful feed fetch)", feedInfo->GetOutputFilename());
 		feedFile.reset();
 	}
-	return std::move(feedFile);
+	return feedFile;
 }
 
 void FeedCoordinator::DownloadQueueUpdate(Subject* caller, void* aspect)
@@ -620,11 +658,11 @@ bool FeedCoordinator::HasActiveDownloads()
 
 void FeedCoordinator::CheckSaveFeeds()
 {
-	debug("CheckSaveFeeds");
 	Guard guard(m_downloadsMutex);
 	if (m_save)
 	{
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		debug("CheckSaveFeeds: save");
+		if (g_Options->GetServerMode())
 		{
 			g_DiskState->SaveFeeds(&m_feeds, &m_feedHistory);
 		}

daemon/feed/FeedCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -67,6 +67,13 @@ private:
 		virtual void Update(Subject* caller, void* aspect) { m_owner->DownloadQueueUpdate(caller, aspect); }
 	};
 
+	class WorkStateObserver: public Observer
+	{
+	public:
+		FeedCoordinator* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	class FeedCacheItem
 	{
 	public:
@@ -106,9 +113,12 @@ private:
 	FeedHistory m_feedHistory;
 	Mutex m_downloadsMutex;
 	DownloadQueueObserver m_downloadQueueObserver;
+	WorkStateObserver m_workStateObserver;
 	bool m_force = false;
 	bool m_save = false;
 	FeedCache m_feedCache;
+	ConditionVar m_waitCond;
+	bool m_wokenUp = false;
 
 	void StartFeedDownload(FeedInfo* feedInfo, bool force);
 	void FeedCompleted(FeedDownloader* feedDownloader);
@@ -121,6 +131,8 @@ private:
 	void CleanupCache();
 	void CheckSaveFeeds();
 	std::unique_ptr<FeedFile> parseFeed(FeedInfo* feedInfo);
+	void SchedulerNextUpdate(FeedInfo* feedInfo, bool success);
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 extern FeedCoordinator* g_FeedCoordinator;

daemon/feed/FeedFile.cpp

@@ -360,6 +360,10 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 
 bool FeedFile::Parse()
 {
+#ifdef DISABLE_LIBXML2
+	error("Could not parse rss feed, program was compiled without libxml2 support");
+	return false;
+#else
 	xmlSAXHandler SAX_handler = {0};
 	SAX_handler.startElement = reinterpret_cast<startElementSAXFunc>(SAX_StartElement);
 	SAX_handler.endElement = reinterpret_cast<endElementSAXFunc>(SAX_EndElement);
@@ -378,6 +382,7 @@ bool FeedFile::Parse()
 	}
 
 	return true;
+#endif
 }
 
 void FeedFile::Parse_StartElement(const char *name, const char **atts)
@@ -566,7 +571,11 @@ void FeedFile::SAX_characters(FeedFile* file, const char * xmlstr, int len)
 
 void* FeedFile::SAX_getEntity(FeedFile* file, const char * name)
 {
+#ifdef DISABLE_LIBXML2
+	void* e = nullptr;
+#else
 	xmlEntityPtr e = xmlGetPredefinedEntity((xmlChar* )name);
+#endif
 	if (!e)
 	{
 		warn("entity not found");

daemon/feed/FeedFilter.cpp

@@ -51,7 +51,7 @@ bool FeedFilter::Term::MatchValue(const char* strValue, int64 intValue)
 
 	if (m_command < fcEqual && !strValue)
 	{
-		intBuf.Format("%lld", intValue);
+		intBuf.Format("%" PRId64, intValue);
 		strValue = intBuf;
 	}
 
@@ -897,7 +897,7 @@ void FeedFilter::Rule::ExpandRefValues(FeedItemInfo& feedItemInfo, CString* dest
 			break; // error
 		}
 
-		curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+		curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 	}
 
 	*destStr = std::move(curvalue);

daemon/feed/FeedInfo.h

@@ -51,6 +51,10 @@ public:
 	const char* GetExtensions() { return m_extensions; }
 	time_t GetLastUpdate() { return m_lastUpdate; }
 	void SetLastUpdate(time_t lastUpdate) { m_lastUpdate = lastUpdate; }
+	time_t GetNextUpdate() { return m_nextUpdate; }
+	void SetNextUpdate(time_t nextUpdate) { m_nextUpdate = nextUpdate; }
+	int GetLastInterval() { return m_lastInterval; }
+	void SetLastInterval(int lastInterval) { m_lastInterval = lastInterval; }
 	bool GetPreview() { return m_preview; }
 	void SetPreview(bool preview) { m_preview = preview; }
 	EStatus GetStatus() { return m_status; }
@@ -68,6 +72,7 @@ private:
 	int m_id;
 	CString m_name;
 	CString m_url;
+	bool m_backlog;
 	int m_interval;
 	CString m_filter;
 	uint32 m_filterHash;
@@ -76,12 +81,13 @@ private:
 	CString m_extensions;
 	int m_priority;
 	time_t m_lastUpdate = 0;
+	time_t m_nextUpdate = 0;
+	int m_lastInterval = 0;
 	bool m_preview = false;
 	EStatus m_status = fsUndefined;
 	CString m_outputFilename;
 	bool m_fetch = false;
 	bool m_force = false;
-	bool m_backlog;
 };
 
 typedef std::deque<std::unique_ptr<FeedInfo>> Feeds;

daemon/frontend/Frontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "nzbget.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Frontend.h"
 #include "Log.h"
 #include "Connection.h"
@@ -33,9 +34,24 @@ Frontend::Frontend()
 {
 	debug("Creating Frontend");
 
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
+
 	m_updateInterval = g_Options->GetUpdateInterval();
 }
 
+void Frontend::Stop()
+{
+	Thread::Stop();
+
+	m_waitCond.NotifyAll();
+}
+
+void Frontend::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_waitCond.NotifyAll();
+}
+
 bool Frontend::PrepareData()
 {
 	if (IsRemoteMode())
@@ -57,8 +73,8 @@ bool Frontend::PrepareData()
 		if (m_summary)
 		{
 			m_currentDownloadSpeed = g_StatMeter->CalcCurrentDownloadSpeed();
-			m_pauseDownload = g_Options->GetPauseDownload();
-			m_downloadLimit = g_Options->GetDownloadRate();
+			m_pauseDownload = g_WorkState->GetPauseDownload();
+			m_downloadLimit = g_WorkState->GetSpeedLimit();
 			m_threadCount = Thread::GetThreadCount();
 			g_StatMeter->CalcTotalStat(&m_upTimeSec, &m_dnTimeSec, &m_allBytes, &m_standBy);
 
@@ -108,8 +124,8 @@ void Frontend::ServerPauseUnpause(bool pause)
 	}
 	else
 	{
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(pause);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(pause);
 	}
 }
 
@@ -121,7 +137,7 @@ void Frontend::ServerSetDownloadRate(int rate)
 	}
 	else
 	{
-		g_Options->SetDownloadRate(rate);
+		g_WorkState->SetSpeedLimit(rate);
 	}
 }
 
@@ -307,3 +323,16 @@ bool Frontend::RequestEditQueue(DownloadQueue::EEditAction action, int offset, i
 	IdList ids = { id };
 	return client.RequestServerEditQueue(action, offset, nullptr, &ids, nullptr, rmId);
 }
+
+void Frontend::Wait(int milliseconds)
+{
+	if (g_WorkState->GetPauseFrontend())
+	{
+		Guard guard(m_waitMutex);
+		m_waitCond.WaitFor(m_waitMutex, 2000);
+	}
+	else
+	{
+		Util::Sleep(milliseconds);
+	}
+}

daemon/frontend/Frontend.h

@@ -27,6 +27,7 @@
 #include "DownloadInfo.h"
 #include "MessageBase.h"
 #include "QueueEditor.h"
+#include "Observer.h"
 
 class Frontend : public Thread
 {
@@ -51,7 +52,10 @@ protected:
 	int m_dnTimeSec = 0;
 	int64 m_allBytes = 0;
 	bool m_standBy = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
+	virtual void Stop();
 	bool PrepareData();
 	void FreeData();
 	GuardedMessageList GuardMessages();
@@ -63,12 +67,22 @@ protected:
 	bool RequestSetDownloadRate(int rate);
 	bool ServerEditQueue(DownloadQueue::EEditAction action, int offset, int entry);
 	bool RequestEditQueue(DownloadQueue::EEditAction action, int offset, int id);
+	void Wait(int milliseconds);
 
 private:
+	class WorkStateObserver : public Observer
+	{
+	public:
+		Frontend* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	MessageList m_remoteMessages;
+	WorkStateObserver m_workStateObserver;
 
 	bool RequestMessages();
 	bool RequestFileList();
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 #endif

daemon/frontend/LoggableFrontend.cpp

@@ -20,6 +20,7 @@
 
 
 #include "nzbget.h"
+#include "Util.h"
 #include "LoggableFrontend.h"
 #include "Log.h"
 
@@ -30,7 +31,7 @@ void LoggableFrontend::Run()
 	while (!IsStopped())
 	{
 		Update();
-		usleep(m_updateInterval * 1000);
+		Wait(m_updateInterval);
 	}
 	// Printing the last messages
 	Update();

daemon/frontend/NCursesFrontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -213,8 +213,10 @@ void NCursesFrontend::Run()
 			m_dataUpdatePos = m_updateInterval;
 		}
 
-		usleep(10 * 1000);
-		m_dataUpdatePos -= 10;
+		// update more often (sleep shorter) if need faster reaction on user input
+		int sleepInterval = m_inputMode == normal ? 100 : 10;
+		Wait(sleepInterval);
+		m_dataUpdatePos -= sleepInterval;
 	}
 
 	FreeData();
@@ -738,7 +740,7 @@ void NCursesFrontend::PrintFilename(FileInfo * fileInfo, int row, bool selected)
 	BString<1024> nzbNiceName;
 	if (m_showNzbname)
 	{
-		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR);
+		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR);
 	}
 
 	BString<1024> text("%s%i%s%s%s %s%s (%s%s)%s", Brace1, fileInfo->GetId(),

daemon/main/CommandLineParser.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -517,6 +517,10 @@ void CommandLineParser::InitCommandLine(int argc, const char* const_argv[])
 					{
 						m_editQueueAction = DownloadQueue::eaGroupParkDelete;
 					}
+					else if (!strcasecmp(optarg, "SF"))
+					{
+						m_editQueueAction = DownloadQueue::eaGroupSortFiles;
+					}
 					else if (!strcasecmp(optarg, "C") || !strcasecmp(optarg, "K") || !strcasecmp(optarg, "CP"))
 					{
 						// switch "K" is provided for compatibility with v. 0.8.0 and can be removed in future versions
@@ -741,11 +745,11 @@ void CommandLineParser::PrintUsage(const char* com)
 		"                 S          Print only server status\n"
 		"    <RegEx>                 Regular expression (only with options \"FR\", \"GR\")\n"
 		"                            using POSIX Extended Regular Expression Syntax\n"
-		"  -P, --pause   [D|O|S]  Pause server\n"
+		"  -P, --pause   [D|O|S]     Pause server\n"
 		"                 D          Pause download queue (default)\n"
 		"                 O          Pause post-processor queue\n"
 		"                 S          Pause scan of incoming nzb-directory\n"
-		"  -U, --unpause [D|O|S]  Unpause server\n"
+		"  -U, --unpause [D|O|S]     Unpause server\n"
 		"                 D          Unpause download queue (default)\n"
 		"                 O          Unpause post-processor queue\n"
 		"                 S          Unpause scan of incoming nzb-directory\n"
@@ -772,7 +776,6 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       T                    Move to top of queue\n"
 		"       B                    Move to bottom of queue\n"
 		"       D                    Delete\n"
-		"    - for files (F) and groups (G):\n"
 		"       P                    Pause\n"
 		"       U                    Resume (unpause)\n"
 		"    - for groups (G):\n"
@@ -784,6 +787,7 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       CP <name>            Set category and apply post-process parameters\n"
 		"       N <name>             Rename\n"
 		"       M                    Merge\n"
+		"       SF                   Sort inner files for optimal order\n"
 		"       S <name>             Split - create new group from selected files\n"
 		"       O <name>=<value>     Set post-process parameter\n"
 		"    - for post-jobs (O):\n"
@@ -890,7 +894,7 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			if (p)
 			{
 				BString<100> buf;
-				buf.Set(optarg, p - optarg);
+				buf.Set(optarg, (int)(p - optarg));
 				editQueueIdFrom = atoi(buf);
 				editQueueIdTo = atoi(p + 1);
 				if (editQueueIdFrom <= 0 || editQueueIdTo <= 0)
@@ -911,25 +915,18 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			}
 
 			int editQueueIdCount = 0;
-			if (editQueueIdTo != 0)
+			if (editQueueIdFrom < editQueueIdTo)
 			{
-				if (editQueueIdFrom < editQueueIdTo)
-				{
-					editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
-				}
-				else
-				{
-					editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
-				}
+				editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
 			}
 			else
 			{
-				editQueueIdCount = 1;
+				editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
 			}
 
 			for (int i = 0; i < editQueueIdCount; i++)
 			{
-				if (editQueueIdFrom < editQueueIdTo || editQueueIdTo == 0)
+				if (editQueueIdFrom < editQueueIdTo)
 				{
 					m_editQueueIdList.push_back(editQueueIdFrom + i);
 				}

daemon/main/CommandLineParser.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by

daemon/main/DiskService.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,23 +21,39 @@
 #include "nzbget.h"
 #include "DiskService.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "StatMeter.h"
 #include "Log.h"
 #include "Util.h"
 #include "FileSystem.h"
 
+DiskService::DiskService()
+{
+	g_WorkState->Attach(this);
+}
+
+void DiskService::Update(Subject* caller, void* aspect)
+{
+	WakeUp();
+}
+
+int DiskService::ServiceInterval()
+{
+	return m_waitingRequiredDir ? 1 :
+		g_Options->GetDiskSpace() <= 0 ? Service::Sleep :
+		// notifications from 'WorkState' are not 100% reliable due to race conditions
+		!g_WorkState->GetDownloading() ? 10 :
+		1;
+}
+
 void DiskService::ServiceWork()
 {
-	m_interval++;
-	if (m_interval == 5)
+	debug("Disk service work");
+
+	if (g_Options->GetDiskSpace() > 0 && g_WorkState->GetDownloading())
 	{
-		if (!g_Options->GetPauseDownload() &&
-			g_Options->GetDiskSpace() > 0 && !g_StatMeter->GetStandBy())
-		{
-			// check free disk space every 1 second
-			CheckDiskSpace();
-		}
-		m_interval = 0;
+		// check free disk space every 1 second
+		CheckDiskSpace();
 	}
 
 	if (m_waitingRequiredDir)
@@ -48,11 +64,13 @@ void DiskService::ServiceWork()
 
 void DiskService::CheckDiskSpace()
 {
+	debug("Disk service work: check disk space");
+
 	int64 freeSpace = FileSystem::FreeDiskSize(g_Options->GetDestDir());
 	if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 	{
 		warn("Low disk space on %s. Pausing download", g_Options->GetDestDir());
-		g_Options->SetPauseDownload(true);
+		g_WorkState->SetPauseDownload(true);
 	}
 
 	if (!Util::EmptyStr(g_Options->GetInterDir()))
@@ -61,13 +79,15 @@ void DiskService::CheckDiskSpace()
 		if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 		{
 			warn("Low disk space on %s. Pausing download", g_Options->GetInterDir());
-			g_Options->SetPauseDownload(true);
+			g_WorkState->SetPauseDownload(true);
 		}
 	}
 }
 
 void DiskService::CheckRequiredDir()
 {
+	debug("Disk service work: check required dir");
+
 	if (!Util::EmptyStr(g_Options->GetRequiredDir()))
 	{
 		bool allExist = true;
@@ -97,7 +117,7 @@ void DiskService::CheckRequiredDir()
 		info("All required directories available");
 	}
 
-	g_Options->SetTempPauseDownload(false);
-	g_Options->SetTempPausePostprocess(false);
+	g_WorkState->SetTempPauseDownload(false);
+	g_WorkState->SetTempPausePostprocess(false);
 	m_waitingRequiredDir = false;
 }

daemon/main/DiskService.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,15 +22,19 @@
 #define DISKSERVICE_H
 
 #include "Service.h"
+#include "Observer.h"
 
-class DiskService : public Service
+class DiskService : public Service, public Observer
 {
+public:
+	DiskService();
+
 protected:
-	virtual int ServiceInterval() { return 200; }
+	virtual int ServiceInterval();
 	virtual void ServiceWork();
+	virtual void Update(Subject* caller, void* aspect);
 
 private:
-	int m_interval = 0;
 	bool m_waitingRequiredDir = true;
 	bool m_waitingReported = false;
 

daemon/main/Maintenance.cpp

@@ -68,7 +68,7 @@ Maintenance::~Maintenance()
 	{
 		while (m_updateScriptController)
 		{
-			usleep(20*1000);
+			Util::Sleep(20);
 		}
 	}
 }
@@ -179,7 +179,7 @@ bool Maintenance::ReadPackageInfoStr(const char* key, CString& value)
 		return false;
 	}
 
-	int len = pend - p;
+	size_t len = pend - p;
 	if (len >= sizeof(fileName))
 	{
 		error("Could not parse file %s", *fileName);
@@ -373,7 +373,7 @@ bool Signature::ReadSignature()
 			{
 				hexSig[sigLen - 2] = '\0'; // trim trailing ",
 			}
-			for (; *hexSig && *(hexSig+1);)
+			while (*hexSig && *(hexSig+1) && output != m_signature + sizeof(m_signature))
 			{
 				uchar c1 = *hexSig++;
 				uchar c2 = *hexSig++;

daemon/main/Options.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -59,27 +59,28 @@ static const char* OPTION_RESTRICTEDUSERNAME	= "RestrictedUsername";
 static const char* OPTION_RESTRICTEDPASSWORD	= "RestrictedPassword";
 static const char* OPTION_ADDUSERNAME			= "AddUsername";
 static const char* OPTION_ADDPASSWORD			= "AddPassword";
+static const char* OPTION_FORMAUTH				= "FormAuth";
 static const char* OPTION_SECURECONTROL			= "SecureControl";
 static const char* OPTION_SECUREPORT			= "SecurePort";
 static const char* OPTION_SECURECERT			= "SecureCert";
 static const char* OPTION_SECUREKEY				= "SecureKey";
+static const char* OPTION_CERTSTORE				= "CertStore";
+static const char* OPTION_CERTCHECK				= "CertCheck";
 static const char* OPTION_AUTHORIZEDIP			= "AuthorizedIP";
 static const char* OPTION_ARTICLETIMEOUT		= "ArticleTimeout";
 static const char* OPTION_URLTIMEOUT			= "UrlTimeout";
-static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_REMOTETIMEOUT			= "RemoteTimeout";
 static const char* OPTION_FLUSHQUEUE			= "FlushQueue";
-static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
-static const char* OPTION_BROKENLOG				= "BrokenLog";
 static const char* OPTION_NZBLOG				= "NzbLog";
-static const char* OPTION_DECODE				= "Decode";
+static const char* OPTION_RAWARTICLE			= "RawArticle";
+static const char* OPTION_SKIPWRITE				= "SkipWrite";
 static const char* OPTION_ARTICLERETRIES		= "ArticleRetries";
 static const char* OPTION_ARTICLEINTERVAL		= "ArticleInterval";
 static const char* OPTION_URLRETRIES			= "UrlRetries";
 static const char* OPTION_URLINTERVAL			= "UrlInterval";
-static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
 static const char* OPTION_CONTINUEPARTIAL		= "ContinuePartial";
 static const char* OPTION_URLCONNECTIONS		= "UrlConnections";
-static const char* OPTION_LOGBUFFERSIZE			= "LogBufferSize";
+static const char* OPTION_LOGBUFFER				= "LogBuffer";
 static const char* OPTION_INFOTARGET			= "InfoTarget";
 static const char* OPTION_WARNINGTARGET			= "WarningTarget";
 static const char* OPTION_ERRORTARGET			= "ErrorTarget";
@@ -90,11 +91,13 @@ static const char* OPTION_PARREPAIR				= "ParRepair";
 static const char* OPTION_PARSCAN				= "ParScan";
 static const char* OPTION_PARQUICK				= "ParQuick";
 static const char* OPTION_POSTSTRATEGY			= "PostStrategy";
+static const char* OPTION_FILENAMING			= "FileNaming";
 static const char* OPTION_PARRENAME				= "ParRename";
 static const char* OPTION_PARBUFFER				= "ParBuffer";
 static const char* OPTION_PARTHREADS			= "ParThreads";
 static const char* OPTION_RARRENAME				= "RarRename";
 static const char* OPTION_HEALTHCHECK			= "HealthCheck";
+static const char* OPTION_DIRECTRENAME			= "DirectRename";
 static const char* OPTION_UMASK					= "UMask";
 static const char* OPTION_UPDATEINTERVAL		= "UpdateInterval";
 static const char* OPTION_CURSESNZBNAME			= "CursesNzbName";
@@ -106,14 +109,15 @@ static const char* OPTION_WRITEBUFFER			= "WriteBuffer";
 static const char* OPTION_NZBDIRINTERVAL		= "NzbDirInterval";
 static const char* OPTION_NZBDIRFILEAGE			= "NzbDirFileAge";
 static const char* OPTION_DISKSPACE				= "DiskSpace";
-static const char* OPTION_DUMPCORE				= "DumpCore";
+static const char* OPTION_CRASHTRACE			= "CrashTrace";
+static const char* OPTION_CRASHDUMP				= "CrashDump";
 static const char* OPTION_PARPAUSEQUEUE			= "ParPauseQueue";
 static const char* OPTION_SCRIPTPAUSEQUEUE		= "ScriptPauseQueue";
 static const char* OPTION_NZBCLEANUPDISK		= "NzbCleanupDisk";
 static const char* OPTION_PARTIMELIMIT			= "ParTimeLimit";
 static const char* OPTION_KEEPHISTORY			= "KeepHistory";
-static const char* OPTION_ACCURATERATE			= "AccurateRate";
 static const char* OPTION_UNPACK				= "Unpack";
+static const char* OPTION_DIRECTUNPACK			= "DirectUnpack";
 static const char* OPTION_UNPACKCLEANUPDISK		= "UnpackCleanupDisk";
 static const char* OPTION_UNRARCMD				= "UnrarCmd";
 static const char* OPTION_SEVENZIPCMD			= "SevenZipCmd";
@@ -134,6 +138,8 @@ static const char* OPTION_SHELLOVERRIDE			= "ShellOverride";
 static const char* OPTION_MONTHLYQUOTA			= "MonthlyQuota";
 static const char* OPTION_QUOTASTARTDAY			= "QuotaStartDay";
 static const char* OPTION_DAILYQUOTA			= "DailyQuota";
+static const char* OPTION_REORDERFILES			= "ReorderFiles";
+static const char* OPTION_UPDATECHECK			= "UpdateCheck";
 
 // obsolete options
 static const char* OPTION_POSTLOGKIND			= "PostLogKind";
@@ -160,6 +166,13 @@ static const char* OPTION_HISTORYCLEANUPDISK	= "HistoryCleanupDisk";
 static const char* OPTION_SCANSCRIPT			= "ScanScript";
 static const char* OPTION_QUEUESCRIPT			= "QueueScript";
 static const char* OPTION_FEEDSCRIPT			= "FeedScript";
+static const char* OPTION_DECODE				= "Decode";
+static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
+static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
+static const char* OPTION_ACCURATERATE			= "AccurateRate";
+static const char* OPTION_CREATEBROKENLOG		= "CreateBrokenLog";
+static const char* OPTION_BROKENLOG				= "BrokenLog";
 
 const char* BoolNames[] = { "yes", "no", "true", "false", "1", "0", "on", "off", "enable", "disable", "enabled", "disabled" };
 const int BoolValues[] = { 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
@@ -193,10 +206,13 @@ bool Options::OptEntry::Restricted()
 
 	bool restricted = !strcasecmp(m_name, OPTION_CONTROLIP) ||
 		!strcasecmp(m_name, OPTION_CONTROLPORT) ||
+		!strcasecmp(m_name, OPTION_FORMAUTH) ||
 		!strcasecmp(m_name, OPTION_SECURECONTROL) ||
 		!strcasecmp(m_name, OPTION_SECUREPORT) ||
 		!strcasecmp(m_name, OPTION_SECURECERT) ||
 		!strcasecmp(m_name, OPTION_SECUREKEY) ||
+		!strcasecmp(m_name, OPTION_CERTSTORE) ||
+		!strcasecmp(m_name, OPTION_CERTCHECK) ||
 		!strcasecmp(m_name, OPTION_AUTHORIZEDIP) ||
 		!strcasecmp(m_name, OPTION_DAEMONUSERNAME) ||
 		!strcasecmp(m_name, OPTION_UMASK) ||
@@ -405,7 +421,7 @@ void Options::InitDefaults()
 	SetOption(OPTION_QUEUEDIR, "${MainDir}/queue");
 	SetOption(OPTION_NZBDIR, "${MainDir}/nzb");
 	SetOption(OPTION_LOCKFILE, "${MainDir}/nzbget.lock");
-	SetOption(OPTION_LOGFILE, "${DestDir}/nzbget.log");
+	SetOption(OPTION_LOGFILE, "${MainDir}/nzbget.log");
 	SetOption(OPTION_SCRIPTDIR, "${MainDir}/scripts");
 	SetOption(OPTION_REQUIREDDIR, "");
 	SetOption(OPTION_WRITELOG, "append");
@@ -422,27 +438,28 @@ void Options::InitDefaults()
 	SetOption(OPTION_ADDUSERNAME, "");
 	SetOption(OPTION_ADDPASSWORD, "");
 	SetOption(OPTION_CONTROLPORT, "6789");
+	SetOption(OPTION_FORMAUTH, "no");
 	SetOption(OPTION_SECURECONTROL, "no");
 	SetOption(OPTION_SECUREPORT, "6791");
 	SetOption(OPTION_SECURECERT, "");
 	SetOption(OPTION_SECUREKEY, "");
+	SetOption(OPTION_CERTSTORE, "");
+	SetOption(OPTION_CERTCHECK, "no");
 	SetOption(OPTION_AUTHORIZEDIP, "");
 	SetOption(OPTION_ARTICLETIMEOUT, "60");
 	SetOption(OPTION_URLTIMEOUT, "60");
-	SetOption(OPTION_SAVEQUEUE, "yes");
+	SetOption(OPTION_REMOTETIMEOUT, "90");
 	SetOption(OPTION_FLUSHQUEUE, "yes");
-	SetOption(OPTION_RELOADQUEUE, "yes");
-	SetOption(OPTION_BROKENLOG, "yes");
 	SetOption(OPTION_NZBLOG, "yes");
-	SetOption(OPTION_DECODE, "yes");
+	SetOption(OPTION_RAWARTICLE, "no");
+	SetOption(OPTION_SKIPWRITE, "no");
 	SetOption(OPTION_ARTICLERETRIES, "3");
 	SetOption(OPTION_ARTICLEINTERVAL, "10");
 	SetOption(OPTION_URLRETRIES, "3");
 	SetOption(OPTION_URLINTERVAL, "10");
-	SetOption(OPTION_TERMINATETIMEOUT, "600");
 	SetOption(OPTION_CONTINUEPARTIAL, "no");
 	SetOption(OPTION_URLCONNECTIONS, "4");
-	SetOption(OPTION_LOGBUFFERSIZE, "1000");
+	SetOption(OPTION_LOGBUFFER, "1000");
 	SetOption(OPTION_INFOTARGET, "both");
 	SetOption(OPTION_WARNINGTARGET, "both");
 	SetOption(OPTION_ERRORTARGET, "both");
@@ -453,11 +470,13 @@ void Options::InitDefaults()
 	SetOption(OPTION_PARSCAN, "extended");
 	SetOption(OPTION_PARQUICK, "yes");
 	SetOption(OPTION_POSTSTRATEGY, "sequential");
+	SetOption(OPTION_FILENAMING, "article");
 	SetOption(OPTION_PARRENAME, "yes");
 	SetOption(OPTION_PARBUFFER, "16");
 	SetOption(OPTION_PARTHREADS, "1");
 	SetOption(OPTION_RARRENAME, "yes");
 	SetOption(OPTION_HEALTHCHECK, "none");
+	SetOption(OPTION_DIRECTRENAME, "no");
 	SetOption(OPTION_SCRIPTORDER, "");
 	SetOption(OPTION_EXTENSIONS, "");
 	SetOption(OPTION_DAEMONUSERNAME, "root");
@@ -472,14 +491,15 @@ void Options::InitDefaults()
 	SetOption(OPTION_NZBDIRINTERVAL, "5");
 	SetOption(OPTION_NZBDIRFILEAGE, "60");
 	SetOption(OPTION_DISKSPACE, "250");
-	SetOption(OPTION_DUMPCORE, "no");
+	SetOption(OPTION_CRASHTRACE, "no");
+	SetOption(OPTION_CRASHDUMP, "no");
 	SetOption(OPTION_PARPAUSEQUEUE, "no");
 	SetOption(OPTION_SCRIPTPAUSEQUEUE, "no");
 	SetOption(OPTION_NZBCLEANUPDISK, "no");
 	SetOption(OPTION_PARTIMELIMIT, "0");
 	SetOption(OPTION_KEEPHISTORY, "7");
-	SetOption(OPTION_ACCURATERATE, "no");
 	SetOption(OPTION_UNPACK, "no");
+	SetOption(OPTION_DIRECTUNPACK, "no");
 	SetOption(OPTION_UNPACKCLEANUPDISK, "no");
 #ifdef WIN32
 	SetOption(OPTION_UNRARCMD, "unrar.exe");
@@ -503,6 +523,8 @@ void Options::InitDefaults()
 	SetOption(OPTION_MONTHLYQUOTA, "0");
 	SetOption(OPTION_QUOTASTARTDAY, "1");
 	SetOption(OPTION_DAILYQUOTA, "0");
+	SetOption(OPTION_REORDERFILES, "no");
+	SetOption(OPTION_UPDATECHECK, "none");
 }
 
 void Options::InitOptFile()
@@ -620,8 +642,6 @@ void Options::CheckDir(CString& dir, const char* optionName,
 
 		FileSystem::NormalizePathSeparators((char*)usedir2);
 		dir = usedir2;
-
-		usedir2[usedir2.Length() - 1] = '\0';
 		SetOption(optionName, usedir2);
 	}
 
@@ -659,6 +679,7 @@ void Options::InitOptions()
 	m_addPassword			= GetOption(OPTION_ADDPASSWORD);
 	m_secureCert			= GetOption(OPTION_SECURECERT);
 	m_secureKey				= GetOption(OPTION_SECUREKEY);
+	m_certStore				= GetOption(OPTION_CERTSTORE);
 	m_authorizedIp			= GetOption(OPTION_AUTHORIZEDIP);
 	m_lockFile				= GetOption(OPTION_LOCKFILE);
 	m_daemonUsername		= GetOption(OPTION_DAEMONUSERNAME);
@@ -674,7 +695,7 @@ void Options::InitOptions()
 	m_downloadRate			= ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024;
 	m_articleTimeout		= ParseIntValue(OPTION_ARTICLETIMEOUT, 10);
 	m_urlTimeout			= ParseIntValue(OPTION_URLTIMEOUT, 10);
-	m_terminateTimeout		= ParseIntValue(OPTION_TERMINATETIMEOUT, 10);
+	m_remoteTimeout			= ParseIntValue(OPTION_REMOTETIMEOUT, 10);
 	m_articleRetries		= ParseIntValue(OPTION_ARTICLERETRIES, 10);
 	m_articleInterval		= ParseIntValue(OPTION_ARTICLEINTERVAL, 10);
 	m_urlRetries			= ParseIntValue(OPTION_URLRETRIES, 10);
@@ -682,7 +703,7 @@ void Options::InitOptions()
 	m_controlPort			= ParseIntValue(OPTION_CONTROLPORT, 10);
 	m_securePort			= ParseIntValue(OPTION_SECUREPORT, 10);
 	m_urlConnections		= ParseIntValue(OPTION_URLCONNECTIONS, 10);
-	m_logBufferSize			= ParseIntValue(OPTION_LOGBUFFERSIZE, 10);
+	m_logBuffer				= ParseIntValue(OPTION_LOGBUFFER, 10);
 	m_rotateLog				= ParseIntValue(OPTION_ROTATELOG, 10);
 	m_umask					= ParseIntValue(OPTION_UMASK, 8);
 	m_updateInterval		= ParseIntValue(OPTION_UPDATEINTERVAL, 10);
@@ -708,34 +729,37 @@ void Options::InitOptions()
 	m_quotaStartDay			= ParseIntValue(OPTION_QUOTASTARTDAY, 10);
 	m_dailyQuota			= ParseIntValue(OPTION_DAILYQUOTA, 10);
 
-	m_brokenLog				= (bool)ParseEnumValue(OPTION_BROKENLOG, BoolCount, BoolNames, BoolValues);
 	m_nzbLog				= (bool)ParseEnumValue(OPTION_NZBLOG, BoolCount, BoolNames, BoolValues);
 	m_appendCategoryDir		= (bool)ParseEnumValue(OPTION_APPENDCATEGORYDIR, BoolCount, BoolNames, BoolValues);
 	m_continuePartial		= (bool)ParseEnumValue(OPTION_CONTINUEPARTIAL, BoolCount, BoolNames, BoolValues);
-	m_saveQueue				= (bool)ParseEnumValue(OPTION_SAVEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_flushQueue			= (bool)ParseEnumValue(OPTION_FLUSHQUEUE, BoolCount, BoolNames, BoolValues);
 	m_dupeCheck				= (bool)ParseEnumValue(OPTION_DUPECHECK, BoolCount, BoolNames, BoolValues);
 	m_parRepair				= (bool)ParseEnumValue(OPTION_PARREPAIR, BoolCount, BoolNames, BoolValues);
 	m_parQuick				= (bool)ParseEnumValue(OPTION_PARQUICK, BoolCount, BoolNames, BoolValues);
 	m_parRename				= (bool)ParseEnumValue(OPTION_PARRENAME, BoolCount, BoolNames, BoolValues);
 	m_rarRename				= (bool)ParseEnumValue(OPTION_RARRENAME, BoolCount, BoolNames, BoolValues);
-	m_reloadQueue			= (bool)ParseEnumValue(OPTION_RELOADQUEUE, BoolCount, BoolNames, BoolValues);
+	m_directRename			= (bool)ParseEnumValue(OPTION_DIRECTRENAME, BoolCount, BoolNames, BoolValues);
 	m_cursesNzbName			= (bool)ParseEnumValue(OPTION_CURSESNZBNAME, BoolCount, BoolNames, BoolValues);
 	m_cursesTime			= (bool)ParseEnumValue(OPTION_CURSESTIME, BoolCount, BoolNames, BoolValues);
 	m_cursesGroup			= (bool)ParseEnumValue(OPTION_CURSESGROUP, BoolCount, BoolNames, BoolValues);
 	m_crcCheck				= (bool)ParseEnumValue(OPTION_CRCCHECK, BoolCount, BoolNames, BoolValues);
 	m_directWrite			= (bool)ParseEnumValue(OPTION_DIRECTWRITE, BoolCount, BoolNames, BoolValues);
-	m_decode				= (bool)ParseEnumValue(OPTION_DECODE, BoolCount, BoolNames, BoolValues);
-	m_dumpCore				= (bool)ParseEnumValue(OPTION_DUMPCORE, BoolCount, BoolNames, BoolValues);
+	m_rawArticle			= (bool)ParseEnumValue(OPTION_RAWARTICLE, BoolCount, BoolNames, BoolValues);
+	m_skipWrite				= (bool)ParseEnumValue(OPTION_SKIPWRITE, BoolCount, BoolNames, BoolValues);
+	m_crashTrace			= (bool)ParseEnumValue(OPTION_CRASHTRACE, BoolCount, BoolNames, BoolValues);
+	m_crashDump				= (bool)ParseEnumValue(OPTION_CRASHDUMP, BoolCount, BoolNames, BoolValues);
 	m_parPauseQueue			= (bool)ParseEnumValue(OPTION_PARPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_scriptPauseQueue		= (bool)ParseEnumValue(OPTION_SCRIPTPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_nzbCleanupDisk		= (bool)ParseEnumValue(OPTION_NZBCLEANUPDISK, BoolCount, BoolNames, BoolValues);
-	m_accurateRate			= (bool)ParseEnumValue(OPTION_ACCURATERATE, BoolCount, BoolNames, BoolValues);
+	m_formAuth				= (bool)ParseEnumValue(OPTION_FORMAUTH, BoolCount, BoolNames, BoolValues);
 	m_secureControl			= (bool)ParseEnumValue(OPTION_SECURECONTROL, BoolCount, BoolNames, BoolValues);
 	m_unpack				= (bool)ParseEnumValue(OPTION_UNPACK, BoolCount, BoolNames, BoolValues);
+	m_directUnpack			= (bool)ParseEnumValue(OPTION_DIRECTUNPACK, BoolCount, BoolNames, BoolValues);
 	m_unpackCleanupDisk		= (bool)ParseEnumValue(OPTION_UNPACKCLEANUPDISK, BoolCount, BoolNames, BoolValues);
 	m_unpackPauseQueue		= (bool)ParseEnumValue(OPTION_UNPACKPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_urlForce				= (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues);
+	m_certCheck				= (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues);
+	m_reorderFiles			= (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues);
 
 	const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" };
 	const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses };
@@ -757,6 +781,11 @@ void Options::InitOptions()
 	const int PostStrategyCount = 4;
 	m_postStrategy = (EPostStrategy)ParseEnumValue(OPTION_POSTSTRATEGY, PostStrategyCount, PostStrategyNames, PostStrategyValues);
 
+	const char* FileNamingNames[] = { "auto", "article", "nzb" };
+	const int FileNamingValues[] = { nfAuto, nfArticle, nfNzb };
+	const int FileNamingCount = 4;
+	m_fileNaming = (EFileNaming)ParseEnumValue(OPTION_FILENAMING, FileNamingCount, FileNamingNames, FileNamingValues);
+
 	const char* HealthCheckNames[] = { "pause", "delete", "park", "none" };
 	const int HealthCheckValues[] = { hcPause, hcDelete, hcPark, hcNone };
 	const int HealthCheckCount = 4;
@@ -886,7 +915,7 @@ void Options::SetOption(const char* optname, const char* value)
 			const char* varvalue = GetOption(variable);
 			if (varvalue)
 			{
-				curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+				curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 			}
 			else
 			{
@@ -979,6 +1008,16 @@ void Options::InitServers()
 			m_tls |= tls;
 		}
 
+		const char* nipversion = GetOption(BString<100>("Server%i.IpVersion", n));
+		int ipversion = 0;
+		if (nipversion)
+		{
+			const char* IpVersionNames[] = {"auto", "ipv4", "ipv6"};
+			const int IpVersionValues[] = {0, 4, 6};
+			const int IpVersionCount = 3;
+			ipversion = ParseEnumValue(BString<100>("Server%i.IpVersion", n), IpVersionCount, IpVersionNames, IpVersionValues);
+		}
+
 		const char* ncipher = GetOption(BString<100>("Server%i.Cipher", n));
 		const char* nconnections = GetOption(BString<100>("Server%i.Connections", n));
 		const char* nretention = GetOption(BString<100>("Server%i.Retention", n));
@@ -999,6 +1038,7 @@ void Options::InitServers()
 				m_extender->AddNewsServer(n, active, nname,
 					nhost,
 					nport ? atoi(nport) : 119,
+					ipversion,
 					nusername, npassword,
 					joinGroup, tls, ncipher,
 					nconnections ? atoi(nconnections) : 1,
@@ -1460,7 +1500,7 @@ bool Options::SplitOptionString(const char* option, CString& optName, CString& o
 		return false;
 	}
 
-	optName.Set(option, eq - option);
+	optName.Set(option, (int)(eq - option));
 	optValue.Set(eq + 1);
 
 	ConvertOldOption(optName, optValue);
@@ -1495,7 +1535,8 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 			!strcasecmp(p, ".password") || !strcasecmp(p, ".joingroup") ||
 			!strcasecmp(p, ".encryption") || !strcasecmp(p, ".connections") ||
 			!strcasecmp(p, ".cipher") || !strcasecmp(p, ".group") ||
-			!strcasecmp(p, ".retention") || !strcasecmp(p, ".optional")))
+			!strcasecmp(p, ".retention") || !strcasecmp(p, ".optional") ||
+			!strcasecmp(p, ".notes") || !strcasecmp(p, ".ipversion")))
 		{
 			return true;
 		}
@@ -1558,7 +1599,13 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 		!strcasecmp(optname, OPTION_RELOADPOSTQUEUE) ||
 		!strcasecmp(optname, OPTION_PARCLEANUPQUEUE) ||
 		!strcasecmp(optname, OPTION_DELETECLEANUPDISK) ||
-		!strcasecmp(optname, OPTION_HISTORYCLEANUPDISK))
+		!strcasecmp(optname, OPTION_HISTORYCLEANUPDISK) ||
+		!strcasecmp(optname, OPTION_SAVEQUEUE) ||
+		!strcasecmp(optname, OPTION_RELOADQUEUE) ||
+		!strcasecmp(optname, OPTION_TERMINATETIMEOUT) ||
+		!strcasecmp(optname, OPTION_ACCURATERATE) ||
+		!strcasecmp(optname, OPTION_CREATEBROKENLOG) ||
+		!strcasecmp(optname, OPTION_BROKENLOG))
 	{
 		ConfigWarn("Option \"%s\" is obsolete, ignored", optname);
 		return true;
@@ -1678,9 +1725,20 @@ void Options::ConvertOldOption(CString& option, CString& value)
 		option = "ArticleInterval";
 	}
 
-	if (!strcasecmp(option, "CreateBrokenLog"))
+	if (!strcasecmp(option, "DumpCore"))
 	{
-		option = "BrokenLog";
+		option = OPTION_CRASHDUMP;
+	}
+
+	if (!strcasecmp(option, OPTION_DECODE))
+	{
+		option = OPTION_RAWARTICLE;
+		value = !strcasecmp(value, "no") ? "yes" : "no";
+	}
+
+	if (!strcasecmp(option, "LogBufferSize"))
+	{
+		option = OPTION_LOGBUFFER;
 	}
 }
 
@@ -1697,6 +1755,11 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_PARRENAME);
 		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_PARRENAME);
 	}
+	if (m_directRename)
+	{
+		LocateOptionSrcPos(OPTION_DIRECTRENAME);
+		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_DIRECTRENAME);
+	}
 #endif
 
 #ifdef DISABLE_CURSES
@@ -1713,13 +1776,53 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_SECURECONTROL);
 		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_SECURECONTROL);
 	}
+
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_CERTCHECK);
+	}
 #endif
 
-	if (!m_decode)
+#ifdef HAVE_OPENSSL
+#ifndef HAVE_X509_CHECK_HOST
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is limited because the program "
+			"was compiled with older OpenSSL version not supporting hostname validation (at least OpenSSL 1.0.2d is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER < 0x030104
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is disabled because the program "
+			"was compiled with older GnuTLS version not supporting certificate validation (at least GnuTLS 3.1.4 is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+	if (m_certCheck && m_certStore.Empty())
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Option \"%s\" requires proper configuration of option \"%s\"", OPTION_CERTCHECK, OPTION_CERTSTORE);
+		m_certCheck = false;
+	}
+
+	if (m_rawArticle)
 	{
 		m_directWrite = false;
 	}
 
+	if (m_skipWrite)
+	{
+		m_directRename = false;
+	}
+
 	// if option "ConfigTemplate" is not set, use "WebDir" as default location for template
 	// (for compatibility with versions 9 and 10).
 	if (m_configTemplate.Empty() && !m_noDiskAccess)
@@ -1801,10 +1904,10 @@ void Options::MergeOldScriptOption(OptEntries* optEntries, const char* optname,
 		{
 			for (OptEntry& opt : optEntries)
 			{
-				const char* optname = opt.GetName();
-				if (!strncasecmp(optname, "category", 8))
+				const char* catoptname = opt.GetName();
+				if (!strncasecmp(catoptname, "category", 8))
 				{
-					char* p = (char*)optname + 8;
+					char* p = (char*)catoptname + 8;
 					while (*p >= '0' && *p <= '9') p++;
 					if (p && (!strcasecmp(p, ".extensions")))
 					{

daemon/main/Options.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -92,6 +92,12 @@ public:
 		ppAggressive,
 		ppRocket
 	};
+	enum EFileNaming
+	{
+		nfAuto,
+		nfArticle,
+		nfNzb
+	};
 
 	class OptEntry
 	{
@@ -161,7 +167,7 @@ public:
 	{
 	public:
 		virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-			int port, const char* user, const char* pass, bool joinGroup,
+			int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 			bool tls, const char* cipher, int maxConnections, int retention,
 			int level, int group, bool optional) = 0;
 		virtual void AddFeed(int id, const char* name, const char* url, int interval,
@@ -197,7 +203,6 @@ public:
 	const char* GetConfigTemplate() { return m_configTemplate; }
 	const char* GetScriptDir() { return m_scriptDir; }
 	const char* GetRequiredDir() { return m_requiredDir; }
-	bool GetBrokenLog() const { return m_brokenLog; }
 	bool GetNzbLog() const { return m_nzbLog; }
 	EMessageTarget GetInfoTarget() const { return m_infoTarget; }
 	EMessageTarget GetWarningTarget() const { return m_warningTarget; }
@@ -206,15 +211,15 @@ public:
 	EMessageTarget GetDetailTarget() const { return m_detailTarget; }
 	int GetArticleTimeout() { return m_articleTimeout; }
 	int GetUrlTimeout() { return m_urlTimeout; }
-	int GetTerminateTimeout() { return m_terminateTimeout; }
-	bool GetDecode() { return m_decode; };
+	int GetRemoteTimeout() { return m_remoteTimeout; }
+	bool GetRawArticle() { return m_rawArticle; };
+	bool GetSkipWrite() { return m_skipWrite; };
 	bool GetAppendCategoryDir() { return m_appendCategoryDir; }
 	bool GetContinuePartial() { return m_continuePartial; }
 	int GetArticleRetries() { return m_articleRetries; }
 	int GetArticleInterval() { return m_articleInterval; }
 	int GetUrlRetries() { return m_urlRetries; }
 	int GetUrlInterval() { return m_urlInterval; }
-	bool GetSaveQueue() { return m_saveQueue; }
 	bool GetFlushQueue() { return m_flushQueue; }
 	bool GetDupeCheck() { return m_dupeCheck; }
 	const char* GetControlIp() { return m_controlIp; }
@@ -225,17 +230,19 @@ public:
 	const char* GetAddUsername() { return m_addUsername; }
 	const char* GetAddPassword() { return m_addPassword; }
 	int GetControlPort() { return m_controlPort; }
+	bool GetFormAuth() { return m_formAuth; }
 	bool GetSecureControl() { return m_secureControl; }
 	int GetSecurePort() { return m_securePort; }
 	const char* GetSecureCert() { return m_secureCert; }
 	const char* GetSecureKey() { return m_secureKey; }
+	const char* GetCertStore() { return m_certStore; }
+	bool GetCertCheck() { return m_certCheck; }
 	const char* GetAuthorizedIp() { return m_authorizedIp; }
 	const char* GetLockFile() { return m_lockFile; }
 	const char* GetDaemonUsername() { return m_daemonUsername; }
 	EOutputMode GetOutputMode() { return m_outputMode; }
-	bool GetReloadQueue() { return m_reloadQueue; }
 	int GetUrlConnections() { return m_urlConnections; }
-	int GetLogBufferSize() { return m_logBufferSize; }
+	int GetLogBuffer() { return m_logBuffer; }
 	EWriteLog GetWriteLog() { return m_writeLog; }
 	const char* GetLogFile() { return m_logFile; }
 	int GetRotateLog() { return m_rotateLog; }
@@ -263,14 +270,15 @@ public:
 	int GetNzbDirFileAge() { return m_nzbDirFileAge; }
 	int GetDiskSpace() { return m_diskSpace; }
 	bool GetTls() { return m_tls; }
-	bool GetDumpCore() { return m_dumpCore; }
+	bool GetCrashTrace() { return m_crashTrace; }
+	bool GetCrashDump() { return m_crashDump; }
 	bool GetParPauseQueue() { return m_parPauseQueue; }
 	bool GetScriptPauseQueue() { return m_scriptPauseQueue; }
 	bool GetNzbCleanupDisk() { return m_nzbCleanupDisk; }
 	int GetParTimeLimit() { return m_parTimeLimit; }
 	int GetKeepHistory() { return m_keepHistory; }
-	bool GetAccurateRate() { return m_accurateRate; }
 	bool GetUnpack() { return m_unpack; }
+	bool GetDirectUnpack() { return m_directUnpack; }
 	bool GetUnpackCleanupDisk() { return m_unpackCleanupDisk; }
 	const char* GetUnrarCmd() { return m_unrarCmd; }
 	const char* GetSevenZipCmd() { return m_sevenZipCmd; }
@@ -289,6 +297,10 @@ public:
 	int GetMonthlyQuota() { return m_monthlyQuota; }
 	int GetQuotaStartDay() { return m_quotaStartDay; }
 	int GetDailyQuota() { return m_dailyQuota; }
+	bool GetDirectRename() { return m_directRename; }
+	bool GetReorderFiles() { return m_reorderFiles; }
+	EFileNaming GetFileNaming() { return m_fileNaming; }
+	int GetDownloadRate() const { return m_downloadRate; }
 
 	Categories* GetCategories() { return &m_categories; }
 	Category* FindCategory(const char* name, bool searchAliases) { return m_categories.FindCategory(name, searchAliases); }
@@ -300,24 +312,6 @@ public:
 	bool GetDaemonMode() { return m_daemonMode; }
 	void SetRemoteClientMode(bool remoteClientMode) { m_remoteClientMode = remoteClientMode; }
 	bool GetRemoteClientMode() { return m_remoteClientMode; }
-	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; }
-	bool GetPauseDownload() const { return m_pauseDownload; }
-	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; }
-	bool GetPausePostProcess() const { return m_pausePostProcess; }
-	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; }
-	bool GetPauseScan() const { return m_pauseScan; }
-	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; }
-	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
-	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
-	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; }
-	void SetDownloadRate(int rate) { m_downloadRate = rate; }
-	int GetDownloadRate() const { return m_downloadRate; }
-	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; }
-	time_t GetResumeTime() const { return m_resumeTime; }
-	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; }
-	int GetLocalTimeOffset() { return m_localTimeOffset; }
-	void SetQuotaReached(bool quotaReached) { m_quotaReached = quotaReached; }
-	bool GetQuotaReached() { return m_quotaReached; }
 
 private:
 	OptEntries m_optEntries;
@@ -347,19 +341,18 @@ private:
 	EMessageTarget m_errorTarget = mtScreen;
 	EMessageTarget m_debugTarget = mtNone;
 	EMessageTarget m_detailTarget = mtScreen;
-	bool m_decode = true;
-	bool m_brokenLog = false;
+	bool m_skipWrite = false;
+	bool m_rawArticle = false;
 	bool m_nzbLog = false;
 	int m_articleTimeout = 0;
 	int m_urlTimeout = 0;
-	int m_terminateTimeout = 0;
+	int m_remoteTimeout = 0;
 	bool m_appendCategoryDir = false;
 	bool m_continuePartial = false;
 	int m_articleRetries = 0;
 	int m_articleInterval = 0;
 	int m_urlRetries = 0;
 	int m_urlInterval = 0;
-	bool m_saveQueue = false;
 	bool m_flushQueue = false;
 	bool m_dupeCheck = false;
 	CString m_controlIp;
@@ -369,18 +362,20 @@ private:
 	CString m_restrictedPassword;
 	CString m_addUsername;
 	CString m_addPassword;
+	bool m_formAuth = false;
 	int m_controlPort = 0;
 	bool m_secureControl = false;
 	int m_securePort = 0;
 	CString m_secureCert;
 	CString m_secureKey;
+	CString m_certStore;
+	bool m_certCheck = false;
 	CString m_authorizedIp;
 	CString m_lockFile;
 	CString m_daemonUsername;
 	EOutputMode m_outputMode = omLoggable;
-	bool m_reloadQueue = false;
 	int m_urlConnections = 0;
-	int m_logBufferSize = 0;
+	int m_logBuffer = 0;
 	EWriteLog m_writeLog = wlAppend;
 	int m_rotateLog = 0;
 	CString m_logFile;
@@ -393,6 +388,7 @@ private:
 	int m_parBuffer = 0;
 	int m_parThreads = 0;
 	bool m_rarRename = false;
+	bool m_directRename = false;
 	EHealthCheck m_healthCheck = hcNone;
 	CString m_extensions;
 	CString m_scriptOrder;
@@ -408,14 +404,15 @@ private:
 	int m_nzbDirFileAge = 0;
 	int m_diskSpace = 0;
 	bool m_tls = false;
-	bool m_dumpCore = false;
+	bool m_crashTrace = false;
+	bool m_crashDump = false;
 	bool m_parPauseQueue = false;
 	bool m_scriptPauseQueue = false;
 	bool m_nzbCleanupDisk = false;
 	int m_parTimeLimit = 0;
 	int m_keepHistory = 0;
-	bool m_accurateRate = false;
 	bool m_unpack = false;
+	bool m_directUnpack = false;
 	bool m_unpackCleanupDisk = false;
 	CString m_unrarCmd;
 	CString m_sevenZipCmd;
@@ -434,20 +431,14 @@ private:
 	int m_monthlyQuota = 0;
 	int m_quotaStartDay = 0;
 	int m_dailyQuota = 0;
+	bool m_reorderFiles = false;
+	EFileNaming m_fileNaming = nfArticle;
+	int m_downloadRate = 0;
 
-	// Current state
+	// Application mode
 	bool m_serverMode = false;
 	bool m_daemonMode = false;
 	bool m_remoteClientMode = false;
-	bool m_pauseDownload = false;
-	bool m_pausePostProcess = false;
-	bool m_pauseScan = false;
-	bool m_tempPauseDownload = true;
-	bool m_tempPausePostprocess = true;
-	int m_downloadRate = 0;
-	time_t m_resumeTime = 0;
-	int m_localTimeOffset = 0;
-	bool m_quotaReached = false;
 
 	void Init(const char* exeName, const char* configFilename, bool noConfig,
 		CmdOptList* commandLineOptions, bool noDiskAccess, Extender* extender);

daemon/main/Scheduler.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Scheduler.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "NewsServer.h"
 #include "ServerPool.h"
@@ -51,13 +52,33 @@ void Scheduler::FirstCheck()
 	CheckTasks();
 }
 
+void Scheduler::ScheduleNextWork()
+{
+	// Ideally we should calculate wait time until next scheduler task or until resume time.
+	// The first isn't trivial and the second requires watching/reaction on changed scheduled resume time.
+	// We do it simpler instead: check once per minute, when seconds are changing from 59 to 00.
+
+	time_t curTime = Util::CurrentTime();
+	tm sched;
+	gmtime_r(&curTime, &sched);
+	sched.tm_min++;
+	sched.tm_sec = 0;
+	time_t nextMinute = Util::Timegm(&sched);
+
+	m_serviceInterval = nextMinute - curTime;
+}
+
 void Scheduler::ServiceWork()
 {
+	debug("Scheduler service work");
+
 	if (!DownloadQueue::IsLoaded())
 	{
 		return;
 	}
 
+	debug("Scheduler service work: doing work");
+
 	if (!m_firstChecked)
 	{
 		FirstCheck();
@@ -68,6 +89,7 @@ void Scheduler::ServiceWork()
 	m_executeProcess = true;
 	CheckTasks();
 	CheckScheduledResume();
+	ScheduleNextWork();
 }
 
 void Scheduler::CheckTasks()
@@ -93,12 +115,15 @@ void Scheduler::CheckTasks()
 
 				for (Task* task : &m_taskList)
 				{
-					task->m_lastExecuted = 0;
+					if (task->m_hours != Task::STARTUP_TASK)
+					{
+						task->m_lastExecuted = 0;
+					}
 				}
 			}
 
-			time_t localCurrent = current + g_Options->GetLocalTimeOffset();
-			time_t localLastCheck = m_lastCheck + g_Options->GetLocalTimeOffset();
+			time_t localCurrent = current + g_WorkState->GetLocalTimeOffset();
+			time_t localLastCheck = m_lastCheck + g_WorkState->GetLocalTimeOffset();
 
 			tm tmCurrent;
 			gmtime_r(&localCurrent, &tmCurrent);
@@ -156,10 +181,12 @@ void Scheduler::CheckTasks()
 
 void Scheduler::ExecuteTask(Task* task)
 {
+#ifdef DEBUG
 	const char* commandName[] = { "Pause", "Unpause", "Pause Post-processing", "Unpause Post-processing",
 		"Set download rate", "Execute process", "Execute script",
 		"Pause Scan", "Unpause Scan", "Enable Server", "Disable Server", "Fetch Feed" };
 	debug("Executing scheduled command: %s", commandName[task->m_command]);
+#endif
 
 	bool executeProcess = m_executeProcess || task->m_hours == Task::STARTUP_TASK;
 
@@ -168,26 +195,26 @@ void Scheduler::ExecuteTask(Task* task)
 		case scDownloadRate:
 			if (!task->m_param.Empty())
 			{
-				g_Options->SetDownloadRate(atoi(task->m_param) * 1024);
+				g_WorkState->SetSpeedLimit(atoi(task->m_param) * 1024);
 				m_downloadRateChanged = true;
 			}
 			break;
 
 		case scPauseDownload:
 		case scUnpauseDownload:
-			g_Options->SetPauseDownload(task->m_command == scPauseDownload);
+			g_WorkState->SetPauseDownload(task->m_command == scPauseDownload);
 			m_pauseDownloadChanged = true;
 			break;
 
 		case scPausePostProcess:
 		case scUnpausePostProcess:
-			g_Options->SetPausePostProcess(task->m_command == scPausePostProcess);
+			g_WorkState->SetPausePostProcess(task->m_command == scPausePostProcess);
 			m_pausePostProcessChanged = true;
 			break;
 
 		case scPauseScan:
 		case scUnpauseScan:
-			g_Options->SetPauseScan(task->m_command == scPauseScan);
+			g_WorkState->SetPauseScan(task->m_command == scPauseScan);
 			m_pauseScanChanged = true;
 			break;
 
@@ -226,19 +253,19 @@ void Scheduler::PrintLog()
 {
 	if (m_downloadRateChanged)
 	{
-		info("Scheduler: setting download rate to %i KB/s", g_Options->GetDownloadRate() / 1024);
+		info("Scheduler: setting download rate to %i KB/s", g_WorkState->GetSpeedLimit() / 1024);
 	}
 	if (m_pauseDownloadChanged)
 	{
-		info("Scheduler: %s download", g_Options->GetPauseDownload() ? "pausing" : "unpausing");
+		info("Scheduler: %s download", g_WorkState->GetPauseDownload() ? "pausing" : "unpausing");
 	}
 	if (m_pausePostProcessChanged)
 	{
-		info("Scheduler: %s post-processing", g_Options->GetPausePostProcess() ? "pausing" : "unpausing");
+		info("Scheduler: %s post-processing", g_WorkState->GetPausePostProcess() ? "pausing" : "unpausing");
 	}
 	if (m_pauseScanChanged)
 	{
-		info("Scheduler: %s scan", g_Options->GetPauseScan() ? "pausing" : "unpausing");
+		info("Scheduler: %s scan", g_WorkState->GetPauseScan() ? "pausing" : "unpausing");
 	}
 	if (m_serverChanged)
 	{
@@ -305,14 +332,14 @@ void Scheduler::FetchFeed(const char* feedList)
 
 void Scheduler::CheckScheduledResume()
 {
-	time_t resumeTime = g_Options->GetResumeTime();
+	time_t resumeTime = g_WorkState->GetResumeTime();
 	time_t currentTime = Util::CurrentTime();
 	if (resumeTime > 0 && currentTime >= resumeTime)
 	{
 		info("Autoresume");
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(false);
-		g_Options->SetPausePostProcess(false);
-		g_Options->SetPauseScan(false);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(false);
+		g_WorkState->SetPausePostProcess(false);
+		g_WorkState->SetPauseScan(false);
 	}
 }

daemon/main/Scheduler.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -66,7 +66,7 @@ public:
 	void AddTask(std::unique_ptr<Task> task);
 
 protected:
-	virtual int ServiceInterval() { return 1000; }
+	virtual int ServiceInterval() { return m_serviceInterval; }
 	virtual void ServiceWork();
 
 private:
@@ -84,6 +84,7 @@ private:
 	bool m_serverChanged;
 	ServerStatusList m_serverStatusList;
 	bool m_firstChecked = false;
+	int m_serviceInterval = 1;
 
 	void ExecuteTask(Task* task);
 	void CheckTasks();
@@ -93,6 +94,7 @@ private:
 	void FetchFeed(const char* feedList);
 	void CheckScheduledResume();
 	void FirstCheck();
+	void ScheduleNextWork();
 };
 
 #endif

daemon/main/StackTrace.cpp

@@ -147,17 +147,20 @@ LONG __stdcall ExceptionFilter(EXCEPTION_POINTERS* exPtrs)
 
 #ifdef DEBUG
 	PrintBacktrace(exPtrs->ContextRecord);
-#else
-	 info("Detailed exception information can be printed by debug version of NZBGet (available from download page)");
-#endif
-
-	ExitProcess(-1);
 	return EXCEPTION_CONTINUE_SEARCH;
+#else
+	info("Detailed crash information can be printed by debug version of NZBGet."
+		" For more info visit http://nzbget.net/crash-dump");
+	return EXCEPTION_EXECUTE_HANDLER;
+#endif
 }
 
 void InstallErrorHandler()
 {
-	SetUnhandledExceptionFilter(ExceptionFilter);
+	if (g_Options->GetCrashTrace())
+	{
+		SetUnhandledExceptionFilter(ExceptionFilter);
+	}
 }
 
 #else
@@ -171,7 +174,7 @@ std::vector<sighandler> SignalProcList;
 /**
 * activates the creation of core-files
 */
-void EnableDumpCore()
+void EnableCoreDump()
 {
 	rlimit rlim;
 	rlim.rlim_cur= RLIM_INFINITY;
@@ -248,9 +251,9 @@ void SignalProc(int signum)
 void InstallErrorHandler()
 {
 #ifdef HAVE_SYS_PRCTL_H
-	if (g_Options->GetDumpCore())
+	if (g_Options->GetCrashDump())
 	{
-		EnableDumpCore();
+		EnableCoreDump();
 	}
 #endif
 
@@ -258,7 +261,10 @@ void InstallErrorHandler()
 	signal(SIGTERM, SignalProc);
 	signal(SIGPIPE, SIG_IGN);
 #ifdef DEBUG
-	signal(SIGSEGV, SignalProc);
+	if (g_Options->GetCrashTrace())
+	{
+		signal(SIGSEGV, SignalProc);
+	}
 #endif
 #ifdef SIGCHLD_HANDLER
 	// it could be necessary on some systems to activate a handler for SIGCHLD
@@ -276,7 +282,7 @@ public:
 	void DoSegFault()
 	{
 		char* N = nullptr;
-		strcpy(N, "");
+		*N = '\0';
 	}
 };
 

daemon/main/WorkState.cpp

@@ -0,0 +1,27 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "WorkState.h"
+
+void WorkState::Changed()
+{
+	Notify(nullptr);
+}

daemon/main/WorkState.h

@@ -0,0 +1,76 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef WORKSTATE_H
+#define WORKSTATE_H
+
+#include "Observer.h"
+
+// WorkState is observable but notifications are not 100% reliable.
+// The changes via Set-methods and readings via Get-methods are not synchronized throughout the program.
+// As result race conditions may occur and some changes may go unnoticed.
+// When waiting for changes don't wait too long to avoid lock ups.
+
+class WorkState : public Subject
+{
+public:
+	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; Changed(); }
+	bool GetPauseDownload() const { return m_pauseDownload; }
+	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; Changed(); }
+	bool GetPausePostProcess() const { return m_pausePostProcess; }
+	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; Changed(); }
+	bool GetPauseScan() const { return m_pauseScan; }
+	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; Changed(); }
+	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
+	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; Changed(); }
+	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
+	void SetPauseFrontend(bool pauseFrontend) { m_pauseFrontend = pauseFrontend; Changed(); }
+	bool GetPauseFrontend() const { return m_pauseFrontend; }
+	void SetSpeedLimit(int speedLimit) { m_speedLimit = speedLimit; Changed(); }
+	int GetSpeedLimit() const { return m_speedLimit; }
+	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; Changed(); }
+	time_t GetResumeTime() const { return m_resumeTime; }
+	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; Changed(); }
+	int GetLocalTimeOffset() { return m_localTimeOffset; }
+	void SetQuotaReached(bool quotaReached) { m_quotaReached = quotaReached; Changed(); }
+	bool GetQuotaReached() { return m_quotaReached; }
+	void SetDownloading(bool downloading) { m_downloading = downloading; Changed(); }
+	bool GetDownloading() { return m_downloading; }
+
+private:
+	bool m_pauseDownload = false;
+	bool m_pausePostProcess = false;
+	bool m_pauseScan = false;
+	bool m_tempPauseDownload = true;
+	bool m_tempPausePostprocess = true;
+	bool m_pauseFrontend = false;
+	int m_downloadRate = 0;
+	time_t m_resumeTime = 0;
+	int m_localTimeOffset = 0;
+	bool m_quotaReached = false;
+	int m_speedLimit = 0;
+	bool m_downloading = false;
+
+	void Changed();
+};
+
+extern WorkState* g_WorkState;
+
+#endif

daemon/main/nzbget.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "Log.h"
 #include "NzbFile.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "CommandLineParser.h"
 #include "ScriptConfig.h"
 #include "Thread.h"
@@ -39,7 +40,6 @@
 #include "PrePostProcessor.h"
 #include "HistoryCoordinator.h"
 #include "DupeCoordinator.h"
-#include "ParChecker.h"
 #include "Scheduler.h"
 #include "Scanner.h"
 #include "FeedCoordinator.h"
@@ -52,6 +52,8 @@
 #include "Util.h"
 #include "FileSystem.h"
 #include "StackTrace.h"
+#include "CommandScript.h"
+#include "YEncode.h"
 #ifdef WIN32
 #include "WinService.h"
 #include "WinConsole.h"
@@ -70,6 +72,7 @@ void RunMain();
 // Globals
 Log* g_Log;
 Options* g_Options;
+WorkState* g_WorkState;
 ServerPool* g_ServerPool;
 QueueCoordinator* g_QueueCoordinator;
 UrlCoordinator* g_UrlCoordinator;
@@ -85,6 +88,7 @@ ArticleCache* g_ArticleCache;
 QueueScriptCoordinator* g_QueueScriptCoordinator;
 ServiceCoordinator* g_ServiceCoordinator;
 ScriptConfig* g_ScriptConfig;
+CommandScriptLog* g_CommandScriptLog; 
 #ifdef WIN32
 WinConsole* g_WinConsole;
 #endif
@@ -111,6 +115,7 @@ int main(int argc, char *argv[], char *argp[])
 #endif
 
 	Util::Init();
+	YEncode::init();
 
 	g_ArgumentCount = argc;
 	g_Arguments = (char*(*)[])argv;
@@ -144,7 +149,7 @@ int main(int argc, char *argv[], char *argp[])
 	InstallUninstallServiceCheck(argc, argv);
 #endif
 
-	srand(Util::CurrentTime());
+	srand((unsigned int)Util::CurrentTime());
 
 #ifdef WIN32
 	for (int i=0; i < argc; i++)
@@ -174,7 +179,7 @@ public:
 
 	// Options::Extender
 	virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-		int port, const char* user, const char* pass, bool joinGroup,
+		int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
 		int level, int group, bool optional);
 	virtual void AddFeed(int id, const char* name, const char* url, int interval,
@@ -190,6 +195,7 @@ private:
 	// globals
 	std::unique_ptr<Log> m_log;
 	std::unique_ptr<Options> m_options;
+	std::unique_ptr<WorkState> m_workState;
 	std::unique_ptr<ServerPool> m_serverPool;
 	std::unique_ptr<QueueCoordinator> m_queueCoordinator;
 	std::unique_ptr<UrlCoordinator> m_urlCoordinator;
@@ -205,6 +211,7 @@ private:
 	std::unique_ptr<QueueScriptCoordinator> m_queueScriptCoordinator;
 	std::unique_ptr<ServiceCoordinator> m_serviceCoordinator;
 	std::unique_ptr<ScriptConfig> m_scriptConfig;
+	std::unique_ptr<CommandScriptLog> m_commandScriptLog;
 #ifdef WIN32
 	std::unique_ptr<WinConsole> m_winConsole;
 #endif
@@ -219,13 +226,17 @@ private:
 
 	bool m_reloading = false;
 	bool m_daemonized = false;
+	bool m_stopped = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	void Init();
+	void Final();
 	void BootConfig();
 	void CreateGlobals();
 	void Cleanup();
 	void PrintOptions();
-	bool ProcessDirect();
+	void ProcessDirect();
 	void ProcessClientRequest();
 	void ProcessWebGet();
 	void ProcessSigVerify();
@@ -280,6 +291,9 @@ void NZBGet::Init()
 
 	m_scanner->InitOptions();
 	m_queueScriptCoordinator->InitOptions();
+#ifndef DISABLE_TLS
+	TlsSocket::InitOptions(g_Options->GetCertCheck() ? g_Options->GetCertStore() : nullptr);
+#endif
 
 	if (m_commandLineParser->GetDaemonMode())
 	{
@@ -313,6 +327,17 @@ void NZBGet::Init()
 	InstallErrorHandler();
 }
 
+void NZBGet::Final()
+{
+	if (!m_reloading)
+	{
+#ifndef DISABLE_TLS
+		TlsSocket::Final();
+#endif
+		Connection::Final();
+	}
+}
+
 void NZBGet::CreateGlobals()
 {
 #ifdef WIN32
@@ -320,6 +345,9 @@ void NZBGet::CreateGlobals()
 	g_WinConsole = m_winConsole.get();
 #endif
 
+	m_workState = std::make_unique<WorkState>();
+	g_WorkState = m_workState.get();
+
 	m_serviceCoordinator = std::make_unique<ServiceCoordinator>();
 	g_ServiceCoordinator = m_serviceCoordinator.get();
 
@@ -365,6 +393,9 @@ void NZBGet::CreateGlobals()
 	m_scriptConfig = std::make_unique<ScriptConfig>();
 	g_ScriptConfig = m_scriptConfig.get();
 
+	m_commandScriptLog = std::make_unique<CommandScriptLog>();
+	g_CommandScriptLog = m_commandScriptLog.get();
+
 	m_scheduler = std::make_unique<Scheduler>();
 
 	m_diskService = std::make_unique<DiskService>();
@@ -382,7 +413,7 @@ void NZBGet::BootConfig()
 	if (m_commandLineParser->GetPrintUsage() || m_commandLineParser->GetErrors() || g_ArgumentCount <= 1)
 	{
 		m_commandLineParser->PrintUsage(((const char**)(*g_Arguments))[0]);
-		exit(0);
+		exit(m_commandLineParser->GetPrintUsage() ? 0 : 1);
 	}
 
 	debug("Reading options");
@@ -390,7 +421,8 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetNoConfig(), (Options::CmdOptList*)m_commandLineParser->GetOptionList(), this);
 	m_options->SetRemoteClientMode(m_commandLineParser->GetRemoteClientMode());
 	m_options->SetServerMode(m_commandLineParser->GetServerMode());
-	m_options->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetSpeedLimit(g_Options->GetDownloadRate());
 
 	m_log->InitOptions();
 
@@ -402,9 +434,9 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetClientOperation() == CommandLineParser::opClientNoOperation)
 	{
 		info("Pausing all activities due to errors in configuration");
-		m_options->SetPauseDownload(true);
-		m_options->SetPausePostProcess(true);
-		m_options->SetPauseScan(true);
+		m_workState->SetPauseDownload(true);
+		m_workState->SetPausePostProcess(true);
+		m_workState->SetPauseScan(true);
 	}
 
 	m_serverPool->SetTimeout(m_options->GetArticleTimeout());
@@ -437,46 +469,41 @@ void NZBGet::Cleanup()
 	g_QueueScriptCoordinator = nullptr;
 	g_Maintenance = nullptr;
 	g_StatMeter = nullptr;
+	g_CommandScriptLog = nullptr;
 #ifdef WIN32
 	g_WinConsole = nullptr;
 #endif
 }
 
-bool NZBGet::ProcessDirect()
+void NZBGet::ProcessDirect()
 {
 #ifdef DEBUG
 	if (m_commandLineParser->GetTestBacktrace())
 	{
-		TestSegFault();
+		TestSegFault(); // never returns
 	}
 #endif
 
 	if (m_commandLineParser->GetWebGet())
 	{
-		ProcessWebGet();
-		return true;
+		ProcessWebGet(); // never returns
 	}
 
 	if (m_commandLineParser->GetSigVerify())
 	{
-		ProcessSigVerify();
-		return true;
+		ProcessSigVerify(); // never returns
 	}
 
 	// client request
 	if (m_commandLineParser->GetClientOperation() != CommandLineParser::opClientNoOperation)
 	{
-		ProcessClientRequest();
-		return true;
+		ProcessClientRequest(); // never returns
 	}
 
 	if (m_commandLineParser->GetPrintOptions())
 	{
-		PrintOptions();
-		return true;
+		PrintOptions(); // never returns
 	}
-
-	return false;
 }
 
 void NZBGet::StartRemoteServer()
@@ -490,7 +517,11 @@ void NZBGet::StartRemoteServer()
 	m_remoteServer = std::make_unique<RemoteServer>(false);
 	m_remoteServer->Start();
 
-	if (m_options->GetSecureControl())
+	if (m_options->GetSecureControl()
+#ifndef WIN32
+		&& !(m_options->GetControlIp() && m_options->GetControlIp()[0] == '/')
+#endif
+)
 	{
 		m_remoteSecureServer = std::make_unique<RemoteServer>(true);
 		m_remoteSecureServer->Start();
@@ -503,37 +534,55 @@ void NZBGet::StopRemoteServer()
 	{
 		debug("stopping RemoteServer");
 		m_remoteServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteServer->IsRunning())
-		{
-			debug("Killing RemoteServer");
-			m_remoteServer->Kill();
-		}
-		debug("RemoteServer stopped");
 	}
 
 	if (m_remoteSecureServer)
 	{
 		debug("stopping RemoteSecureServer");
 		m_remoteSecureServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteSecureServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteSecureServer->IsRunning())
-		{
-			debug("Killing RemoteSecureServer");
-			m_remoteSecureServer->Kill();
-		}
-		debug("RemoteSecureServer stopped");
 	}
+
+	int maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		Util::Sleep(100);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		m_remoteServer->ForceStop();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		m_remoteSecureServer->ForceStop();
+	}
+
+	maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		Util::Sleep(100);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		debug("Killing RemoteServer");
+		m_remoteServer->Kill();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		debug("Killing RemoteSecureServer");
+		m_remoteSecureServer->Kill();
+	}
+
+	debug("RemoteServer stopped");
 }
 
 void NZBGet::StartFrontend()
@@ -575,7 +624,7 @@ void NZBGet::StopFrontend()
 		}
 		while (m_frontend->IsRunning())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 		}
 		debug("Frontend stopped");
 	}
@@ -655,7 +704,14 @@ void NZBGet::DoMainLoop()
 				m_serviceCoordinator->Stop();
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
+
+		if (m_options->GetServerMode() && !m_stopped)
+		{
+			// wait for stop signal
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&]{ return m_stopped; });
+		}
 	}
 
 	debug("Main program loop terminated");
@@ -667,10 +723,7 @@ void NZBGet::Run(bool reload)
 
 	Init();
 
-	if (ProcessDirect())
-	{
-		return;
-	}
+	ProcessDirect();
 
 	StartRemoteServer();
 	StartFrontend();
@@ -689,111 +742,116 @@ void NZBGet::Run(bool reload)
 
 	StopRemoteServer();
 	StopFrontend();
+
+	Final();
 }
 
 void NZBGet::ProcessClientRequest()
 {
 	RemoteClient Client;
+	bool ok = false;
 
 	switch (m_commandLineParser->GetClientOperation())
 	{
 		case CommandLineParser::opClientRequestListFiles:
-			Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListGroups:
-			Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListStatus:
-			Client.RequestServerList(false, false, nullptr);
+			ok = Client.RequestServerList(false, false, nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadPause:
-			Client.RequestServerPauseUnpause(true, rpDownload);
+			ok = Client.RequestServerPauseUnpause(true, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadUnpause:
-			Client.RequestServerPauseUnpause(false, rpDownload);
+			ok = Client.RequestServerPauseUnpause(false, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestSetRate:
-			Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
+			ok = Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
 			break;
 
 		case CommandLineParser::opClientRequestDumpDebug:
-			Client.RequestServerDumpDebug();
+			ok = Client.RequestServerDumpDebug();
 			break;
 
 		case CommandLineParser::opClientRequestEditQueue:
-			Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
+			ok = Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
 				m_commandLineParser->GetEditQueueOffset(), m_commandLineParser->GetEditQueueText(),
 				m_commandLineParser->GetEditQueueIdList(), m_commandLineParser->GetEditQueueNameList(),
 				(ERemoteMatchMode)m_commandLineParser->GetMatchMode());
 			break;
 
 		case CommandLineParser::opClientRequestLog:
-			Client.RequestServerLog(m_commandLineParser->GetLogLines());
+			ok = Client.RequestServerLog(m_commandLineParser->GetLogLines());
 			break;
 
 		case CommandLineParser::opClientRequestShutdown:
-			Client.RequestServerShutdown();
+			ok = Client.RequestServerShutdown();
 			break;
 
 		case CommandLineParser::opClientRequestReload:
-			Client.RequestServerReload();
+			ok = Client.RequestServerReload();
 			break;
 
 		case CommandLineParser::opClientRequestDownload:
-			Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
+			ok = Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
 				m_commandLineParser->GetAddCategory(), m_commandLineParser->GetAddTop(), m_commandLineParser->GetAddPaused(), m_commandLineParser->GetAddPriority(),
 				m_commandLineParser->GetAddDupeKey(), m_commandLineParser->GetAddDupeMode(), m_commandLineParser->GetAddDupeScore());
 			break;
 
 		case CommandLineParser::opClientRequestVersion:
-			Client.RequestServerVersion();
+			ok = Client.RequestServerVersion();
 			break;
 
 		case CommandLineParser::opClientRequestPostQueue:
-			Client.RequestPostQueue();
+			ok = Client.RequestPostQueue();
 			break;
 
 		case CommandLineParser::opClientRequestWriteLog:
-			Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
+			ok = Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
 			break;
 
 		case CommandLineParser::opClientRequestScanAsync:
-			Client.RequestScan(false);
+			ok = Client.RequestScan(false);
 			break;
 
 		case CommandLineParser::opClientRequestScanSync:
-			Client.RequestScan(true);
+			ok = Client.RequestScan(true);
 			break;
 
 		case CommandLineParser::opClientRequestPostPause:
-			Client.RequestServerPauseUnpause(true, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(true, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestPostUnpause:
-			Client.RequestServerPauseUnpause(false, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(false, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestScanPause:
-			Client.RequestServerPauseUnpause(true, rpScan);
+			ok = Client.RequestServerPauseUnpause(true, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestScanUnpause:
-			Client.RequestServerPauseUnpause(false, rpScan);
+			ok = Client.RequestServerPauseUnpause(false, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestHistory:
 		case CommandLineParser::opClientRequestHistoryAll:
-			Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
+			ok = Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
 			break;
 
 		case CommandLineParser::opClientNoOperation:
-			break;
+			return;
 	}
+
+	exit(ok ? 0 : 1);
 }
 
 void NZBGet::ProcessWebGet()
@@ -856,6 +914,11 @@ void NZBGet::Stop(bool reload)
 #endif
 		}
 	}
+
+	// trigger stop/reload signal
+	Guard guard(m_waitMutex);
+	m_stopped = true;
+	m_waitCond.NotifyAll();
 }
 
 void NZBGet::PrintOptions()
@@ -864,6 +927,7 @@ void NZBGet::PrintOptions()
 	{
 		printf("%s = \"%s\"\n", optEntry.GetName(), optEntry.GetValue());
 	}
+	exit(0);
 }
 
 #ifndef WIN32
@@ -896,7 +960,12 @@ void NZBGet::Daemonize()
 			error("Starting daemon failed: could not create lock-file %s", m_options->GetLockFile());
 			exit(1);
 		}
+
+#ifdef HAVE_LOCKF
 		if (lockf(lfp, F_TLOCK, 0) < 0)
+#else
+		if (flock(lfp, LOCK_EX) < 0)
+#endif
 		{
 			error("Starting daemon failed: could not acquire lock on lock-file %s", m_options->GetLockFile());
 			exit(1);
@@ -938,10 +1007,10 @@ void NZBGet::Daemonize()
 #endif
 
 void NZBGet::AddNewsServer(int id, bool active, const char* name, const char* host,
-	int port, const char* user, const char* pass, bool joinGroup, bool tls,
+	int port, int ipVersion, const char* user, const char* pass, bool joinGroup, bool tls,
 	const char* cipher, int maxConnections, int retention, int level, int group, bool optional)
 {
-	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, user, pass, joinGroup,
+	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, ipVersion, user, pass, joinGroup,
 		tls, cipher, maxConnections, retention, level, group, optional));
 }
 

daemon/main/nzbget.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -40,6 +40,9 @@
 //#define HAVE_LIBGNUTLS
 #endif
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#define HAVE_X509_CHECK_HOST 1
+
 /* Define to the name of macro which returns the name of function being
 compiled */
 #define FUNCTION_MACRO_NAME __FUNCTION__
@@ -56,9 +59,6 @@ compiled */
 /* Define to 1 if variadic macros are supported */
 #define HAVE_VARIADIC_MACROS
 
-/* Define to 1 if libpar2 supports cancelling (needs a special patch) */
-#define HAVE_PAR2_CANCEL
-
 /* Define to 1 if function GetAddrInfo is supported */
 #define HAVE_GETADDRINFO
 
@@ -66,7 +66,16 @@ compiled */
 #define SOCKLEN_T socklen_t
 
 /* Define to 1 if you have the <regex.h> header file. */
+#ifndef DISABLE_REGEX
 #define HAVE_REGEX_H 1
+// Static linking to regex library
+#define REGEX_STATIC
+#endif
+
+#ifndef DISABLE_GZIP
+// Static linking to zlib library
+#define ZLIB_WINAPI
+#endif
 
 /* Suppress warnings */
 #define _CRT_SECURE_NO_DEPRECATE
@@ -74,13 +83,21 @@ compiled */
 /* Suppress warnings */
 #define _CRT_NONSTDC_NO_WARNINGS
 
+#ifndef _WIN64
 #define _USE_32BIT_TIME_T
+#endif
 
 #if _WIN32_WINNT < 0x0501
 #undef _WIN32_WINNT
 #define _WIN32_WINNT 0x0501
 #endif
 
+#ifdef _WIN64
+#define __amd64__
+#else
+#define __i686__
+#endif
+
 #ifdef _DEBUG
 // detection of memory leaks
 #define _CRTDBG_MAP_ALLOC
@@ -98,7 +115,7 @@ compiled */
 
 // WINDOWS INCLUDES
 
-// Using "WIN32_LEAN_AND_MEAN" to disable including on many unneeded headers
+// Using "WIN32_LEAN_AND_MEAN" to disable including of many unneeded headers
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 
@@ -151,21 +168,29 @@ using namespace MSXML;
 #include <sys/resource.h>
 #include <sys/statvfs.h>
 #include <sys/wait.h>
+#include <sys/un.h>
+#include <sys/file.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <stdint.h>
 #include <pwd.h>
 #include <dirent.h>
 
+#ifndef DISABLE_LIBXML2
 #include <libxml/parser.h>
 #include <libxml/xmlreader.h>
 #include <libxml/xmlerror.h>
 #include <libxml/entities.h>
+#endif
 
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
 #endif
 
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#endif
+
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
 #endif
@@ -184,6 +209,7 @@ using namespace MSXML;
 #include <stdarg.h>
 #include <time.h>
 #include <ctype.h>
+#include <inttypes.h>
 
 #include <string>
 #include <vector>
@@ -191,12 +217,22 @@ using namespace MSXML;
 #include <list>
 #include <set>
 #include <map>
+#include <unordered_map>
 #include <iterator>
 #include <algorithm>
-#include <iostream>
 #include <fstream>
 #include <memory>
 #include <functional>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <chrono>
+
+// NOTE: do not include <iostream> in "nzbget.h". <iostream> contains objects requiring
+// intialization, causing every unit in nzbget to have initialization routine. This in particular
+// is causing fatal problems in SIMD units which must not have static initialization because
+// they contain code with runtime CPU dispatching.
+//#include <iostream>
 
 #ifdef HAVE_LIBGNUTLS
 #ifdef WIN32
@@ -205,6 +241,7 @@ typedef SSIZE_T ssize_t;
 typedef int pid_t;
 #endif
 #include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
 #if GNUTLS_VERSION_NUMBER <= 0x020b00
 #define NEED_GCRYPT_LOCKING
 #endif
@@ -222,6 +259,8 @@ typedef int pid_t;
 #include <openssl/rsa.h>
 #include <openssl/sha.h>
 #include <openssl/pem.h>
+#include <openssl/x509v3.h>
+#include <openssl/comp.h>
 #endif /* HAVE_OPENSSL */
 
 #ifdef HAVE_REGEX_H
@@ -239,9 +278,6 @@ typedef int pid_t;
 #ifdef HAVE_MEMORY_H
 # include <memory.h>
 #endif
-#ifdef HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
 #endif /* NOT DISABLE_PARCHECK */
 
 
@@ -268,7 +304,6 @@ typedef int pid_t;
 #define S_ISDIR(mode) __S_ISTYPE((mode), _S_IFDIR)
 #define S_ISREG(mode) __S_ISTYPE((mode), _S_IFREG)
 #define S_DIRMODE nullptr
-#define usleep(usec) Sleep((usec) / 1000)
 #define socklen_t int
 #define SHUT_WR 0x01
 #define SHUT_RDWR 0x02
@@ -290,11 +325,22 @@ typedef int pid_t;
 #define FOPEN_WB "wbN"
 #define FOPEN_AB "abN"
 
+#define __SSE2__
+#define __SSSE3__
+#define __PCLMUL__
+
 #ifdef DEBUG
 // redefine "exit" to avoid printing memory leaks report when terminated because of wrong command line switches
 #define exit(code) ExitProcess(code)
 #endif
 
+#ifdef HAVE_OPENSSL
+FILE _iob[] = {*stdin, *stdout, *stderr};
+extern "C" FILE * __cdecl __iob_func(void) { return _iob; }
+// For static linking of OpenSSL libraries:
+#pragma comment (lib, "legacy_stdio_definitions.lib")
+#endif /* HAVE_OPENSSL */
+
 #else
 
 // POSIX
@@ -338,8 +384,25 @@ typedef signed long long int64;
 typedef unsigned long long uint64;
 #endif
 
+#ifndef PRId64
+#define PRId64 "lld"
+#endif
+#ifndef PRIi64
+#define PRIi64 "lli"
+#endif
+#ifndef PRIu64
+#define PRIu64 "llu"
+#endif
+
 typedef unsigned char uchar;
 
+// Assume little endian if byte order is not defined
+#ifndef __BYTE_ORDER
+#define __LITTLE_ENDIAN 1234
+#define __BIG_ENDIAN    4321
+#define __BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
 #ifdef __GNUC__
 #define PRINTF_SYNTAX(strindex) __attribute__ ((format (printf, strindex, strindex+1)))
 #define SCANF_SYNTAX(strindex) __attribute__ ((format (scanf, strindex, strindex+1)))

daemon/nntp/ArticleDownloader.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "StatMeter.h"
 #include "Util.h"
@@ -33,7 +34,6 @@ ArticleDownloader::ArticleDownloader()
 {
 	debug("Creating ArticleDownloader");
 
-	m_articleWriter.SetOwner(this);
 	SetLastUpdateTimeNow();
 }
 
@@ -98,13 +98,13 @@ void ArticleDownloader::Run()
 		while (!m_connection && !(IsStopped() || serverConfigGeneration != g_ServerPool->GetGeneration()))
 		{
 			m_connection = g_ServerPool->GetConnection(level, wantServer, &failedServers);
-			usleep(5 * 1000);
+			Util::Sleep(5);
 		}
 		SetLastUpdateTimeNow();
 		SetStatus(adRunning);
 
-		if (IsStopped() || ((g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
@@ -195,8 +195,8 @@ void ArticleDownloader::Run()
 			break;
 		}
 
-		if (IsStopped() || ((g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
@@ -296,6 +296,11 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	m_writingStarted = false;
 	m_articleInfo->SetCrc(0);
 
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Reset();
+	}
+
 	if (m_connection->GetNewsServer()->GetJoinGroup())
 	{
 		// change group
@@ -316,14 +321,8 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	}
 
 	// retrieve article
-	for (int retry = 3; retry > 0; retry--)
-	{
-		response = m_connection->Request(BString<1024>("ARTICLE %s\r\n", m_articleInfo->GetMessageId()));
-		if ((response && !strncmp(response, "2", 1)) || m_connection->GetAuthError())
-		{
-			break;
-		}
-	}
+	response = m_connection->Request(BString<1024>("%s %s\r\n",
+		g_Options->GetRawArticle() ? "ARTICLE" : "BODY", m_articleInfo->GetMessageId()));
 
 	status = CheckResponse(response, "could not fetch article");
 	if (status != adFinished)
@@ -331,47 +330,35 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 		return status;
 	}
 
-	if (g_Options->GetDecode())
-	{
-		m_yDecoder.Clear();
-		m_yDecoder.SetCrcCheck(g_Options->GetCrcCheck());
-		m_uDecoder.Clear();
-	}
+	m_decoder.Clear();
+	m_decoder.SetCrcCheck(g_Options->GetCrcCheck());
+	m_decoder.SetRawMode(g_Options->GetRawArticle());
 
-	bool body = false;
-	bool end = false;
-	CharBuffer lineBuf(1024*10);
 	status = adRunning;
+	CharBuffer lineBuf(1024*4);
 
-	while (!IsStopped())
+	while (!IsStopped() && !m_decoder.GetEof())
 	{
-		time_t oldTime = m_lastUpdateTime;
-		SetLastUpdateTimeNow();
-		if (oldTime != m_lastUpdateTime)
-		{
-			AddServerData();
-		}
-
-		// Throttle the bandwidth
-		while (!IsStopped() && (g_Options->GetDownloadRate() > 0.0f) &&
-			(g_StatMeter->CalcCurrentDownloadSpeed() > g_Options->GetDownloadRate() ||
-			g_StatMeter->CalcMomentaryDownloadSpeed() > g_Options->GetDownloadRate()))
+		// throttle the bandwidth
+		while (!IsStopped() && (g_WorkState->GetSpeedLimit() > 0.0f) &&
+			(g_StatMeter->CalcCurrentDownloadSpeed() > g_WorkState->GetSpeedLimit() ||
+			g_StatMeter->CalcMomentaryDownloadSpeed() > g_WorkState->GetSpeedLimit()))
 		{
 			SetLastUpdateTimeNow();
-			usleep(10 * 1000);
+			Util::Sleep(10);
 		}
 
-		int len = 0;
-		char* line = m_connection->ReadLine(lineBuf, lineBuf.Size(), &len);
-
-		g_StatMeter->AddSpeedReading(len);
-		if (g_Options->GetAccurateRate())
+		char* buffer;
+		int len;
+		m_connection->ReadBuffer(&buffer, &len);
+		if (len == 0)
 		{
-			AddServerData();
+			len = m_connection->TryRecv(lineBuf, lineBuf.Size());
+			buffer = lineBuf;
 		}
 
-		// Have we encountered a timeout?
-		if (!line)
+		// have we encountered a timeout?
+		if (len <= 0)
 		{
 			if (!IsStopped())
 			{
@@ -381,67 +368,25 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 			break;
 		}
 
-		//detect end of article
-		if (!strcmp(line, ".\r\n") || !strcmp(line, ".\n"))
+		g_StatMeter->AddSpeedReading(len);
+		time_t oldTime = m_lastUpdateTime;
+		SetLastUpdateTimeNow();
+		if (oldTime != m_lastUpdateTime)
 		{
-			end = true;
-			break;
+			AddServerData();
 		}
 
-		//detect lines starting with "." (marked as "..")
-		if (!strncmp(line, "..", 2))
-		{
-			line++;
-			len--;
-		}
-
-		if (!body)
-		{
-			// detect body of article
-			if (*line == '\r' || *line == '\n')
-			{
-				body = true;
-			}
-			// check id of returned article
-			else if (!strncmp(line, "Message-ID: ", 12))
-			{
-				char* p = line + 12;
-				if (strncmp(p, m_articleInfo->GetMessageId(), strlen(m_articleInfo->GetMessageId())))
-				{
-					if (char* e = strrchr(p, '\r')) *e = '\0'; // remove trailing CR-character
-					detail("Article %s @ %s failed: Wrong message-id, expected %s, returned %s", *m_infoName,
-						*m_connectionName, m_articleInfo->GetMessageId(), p);
-					status = adFailed;
-					break;
-				}
-			}
-		}
-
-		if (m_format == Decoder::efUnknown && g_Options->GetDecode())
-		{
-			m_format = Decoder::DetectFormat(line, len, body);
-			if (m_format != Decoder::efUnknown)
-			{
-				// sometimes news servers misbehave and send article body without new line separator between headers and body
-				// if we found decoder signature we know the body is already arrived
-				body = true;
-			}
-		}
+		// decode article data
+		len = m_decoder.DecodeBuffer(buffer, len);
 
 		// write to output file
-		if (((body && m_format != Decoder::efUnknown) || !g_Options->GetDecode()) && !Write(line, len))
+		if (len > 0 && !Write(buffer, len))
 		{
 			status = adFatalError;
 			break;
 		}
 	}
 
-	if (!end && status == adRunning && !IsStopped())
-	{
-		detail("Article %s @ %s failed: article incomplete", *m_infoName, *m_connectionName);
-		status = adFailed;
-	}
-
 	if (IsStopped())
 	{
 		status = adFailed;
@@ -500,89 +445,69 @@ ArticleDownloader::EStatus ArticleDownloader::CheckResponse(const char* response
 	}
 }
 
-bool ArticleDownloader::Write(char* line, int len)
+bool ArticleDownloader::Write(char* buffer, int len)
 {
 	const char* articleFilename = nullptr;
 	int64 articleFileSize = 0;
 	int64 articleOffset = 0;
 	int articleSize = 0;
 
-	if (g_Options->GetDecode())
+	if (!m_writingStarted)
 	{
-		if (m_format == Decoder::efYenc)
+		if (!g_Options->GetRawArticle())
 		{
-			len = m_yDecoder.DecodeBuffer(line, len);
-			articleFilename = m_yDecoder.GetArticleFilename();
-			articleFileSize = m_yDecoder.GetSize();
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			len = m_uDecoder.DecodeBuffer(line, len);
-			articleFilename = m_uDecoder.GetArticleFilename();
-		}
-		else
-		{
-			detail("Decoding %s failed: unsupported encoding", *m_infoName);
-			return false;
-		}
-
-		if (len > 0 && m_format == Decoder::efYenc)
-		{
-			if (m_yDecoder.GetBegin() == 0 || m_yDecoder.GetEnd() == 0)
+			articleFilename = m_decoder.GetArticleFilename();
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
-				return false;
+				if (m_decoder.GetBeginPos() == 0 || m_decoder.GetEndPos() == 0)
+				{
+					return false;
+				}
+				articleFileSize = m_decoder.GetSize();
+				articleOffset = m_decoder.GetBeginPos() - 1;
+				articleSize = (int)(m_decoder.GetEndPos() - m_decoder.GetBeginPos() + 1);
+				if (articleSize <= 0 || articleSize > 1024*1024*1024)
+				{
+					warn("Malformed article %s: size %i out of range", *m_infoName, articleSize);
+					return false;
+				}
 			}
-			articleOffset = m_yDecoder.GetBegin() - 1;
-			articleSize = (int)(m_yDecoder.GetEnd() - m_yDecoder.GetBegin() + 1);
 		}
-	}
 
-	if (!m_writingStarted && len > 0)
-	{
-		if (!m_articleWriter.Start(m_format, articleFilename, articleFileSize, articleOffset, articleSize))
+		if (!m_articleWriter.Start(m_decoder.GetFormat(), articleFilename, articleFileSize, articleOffset, articleSize))
 		{
 			return false;
 		}
 		m_writingStarted = true;
 	}
 
-	bool ok = len == 0 || m_articleWriter.Write(line, len);
+	bool ok = m_articleWriter.Write(buffer, len);
+
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Append(buffer, len);
+	}
 
 	return ok;
 }
 
 ArticleDownloader::EStatus ArticleDownloader::DecodeCheck()
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
-		Decoder* decoder = nullptr;
-		if (m_format == Decoder::efYenc)
-		{
-			decoder = &m_yDecoder;
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			decoder = &m_uDecoder;
-		}
-		else
-		{
-			detail("Decoding %s failed: no binary data or unsupported encoding format", *m_infoName);
-			return adFailed;
-		}
-
-		Decoder::EStatus status = decoder->Check();
+		Decoder::EStatus status = m_decoder.Check();
 
 		if (status == Decoder::dsFinished)
 		{
-			if (decoder->GetArticleFilename())
+			if (m_decoder.GetArticleFilename())
 			{
-				m_articleFilename = decoder->GetArticleFilename();
+				m_articleFilename = m_decoder.GetArticleFilename();
 			}
 
-			if (m_format == Decoder::efYenc)
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
 				m_articleInfo->SetCrc(g_Options->GetCrcCheck() ?
-					m_yDecoder.GetCalculatedCrc() : m_yDecoder.GetExpectedCrc());
+					m_decoder.GetCalculatedCrc() : m_decoder.GetExpectedCrc());
 			}
 
 			return adFinished;
@@ -643,22 +568,6 @@ void ArticleDownloader::Stop()
 	debug("ArticleDownloader stopped successfully");
 }
 
-bool ArticleDownloader::Terminate()
-{
-	NntpConnection* connection = m_connection;
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		g_StatMeter->AddServerData(connection->FetchTotalBytesRead(), connection->GetNewsServer()->GetId());
-		g_ServerPool->FreeConnection(connection, true);
-	}
-	return terminated;
-}
-
 void ArticleDownloader::FreeConnection(bool keepConnected)
 {
 	if (m_connection)

daemon/nntp/ArticleDownloader.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -29,6 +29,15 @@
 #include "NntpConnection.h"
 #include "Decoder.h"
 #include "ArticleWriter.h"
+#include "Util.h"
+
+class ArticleContentAnalyzer
+{
+public:
+	virtual ~ArticleContentAnalyzer() {};
+	virtual void Reset() = 0;
+	virtual void Append(const void* buffer, int len) = 0;
+};
 
 class ArticleDownloader : public Thread, public Subject
 {
@@ -47,16 +56,6 @@ public:
 		adFatalError
 	};
 
-	class ArticleWriterImpl : public ArticleWriter
-	{
-	public:
-		void SetOwner(ArticleDownloader* owner) { m_owner = owner; }
-	protected:
-		virtual void SetLastUpdateTimeNow() { m_owner->SetLastUpdateTimeNow(); }
-	private:
-		ArticleDownloader* m_owner;
-	};
-
 	ArticleDownloader();
 	virtual ~ArticleDownloader();
 	void SetFileInfo(FileInfo* fileInfo) { m_fileInfo = fileInfo; }
@@ -67,7 +66,6 @@ public:
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 	virtual void Run();
 	virtual void Stop();
-	bool Terminate();
 	time_t GetLastUpdateTime() { return m_lastUpdateTime; }
 	void SetLastUpdateTimeNow();
 	const char* GetArticleFilename() { return m_articleFilename; }
@@ -77,6 +75,8 @@ public:
 	void SetConnection(NntpConnection* connection) { m_connection = connection; }
 	void CompleteFileParts() { m_articleWriter.CompleteFileParts(); }
 	int GetDownloadedSize() { return m_downloadedSize; }
+	void SetContentAnalyzer(std::unique_ptr<ArticleContentAnalyzer> contentAnalyzer) { m_contentAnalyzer = std::move(contentAnalyzer); }
+	ArticleContentAnalyzer* GetContentAnalyzer() { return m_contentAnalyzer.get(); }
 
 	void LogDebugInfo();
 
@@ -90,20 +90,19 @@ private:
 	CString m_connectionName;
 	CString m_articleFilename;
 	time_t m_lastUpdateTime;
-	Decoder::EFormat m_format = Decoder::efUnknown;
-	YDecoder m_yDecoder;
-	UDecoder m_uDecoder;
-	ArticleWriterImpl m_articleWriter;
+	Decoder m_decoder;
+	ArticleWriter m_articleWriter;
 	ServerStatList m_serverStats;
 	bool m_writingStarted;
 	int m_downloadedSize = 0;
+	std::unique_ptr<ArticleContentAnalyzer> m_contentAnalyzer;
 
 	EStatus Download();
 	EStatus DecodeCheck();
 	void FreeConnection(bool keepConnected);
 	EStatus CheckResponse(const char* response, const char* comment);
 	void SetStatus(EStatus status) { m_status = status; }
-	bool Write(char* line, int len);
+	bool Write(char* buffer, int len);
 	void AddServerData();
 };
 

daemon/nntp/ArticleWriter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -84,7 +84,7 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 			}
 
 			if (!outputInitialized && filename &&
-				FileSystem::FileExists(m_fileInfo->GetNzbInfo()->GetDestDir(), filename))
+				FileSystem::FileExists(BString<1024>("%s%c%s", m_fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, filename)))
 			{
 				m_duplicate = true;
 				return false;
@@ -106,14 +106,14 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 	}
 
 	// allocate cache buffer
-	if (g_Options->GetArticleCache() > 0 && g_Options->GetDecode() &&
+	if (g_Options->GetArticleCache() > 0 && !g_Options->GetRawArticle() &&
 		(!g_Options->GetDirectWrite() || m_format == Decoder::efYenc))
 	{
 		m_articleData = g_ArticleCache->Alloc(m_articleSize);
 
 		while (!m_articleData.GetData() && g_ArticleCache->GetFlushing())
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			m_articleData = g_ArticleCache->Alloc(m_articleSize);
 		}
 
@@ -126,11 +126,11 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 	if (!m_articleData.GetData())
 	{
 		bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
-		const char* filename = directWrite ? m_outputFilename : m_tempFilename;
-		if (!m_outFile.Open(filename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
+		const char* outFilename = directWrite ? m_outputFilename : m_tempFilename;
+		if (!m_outFile.Open(outFilename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
 		{
 			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-				"Could not %s file %s: %s", directWrite ? "open" : "create", filename,
+				"Could not %s file %s: %s", directWrite ? "open" : "create", outFilename,
 				*FileSystem::GetLastErrorMessage());
 			return false;
 		}
@@ -147,22 +147,31 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 
 bool ArticleWriter::Write(char* buffer, int len)
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		m_articlePtr += len;
 	}
 
-	if (g_Options->GetDecode() && m_articleData.GetData())
+	if (m_articlePtr > m_articleSize)
+	{
+		// An attempt to write beyond article border is detected.
+		// That's an error condition (damaged article).
+		// We return 'false' since this isn't a fatal disk error and
+		// article size mismatch will be detected in decoder check anyway.
+		return true;
+	}
+
+	if (!g_Options->GetRawArticle() && m_articleData.GetData())
 	{
-		if (m_articlePtr > m_articleSize)
-		{
-			detail("Decoding %s failed: article size mismatch", *m_infoName);
-			return false;
-		}
 		memcpy(m_articleData.GetData() + m_articlePtr - len, buffer, len);
 		return true;
 	}
 
+	if (g_Options->GetSkipWrite())
+	{
+		return true;
+	}
+
 	return m_outFile.Write(buffer, len) > 0;
 }
 
@@ -179,7 +188,7 @@ void ArticleWriter::Finish(bool success)
 
 	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
 
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		if (!directWrite && !m_articleData.GetData())
 		{
@@ -189,10 +198,9 @@ void ArticleWriter::Finish(bool success)
 					"Could not rename file %s to %s: %s", *m_tempFilename, m_resultFilename,
 					*FileSystem::GetLastErrorMessage());
 			}
+			FileSystem::DeleteFile(m_tempFilename);
 		}
 
-		FileSystem::DeleteFile(m_tempFilename);
-
 		if (m_articleData.GetData())
 		{
 			if (m_articleSize != m_articlePtr)
@@ -224,19 +232,20 @@ void ArticleWriter::Finish(bool success)
 /* creates output file and subdirectores */
 bool ArticleWriter::CreateOutputFile(int64 size)
 {
-	if (g_Options->GetDirectWrite() && FileSystem::FileExists(m_outputFilename) &&
-		FileSystem::FileSize(m_outputFilename) == size)
+	if (FileSystem::FileExists(m_outputFilename))
 	{
-		// keep existing old file from previous program session
-		return true;
+		if (FileSystem::FileSize(m_outputFilename) == size)
+		{
+			// keep existing old file from previous program session
+			return true;
+		}
+		// delete existing old file from previous program session
+		FileSystem::DeleteFile(m_outputFilename);
 	}
 
-	// delete eventually existing old file from previous program session
-	FileSystem::DeleteFile(m_outputFilename);
-
 	// ensure the directory exist
 	BString<1024> destDir;
-	destDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+	destDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 	CString errmsg;
 
 	if (!FileSystem::ForceDirectories(destDir, errmsg))
@@ -275,7 +284,7 @@ void ArticleWriter::BuildOutputFilename()
 		else
 		{
 			filename.Format("%s%c%i.out.tmp", m_fileInfo->GetNzbInfo()->GetDestDir(),
-				(int)PATH_SEPARATOR, m_fileInfo->GetId());
+				PATH_SEPARATOR, m_fileInfo->GetId());
 			m_fileInfo->SetOutputFilename(filename);
 		}
 
@@ -292,18 +301,20 @@ void ArticleWriter::CompleteFileParts()
 
 	BString<1024> nzbName;
 	BString<1024> nzbDestDir;
+	BString<1024> filename;
 
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
 		nzbName = m_fileInfo->GetNzbInfo()->GetName();
 		nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir();
+		filename = m_fileInfo->GetFilename();
 	}
 
-	BString<1024> infoFilename("%s%c%s", *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
+	BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename);
 
 	bool cached = m_fileInfo->GetCachedArticles() > 0;
 
-	if (!g_Options->GetDecode())
+	if (g_Options->GetRawArticle())
 	{
 		detail("Moving articles for %s", *infoFilename);
 	}
@@ -332,17 +343,17 @@ void ArticleWriter::CompleteFileParts()
 	CString ofn;
 	if (m_fileInfo->GetForceDirectWrite())
 	{
-		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, m_fileInfo->GetFilename());
+		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename);
 	}
 	else
 	{
-		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename);
 	}
 
 	DiskFile outfile;
 	BString<1024> tmpdestfile("%s.tmp", *ofn);
 
-	if (g_Options->GetDecode() && !directWrite)
+	if (!g_Options->GetRawArticle() && !directWrite)
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!outfile.Open(tmpdestfile, DiskFile::omWrite))
@@ -362,7 +373,7 @@ void ArticleWriter::CompleteFileParts()
 		}
 		tmpdestfile = *m_outputFilename;
 	}
-	else if (!g_Options->GetDecode())
+	else if (g_Options->GetRawArticle())
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!FileSystem::CreateDirectory(ofn))
@@ -390,7 +401,7 @@ void ArticleWriter::CompleteFileParts()
 		CharBuffer buffer;
 		bool firstArticle = true;
 
-		if (g_Options->GetDecode() && !directWrite)
+		if (!g_Options->GetRawArticle() && !directWrite)
 		{
 			buffer.Reserve(1024 * 64);
 		}
@@ -402,22 +413,27 @@ void ArticleWriter::CompleteFileParts()
 				continue;
 			}
 
-			if (g_Options->GetDecode() && !directWrite && pa->GetSegmentOffset() > -1 &&
+			if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 &&
 				pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1)
 			{
 				memset(buffer, 0, buffer.Size());
-				while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
-					outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				if (!g_Options->GetSkipWrite())
+				{
+					while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
+						outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				}
 			}
 
 			if (pa->GetSegmentContent())
 			{
-				outfile.Seek(pa->GetSegmentOffset());
-				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				if (!g_Options->GetSkipWrite())
+				{
+					outfile.Seek(pa->GetSegmentOffset());
+					outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				}
 				pa->DiscardSegment();
-				SetLastUpdateTimeNow();
 			}
-			else if (g_Options->GetDecode() && !directWrite)
+			else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite())
 			{
 				DiskFile infile;
 				if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead))
@@ -427,7 +443,6 @@ void ArticleWriter::CompleteFileParts()
 					{
 						cnt = (int)infile.Read(buffer, buffer.Size());
 						outfile.Write(buffer, cnt);
-						SetLastUpdateTimeNow();
 					}
 					infile.Close();
 				}
@@ -436,14 +451,14 @@ void ArticleWriter::CompleteFileParts()
 					m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1);
 					m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1);
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-						"Could not find file %s for %s%c%s [%i/%i]",
-						pa->GetResultFilename(), *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename(),
-						pa->GetPartNumber(), (int)m_fileInfo->GetArticles()->size());
+						"Could not find file %s for %s [%i/%i]",
+						pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(),
+						(int)m_fileInfo->GetArticles()->size());
 				}
 			}
-			else if (!g_Options->GetDecode())
+			else if (g_Options->GetRawArticle())
 			{
-				BString<1024> dstFileName("%s%c%03i", *ofn, (int)PATH_SEPARATOR, pa->GetPartNumber());
+				BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber());
 				if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName))
 				{
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
@@ -454,7 +469,7 @@ void ArticleWriter::CompleteFileParts()
 
 			if (m_format == Decoder::efYenc)
 			{
-				crc = firstArticle ? pa->GetCrc() : Util::Crc32Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
+				crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
 				firstArticle = false;
 			}
 		}
@@ -490,7 +505,7 @@ void ArticleWriter::CompleteFileParts()
 		{
 			debug("Checking old dir for: %s", *m_outputFilename);
 			BString<1024> oldDestDir;
-			oldDestDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+			oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 			if (FileSystem::DirEmpty(oldDestDir))
 			{
 				debug("Deleting old dir: %s", *oldDestDir);
@@ -520,29 +535,31 @@ void ArticleWriter::CompleteFileParts()
 			"%i of %i article downloads failed for \"%s\"",
 			m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(),
 			m_fileInfo->GetTotalArticles(), *infoFilename);
-
-		if (g_Options->GetBrokenLog())
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", *nzbDestDir, (int)PATH_SEPARATOR);
-			DiskFile file;
-			if (file.Open(brokenLogName, DiskFile::omAppend))
-			{
-				file.Print("%s (%i/%i)%s", m_fileInfo->GetFilename(), m_fileInfo->GetSuccessArticles(),
-					m_fileInfo->GetTotalArticles(), LINE_ENDING);
-				file.Close();
-			}
-		}
 	}
 	else
 	{
 		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename);
 	}
 
-	m_fileInfo->SetCrc(crc);
-	m_fileInfo->SetOutputFilename(ofn);
-
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
+
+		m_fileInfo->SetCrc(crc);
+		m_fileInfo->SetOutputFilename(ofn);
+
+		if (strcmp(m_fileInfo->GetFilename(), filename))
+		{
+			// file was renamed during completion, need to move the file
+			ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+			if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn))
+			{
+				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
+					"Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(),
+					*ofn, *FileSystem::GetLastErrorMessage());
+			}
+			m_fileInfo->SetOutputFilename(ofn);
+		}
+
 		if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir))
 		{
 			// destination directory was changed during completion, need to move the file
@@ -565,10 +582,18 @@ void ArticleWriter::FlushCache()
 		ArticleCache::FlushGuard flushGuard = g_ArticleCache->GuardFlush();
 
 		std::vector<ArticleInfo*> cachedArticles;
-		cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
+
+			if (m_fileInfo->GetFlushLocked())
+			{
+				return;
+			}
+
+			m_fileInfo->SetFlushLocked(true);
+
+			cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 			for (ArticleInfo* pa : m_fileInfo->GetArticles())
 			{
 				if (pa->GetSegmentContent())
@@ -593,6 +618,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not open file %s: %s", m_fileInfo->GetOutputFilename(),
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -608,6 +636,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not create file %s: %s", *destFile,
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -624,7 +655,10 @@ void ArticleWriter::FlushCache()
 				outfile.Seek(pa->GetSegmentOffset());
 			}
 
-			outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			if (!g_Options->GetSkipWrite())
+			{
+				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			}
 
 			flushedSize += pa->GetSegmentSize();
 			flushedArticles++;
@@ -649,6 +683,7 @@ void ArticleWriter::FlushCache()
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
 			m_fileInfo->SetCachedArticles(m_fileInfo->GetCachedArticles() - flushedArticles);
+			m_fileInfo->SetFlushLocked(false);
 		}
 	}
 
@@ -674,14 +709,14 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 	// move already downloaded files to new destination
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		BString<1024> oldFileName("%s%c%s", oldDestDir, (int)PATH_SEPARATOR, completedFile.GetFileName());
-		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName());
+		BString<1024> oldFileName("%s%c%s", oldDestDir, PATH_SEPARATOR, completedFile.GetFilename());
+		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename());
 
 		// check if file was not moved already
 		if (strcmp(oldFileName, newFileName))
 		{
 			// prevent overwriting of existing files
-			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFileName());
+			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFilename());
 
 			detail("Moving file %s to %s", *oldFileName, *newFileName);
 			if (!FileSystem::MoveFile(oldFileName, newFileName))
@@ -692,57 +727,6 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 		}
 	}
 
-	// move brokenlog.txt
-	if (g_Options->GetBrokenLog())
-	{
-		BString<1024> oldBrokenLogName("%s%c_brokenlog.txt", oldDestDir, (int)PATH_SEPARATOR);
-		if (FileSystem::FileExists(oldBrokenLogName))
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR);
-
-			detail("Moving file %s to %s", *oldBrokenLogName, *brokenLogName);
-			if (FileSystem::FileExists(brokenLogName))
-			{
-				// copy content to existing new file, then delete old file
-				DiskFile outfile;
-				if (outfile.Open(brokenLogName, DiskFile::omAppend))
-				{
-					DiskFile infile;
-					if (infile.Open(oldBrokenLogName, DiskFile::omRead))
-					{
-						CharBuffer buffer(1024 * 50);
-						int cnt = buffer.Size();
-						while (cnt == buffer.Size())
-						{
-							cnt = (int)infile.Read(buffer, buffer.Size());
-							outfile.Write(buffer, cnt);
-						}
-						infile.Close();
-						FileSystem::DeleteFile(oldBrokenLogName);
-					}
-					else
-					{
-						nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *oldBrokenLogName);
-					}
-					outfile.Close();
-				}
-				else
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
-				}
-			}
-			else
-			{
-				// move to new destination
-				if (!FileSystem::MoveFile(oldBrokenLogName, brokenLogName))
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not move file %s to %s: %s",
-						*oldBrokenLogName, *brokenLogName, *FileSystem::GetLastErrorMessage());
-				}
-			}
-		}
-	}
-
 	// delete old directory (if empty)
 	if (FileSystem::DirEmpty(oldDestDir))
 	{
@@ -787,10 +771,15 @@ CachedSegmentData ArticleCache::Alloc(int size)
 		p = malloc(size);
 		if (p)
 		{
-			if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+			if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 			{
 				g_DiskState->WriteCacheFlag();
 			}
+			if (!m_allocated)
+			{
+				// Resume Run(), the notification arrives later, after releasing m_allocMutex
+				m_allocCond.NotifyAll();
+			}
 			m_allocated += size;
 		}
 	}
@@ -807,6 +796,7 @@ bool ArticleCache::Realloc(CachedSegmentData* segment, int newSize)
 	{
 		m_allocated += newSize - segment->m_size;
 		segment->m_size = newSize;
+		segment->m_data = (char*)p;
 	}
 
 	return p;
@@ -820,7 +810,7 @@ void ArticleCache::Free(CachedSegmentData* segment)
 
 		Guard guard(m_allocMutex);
 		m_allocated -= segment->m_size;
-		if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+		if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 		{
 			g_DiskState->DeleteCacheFlag();
 		}
@@ -836,21 +826,36 @@ void ArticleCache::Run()
 	bool justFlushed = false;
 	while (!IsStopped() || m_allocated > 0)
 	{
-		if ((justFlushed || resetCounter >= 1000  || IsStopped() ||
+		if ((justFlushed || resetCounter >= 1000 || IsStopped() ||
 			 (g_Options->GetDirectWrite() && m_allocated >= fillThreshold)) &&
 			m_allocated > 0)
 		{
 			justFlushed = CheckFlush(m_allocated >= fillThreshold);
 			resetCounter = 0;
 		}
+		else if (!m_allocated)
+		{
+			Guard guard(m_allocMutex);
+			m_allocCond.Wait(m_allocMutex, [&]{ return IsStopped() || m_allocated > 0; });
+			resetCounter = 0;
+		}
 		else
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			resetCounter += 5;
 		}
 	}
 }
 
+void ArticleCache::Stop()
+{
+	Thread::Stop();
+
+	// Resume Run() to exit it
+	Guard guard(m_allocMutex);
+	m_allocCond.NotifyAll();
+}
+
 bool ArticleCache::CheckFlush(bool flushEverything)
 {
 	debug("Checking cache, Allocated: %i, FlushEverything: %i", (int)m_allocated, (int)flushEverything);
@@ -871,7 +876,7 @@ bool ArticleCache::CheckFlush(bool flushEverything)
 				if (fileInfo->GetCachedArticles() > 0 && (fileInfo->GetActiveDownloads() == 0 || flushEverything))
 				{
 					m_fileInfo = fileInfo;
-					infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
+					infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR, m_fileInfo->GetFilename());
 					break;
 				}
 			}

daemon/nntp/ArticleWriter.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -60,9 +60,6 @@ public:
 	static bool MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir);
 	void FlushCache();
 
-protected:
-	virtual void SetLastUpdateTimeNow() {}
-
 private:
 	FileInfo* m_fileInfo;
 	ArticleInfo* m_articleInfo;
@@ -98,6 +95,7 @@ public:
 	};
 
 	virtual void Run();
+	virtual void Stop();
 	CachedSegmentData Alloc(int size);
 	bool Realloc(CachedSegmentData* segment, int newSize);
 	void Free(CachedSegmentData* segment);
@@ -114,6 +112,7 @@ private:
 	Mutex m_flushMutex;
 	Mutex m_contentMutex;
 	FileInfo* m_fileInfo = nullptr;
+	ConditionVar m_allocCond;
 
 	bool CheckFlush(bool flushEverything);
 };

daemon/nntp/Decoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,22 +22,129 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Util.h"
+#include "YEncode.h"
 
-const char* Decoder::FormatNames[] = { "Unknown", "yEnc", "UU" };
+Decoder::Decoder()
+{
+	debug("%s", YEncode::decode_simd ? "SIMD yEnc decoder can be used" : "SIMD yEnc decoder isn't available for this CPU");
+	debug("%s", YEncode::crc_simd ? "SIMD Crc routine can be used" : "SIMD Crc routine isn't available for this CPU");
+
+	Clear();
+}
 
 void Decoder::Clear()
 {
 	m_articleFilename.Clear();
+	m_body = false;
+	m_begin = false;
+	m_part = false;
+	m_end = false;
+	m_crc = false;
+	m_eof = false;
+	m_expectedCRC = 0;
+	m_crc32.Reset();
+	m_beginPos = 0;
+	m_endPos = 0;
+	m_size = 0;
+	m_endSize = 0;
+	m_outSize = 0;
+	m_state = 0;
+	m_crcCheck = false;
+	m_lineBuf.Reserve(1024*8);
+	m_lineBuf.SetLength(0);
 }
 
-Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
+/* At the beginning of article the processing goes line by line to find '=ybegin'-marker.
+ * Once the yEnc-data is started switches to blockwise processing.
+ * At the end of yEnc-data switches back to line by line mode to
+ * process '=yend'-marker and EOF-marker.
+ * UU-encoded articles are processed completely in line by line mode.
+ */
+int Decoder::DecodeBuffer(char* buffer, int len)
+{
+	if (m_rawMode)
+	{
+		ProcessRaw(buffer, len);
+		return len;
+	}
+
+	int outlen = 0;
+
+	if (m_body && m_format == efYenc)
+	{
+		outlen = DecodeYenc(buffer, buffer, len);
+		if (m_body)
+		{
+			return outlen;
+		}
+	}
+	else
+	{
+		m_lineBuf.Append(buffer, len);
+	}
+
+	char* line = (char*)m_lineBuf;
+	while (char* end = strchr(line, '\n'))
+	{
+		int llen = (int)(end - line + 1);
+
+		if (line[0] == '.' && line[1] == '\r')
+		{
+			m_eof = true;
+			m_lineBuf.SetLength(0);
+			return outlen;
+		}
+
+		if (m_format == efUnknown)
+		{
+			m_format = DetectFormat(line, llen);
+		}
+
+		if (m_format == efYenc)
+		{
+			ProcessYenc(line, llen);
+			if (m_body)
+			{
+				outlen = DecodeYenc(end + 1, buffer, m_lineBuf.Length() - (int)(end + 1 - m_lineBuf));
+				if (m_body)
+				{
+					m_lineBuf.SetLength(0);
+					return outlen;
+				}
+				line = (char*)m_lineBuf;
+				continue;
+			}
+		}
+		else if (m_format == efUx)
+		{
+			outlen += DecodeUx(line, llen);
+		}
+
+		line = end + 1;
+	}
+
+	if (*line)
+	{
+		len = m_lineBuf.Length() - (int)(line - m_lineBuf);
+		memmove((char*)m_lineBuf, line, len);
+		m_lineBuf.SetLength(len);
+	}
+	else
+	{
+		m_lineBuf.SetLength(0);
+	}
+
+	return outlen;
+}
+
+Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len)
 {
 	if (!strncmp(buffer, "=ybegin ", 8))
 	{
 		return efYenc;
 	}
 
-	if (inBody && (len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
+	if ((len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
 	{
 		return efUx;
 	}
@@ -64,139 +171,122 @@ Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
 	return efUnknown;
 }
 
-/**
-  * YDecoder: fast implementation of yEnc-Decoder
-  */
-
-YDecoder::YDecoder()
+void Decoder::ProcessYenc(char* buffer, int len)
 {
-	Clear();
-}
-
-void YDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_begin = false;
-	m_part = false;
-	m_end = false;
-	m_crc = false;
-	m_expectedCRC = 0;
-	m_calculatedCRC = 0xFFFFFFFF;
-	m_beginPos = 0;
-	m_endPos = 0;
-	m_size = 0;
-	m_endSize = 0;
-	m_crcCheck = false;
-}
-
-int YDecoder::DecodeBuffer(char* buffer, int len)
-{
-	if (m_body && !m_end)
+	if (!strncmp(buffer, "=ybegin ", 8))
 	{
-		if (!strncmp(buffer, "=yend ", 6))
+		m_begin = true;
+		char* pb = strstr(buffer, " name=");
+		if (pb)
 		{
-			m_end = true;
-			char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
-			if (pb)
-			{
-				m_crc = true;
-				pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
-				m_expectedCRC = strtoul(pb, nullptr, 16);
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_endSize = (int64)atoll(pb);
-			}
-			return 0;
+			pb += 6; //=strlen(" name=")
+			char* pe;
+			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++);
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 		}
-
-		char* iptr = buffer;
-		char* optr = buffer;
-		while (true)
+		pb = strstr(buffer, " size=");
+		if (pb)
 		{
-			switch (*iptr)
-			{
-				case '=':	//escape-sequence
-					iptr++;
-					*optr = *iptr - 64 - 42;
-					optr++;
-					break;
-				case '\n':	// ignored char
-				case '\r':	// ignored char
-					break;
-				case '\0':
-					goto BreakLoop;
-				default:	// normal char
-					*optr = *iptr - 42;
-					optr++;
-					break;
-			}
-			iptr++;
+			pb += 6; //=strlen(" size=")
+			m_size = (int64)atoll(pb);
 		}
-BreakLoop:
-
-		if (m_crcCheck)
+		m_part = strstr(buffer, " part=");
+		if (!m_part)
 		{
-			m_calculatedCRC = Util::Crc32m(m_calculatedCRC, (uchar *)buffer, (uint32)(optr - buffer));
-		}
-		return optr - buffer;
-	}
-	else
-	{
-		if (!m_part && !strncmp(buffer, "=ybegin ", 8))
-		{
-			m_begin = true;
-			char* pb = strstr(buffer, " name=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" name=")
-				char* pe;
-				for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-				m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_size = (int64)atoll(pb);
-			}
-			m_part = strstr(buffer, " part=");
-			if (!m_part)
-			{
-				m_body = true;
-				m_beginPos = 1;
-				m_endPos = m_size;
-			}
-		}
-		else if (m_part && !strncmp(buffer, "=ypart ", 7))
-		{
-			m_part = true;
 			m_body = true;
-			char* pb = strstr(buffer, " begin=");
-			if (pb)
-			{
-				pb += 7; //=strlen(" begin=")
-				m_beginPos = (int64)atoll(pb);
-			}
-			pb = strstr(buffer, " end=");
-			if (pb)
-			{
-				pb += 5; //=strlen(" end=")
-				m_endPos = (int64)atoll(pb);
-			}
+			m_beginPos = 1;
+			m_endPos = m_size;
+		}
+	}
+	else if (!strncmp(buffer, "=ypart ", 7))
+	{
+		m_part = true;
+		m_body = true;
+		char* pb = strstr(buffer, " begin=");
+		if (pb)
+		{
+			pb += 7; //=strlen(" begin=")
+			m_beginPos = (int64)atoll(pb);
+		}
+		pb = strstr(buffer, " end=");
+		if (pb)
+		{
+			pb += 5; //=strlen(" end=")
+			m_endPos = (int64)atoll(pb);
+		}
+	}
+	else if (!strncmp(buffer, "=yend ", 6))
+	{
+		m_end = true;
+		char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
+		if (pb)
+		{
+			m_crc = true;
+			pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
+			m_expectedCRC = strtoul(pb, nullptr, 16);
+		}
+		pb = strstr(buffer, " size=");
+		if (pb)
+		{
+			pb += 6; //=strlen(" size=")
+			m_endSize = (int64)atoll(pb);
 		}
 	}
-
-	return 0;
 }
 
-Decoder::EStatus YDecoder::Check()
+int Decoder::DecodeYenc(char* buffer, char* outbuf, int len)
 {
-	m_calculatedCRC ^= 0xFFFFFFFF;
+	const unsigned char* src = (unsigned char*)buffer;
+	unsigned char* dst = (unsigned char*)outbuf;
+
+	int endseq = YEncode::decode(&src, &dst, len, (YEncode::YencDecoderState*)&m_state);
+	int outlen = (int)((char*)dst - outbuf);
+
+	// endseq:
+	//   0: no end sequence found
+	//   1: \r\n=y sequence found, src points to byte after 'y'
+	//   2: \r\n.\r\n sequence found, src points to byte after last '\n'
+	if (endseq != 0)
+	{
+		// switch back to line mode to process '=yend'- or eof- marker
+		m_lineBuf.SetLength(0);
+		m_lineBuf.Append(endseq == 1 ? "=y" : ".\r\n");
+		int rem = len - (int)((const char*)src - buffer);
+		if (rem > 0)
+		{
+			m_lineBuf.Append((const char*)src, rem);
+		}
+		m_body = false;
+	}
+
+	if (m_crcCheck)
+	{
+		m_crc32.Append((uchar*)outbuf, (uint32)outlen);
+	}
+
+	m_outSize += outlen;
+
+	return outlen;
+}
+
+Decoder::EStatus Decoder::Check()
+{
+	switch (m_format)
+	{
+		case efYenc:
+			return CheckYenc();
+			 
+		case efUx:
+			return CheckUx();
+
+		default:
+			return dsUnknownError;
+	}
+}
+
+Decoder::EStatus Decoder::CheckYenc()
+{
+	m_calculatedCRC = m_crc32.Finish();
 
 	debug("Expected crc32=%x", m_expectedCRC);
 	debug("Calculated crc32=%x", m_calculatedCRC);
@@ -209,7 +299,7 @@ Decoder::EStatus YDecoder::Check()
 	{
 		return dsArticleIncomplete;
 	}
-	else if (!m_part && m_size != m_endSize)
+	else if ((!m_part && m_size != m_endSize) || (m_endSize != m_outSize))
 	{
 		return dsInvalidSize;
 	}
@@ -222,24 +312,7 @@ Decoder::EStatus YDecoder::Check()
 }
 
 
-/**
-  * UDecoder: supports UU encoding formats
-  */
-
-UDecoder::UDecoder()
-{
-	Clear();
-}
-
-void UDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_end = false;
-}
-
-/* DecodeBuffer-function uses portions of code from tool UUDECODE by Clem Dye
+/* DecodeUx-function uses portions of code from tool UUDECODE by Clem Dye
  * UUDECODE.c (http://www.bastet.com/uue.zip)
  * Copyright (C) 1998 Clem Dye
  *
@@ -248,7 +321,7 @@ void UDecoder::Clear()
 
 #define UU_DECODE_CHAR(c) (c == '`' ? 0 : (((c) - ' ') & 077))
 
-int UDecoder::DecodeBuffer(char* buffer, int len)
+int Decoder::DecodeUx(char* buffer, int len)
 {
 	if (!m_body)
 	{
@@ -264,7 +337,7 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			// extracting filename
 			char* pe;
 			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 
 			m_body = true;
 			return 0;
@@ -301,10 +374,7 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			}
 			else
 			{
-				if (effLen >= 1)
-				{
-					*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
-				}
+				*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
 				if (effLen >= 2)
 				{
 					*optr++ = UU_DECODE_CHAR (iptr[1]) << 4 | UU_DECODE_CHAR (iptr[2]) >> 2;
@@ -312,13 +382,13 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			}
 		}
 
-		return optr - buffer;
+		return (int)(optr - buffer);
 	}
 
 	return 0;
 }
 
-Decoder::EStatus UDecoder::Check()
+Decoder::EStatus Decoder::CheckUx()
 {
 	if (!m_body)
 	{
@@ -327,3 +397,50 @@ Decoder::EStatus UDecoder::Check()
 
 	return dsFinished;
 }
+
+void Decoder::ProcessRaw(char* buffer, int len)
+{
+	switch (m_state)
+	{
+		case 1:
+			m_eof = len >= 4 && buffer[0] == '\n' &&
+				buffer[1] == '.' && buffer[2] == '\r' && buffer[3] == '\n';
+			break;
+
+		case 2:
+			m_eof = len >= 3 && buffer[0] == '.' && buffer[1] == '\r' && buffer[2] == '\n';
+			break;
+
+		case 3:
+			m_eof = len >= 2 && buffer[0] == '\r' && buffer[1] == '\n';
+			break;
+
+		case 4:
+			m_eof = len >= 1 && buffer[0] == '\n';
+			break;
+	}
+
+	m_eof |= len >= 5 && strstr(buffer, "\r\n.\r\n");
+
+	if (len >= 4 && buffer[len-4] == '\r' && buffer[len-3] == '\n' &&
+		buffer[len-2] == '.' && buffer[len-1] == '\r')
+	{
+		m_state = 4;
+	}
+	else if (len >= 3 && buffer[len-3] == '\r' && buffer[len-2] == '\n' && buffer[len-1] == '.')
+	{
+		m_state = 3;
+	}
+	else if (len >= 2 && buffer[len-2] == '\r' && buffer[len-1] == '\n')
+	{
+		m_state = 2;
+	}
+	else if (len >= 1 && buffer[len-1] == '\r')
+	{
+		m_state = 1;
+	}
+	else
+	{
+		m_state = 0;
+	}
+}

daemon/nntp/Decoder.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #define DECODER_H
 
 #include "NString.h"
+#include "Util.h"
 
 class Decoder
 {
@@ -43,37 +44,26 @@ public:
 		efUx,
 	};
 
-	static const char* FormatNames[];
-
-	virtual ~Decoder() {}
-	virtual EStatus Check() = 0;
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len) = 0;
-	const char* GetArticleFilename() { return m_articleFilename; }
-	static EFormat DetectFormat(const char* buffer, int len, bool inBody);
-
-protected:
-	CString m_articleFilename;
-};
-
-class YDecoder: public Decoder
-{
-public:
-	YDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
+	Decoder();
+	EStatus Check();
+	void Clear();
+	int DecodeBuffer(char* buffer, int len);
 	void SetCrcCheck(bool crcCheck) { m_crcCheck = crcCheck; }
-	int64 GetBegin() { return m_beginPos; }
-	int64 GetEnd() { return m_endPos; }
+	void SetRawMode(bool rawMode) { m_rawMode = rawMode; }
+	EFormat GetFormat() { return m_format; }
+	int64 GetBeginPos() { return m_beginPos; }
+	int64 GetEndPos() { return m_endPos; }
 	int64 GetSize() { return m_size; }
 	uint32 GetExpectedCrc() { return m_expectedCRC; }
 	uint32 GetCalculatedCrc() { return m_calculatedCRC; }
+	bool GetEof() { return m_eof; }
+	const char* GetArticleFilename() { return m_articleFilename; }
 
-private:
+private: 
+	EFormat m_format = efUnknown;
 	bool m_begin;
 	bool m_part;
-	bool m_body;
+	bool m_body; 
 	bool m_end;
 	bool m_crc;
 	uint32 m_expectedCRC;
@@ -82,20 +72,22 @@ private:
 	int64 m_endPos;
 	int64 m_size;
 	int64 m_endSize;
+	int64 m_outSize;
+	bool m_eof;
 	bool m_crcCheck;
-};
+	char m_state;
+	bool m_rawMode = false;
+	CString m_articleFilename;
+	StringBuilder m_lineBuf;
+	Crc32 m_crc32;
 
-class UDecoder: public Decoder
-{
-public:
-	UDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
-
-private:
-	bool m_body;
-	bool m_end;
+	EFormat DetectFormat(const char* buffer, int len);
+	void ProcessYenc(char* buffer, int len);
+	int DecodeYenc(char* buffer, char* outbuf, int len);
+	EStatus CheckYenc();
+	int DecodeUx(char* buffer, int len);
+	EStatus CheckUx();
+	void ProcessRaw(char* buffer, int len);
 };
 
 #endif

daemon/nntp/NewsServer.cpp

@@ -22,13 +22,13 @@
 #include "nzbget.h"
 #include "NewsServer.h"
 
-NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port,
+NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 	const char* user, const char* pass, bool joinGroup, bool tls, const char* cipher,
 	int maxConnections, int retention, int level, int group, bool optional) :
-		m_id(id), m_active(active), m_port(port), m_level(level), m_normLevel(level),
-		m_group(group), m_maxConnections(maxConnections), m_joinGroup(joinGroup), m_tls(tls),
-		m_name(name), m_host(host ? host : ""), m_user(user ? user : ""), m_password(pass ? pass : ""),
-		m_cipher(cipher ? cipher : ""), m_retention(retention), m_optional(optional)
+		m_id(id), m_active(active), m_name(name), m_host(host ? host : ""), m_port(port), m_ipVersion(ipVersion),
+		m_user(user ? user : ""), m_password(pass ? pass : ""), m_joinGroup(joinGroup), m_tls(tls),
+		m_cipher(cipher ? cipher : ""), m_maxConnections(maxConnections), m_retention(retention),
+		m_level(level), m_normLevel(level), m_group(group), m_optional(optional)
 {
 	if (m_name.Empty())
 	{

daemon/nntp/NewsServer.h

@@ -32,7 +32,7 @@
 class NewsServer
 {
 public:
-	NewsServer(int id, bool active, const char* name, const char* host, int port,
+	NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 		const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
 		int level, int group, bool optional);
@@ -45,6 +45,7 @@ public:
 	int GetGroup() { return m_group; }
 	const char* GetHost() { return m_host; }
 	int GetPort() { return m_port; }
+	int GetIpVersion() { return m_ipVersion; }
 	const char* GetUser() { return m_user; }
 	const char* GetPassword() { return m_password; }
 	int GetMaxConnections() { return m_maxConnections; }
@@ -64,18 +65,19 @@ private:
 	int m_stateId = 0;
 	bool m_active;
 	CString m_name;
-	int m_group;
 	CString m_host;
 	int m_port;
+	int m_ipVersion;
 	CString m_user;
 	CString m_password;
-	int m_maxConnections;
-	int m_level;
-	int m_normLevel;
 	bool m_joinGroup;
 	bool m_tls;
 	CString m_cipher;
+	int m_maxConnections;
 	int m_retention;
+	int m_level;
+	int m_normLevel;
+	int m_group;
 	bool m_optional = false;
 	time_t m_blockTime = 0;
 };

daemon/nntp/NntpConnection.cpp

@@ -27,10 +27,13 @@
 
 static const int CONNECTION_LINEBUFFER_SIZE = 1024*10;
 
-NntpConnection::NntpConnection(NewsServer* newsServer) : Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
+NntpConnection::NntpConnection(NewsServer* newsServer) :
+	Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
 {
 	m_lineBuf.Reserve(CONNECTION_LINEBUFFER_SIZE);
 	SetCipher(newsServer->GetCipher());
+	SetIPVersion(newsServer->GetIpVersion() == 4 ? Connection::ipV4 :
+		newsServer->GetIpVersion() == 6 ? Connection::ipV6 : Connection::ipAuto);
 }
 
 const char* NntpConnection::Request(const char* req)
@@ -224,10 +227,7 @@ bool NntpConnection::Disconnect()
 {
 	if (m_status == csConnected)
 	{
-		if (!m_broken)
-		{
-			Request("quit\r\n");
-		}
+		Request("quit\r\n");
 		m_activeGroup = nullptr;
 	}
 	return Connection::Disconnect();

daemon/nntp/StatMeter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "StatMeter.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "DiskState.h"
 #include "Util.h"
@@ -57,8 +58,8 @@ void ServerVolume::CalcSlots(time_t locCurTime)
 void ServerVolume::AddData(int bytes)
 {
 	time_t curTime = Util::CurrentTime();
-	time_t locCurTime = curTime + g_Options->GetLocalTimeOffset();
-	time_t locDataTime = m_dataTime + g_Options->GetLocalTimeOffset();
+	time_t locCurTime = curTime + g_WorkState->GetLocalTimeOffset();
+	time_t locDataTime = m_dataTime + g_WorkState->GetLocalTimeOffset();
 
 	int lastMinSlot = m_minSlot;
 	int lastHourSlot = m_hourSlot;
@@ -138,28 +139,28 @@ void ServerVolume::LogDebugInfo()
 
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerSeconds[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerSeconds[i]);
 	}
 	info("Secs: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerMinutes[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerMinutes[i]);
 	}
 	info("Mins: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 24; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerHours[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerHours[i]);
 	}
 	info("Hours: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < (int)m_bytesPerDays.size(); i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", m_firstDay + i, m_bytesPerDays[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", m_firstDay + i, m_bytesPerDays[i]);
 	}
 	info("Days: %s", *msg);
 }
@@ -188,10 +189,10 @@ void StatMeter::AdjustTimeOffset()
 	tmSplittedTime.tm_isdst = -1;
 	time_t locTime = mktime(&tmSplittedTime);
 	time_t localTimeDelta = utcTime - locTime;
-	g_Options->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
+	g_WorkState->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
 	m_lastTimeOffset = utcTime;
 
-	debug("UTC delta: %i (%i+%i)", g_Options->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
+	debug("UTC delta: %i (%i+%i)", g_WorkState->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
 }
 
 /*
@@ -308,8 +309,6 @@ void StatMeter::AddSpeedReading(int bytes)
 	time_t curTime = Util::CurrentTime();
 	int nowSlot = (int)curTime / SPEEDMETER_SLOTSIZE;
 
-	Guard guard(g_Options->GetAccurateRate() ? &m_speedMutex : nullptr);
-
 	if (curTime != m_curSecTime)
 	{
 		m_curSecTime =	curTime;
@@ -382,9 +381,9 @@ void StatMeter::LogDebugInfo()
 	int timeDiff = (int)Util::CurrentTime() - m_speedStartTime * SPEEDMETER_SLOTSIZE;
 	info("      Speed: %i", speed);
 	info("      SpeedStartTime: %i", m_speedStartTime);
-	info("      SpeedTotalBytes: %lli", m_speedTotalBytes);
+	info("      SpeedTotalBytes: %" PRIi64, m_speedTotalBytes);
 	info("      SpeedBytesIndex: %i", m_speedBytesIndex);
-	info("      AllBytes: %lli", m_allBytes);
+	info("      AllBytes: %" PRIi64, m_allBytes);
 	info("      Time: %i", (int)Util::CurrentTime());
 	info("      TimeDiff: %i", timeDiff);
 	for (int i=0; i < SPEEDMETER_SLOTS; i++)
@@ -448,7 +447,7 @@ bool StatMeter::Load(bool* perfectServerMatch)
 
 	for (ServerVolume& serverVolume : m_serverVolumes)
 	{
-		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_Options->GetLocalTimeOffset());
+		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_WorkState->GetLocalTimeOffset());
 	}
 
 	return ok;
@@ -467,20 +466,20 @@ void StatMeter::CheckQuota()
 	bool monthlyQuotaReached = g_Options->GetMonthlyQuota() > 0 && monthBytes >= (int64)g_Options->GetMonthlyQuota() * 1024 * 1024;
 	bool dailyQuotaReached = g_Options->GetDailyQuota() > 0 && dayBytes >= (int64)g_Options->GetDailyQuota() * 1024 * 1024;
 
-	if (monthlyQuotaReached && !g_Options->GetQuotaReached())
+	if (monthlyQuotaReached && !g_WorkState->GetQuotaReached())
 	{
 		warn("Monthly quota reached at %s", *Util::FormatSize(monthBytes));
 	}
-	else if (dailyQuotaReached && !g_Options->GetQuotaReached())
+	else if (dailyQuotaReached && !g_WorkState->GetQuotaReached())
 	{
 		warn("Daily quota reached at %s", *Util::FormatSize(dayBytes));
 	}
-	else if (!monthlyQuotaReached && !dailyQuotaReached && g_Options->GetQuotaReached())
+	else if (!monthlyQuotaReached && !dailyQuotaReached && g_WorkState->GetQuotaReached())
 	{
 		info("Quota lifted");
 	}
 
-	g_Options->SetQuotaReached(monthlyQuotaReached || dailyQuotaReached);
+	g_WorkState->SetQuotaReached(monthlyQuotaReached || dailyQuotaReached);
 }
 
 void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
@@ -489,8 +488,8 @@ void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
 
 	ServerVolume totalVolume = m_serverVolumes[0];
 
-	time_t locTime = Util::CurrentTime() + g_Options->GetLocalTimeOffset();
-	int daySlot = locTime / 86400 - totalVolume.GetFirstDay();
+	time_t locTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
+	int daySlot = (int)(locTime / 86400) - totalVolume.GetFirstDay();
 
 	dayBytes = 0;
 	if (daySlot < (int)totalVolume.BytesPerDays()->size())
@@ -517,7 +516,7 @@ int StatMeter::CalcMonthSlots(ServerVolume& volume)
 {
 	int elapsedDays;
 
-	time_t locCurTime = Util::CurrentTime() + g_Options->GetLocalTimeOffset();
+	time_t locCurTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
 	tm dayparts;
 	gmtime_r(&locCurTime, &dayparts);
 
@@ -534,7 +533,7 @@ int StatMeter::CalcMonthSlots(ServerVolume& volume)
 			dayparts.tm_mon++;
 			prevMonth = Util::Timegm(&dayparts);
 		}
-		elapsedDays = (locCurTime - prevMonth) / 60 / 60 / 24 + 1;
+		elapsedDays = (int)(locCurTime - prevMonth) / 60 / 60 / 24 + 1;
 	}
 	else
 	{

daemon/nntp/StatMeter.h

@@ -84,7 +84,6 @@ public:
 	void AddServerData(int bytes, int serverId);
 	void CalcTotalStat(int* upTimeSec, int* dnTimeSec, int64* allBytes, bool* standBy);
 	void CalcQuotaUsage(int64& monthBytes, int64& dayBytes);
-	bool GetStandBy() { return m_standBy; }
 	void IntervalCheck();
 	void EnterLeaveStandBy(bool enter);
 	GuardedServerVolumes GuardServerVolumes();
@@ -106,7 +105,6 @@ private:
 	int m_speedBytesIndex;
 	int m_curSecBytes;
 	time_t m_curSecTime;
-	Mutex m_speedMutex;
 
 	// time
 	int64 m_allBytes = 0;

daemon/nserv/NServFrontend.cpp

@@ -34,7 +34,7 @@ void NServFrontend::Run()
 	while (!IsStopped())
 	{
 		Update();
-		usleep(100 * 1000);
+		Util::Sleep(100);
 	}
 	// Printing the last messages
 	Update();

daemon/nserv/NServMain.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -44,6 +44,10 @@ struct NServOpts
 	bool generateNzb;
 	int segmentSize;
 	bool quit;
+	int latency;
+	int speed;
+	bool memCache;
+	bool paramError;
 
 	NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts);
 };
@@ -59,7 +63,7 @@ int NServMain(int argc, char* argv[])
 	Options::CmdOptList cmdOpts;
 	NServOpts opts(argc, argv, cmdOpts);
 
-	if (opts.dataDir.Empty())
+	if (opts.dataDir.Empty() || opts.paramError)
 	{
 		NServPrintUsage(argv[0]);
 		return 1;
@@ -85,6 +89,10 @@ int NServMain(int argc, char* argv[])
 	TlsSocket::Init();
 #endif
 
+#ifndef WIN32
+	signal(SIGPIPE, SIG_IGN);
+#endif
+
 	NServFrontend frontend;
 	frontend.Start();
 
@@ -105,16 +113,18 @@ int NServMain(int argc, char* argv[])
 	}
 
 	std::vector<std::unique_ptr<NntpServer>> instances;
+	NntpCache cache;
 
 	for (int i = 0; i < opts.instances; i++)
 	{
 		instances.emplace_back(std::make_unique<NntpServer>(i + 1, opts.bindAddress,
-			opts.firstPort + i, opts.secureCert, opts.secureKey, opts.dataDir, opts.cacheDir));
+			opts.firstPort + i, opts.secureCert, opts.secureKey, opts.dataDir, opts.cacheDir,
+			opts.latency, opts.speed, opts.memCache ? &cache : nullptr));
 		instances.back()->Start();
 	}
 
 	info("Press Ctrl+C to quit");
-	while (getchar()) usleep(1000*200);
+	while (getchar()) Util::Sleep(200);
 
 	for (std::unique_ptr<NntpServer>& serv: instances)
 	{
@@ -130,7 +140,7 @@ int NServMain(int argc, char* argv[])
 		{
 			hasRunning |= serv->IsRunning();
 		}
-		usleep(50 * 1000);
+		Util::Sleep(50);
 	} while (hasRunning);
 
 	return 0;
@@ -143,12 +153,15 @@ void NServPrintUsage(const char* com)
 		"    -d <data-dir>   - directory whose files will be served\n"
 		"  Optional switches:\n"
 		"    -c <cache-dir>  - directory to store encoded articles\n"
+		"    -m              - in-memory cache (unlimited, use with care)\n"
 		"    -l <log-file>   - write into log-file (disabled by default)\n"
 		"    -i <instances>  - number of server instances (default is 1)\n"
 		"    -b <address>    - ip address to bind to (default is 0.0.0.0)\n"
 		"    -p <port>       - port number for the first instance (default is 6791)\n"
 		"    -s <cert> <key> - paths to SSL certificate and key files\n"
 		"    -v <verbose>    - verbosity level 0..3 (default is 2)\n"
+		"    -w <msec>       - response latency (in milliseconds)\n"
+		"    -r <KB/s>       - speed throttling (in kilobytes per second)\n"
 		"    -z <seg-size>   - generate nzbs for all files in data-dir (size in bytes)\n"
 		"    -q              - quit after generating nzbs (in combination with -z)\n"
 		, FileSystem::BaseFileName(com));
@@ -162,9 +175,13 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 	generateNzb = false;
 	segmentSize = 500000;
 	quit = false;
+	latency = 0;
+	memCache = false;
+	speed = 0;
+	paramError = false;
 	int verbosity = 2;
 
-	char short_options[] = "b:c:d:l:p:i:s:v:z:q";
+	char short_options[] = "b:c:d:l:p:i:ms:v:w:r:z:q";
 
 	optind = 2;
 	while (true)
@@ -181,6 +198,10 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 				cacheDir = optind > argc ? nullptr : argv[optind - 1];
 				break;
 
+			case 'm':
+				memCache = true;
+				break;
+
 			case 'l':
 				logFile = optind > argc ? nullptr : argv[optind - 1];
 				break;
@@ -207,6 +228,14 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 				verbosity = atoi(optind > argc ? "1" : argv[optind - 1]);
 				break;
 
+			case 'w':
+				latency = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
+			case 'r':
+				speed = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
 			case 'z':
 				generateNzb = true;
 				segmentSize = atoi(optind > argc ? "500000" : argv[optind - 1]);
@@ -218,6 +247,11 @@ NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
 		}
 	}
 
+	if (optind < argc)
+	{
+		paramError = true;
+	}
+
 	if (logFile.Empty())
 	{
 		cmdOpts.push_back("WriteLog=none");

daemon/nserv/NntpServer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -28,9 +28,10 @@ class NntpProcessor : public Thread
 {
 public:
 	NntpProcessor(int id, int serverId, const char* dataDir, const char* cacheDir,
-		const char* secureCert, const char* secureKey) :
+		const char* secureCert, const char* secureKey, int latency, int speed, NntpCache* cache) :
 		m_id(id), m_serverId(serverId), m_dataDir(dataDir), m_cacheDir(cacheDir),
-		m_secureCert(secureCert), m_secureKey(secureKey) {}
+		m_secureCert(secureCert), m_secureKey(secureKey), m_latency(latency),
+		m_speed(speed), m_cache(cache) {}
 	~NntpProcessor() { m_connection->Disconnect(); }
 	virtual void Run();
 	void SetConnection(std::unique_ptr<Connection>&& connection) { m_connection = std::move(connection); }
@@ -43,24 +44,37 @@ private:
 	const char* m_cacheDir;
 	const char* m_secureCert;
 	const char* m_secureKey;
+	int m_latency;
+	int m_speed;
 	const char* m_messageid;
 	CString m_filename;
 	int m_part;
 	int64 m_offset;
 	int m_size;
 	bool m_sendHeaders;
+	int64 m_start;
+	NntpCache* m_cache;
 
 	void ServArticle();
 	void SendSegment();
 	bool ServerInList(const char* servList);
+	void SendData(const char* buffer, int size);
 };
 
+
 void NntpServer::Run()
 {
 	debug("Entering NntpServer-loop");
 
 	info("Listening on port %i", m_port);
 
+#ifdef WIN32
+	if (m_speed > 0)
+	{
+		timeBeginPeriod(1);
+	}
+#endif
+
 	int num = 1;
 
 	while (!IsStopped())
@@ -89,12 +103,12 @@ void NntpServer::Run()
 				break;
 			}
 			m_connection.reset();
-			usleep(500 * 1000);
+			Util::Sleep(500);
 			continue;
 		}
 		
-		NntpProcessor* commandThread = new NntpProcessor(num++, m_id,
-			m_dataDir, m_cacheDir, m_secureCert, m_secureKey);
+		NntpProcessor* commandThread = new NntpProcessor(num++, m_id, m_dataDir,
+			m_cacheDir, m_secureCert, m_secureKey, m_latency, m_speed, m_cache);
 		commandThread->SetAutoDestroy(true);
 		commandThread->SetConnection(std::move(acceptedConnection));
 		commandThread->Start();
@@ -134,6 +148,7 @@ void NntpProcessor::Run()
 	}
 #endif
 
+	info("[%i] Incoming connection from: %s", m_id, m_connection->GetHost() );
 	m_connection->WriteLine("200 Welcome (NServ)\r\n");
 
 	CharBuffer buf(1024);
@@ -157,7 +172,7 @@ void NntpProcessor::Run()
 		}
 		else if (!strncasecmp(line, "GROUP ", 6))
 		{
-			m_connection->WriteLine(CString::FormatStr("211 0 0 0 %s\r\n", line + 7));
+			m_connection->WriteLine(CString::FormatStr("211 0 0 0 %s\r\n", line + 6));
 		}
 		else if (!strncasecmp(line, "AUTHINFO ", 9))
 		{
@@ -199,6 +214,11 @@ void NntpProcessor::ServArticle()
 {
 	detail("[%i] Serving: %s", m_id, m_messageid);
 
+	if (m_latency)
+	{
+		Util::Sleep(m_latency);
+	}
+
 	bool ok = false;
 
 	const char* from = strchr(m_messageid, '?');
@@ -209,7 +229,7 @@ void NntpProcessor::ServArticle()
 
 	if (from && off && to && end)
 	{
-		m_filename.Set(m_messageid + 1, from - m_messageid - 1);
+		m_filename.Set(m_messageid + 1, (int)(from - m_messageid - 1));
 		m_part = atoi(from + 1);
 		m_offset = atoll(off + 1);
 		m_size = atoi(to + 1);
@@ -248,16 +268,34 @@ bool NntpProcessor::ServerInList(const char* servList)
 
 void NntpProcessor::SendSegment()
 {
-	detail("[%i] Sending segment %s (%i=%lli:%i)", m_id, *m_filename, m_part, (long long)m_offset, m_size);
+	detail("[%i] Sending segment %s (%i=%" PRIi64 ":%i)", m_id, *m_filename, m_part, m_offset, m_size);
+
+	if (m_speed > 0)
+	{
+		m_start = Util::CurrentTicks();
+	}
 
 	BString<1024> fullFilename("%s/%s", m_dataDir, *m_filename);
 	BString<1024> cacheFileDir("%s/%s", m_cacheDir, *m_filename);
-	BString<1024> cacheFileName("%i=%lli-%i", m_part, (long long)m_offset, m_size);
+	BString<1024> cacheFileName("%i=%" PRIi64 "-%i", m_part, m_offset, m_size);
 	BString<1024> cacheFullFilename("%s/%s", *cacheFileDir, *cacheFileName);
+	BString<1024> cacheKey("%s/%s", *m_filename, *cacheFileName);
+
+	const char* cachedData = nullptr;
+	int cachedSize;
+	if (m_cache)
+	{
+		m_cache->Find(cacheKey, cachedData, cachedSize);
+	}
 
 	DiskFile cacheFile;
-	bool readCache = m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
-	bool writeCache = m_cacheDir && !readCache;
+	bool readCache = !cachedData && m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
+	bool writeCache = !cachedData && m_cacheDir && !readCache;
+	StringBuilder cacheMem;
+	if (m_cache && !cachedData)
+	{
+		cacheMem.Reserve((int)(m_size * 1.1));
+	}
 
 	CString errmsg;
 	if (writeCache && !FileSystem::ForceDirectories(cacheFileDir, errmsg))
@@ -270,23 +308,27 @@ void NntpProcessor::SendSegment()
 		error("Could not create file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
 	}
 
-	if (!readCache && !FileSystem::FileExists(fullFilename))
+	if (!cachedData && !readCache && !FileSystem::FileExists(fullFilename))
 	{
 		m_connection->WriteLine(CString::FormatStr("430 Article not found\r\n"));
 		return;
 	}
 
 	YEncoder encoder(fullFilename, m_part, m_offset, m_size, 
-		[con = m_connection.get(), writeCache, &cacheFile](const char* buf, int size)
+		[proc = this, writeCache, &cacheFile, &cacheMem](const char* buf, int size)
 		{
+			if (proc->m_cache)
+			{
+				cacheMem.Append(buf);
+			}
 			if (writeCache)
 			{
 				cacheFile.Write(buf, size);
 			}
-			con->Send(buf, size);
+			proc->SendData(buf, size);
 		});
 
-	if (!readCache && !encoder.OpenFile(errmsg))
+	if (!cachedData && !readCache && !encoder.OpenFile(errmsg))
 	{
 		m_connection->WriteLine(CString::FormatStr("403 %s\r\n", *errmsg));
 		return;
@@ -300,7 +342,11 @@ void NntpProcessor::SendSegment()
 		m_connection->WriteLine("\r\n");
 	}
 
-	if (readCache)
+	if (cachedData)
+	{
+		SendData(cachedData, cachedSize);
+	}
+	else if (readCache)
 	{
 		cacheFile.Seek(0, DiskFile::soEnd);
 		int size = (int)cacheFile.Position();
@@ -310,12 +356,88 @@ void NntpProcessor::SendSegment()
 		{
 			error("Could not read file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
 		}
-		m_connection->Send(buf, size);
+		if (m_cache)
+		{
+			cacheMem.Append(buf, size);
+		}
+		SendData(buf, size);
 	}
 	else
 	{
 		encoder.WriteSegment();
 	}
 
+	if (!cachedData && cacheMem.Length() > 0)
+	{
+		m_cache->Append(cacheKey, cacheMem, cacheMem.Length());
+	}
+
 	m_connection->WriteLine(".\r\n");
 }
+
+void NntpProcessor::SendData(const char* buffer, int size)
+{
+	if (m_speed == 0)
+	{
+		m_connection->Send(buffer, size);
+		return;
+	}
+
+	int64 expectedTime = (int64)1000 * size / (m_speed * 1024) - (Util::CurrentTicks() - m_start) / 1000;
+
+	int chunkNum = 21;
+	int chunkSize = size;
+	int pause = 0;
+
+	while (pause < 1 && chunkNum > 1)
+	{
+		chunkNum--;
+		chunkSize = size / chunkNum;
+		pause = (int)(expectedTime / chunkNum);
+	}
+
+	int sent = 0;
+	for (int i = 0; i < chunkNum; i++)
+	{
+		int len = sent + chunkSize < size ? chunkSize : size - sent;
+
+		while (sent + len < size && *(buffer + sent + len) != '\r')
+		{
+			len++;
+		}
+		
+		m_connection->Send(buffer + sent, len);
+		int64 now = Util::CurrentTicks();
+		if (now + pause * 1000 < m_start + expectedTime * 1000)
+		{
+			Util::Sleep(pause);
+		}
+		sent += len;
+	}
+}
+
+
+void NntpCache::Append(const char* key, const char* data, int len)
+{
+	Guard guard(m_lock);
+	if (!len)
+	{
+		len = strlen(data);
+	}
+	m_items.emplace(key, std::make_unique<CacheItem>(key, data, len));
+}
+
+bool NntpCache::Find(const char* key, const char*& data, int& size)
+{
+	Guard guard(m_lock);
+
+	CacheMap::iterator pos = m_items.find(key);
+	if (pos != m_items.end())
+	{
+		data = (*pos).second->m_data;
+		size = (*pos).second->m_size;
+		return true;
+	}
+
+	return false;
+}

daemon/nserv/NntpServer.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,14 +23,41 @@
 
 #include "Thread.h"
 #include "Connection.h"
+#include "Util.h"
+
+class NntpCache
+{
+public:
+	void Append(const char* key, const char* data, int len = 0);
+	bool Find(const char* key, const char*& data, int& size);
+
+private:
+	class CacheItem
+	{
+	public:
+		CacheItem(const char* key, const char* data, int size) :
+			m_key(key), m_data(data), m_size(size) {}
+
+		CString m_key;
+		CString m_data;
+		int m_size = 0;
+	};
+
+	typedef std::unordered_map<std::string, std::unique_ptr<CacheItem>> CacheMap;
+
+	CacheMap m_items;
+	Mutex m_lock;
+};
 
 class NntpServer : public Thread
 {
 public:
 	NntpServer(int id, const char* host, int port, const char* secureCert,
-		const char* secureKey, const char* dataDir, const char* cacheDir) :
+		const char* secureKey, const char* dataDir, const char* cacheDir,
+		int latency, int speed, NntpCache* cache) :
 		m_id(id), m_host(host), m_port(port), m_secureCert(secureCert),
-		m_secureKey(secureKey), m_dataDir(dataDir), m_cacheDir(cacheDir) {}
+		m_secureKey(secureKey), m_dataDir(dataDir), m_cacheDir(cacheDir),
+		m_latency(latency), m_speed(speed), m_cache(cache) {}
 	virtual void Run();
 	virtual void Stop();
 
@@ -38,11 +65,14 @@ private:
 	int m_id;
 	CString m_host;
 	int m_port;
-	CString m_dataDir;
-	CString m_cacheDir;
 	CString m_secureCert;
 	CString m_secureKey;
 	std::unique_ptr<Connection> m_connection;
+	CString m_dataDir;
+	CString m_cacheDir;
+	int m_latency;
+	int m_speed;
+	NntpCache* m_cache;
 };
 
 #endif

daemon/nserv/NzbGenerator.cpp

@@ -118,11 +118,11 @@ void NzbGenerator::AppendFile(DiskFile& outfile, const char* filename, const cha
 	for (int segno = 1; segno <= segmentCount; segno++)
 	{
 		int segSize = (int)(segOffset + m_segmentSize < fileSize ? m_segmentSize : fileSize - segOffset);
-		outfile.Print("<segment bytes=\"%i\" number=\"%i\">%s%s%s?%i=%lli:%i</segment>\n",
-			m_segmentSize, segno, 
+		outfile.Print("<segment bytes=\"%i\" number=\"%i\">%s%s%s?%i=%" PRIi64 ":%i</segment>\n",
+			m_segmentSize, segno,
 			relativePath ? relativePath : "",
 			relativePath ? "/" : "",
-			FileSystem::BaseFileName(filename), segno, (long long)segOffset, (int)segSize);
+			FileSystem::BaseFileName(filename), segno, segOffset, segSize);
 		segOffset += segSize;
 	}
 

daemon/nserv/YEncoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -64,10 +64,10 @@ void YEncoder::WriteSegment()
 	StringBuilder outbuf;
 	outbuf.Reserve(std::max(2048, std::min((int)(m_size * 1.1), 16 * 1024 * 1024)));
 
-	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%lli name=%s\r\n", m_part, (long long)m_fileSize, FileSystem::BaseFileName(m_filename)));
-	outbuf.Append(CString::FormatStr("=ypart begin=%lli end=%lli\r\n", (long long)(m_offset + 1), (long long)(m_offset + m_size)));
+	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%" PRIi64 " name=%s\r\n", m_part, m_fileSize, FileSystem::BaseFileName(m_filename)));
+	outbuf.Append(CString::FormatStr("=ypart begin=%" PRIi64 " end=%" PRIi64 "\r\n", m_offset + 1, m_offset + m_size));
 
-	uint32 crc = 0xFFFFFFFF;
+	Crc32 crc;
 	CharBuffer inbuf(std::min(m_size, 16 * 1024 * 1024));
 	int lnsz = 0;
 	char* out = (char*)outbuf + outbuf.Length();
@@ -82,7 +82,7 @@ void YEncoder::WriteSegment()
 			return; // error;
 		}
 
-		crc = Util::Crc32m(crc, (uchar*)(const char*)inbuf, (int)readBytes);
+		crc.Append((uchar*)(const char*)inbuf, (int)readBytes);
 
 		char* in = inbuf;
 		while (readBytes > 0)
@@ -122,10 +122,8 @@ void YEncoder::WriteSegment()
 			}
 		}
 	}
-	crc ^= 0xFFFFFFFF;
-
 	m_diskfile.Close();
 
-	outbuf.Append(CString::FormatStr("=yend size=%i part=0 pcrc32=%08x\r\n", m_size, (unsigned int)crc));
+	outbuf.Append(CString::FormatStr("=yend size=%i part=0 pcrc32=%08x\r\n", m_size, (unsigned int)crc.Finish()));
 	m_writeFunc(outbuf, outbuf.Length());
 }

daemon/postprocess/DirectUnpack.cpp

@@ -0,0 +1,558 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "DirectUnpack.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "Options.h"
+
+void DirectUnpack::StartJob(NzbInfo* nzbInfo)
+{
+	DirectUnpack* directUnpack = new DirectUnpack();
+	directUnpack->m_nzbId = nzbInfo->GetId();
+	directUnpack->SetAutoDestroy(true);
+
+	nzbInfo->SetUnpackThread(directUnpack);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsRunning);
+
+	directUnpack->Start();
+}
+
+void DirectUnpack::Run()
+{
+	debug("Entering DirectUnpack-loop for %i", m_nzbId);
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo)
+		{
+			debug("Could not find NzbInfo for %i", m_nzbId);
+			return;
+		}
+
+		m_name = nzbInfo->GetName();
+		m_destDir = nzbInfo->GetDestDir();
+		m_finalDir = nzbInfo->BuildFinalDirName();
+
+		NzbParameter* parameter = nzbInfo->GetParameters()->Find("*Unpack:Password");
+		if (parameter)
+		{
+			m_password = parameter->GetValue();
+		}
+	}
+
+	m_infoName.Format("direct unpack for %s", *m_name);
+	m_infoNameUp.Format("Direct unpack for %s", *m_name); // first letter in upper case
+
+	FindArchiveFiles();
+
+	SetInfoName(m_infoName);
+	SetWorkingDir(m_destDir);
+
+	while (!IsStopped())
+	{
+		CString archive;
+		{
+			Guard guard(m_volumeMutex);
+			if (!m_archives.empty())
+			{
+				archive = std::move(m_archives.front());
+				m_archives.pop_front();
+			}
+		}
+
+		if (archive)
+		{
+			if (!m_processed)
+			{
+				PrintMessage(Message::mkInfo, "Directly unpacking %s", *m_name);
+				Cleanup();
+				CreateUnpackDir();
+				m_processed = true;
+			}
+
+			ExecuteUnrar(archive);
+
+			if (!m_unpackOk)
+			{
+				break;
+			}
+		}
+		else
+		{
+			if (m_nzbCompleted)
+			{
+				break;
+			}
+			Util::Sleep(100);
+		}
+	}
+
+	if (!m_unpackOk)
+	{
+		Cleanup();
+	}
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo || nzbInfo->GetUnpackThread() != this)
+		{
+			debug("Could not find NzbInfo for %s", *m_infoName);
+			return;
+		}
+
+		nzbInfo->SetUnpackThread(nullptr);
+		nzbInfo->SetDirectUnpackStatus(m_processed ? m_unpackOk && !IsStopped() ? NzbInfo::nsSuccess : NzbInfo::nsFailure : NzbInfo::nsNone);
+
+		if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			nzbInfo->AddMessage(Message::mkInfo, BString<1024>("%s successful", *m_infoNameUp));
+		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsFailure)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("%s failed", *m_infoNameUp));
+		}
+
+		if (nzbInfo->GetPostInfo() && nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			std::move(m_extractedArchives.begin(), m_extractedArchives.end(),
+				std::back_inserter(*nzbInfo->GetPostInfo()->GetExtractedArchives()));
+		}
+
+		AddExtraTime(nzbInfo);
+
+		if (nzbInfo->GetPostInfo())
+		{
+			nzbInfo->GetPostInfo()->SetWorking(false);
+		}
+	}
+
+	debug("Exiting DirectUnpack-loop for %i", m_nzbId);
+}
+
+void DirectUnpack::ExecuteUnrar(const char* archiveName)
+{
+	// Format:
+	//   unrar x -y -p- -o+ -vp <archive.part0001.rar> <dest-dir>/_unpack/
+
+	ParamList params;
+	if (!PrepareCmdParams(g_Options->GetUnrarCmd(), &params, "unrar"))
+	{
+		return;
+	}
+
+	if (!params.Exists("x") && !params.Exists("e"))
+	{
+		params.emplace_back("x");
+	}
+
+	params.emplace_back("-y");
+
+	if (!m_password.Empty())
+	{
+		params.push_back(CString::FormatStr("-p%s", *m_password));
+	}
+	else
+	{
+		params.emplace_back("-p-");
+	}
+
+	if (!params.Exists("-o+") && !params.Exists("-o-"))
+	{
+		params.emplace_back("-o+");
+	}
+
+	params.emplace_back("-vp");
+
+	params.emplace_back(archiveName);
+	m_unpackExtendedDir = FileSystem::MakeExtendedPath(m_unpackDir, true);
+	params.push_back(*BString<1024>("%s%c", *m_unpackExtendedDir, PATH_SEPARATOR));
+	SetArgs(std::move(params));
+	SetLogPrefix("Unrar");
+	ResetEnv();
+
+	m_allOkMessageReceived = false;
+
+	SetNeedWrite(true);
+	m_unpacking = true;
+	int exitCode = Execute();
+	m_unpacking = false;
+	SetLogPrefix(nullptr);
+
+	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
+
+	if (!m_unpackOk && exitCode > 0)
+	{
+		PrintMessage(Message::mkError, "Unrar error code: %i", exitCode);
+	}
+}
+
+bool DirectUnpack::PrepareCmdParams(const char* command, ParamList* params, const char* infoName)
+{
+	if (FileSystem::FileExists(command))
+	{
+		params->emplace_back(command);
+		return true;
+	}
+
+	std::vector<CString> cmdArgs = Util::SplitCommandLine(command);
+	if (cmdArgs.empty())
+	{
+		PrintMessage(Message::mkError, "Could not start %s, failed to parse command line: %s", infoName, command);
+		m_unpackOk = false;
+		return false;
+	}
+
+	std::move(cmdArgs.begin(), cmdArgs.end(), std::back_inserter(*params));
+
+	return true;
+}
+
+void DirectUnpack::CreateUnpackDir()
+{
+	bool useInterDir = !Util::EmptyStr(g_Options->GetInterDir()) &&
+		!strncmp(m_destDir, g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
+		m_destDir[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
+
+	m_finalDirCreated = useInterDir && !FileSystem::DirectoryExists(m_finalDir);
+
+	const char* destDir = useInterDir && !m_finalDir.Empty() ? *m_finalDir : *m_destDir;
+
+	m_unpackDir.Format("%s%c%s", destDir, PATH_SEPARATOR, "_unpack");
+
+	detail("Unpacking into %s", *m_unpackDir);
+
+	CString errmsg;
+	if (!FileSystem::ForceDirectories(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not create directory %s: %s", *m_unpackDir, *errmsg);
+	}
+}
+
+void DirectUnpack::FindArchiveFiles()
+{
+	Guard guard(m_volumeMutex);
+
+	DirBrowser dir(m_destDir);
+	while (const char* filename = dir.Next())
+	{
+		if (IsMainArchive(filename))
+		{
+			BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+			if (!FileSystem::DirectoryExists(fullFilename))
+			{
+				m_archives.emplace_back(filename);
+			}
+		}
+	}
+}
+
+bool DirectUnpack::IsMainArchive(const char* filename)
+{
+	RegEx regExRarPart(".*\\.part([0-9]+)\\.rar$");
+	bool mainPart = Util::EndsWith(filename, ".rar", false) &&
+		(!regExRarPart.Match(filename) || atoi(filename + regExRarPart.GetMatchStart(1)) == 1);
+	return mainPart;
+}
+
+/**
+ * Unrar prints "Insert disk"-message without new line terminator.
+ * In order to become the message we analyze the output after every char.
+ */
+bool DirectUnpack::ReadLine(char* buf, int bufSize, FILE* stream)
+{
+	int i = 0;
+	bool backspace = false;
+
+	for (; i < bufSize - 1; i++)
+	{
+		int ch = fgetc(stream);
+
+		if (ch == '\b')
+		{
+			backspace = true;
+		}
+		if (!backspace)
+		{
+			buf[i] = ch;
+			buf[i+1] = '\0';
+		}
+		if (ch == EOF)
+		{
+			break;
+		}
+		if (ch == '\n')
+		{
+			i++;
+			break;
+		}
+		if (i > 35 && ch == ' ' &&
+			!strncmp(buf, "Insert disk with", 16) && strstr(buf, " [C]ontinue, [Q]uit "))
+		{
+			i++;
+			break;
+		}
+	}
+
+	// skip unnecessary progress messages
+	if (!strncmp(buf, "...", 3))
+	{
+		buf[0] = '\0';
+	}
+
+	buf[i] = '\0';
+
+	return i > 0;
+}
+
+void DirectUnpack::AddMessage(Message::EKind kind, const char* text)
+{
+	debug("%s", text);
+
+	BString<1024> msgText = text;
+	int len = strlen(text);
+
+	// Modify unrar messages for better readability:
+	// remove the destination path part from message "Extracting file.xxx"
+	if (!strncmp(text, "Unrar: Extracting  ", 19) &&
+		!strncmp(text + 19, m_unpackExtendedDir, strlen(m_unpackExtendedDir)))
+	{
+		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackExtendedDir) + 1);
+	}
+
+	if (!strncmp(text, "Unrar: Insert disk with", 23) && strstr(text, " [C]ontinue, [Q]uit"))
+	{
+		BString<1024> filename;
+		filename.Set(text + 24, (int)(strstr(text, " [C]ontinue, [Q]uit") - text - 24));
+		WaitNextVolume(filename);
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (nzbInfo)
+	{
+		nzbInfo->AddMessage(kind, msgText);
+	}
+	else
+	{
+		ScriptController::AddMessage(kind, msgText);
+	}
+
+	if (!strncmp(msgText, "Unrar: Extracting ", 18) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, msgText + 7);
+	}
+
+	if (!strncmp(text, "Unrar: Extracting from ", 23) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, text + 7);
+		m_extractedArchives.emplace_back(text + 23);
+	}
+
+	if (!IsStopped() && (
+		!strncmp(text, "Unrar: Checksum error in the encrypted file", 42) ||
+		!strncmp(text, "Unrar: CRC failed in the encrypted file", 39) ||
+		!strncmp(text, "Unrar: The specified password is incorrect.", 43) ||
+		strstr(text, " : packed data CRC failed in volume") ||
+		strstr(text, " : packed data checksum error in volume") ||
+		(len > 13 && !strncmp(text + len - 13, " - CRC failed", 13)) ||
+		(len > 18 && !strncmp(text + len - 18, " - checksum failed", 18)) ||
+		!strncmp(text, "Unrar: WARNING: You need to start extraction from a previous volume", 67)))
+	{
+		Stop(downloadQueue, nzbInfo);
+	}
+
+	if (!strncmp(text, "Unrar: All OK", 13))
+	{
+		m_allOkMessageReceived = true;
+	}
+}
+
+void DirectUnpack::Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("Stopping direct unpack for %s", *m_infoName);
+	if (m_processed)
+	{
+		if (nzbInfo)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Cancelling %s", *m_infoName));
+		}
+		else
+		{
+			warn("Cancelling %s", *m_infoName);
+		}
+	}
+	AddExtraTime(nzbInfo);
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetWorking(false);
+	}
+	Thread::Stop();
+	if (m_unpacking)
+	{
+		Terminate();
+	}
+}
+
+void DirectUnpack::WaitNextVolume(const char* filename)
+{
+	debug("WaitNextVolume for %s", filename);
+
+	// Stop direct unpack if destination directory was changed during unpack
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (nzbInfo && (strcmp(m_destDir, nzbInfo->GetDestDir()) ||
+			strcmp(m_finalDir, nzbInfo->BuildFinalDirName())))
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Destination directory changed for %s", nzbInfo->GetName()));
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+
+	BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+	if (FileSystem::FileExists(fullFilename))
+	{
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+	else
+	{
+		Guard guard(m_volumeMutex);
+		m_waitingFile = filename;
+		if (m_nzbCompleted)
+		{
+			// nzb completed but unrar waits for another volume
+			PrintMessage(Message::mkWarning, "Could not find volume %s", filename);
+			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+			NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+}
+
+void DirectUnpack::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo)
+{
+	debug("FileDownloaded for %s/%s", fileInfo->GetNzbInfo()->GetName(), fileInfo->GetFilename());
+
+	if (fileInfo->GetNzbInfo()->GetFailedArticles() > 0)
+	{
+		Stop(downloadQueue, fileInfo->GetNzbInfo());
+		return;
+	}
+
+	Guard guard(m_volumeMutex);
+	if (m_waitingFile && !strcasecmp(fileInfo->GetFilename(), m_waitingFile))
+	{
+		m_waitingFile = nullptr;
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+
+	if (IsMainArchive(fileInfo->GetFilename()))
+	{
+		m_archives.emplace_back(fileInfo->GetFilename());
+	}
+}
+
+void DirectUnpack::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDownloaded for %s", nzbInfo->GetName());
+
+	Guard guard(m_volumeMutex);
+	m_nzbCompleted = true;
+	if (m_waitingFile)
+	{
+		// nzb completed but unrar waits for another volume
+		nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Unrar: Could not find volume %s", *m_waitingFile));
+		Stop(downloadQueue, nzbInfo);
+		return;
+	}
+
+	m_extraStartTime = Util::CurrentTime();
+
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(m_progressLabel);
+	}
+}
+
+void DirectUnpack::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDeleted for %s", nzbInfo->GetName());
+
+	nzbInfo->SetUnpackThread(nullptr);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsFailure);
+	Stop(downloadQueue, nzbInfo);
+}
+
+// Remove _unpack-dir
+void DirectUnpack::Cleanup()
+{
+	debug("Cleanup for %s", *m_infoName);
+
+	CString errmsg;
+	if (FileSystem::DirectoryExists(m_unpackDir) &&
+		!FileSystem::DeleteDirectoryWithContent(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not delete temporary directory %s: %s", *m_unpackDir, *errmsg);
+	}
+
+	if (m_finalDirCreated)
+	{
+		FileSystem::DeleteDirectory(m_finalDir);
+	}
+}
+
+void DirectUnpack::SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel)
+{
+	m_progressLabel = progressLabel;
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(progressLabel);
+	}
+}
+
+void DirectUnpack::AddExtraTime(NzbInfo* nzbInfo)
+{
+	if (m_extraStartTime)
+	{
+		int extraTime = (int)(Util::CurrentTime() - m_extraStartTime);
+		nzbInfo->SetUnpackSec(nzbInfo->GetUnpackSec() + extraTime);
+		nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() + extraTime);
+		m_extraStartTime = 0;
+	}
+}
+
+bool DirectUnpack::IsArchiveFilename(const char* filename)
+{
+	if (Util::EndsWith(filename, ".rar", false))
+	{
+		return true;
+	}
+
+	RegEx regExRarMultiSeq(".*\\.[r-z][0-9][0-9]$");
+	return regExRarMultiSeq.Match(filename);
+}

daemon/postprocess/DirectUnpack.h

@@ -0,0 +1,86 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef DIRECTUNPACK_H
+#define DIRECTUNPACK_H
+
+#include "Log.h"
+#include "Thread.h"
+#include "DownloadInfo.h"
+#include "Script.h"
+
+class DirectUnpack : public Thread, public ScriptController
+{
+public:
+	virtual void Run();
+	static void StartJob(NzbInfo* nzbInfo);
+	void Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo);
+	void NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	static bool IsArchiveFilename(const char* filename);
+
+protected:
+	virtual bool ReadLine(char* buf, int bufSize, FILE* stream);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+
+private:
+	class ParamList : public std::vector<CString>
+	{
+	public:
+		bool Exists(const char* param) { return std::find(begin(), end(), param) != end(); }
+	};
+
+	typedef std::deque<CString> ArchiveList;
+
+	int m_nzbId;
+	CString m_name;
+	CString m_infoName;
+	CString m_infoNameUp;
+	CString m_destDir;
+	CString m_finalDir;
+	CString m_unpackDir;
+	CString m_unpackExtendedDir;
+	CString m_password;
+	CString m_waitingFile;
+	CString m_progressLabel;
+	bool m_allOkMessageReceived = false;
+	bool m_unpackOk = false;
+	bool m_finalDirCreated = false;
+	bool m_nzbCompleted = false;
+	Mutex m_volumeMutex;
+	ArchiveList m_archives;
+	bool m_processed = false;
+	bool m_unpacking = false;
+	time_t m_extraStartTime = 0;
+	ArchiveList m_extractedArchives;
+
+	void CreateUnpackDir();
+	void FindArchiveFiles();
+	void ExecuteUnrar(const char* archiveName);
+	bool PrepareCmdParams(const char* command, ParamList* params, const char* infoName);
+	void WaitNextVolume(const char* filename);
+	void Cleanup();
+	bool IsMainArchive(const char* filename);
+	void SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel);
+	void AddExtraTime(NzbInfo* nzbInfo);
+};
+
+#endif

daemon/postprocess/DupeMatcher.cpp

@@ -80,7 +80,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 		curTime + timeoutSec > Util::CurrentTime() &&
 		curTime >= Util::CurrentTime())					// in a case clock was changed
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	if (unrar.IsRunning())
@@ -91,7 +91,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 	// wait until terminated or killed
 	while (unrar.IsRunning())
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	*maxSize = unrar.m_maxSize;
@@ -161,7 +161,7 @@ bool DupeMatcher::Prepare()
 	char filename[1024];
 	FindLargestFile(m_destDir, filename, sizeof(filename), &m_maxSize, &m_compressed);
 	bool sizeOK = SizeDiffOK(m_maxSize, m_expectedSize, 20);
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, sizeOK ? "" : ", size mismatch");
 	return sizeOK;
 }
@@ -173,7 +173,7 @@ bool DupeMatcher::MatchDupeContent(const char* dupeDir)
 	char filename[1024];
 	FindLargestFile(dupeDir, filename, sizeof(filename), &dupeMaxSize, &dupeCompressed);
 	bool ok = dupeMaxSize == m_maxSize && dupeCompressed == m_compressed;
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, ok ? "" : ", size mismatch");
 	return ok;
 }

daemon/postprocess/ParChecker.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -42,14 +42,6 @@ const char* Par2CmdLineErrStr[] = { "OK",
 	"internal error occurred",
 	"out of memory" };
 
-// Sleep interval for synchronisation (microseconds)
-#ifdef WIN32
-// Windows doesn't allow sleep intervals less than one millisecond
-#define SYNC_SLEEP_INTERVAL 1000
-#else
-#define SYNC_SLEEP_INTERVAL 100
-#endif
-
 class RepairThread;
 
 class Repairer : public Par2::Par2Repairer, public ParChecker::AbstractRepairer
@@ -74,8 +66,8 @@ protected:
 private:
 	typedef vector<Thread*> Threads;
 
-	Par2::CommandLine commandLine;
 	ParChecker* m_owner;
+	Par2::CommandLine commandLine;
 	Threads m_threads;
 	bool m_parallel;
 	Mutex progresslock;
@@ -83,6 +75,7 @@ private:
 	virtual void BeginRepair();
 	virtual void EndRepair();
 	void RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength);
+	static void SyncSleep();
 
 	friend class ParChecker;
 	friend class RepairThread;
@@ -106,6 +99,11 @@ private:
 	volatile bool m_working = false;
 };
 
+class RepairCreatorPacket : public Par2::CreatorPacket
+{
+	friend class ParChecker;
+};
+
 Par2::Result Repairer::PreProcess(const char *parFilename)
 {
 	BString<100> memParam("-m%i", g_Options->GetParBuffer());
@@ -248,7 +246,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 
 		if (!jobAdded)
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			SyncSleep();
 		}
 	}
 
@@ -263,7 +261,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 			if (repairThread->IsWorking())
 			{
 				working = true;
-				usleep(SYNC_SLEEP_INTERVAL);
+				SyncSleep();
 				break;
 			}
 		}
@@ -300,6 +298,17 @@ void Repairer::RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t b
 	}
 }
 
+// Sleep for synchronisation
+void Repairer::SyncSleep()
+{
+#ifdef WIN32
+	// Windows doesn't allow sleep intervals less than one millisecond
+	Sleep(1);
+#else
+	usleep(100);
+#endif
+}
+
 void RepairThread::Run()
 {
 	while (!IsStopped())
@@ -311,7 +320,7 @@ void RepairThread::Run()
 		}
 		else
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			Repairer::SyncSleep();
 		}
 	}
 }
@@ -368,6 +377,7 @@ int ParChecker::StreamBuf::overflow(int ch)
 
 void ParChecker::Cleanup()
 {
+	Guard guard(m_repairerMutex);
 	m_repairer.reset();
 	m_queuedParFiles.clear();
 	m_processedFiles.clear();
@@ -408,14 +418,14 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 
 		if (!IsStopped())
 		{
-			BString<1024> fullParFilename( "%s%c%s", *m_destDir, (int)PATH_SEPARATOR, *parFilename);
+			BString<1024> fullParFilename( "%s%c%s", *m_destDir, PATH_SEPARATOR, *parFilename);
 
 			int baseLen = 0;
 			ParParser::ParseParFilename(parFilename, true, &baseLen, nullptr);
 			BString<1024> infoName;
 			infoName.Set(parFilename, baseLen);
 
-			BString<1024> parInfoName("%s%c%s", *m_nzbName, (int)PATH_SEPARATOR, *infoName);
+			BString<1024> parInfoName("%s%c%s", *m_nzbName, PATH_SEPARATOR, *infoName);
 			SetInfoName(parInfoName);
 
 			EStatus status = RunParCheck(fullParFilename);
@@ -425,11 +435,6 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 			{
 				allStatus = status;
 			}
-
-			if (g_Options->GetBrokenLog())
-			{
-				WriteBrokenLog(status);
-			}
 		}
 	}
 
@@ -464,6 +469,9 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		return psFailed;
 	}
 
+	CString creator = GetPacketCreator();
+	info("Recovery files created by: %s", creator.Empty() ? "<unknown program>" : *creator);
+
 	m_stage = ptVerifyingSources;
 	res = GetRepairer()->Process(false);
 
@@ -580,7 +588,8 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		{
 			m_errMsg = Par2CmdLineErrStr[res];
 		}
-		PrintMessage(Message::mkError, "Repair failed for %s: %s", *m_infoName, *m_errMsg);
+		PrintMessage(Message::mkError, "Repair failed for %s: %s. Recovery files created by: %s",
+			*m_infoName, *m_errMsg, creator.Empty() ? "<unknown program>" : *creator);
 	}
 
 	Cleanup();
@@ -594,7 +603,10 @@ int ParChecker::PreProcessPar()
 	{
 		Cleanup();
 
-		m_repairer = std::make_unique<Repairer>(this);
+		{
+			Guard guard(m_repairerMutex);
+			m_repairer = std::make_unique<Repairer>(this);
+		}
 
 		res = GetRepairer()->PreProcess(m_parFilename);
 		debug("ParChecker: PreProcess-result=%i", res);
@@ -682,7 +694,7 @@ bool ParChecker::LoadMainParBak()
 					Guard guard(m_queuedParFilesMutex);
 					queuedParFilesChanged = m_queuedParFilesChanged;
 				}
-				usleep(100 * 1000);
+				Util::Sleep(100);
 			}
 		}
 	}
@@ -748,7 +760,7 @@ int ParChecker::ProcessMorePars()
 						Guard guard(m_queuedParFilesMutex);
 						queuedParFilesChanged = m_queuedParFilesChanged;
 					}
-					usleep(100 * 1000);
+					Util::Sleep(100);
 				}
 			}
 		}
@@ -814,29 +826,25 @@ bool ParChecker::AddSplittedFragments()
 	DirBrowser dir(m_destDir);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") && !IsParredFile(filename) && !IsProcessedFile(filename))
+		if (!IsParredFile(filename) && !IsProcessedFile(filename))
 		{
 			for (Par2::Par2RepairerSourceFile *sourcefile : GetRepairer()->sourcefiles)
 			{
 				std::string target = sourcefile->TargetFileName();
-				const char* filename2 = target.c_str();
-				const char* basename2 = FileSystem::BaseFileName(filename2);
-				int baseLen = strlen(basename2);
+				const char* current = FileSystem::BaseFileName(target.c_str());
 
-				if (!strncasecmp(filename, basename2, baseLen))
+				// if file was renamed by par-renamer we also check the original filename
+				const char* original = FindFileOrigname(current);
+
+				if (MaybeSplittedFragement(filename, current) ||
+					(!Util::EmptyStr(original) && strcasecmp(original, current) &&
+					MaybeSplittedFragement(filename, original)))
 				{
-					const char* p = filename + baseLen;
-					if (*p == '.')
-					{
-						for (p++; *p && strchr("0123456789", *p); p++) ;
-						if (!*p)
-						{
-							debug("Found splitted fragment %s", filename);
-							BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
-							Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
-							extrafiles.push_back(extrafile);
-						}
-					}
+					detail("Found splitted fragment %s", filename);
+					BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+					Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
+					extrafiles.push_back(extrafile);
+					break;
 				}
 			}
 		}
@@ -857,6 +865,40 @@ bool ParChecker::AddSplittedFragments()
 	return fragmentsAdded;
 }
 
+bool ParChecker::MaybeSplittedFragement(const char* filename1, const char* filename2)
+{
+	// check if name is same but the first name has additional numerical extension
+	int len = strlen(filename2);
+	if (!strncasecmp(filename1, filename2, len))
+	{
+		const char* p = filename1 + len;
+		if (*p == '.')
+		{
+			for (p++; *p && strchr("0123456789", *p); p++) ;
+			if (!*p)
+			{
+				return true;
+			}
+		}
+	}
+
+	// check if same name (without extension) and extensions are numerical and exactly 3 characters long
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+	if (ext1 && ext2 && (strlen(ext1) == 4) && (strlen(ext2) == 4) &&
+		!strncasecmp(filename1, filename2, ext1 - filename1))
+	{
+		for (ext1++; *ext1 && strchr("0123456789", *ext1); ext1++) ;
+		for (ext2++; *ext2 && strchr("0123456789", *ext2); ext2++) ;
+		if (!*ext1 && !*ext2)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
 bool ParChecker::AddMissingFiles()
 {
 	return AddExtraFiles(true, false, m_destDir);
@@ -942,8 +984,7 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d
 	DirBrowser dir(directory);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") &&
-			(externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename))))
+		if (externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename)))
 		{
 			BString<1024> fullfilename("%s%c%s", directory, PATH_SEPARATOR, filename);
 			extrafiles.emplace_back(*fullfilename, FileSystem::FileSize(fullfilename));
@@ -1186,49 +1227,14 @@ void ParChecker::CheckEmptyFiles()
 
 void ParChecker::Cancel()
 {
-	GetRepairer()->cancelled = true;
-	QueueChanged();
-}
-
-void ParChecker::WriteBrokenLog(EStatus status)
-{
-	BString<1024> brokenLogName("%s%c_brokenlog.txt", *m_destDir, (int)PATH_SEPARATOR);
-
-	if (status != psRepairNotNeeded || FileSystem::FileExists(brokenLogName))
 	{
-		DiskFile file;
-		if (file.Open(brokenLogName, DiskFile::omAppend))
+		Guard guard(m_repairerMutex);
+		if (m_repairer)
 		{
-			if (status == psFailed)
-			{
-				if (IsStopped())
-				{
-					file.Print("Repair cancelled for %s\n", *m_infoName);
-				}
-				else
-				{
-					file.Print("Repair failed for %s: %s\n", *m_infoName, *m_errMsg);
-				}
-			}
-			else if (status == psRepairPossible)
-			{
-				file.Print("Repair possible for %s\n", *m_infoName);
-			}
-			else if (status == psRepaired)
-			{
-				file.Print("Successfully repaired %s\n", *m_infoName);
-			}
-			else if (status == psRepairNotNeeded)
-			{
-				file.Print("Repair not needed for %s\n", *m_infoName);
-			}
-			file.Close();
-		}
-		else
-		{
-			PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
+			m_repairer->GetRepairer()->cancelled = true;
 		}
 	}
+	QueueChanged();
 }
 
 void ParChecker::SaveSourceList()
@@ -1399,7 +1405,7 @@ bool ParChecker::VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32
 	{
 		const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 		Par2::u32 blockCrc = entry->crc;
-		parCrc = i == 0 ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+		parCrc = i == 0 ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 	}
 	debug("Block-CRC: %x, filename: %s", parCrc, FileSystem::BaseFileName(sourceFile->GetTargetFile()->FileName().c_str()));
 
@@ -1423,7 +1429,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 		int64 blockEnd = blockStart + blocksize < fileSize - 1 ? blockStart + blocksize : fileSize - 1;
 		bool blockOK = false;
 		bool blockEndFound = false;
-		Par2::u64 curOffset = 0;
+		int64 curOffset = 0;
 		for (Segment& segment : segments)
 		{
 			if (!blockOK && segment.GetSuccess() && segment.GetOffset() <= blockStart &&
@@ -1477,7 +1483,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 			}
 			const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 			Par2::u32 blockCrc = entry->crc;
-			parCrc = blockStart == i ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+			parCrc = blockStart == i ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 		}
 		else
 		{
@@ -1537,7 +1543,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 
 		if (segment.GetOffset() >= start && segment.GetOffset() + segment.GetSize() <= end)
 		{
-			downloadCrc = !started ? segment.GetCrc() : Util::Crc32Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
+			downloadCrc = !started ? segment.GetCrc() : Crc32::Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
 			started = true;
 		}
 
@@ -1555,7 +1561,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 				return false;
 			}
 
-			downloadCrc = Util::Crc32Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
+			downloadCrc = Crc32::Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
 
 			break;
 		}
@@ -1576,21 +1582,37 @@ bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, ui
 	}
 
 	CharBuffer buffer(1024 * 64);
-	uint32 downloadCrc = 0xFFFFFFFF;
+	Crc32 downloadCrc;
 
 	int cnt = buffer.Size();
 	while (cnt == buffer.Size() && start < end)
 	{
 		int needBytes = end - start + 1 > buffer.Size() ? buffer.Size() : (int)(end - start + 1);
 		cnt = (int)file.Read(buffer, needBytes);
-		downloadCrc = Util::Crc32m(downloadCrc, (uchar*)(char*)buffer, cnt);
+		downloadCrc.Append((uchar*)(char*)buffer, cnt);
 		start += cnt;
 	}
 
-	downloadCrc ^= 0xFFFFFFFF;
-
-	*downloadCrcOut = downloadCrc;
+	*downloadCrcOut = downloadCrc.Finish();
 	return true;
 }
 
+CString ParChecker::GetPacketCreator()
+{
+	Par2::CREATORPACKET* creatorpacket;
+	if (GetRepairer()->creatorpacket &&
+		(creatorpacket = (Par2::CREATORPACKET*)(((RepairCreatorPacket*)GetRepairer()->creatorpacket)->packetdata)))
+	{
+		int len = (int)(creatorpacket->header.length - sizeof(Par2::PACKET_HEADER));
+		BString<1024> creator;
+		if (len > 0)
+		{
+			creator.Set((const char*)creatorpacket->client, len);
+		}
+		return *creator;
+	}
+
+	return nullptr;
+}
+
 #endif

daemon/postprocess/ParChecker.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -132,6 +132,7 @@ protected:
 	virtual void RegisterParredFile(const char* filename) {}
 	virtual bool IsParredFile(const char* filename) { return false; }
 	virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) { return fsUnknown; }
+	virtual const char* FindFileOrigname(const char* filename) { return nullptr; }
 	virtual void RequestDupeSources(DupeSourceList* dupeSourceList) {}
 	virtual void StatDupeSources(DupeSourceList* dupeSourceList) {}
 	EStage GetStage() { return m_stage; }
@@ -186,6 +187,7 @@ private:
 	StreamBuf m_parErrStream{this, Message::mkError};
 	std::ostream m_parCout{&m_parOutStream};
 	std::ostream m_parCerr{&m_parErrStream};
+	Mutex m_repairerMutex;
 
 	// "m_repairer" should be of type "Par2::Par2Repairer", however to prevent the
 	// including of libpar2-headers into this header-file we use an empty abstract class.
@@ -204,7 +206,6 @@ private:
 	bool AddDupeFiles();
 	bool AddExtraFiles(bool onlyMissing, bool externalDir, const char* directory);
 	bool IsProcessedFile(const char* filename);
-	void WriteBrokenLog(EStatus status);
 	void SaveSourceList();
 	void DeleteLeftovers();
 	void signal_filename(std::string str);
@@ -220,6 +221,8 @@ private:
 		uint32* downloadCrc);
 	bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrc);
 	void CheckEmptyFiles();
+	CString GetPacketCreator();
+	bool MaybeSplittedFragement(const char* filename1, const char* filename2);
 
 	friend class Repairer;
 };

daemon/postprocess/ParParser.cpp

@@ -44,7 +44,7 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 			bool exists = false;
 			for (CString& filename2 : fileList)
 			{
-				exists = SameParCollection(filename, filename2);
+				exists = SameParCollection(filename, filename2, true);
 				if (exists)
 				{
 					break;
@@ -59,11 +59,11 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 	return fileList && !fileList->empty();
 }
 
-bool ParParser::SameParCollection(const char* filename1, const char* filename2)
+bool ParParser::SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames)
 {
 	int baseLen1 = 0, baseLen2 = 0;
-	return ParseParFilename(filename1, false, &baseLen1, nullptr) &&
-		ParseParFilename(filename2, false, &baseLen2, nullptr) &&
+	return ParseParFilename(filename1, confirmedFilenames, &baseLen1, nullptr) &&
+		ParseParFilename(filename2, confirmedFilenames, &baseLen2, nullptr) &&
 		baseLen1 == baseLen2 &&
 		!strncasecmp(filename1, filename2, baseLen1);
 }

daemon/postprocess/ParParser.h

@@ -31,7 +31,7 @@ public:
 
 	static bool FindMainPars(const char* path, ParFileList* fileList);
 	static bool ParseParFilename(const char* parFilename, bool confirmedFilename, int* baseNameLen, int* blocks);
-	static bool SameParCollection(const char* filename1, const char* filename2);
+	static bool SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames);
 };
 
 #endif

daemon/postprocess/ParRenamer.cpp

@@ -160,7 +160,7 @@ void ParRenamer::LoadParFile(const char* parFilename)
 	}
 
 	m_loadedParList.emplace_back(parFilename);
-	PrintMessage(Message::mkInfo, "Loaded par2-file %s for par-rename", parFilename);
+	PrintMessage(Message::mkInfo, "Loaded par2-file %s for par-rename", FileSystem::BaseFileName(parFilename));
 
 	for (std::pair<const Par2::MD5Hash, Par2::Par2RepairerSourceFile*>& entry : repairer.sourcefilemap)
 	{
@@ -172,7 +172,7 @@ void ParRenamer::LoadParFile(const char* parFilename)
 		Par2::Par2RepairerSourceFile* sourceFile = entry.second;
 		if (!sourceFile || !sourceFile->GetDescriptionPacket())
 		{
-			PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", parFilename);
+			PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", FileSystem::BaseFileName(parFilename));
 			m_badParList.emplace_back(parFilename);
 			m_hasDamagedParFiles = true;
 			continue;

daemon/postprocess/PrePostProcessor.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "PrePostProcessor.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "HistoryCoordinator.h"
 #include "DupeCoordinator.h"
@@ -34,6 +35,7 @@
 #include "NzbFile.h"
 #include "QueueScript.h"
 #include "ParParser.h"
+#include "DirectUnpack.h"
 
 PrePostProcessor::PrePostProcessor()
 {
@@ -48,23 +50,35 @@ void PrePostProcessor::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetReloadQueue())
+	if (g_Options->GetServerMode())
 	{
 		SanitisePostQueue();
 	}
 
 	while (!IsStopped())
 	{
-		if (!g_Options->GetTempPausePostprocess() && m_queuedJobs)
+		if (g_WorkState->GetTempPausePostprocess())
+		{
+			// Postprocess is paused: just wait and loop
+			Util::Sleep(200);
+			continue;
+		}
+
+		if (m_queuedJobs)
 		{
 			// check post-queue every 200 msec
 			CheckPostQueue();
+			Util::Sleep(200);
+		}
+		else
+		{
+			// Wait until we get the stop signal or more jobs in the queue
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&]{ return m_queuedJobs || IsStopped(); });
 		}
-
-		usleep(200 * 1000);
 	}
 
 	WaitJobs();
@@ -76,7 +90,7 @@ void PrePostProcessor::WaitJobs()
 {
 	debug("PrePostProcessor: waiting for jobs to complete");
 
-	// wait 5 seconds until all jobs gracefully finish
+	// wait 5 seconds until all post-processing jobs gracefully finish
 	time_t waitStart = Util::CurrentTime();
 	while (Util::CurrentTime() < waitStart + 5)
 	{
@@ -88,10 +102,10 @@ void PrePostProcessor::WaitJobs()
 			}
 		}
 		CheckPostQueue();
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
-	// kill remaining jobs; not safe but we can't wait any longer
+	// kill remaining post-processing jobs; not safe but we can't wait any longer
 	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 		for (NzbInfo* postJob : m_activeJobs)
@@ -107,21 +121,64 @@ void PrePostProcessor::WaitJobs()
 		}
 	}
 
+	// wait 5 seconds until direct unpack threads gracefully finish
+	waitStart = Util::CurrentTime();
+	while (Util::CurrentTime() < waitStart + 5)
+	{
+		{
+			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+			if (std::find_if(downloadQueue->GetQueue()->begin(),
+				downloadQueue->GetQueue()->end(),
+				[](const std::unique_ptr<NzbInfo>& nzbInfo)
+				{
+					return nzbInfo->GetUnpackThread() != nullptr;
+				}) == downloadQueue->GetQueue()->end())
+			{
+				break;
+			}
+		}
+		Util::Sleep(200);
+	}
+
+	// disconnect remaining direct unpack jobs
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			nzbInfo->SetUnpackThread(nullptr);
+		}
+	}
+
 	debug("PrePostProcessor: Jobs are completed");
 }
 
 void PrePostProcessor::Stop()
 {
 	Thread::Stop();
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
 
-	for (NzbInfo* postJob : m_activeJobs)
 	{
-		if (postJob->GetPostInfo() && postJob->GetPostInfo()->GetPostThread())
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		for (NzbInfo* postJob : m_activeJobs)
 		{
-			postJob->GetPostInfo()->GetPostThread()->Stop();
+			if (postJob->GetPostInfo() && postJob->GetPostInfo()->GetPostThread())
+			{
+				postJob->GetPostInfo()->GetPostThread()->Stop();
+			}
+		}
+
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			if (nzbInfo->GetUnpackThread())
+			{
+				((DirectUnpack*)nzbInfo->GetUnpackThread())->Stop(downloadQueue, nzbInfo);
+			}
 		}
 	}
+
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
 }
 
 /**
@@ -160,14 +217,20 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 	}
 
 	DownloadQueue::Aspect* queueAspect = (DownloadQueue::Aspect*)aspect;
-	if (queueAspect->action == DownloadQueue::eaNzbFound)
+	if (queueAspect->action == DownloadQueue::eaNzbFound ||
+		queueAspect->action == DownloadQueue::eaUrlFound)
 	{
 		NzbFound(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
-	else if (queueAspect->action == DownloadQueue::eaNzbAdded)
+	else if (queueAspect->action == DownloadQueue::eaNzbAdded ||
+		queueAspect->action == DownloadQueue::eaUrlAdded)
 	{
 		NzbAdded(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
+	else if (queueAspect->action == DownloadQueue::eaNzbNamed)
+	{
+		g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeNzbNamed);
+	}
 	else if (queueAspect->action == DownloadQueue::eaNzbDeleted &&
 		queueAspect->nzbInfo->GetDeleting() &&
 		!queueAspect->nzbInfo->GetPostInfo() &&
@@ -179,12 +242,20 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			"Collection %s deleted from queue", queueAspect->nzbInfo->GetName());
 		NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
+	else if (queueAspect->action == DownloadQueue::eaUrlDeleted)
+	{
+		NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo);
+	}
+	else if (queueAspect->action == DownloadQueue::eaUrlFailed)
+	{
+		NzbCompleted(queueAspect->downloadQueue, queueAspect->nzbInfo, true);
+	}
 	else if ((queueAspect->action == DownloadQueue::eaFileCompleted ||
 		queueAspect->action == DownloadQueue::eaFileDeleted))
 	{
-		if (queueAspect->action == DownloadQueue::eaFileCompleted && !queueAspect->nzbInfo->GetPostInfo())
+		if (queueAspect->action == DownloadQueue::eaFileCompleted)
 		{
-			g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeFileDownloaded);
+			FileDownloaded(queueAspect->downloadQueue, queueAspect->nzbInfo, queueAspect->fileInfo);
 		}
 
 #ifndef DISABLE_PARCHECK
@@ -206,7 +277,7 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			 queueAspect->fileInfo->GetDupeDeleted()) &&
 			queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() != NzbInfo::dsHealth &&
 			!queueAspect->nzbInfo->GetPostInfo() &&
-			IsNzbFileCompleted(queueAspect->nzbInfo, true))
+			queueAspect->nzbInfo->IsDownloadCompleted(true))
 		{
 			queueAspect->nzbInfo->PrintMessage(Message::mkInfo,
 				"Collection %s completely downloaded", queueAspect->nzbInfo->GetName());
@@ -217,7 +288,7 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			(queueAspect->action == DownloadQueue::eaFileCompleted &&
 			 queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() > NzbInfo::dsNone)) &&
 			!queueAspect->nzbInfo->GetPostInfo() &&
-			IsNzbFileCompleted(queueAspect->nzbInfo, false))
+			queueAspect->nzbInfo->IsDownloadCompleted(false))
 		{
 			queueAspect->nzbInfo->PrintMessage(Message::mkInfo,
 				"Collection %s deleted from queue", queueAspect->nzbInfo->GetName());
@@ -238,8 +309,13 @@ void PrePostProcessor::NzbAdded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 {
 	if (g_Options->GetParCheck() != Options::pcForce)
 	{
-		downloadQueue->EditEntry(nzbInfo->GetId(),
-			DownloadQueue::eaGroupPauseExtraPars, nullptr);
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupPauseExtraPars, nullptr);
+	}
+
+	if (g_Options->GetReorderFiles() && nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Reordering files for %s", nzbInfo->GetName());
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr);
 	}
 
 	if (nzbInfo->GetDeleteStatus() == NzbInfo::dsDupe ||
@@ -263,12 +339,11 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 		g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeNzbDeleted);
 	}
 
-	if (!nzbInfo->GetPostInfo() && g_Options->GetDecode())
+	if (!nzbInfo->GetPostInfo() && !g_Options->GetRawArticle() && !g_Options->GetSkipWrite())
 	{
 		nzbInfo->PrintMessage(Message::mkInfo, "Queueing %s for post-processing", nzbInfo->GetName());
 
 		nzbInfo->EnterPostProcess();
-		m_queuedJobs++;
 
 		if (nzbInfo->GetParStatus() == NzbInfo::psNone &&
 			g_Options->GetParCheck() != Options::pcAlways &&
@@ -277,16 +352,39 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 			nzbInfo->SetParStatus(NzbInfo::psSkipped);
 		}
 
-		downloadQueue->Save();
+		if (nzbInfo->GetUnpackThread())
+		{
+			nzbInfo->GetPostInfo()->SetWorking(true);
+			m_activeJobs.push_back(nzbInfo);
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDownloaded(downloadQueue, nzbInfo);
+		}
+
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
+
+		// We have more jobs in the queue, notify Run()
+		Guard guard(m_waitMutex);
+		m_queuedJobs++;
+		m_waitCond.NotifyAll();
 	}
 	else
 	{
+		if (nzbInfo->GetUnpackThread())
+		{
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDownloaded(downloadQueue, nzbInfo);
+		}
+
 		NzbCompleted(downloadQueue, nzbInfo, true);
 	}
 }
 
 void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 {
+	if (nzbInfo->GetUnpackThread())
+	{
+		((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDeleted(downloadQueue, nzbInfo);
+	}
+
 	if (nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
 	{
 		nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
@@ -308,6 +406,7 @@ void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo
 
 void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool saveQueue)
 {
+	bool downloadDupe = nzbInfo->GetDupeHint() == NzbInfo::dhRedownloadAuto;
 	bool addToHistory = g_Options->GetKeepHistory() > 0 && !nzbInfo->GetAvoidHistory();
 	if (addToHistory)
 	{
@@ -321,7 +420,8 @@ void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbIn
 		(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsHealth ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsBad ||
-		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan))
+		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan ||
+		 (nzbInfo->GetDeleteStatus() == NzbInfo::dsCopy && downloadDupe)))
 	{
 		g_DupeCoordinator->NzbCompleted(downloadQueue, nzbInfo);
 		needSave = true;
@@ -355,20 +455,20 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo)
 		// download was cancelled, deleting already downloaded files from disk
 		for (CompletedFile& completedFile: nzbInfo->GetCompletedFiles())
 		{
-			BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName());
+			BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename());
 			if (FileSystem::FileExists(fullFileName))
 			{
-				detail("Deleting file %s", completedFile.GetFileName());
+				detail("Deleting file %s", completedFile.GetFilename());
 				FileSystem::DeleteFile(fullFileName);
 			}
 		}
 
-		// delete .out.tmp-files and _brokenlog.txt
+		// delete .out.tmp-files
 		DirBrowser dir(nzbInfo->GetDestDir());
 		while (const char* filename = dir.Next())
 		{
 			int len = strlen(filename);
-			if ((len > 8 && !strcmp(filename + len - 8, ".out.tmp")) || !strcmp(filename, "_brokenlog.txt"))
+			if (len > 8 && !strcmp(filename + len - 8, ".out.tmp"))
 			{
 				BString<1024> fullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, filename);
 				detail("Deleting file %s", filename);
@@ -377,10 +477,7 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo)
 		}
 
 		// delete old directory (if empty)
-		if (FileSystem::DirEmpty(nzbInfo->GetDestDir()))
-		{
-			FileSystem::RemoveDirectory(nzbInfo->GetDestDir());
-		}
+		FileSystem::DeleteDirectory(nzbInfo->GetDestDir());
 	}
 }
 
@@ -433,7 +530,8 @@ void PrePostProcessor::CleanupJobs(DownloadQueue* downloadQueue)
 		[processor = this, downloadQueue](NzbInfo* postJob)
 		{
 			PostInfo* postInfo = postJob->GetPostInfo();
-			if (!postInfo->GetWorking())
+			if (!postInfo->GetWorking() &&
+				!(postInfo->GetPostThread() && postInfo->GetPostThread()->IsRunning()))
 			{
 				delete postInfo->GetPostThread();
 				postInfo->SetPostThread(nullptr);
@@ -511,10 +609,12 @@ NzbInfo* PrePostProcessor::PickNextJob(DownloadQueue* downloadQueue, bool allowP
 	{
 		if (nzbInfo1->GetPostInfo() && !nzbInfo1->GetPostInfo()->GetWorking() &&
 			!g_QueueScriptCoordinator->HasJob(nzbInfo1->GetId(), nullptr) &&
+			nzbInfo1->GetDirectUnpackStatus() != NzbInfo::nsRunning &&
 			(!nzbInfo || nzbInfo1->GetPriority() > nzbInfo->GetPriority()) &&
-			(!g_Options->GetPausePostProcess() || nzbInfo1->GetForcePriority()) &&
+			(!g_WorkState->GetPausePostProcess() || nzbInfo1->GetForcePriority()) &&
 			(allowPar || !nzbInfo1->GetPostInfo()->GetNeedParCheck()) &&
-			IsNzbFileCompleted(nzbInfo1, true))
+			(std::find(m_activeJobs.begin(), m_activeJobs.end(), nzbInfo1) == m_activeJobs.end()) &&
+			nzbInfo1->IsDownloadCompleted(true))
 		{
 			nzbInfo = nzbInfo1;
 		}
@@ -541,13 +641,11 @@ void PrePostProcessor::CheckPostQueue()
 			break;
 		}
 
-		Util::SetStandByMode(false);
-
 		m_activeJobs.push_back(postJob);
 
 		PostInfo* postInfo = postJob->GetPostInfo();
 		if (postInfo->GetStage() == PostInfo::ptQueued &&
-			(!g_Options->GetPausePostProcess() || postInfo->GetNzbInfo()->GetForcePriority()))
+			(!g_WorkState->GetPausePostProcess() || postInfo->GetNzbInfo()->GetForcePriority()))
 		{
 			StartJob(downloadQueue, postInfo, allowPar);
 			CheckRequestPar(downloadQueue);
@@ -561,10 +659,14 @@ void PrePostProcessor::CheckPostQueue()
 		downloadQueue->Save();
 		UpdatePauseState();
 	}
+
+	Util::SetStandByMode(m_activeJobs.empty());
 }
 
 void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo, bool allowPar)
 {
+	NzbInfo* nzbInfo = postInfo->GetNzbInfo();
+
 	if (!postInfo->GetStartTime())
 	{
 		postInfo->SetStartTime(Util::CurrentTime());
@@ -574,8 +676,8 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 	postInfo->SetFileProgress(0);
 	postInfo->SetProgressLabel("");
 
-	if (postInfo->GetNzbInfo()->GetParRenameStatus() == NzbInfo::rsNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone &&
+	if (nzbInfo->GetParRenameStatus() == NzbInfo::rsNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
 		g_Options->GetParRename())
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptParRenaming);
@@ -584,10 +686,10 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 	}
 
 #ifndef DISABLE_PARCHECK
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone)
+	if (nzbInfo->GetParStatus() == NzbInfo::psNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
 	{
-		if (ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+		if (ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 		{
 			if (!allowPar)
 			{
@@ -601,54 +703,54 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		}
 		else
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
-				"Nothing to par-check for %s", postInfo->GetNzbInfo()->GetName());
-			postInfo->GetNzbInfo()->SetParStatus(NzbInfo::psSkipped);
+			nzbInfo->PrintMessage(Message::mkInfo,
+				"Nothing to par-check for %s", nzbInfo->GetName());
+			nzbInfo->SetParStatus(NzbInfo::psSkipped);
 		}
 		return;
 	}
 	
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped &&
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
 		((g_Options->GetParScan() != Options::psDupe &&
-		  postInfo->GetNzbInfo()->CalcHealth() < postInfo->GetNzbInfo()->CalcCriticalHealth(false) &&
-		  postInfo->GetNzbInfo()->CalcCriticalHealth(false) < 1000) ||
-		  postInfo->GetNzbInfo()->CalcHealth() == 0) &&
-		ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+		  nzbInfo->CalcHealth() < nzbInfo->CalcCriticalHealth(false) &&
+		  nzbInfo->CalcCriticalHealth(false) < 1000) ||
+		  nzbInfo->CalcHealth() == 0) &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 	{
-		if (postInfo->GetNzbInfo()->CalcHealth() == 0)
+		if (nzbInfo->CalcHealth() == 0)
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
-				"Skipping par-check for %s due to health 0%%", postInfo->GetNzbInfo()->GetName());
+			nzbInfo->PrintMessage(Message::mkWarning,
+				"Skipping par-check for %s due to health 0%%", nzbInfo->GetName());
 		}
 		else
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
+			nzbInfo->PrintMessage(Message::mkWarning,
 				"Skipping par-check for %s due to health %.1f%% below critical %.1f%%",
-				postInfo->GetNzbInfo()->GetName(),
-				postInfo->GetNzbInfo()->CalcHealth() / 10.0, postInfo->GetNzbInfo()->CalcCriticalHealth(false) / 10.0);
+				nzbInfo->GetName(),
+				nzbInfo->CalcHealth() / 10.0, nzbInfo->CalcCriticalHealth(false) / 10.0);
 		}
-		postInfo->GetNzbInfo()->SetParStatus(NzbInfo::psFailure);
+		nzbInfo->SetParStatus(NzbInfo::psFailure);
 		return;
 	}
 	
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped &&
-		postInfo->GetNzbInfo()->GetFailedSize() - postInfo->GetNzbInfo()->GetParFailedSize() > 0 &&
-		ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
+		nzbInfo->GetFailedSize() - nzbInfo->GetParFailedSize() > 0 &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 	{
-		postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
+		nzbInfo->PrintMessage(Message::mkInfo,
 			"Collection %s with health %.1f%% needs par-check",
-			postInfo->GetNzbInfo()->GetName(), postInfo->GetNzbInfo()->CalcHealth() / 10.0);
+			nzbInfo->GetName(), nzbInfo->CalcHealth() / 10.0);
 		postInfo->SetRequestParCheck(true);
 		return;
 	}
 #endif
 
-	NzbParameter* unpackParameter = postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:", false);
+	NzbParameter* unpackParameter = nzbInfo->GetParameters()->Find("*Unpack:");
 	bool wantUnpack = !(unpackParameter && !strcasecmp(unpackParameter->GetValue(), "no"));
-	bool unpack = wantUnpack && postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone;
+	bool unpack = wantUnpack && nzbInfo->GetUnpackStatus() == NzbInfo::usNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone;
 
-	if (postInfo->GetNzbInfo()->GetRarRenameStatus() == NzbInfo::rsNone &&
+	if (nzbInfo->GetRarRenameStatus() == NzbInfo::rsNone &&
 		unpack && g_Options->GetRarRename())
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptRarRenaming);
@@ -656,43 +758,63 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		return;
 	}
 
-	bool parFailed = postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psFailure ||
-		postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psRepairPossible ||
-		postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psManual;
+#ifndef DISABLE_PARCHECK
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+		g_Options->GetParCheck() == Options::pcAuto &&
+		!UnpackController::HasCompletedArchiveFiles(nzbInfo) &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
+	{
+		nzbInfo->PrintMessage(Message::mkInfo,
+			"Requesting par-check for collection %s without archive files",
+			nzbInfo->GetName());
+		postInfo->SetRequestParCheck(true);
+		return;
+	}
+#endif
+
+	bool parFailed = nzbInfo->GetParStatus() == NzbInfo::psFailure ||
+		nzbInfo->GetParStatus() == NzbInfo::psRepairPossible ||
+		nzbInfo->GetParStatus() == NzbInfo::psManual;
 
 	bool cleanup = !unpack && wantUnpack &&
-		postInfo->GetNzbInfo()->GetCleanupStatus() == NzbInfo::csNone &&
+		nzbInfo->GetCleanupStatus() == NzbInfo::csNone &&
 		!Util::EmptyStr(g_Options->GetExtCleanupDisk()) &&
-		((postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSuccess &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usFailure &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usSpace &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usPassword) ||
-		 (postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSuccess &&
-		  postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psFailure) ||
-		 ((postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone ||
-		   postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSkipped) &&
-		  (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psNone ||
-		   postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped) &&
-		  postInfo->GetNzbInfo()->CalcHealth() == 1000));
+		((nzbInfo->GetParStatus() == NzbInfo::psSuccess &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usPassword) ||
+		 (nzbInfo->GetUnpackStatus() == NzbInfo::usSuccess &&
+		  nzbInfo->GetParStatus() != NzbInfo::psFailure) ||
+		 ((nzbInfo->GetUnpackStatus() == NzbInfo::usNone ||
+		   nzbInfo->GetUnpackStatus() == NzbInfo::usSkipped) &&
+		  (nzbInfo->GetParStatus() == NzbInfo::psNone ||
+		   nzbInfo->GetParStatus() == NzbInfo::psSkipped) &&
+		  nzbInfo->CalcHealth() == 1000));
 
 	bool moveInter = !unpack &&
-		postInfo->GetNzbInfo()->GetMoveStatus() == NzbInfo::msNone &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usFailure &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usSpace &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usPassword &&
-		postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psFailure &&
-		postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psManual &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone &&
+		nzbInfo->GetMoveStatus() == NzbInfo::msNone &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usPassword &&
+		nzbInfo->GetParStatus() != NzbInfo::psFailure &&
+		nzbInfo->GetParStatus() != NzbInfo::psManual &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+		!(((nzbInfo->GetUnpackStatus() == NzbInfo::usNone ||
+		   nzbInfo->GetUnpackStatus() == NzbInfo::usSkipped) &&
+		  (nzbInfo->GetParStatus() == NzbInfo::psNone ||
+		   nzbInfo->GetParStatus() == NzbInfo::psSkipped) &&
+		  nzbInfo->CalcHealth() < 1000)) &&
 		!Util::EmptyStr(g_Options->GetInterDir()) &&
-		!strncmp(postInfo->GetNzbInfo()->GetDestDir(), g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
-		postInfo->GetNzbInfo()->GetDestDir()[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
+		!strncmp(nzbInfo->GetDestDir(), g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
+		nzbInfo->GetDestDir()[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
 
 	if (unpack && parFailed)
 	{
-		postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
-			"Skipping unpack for %s due to %s", postInfo->GetNzbInfo()->GetName(),
-			postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psManual ? "required par-repair" : "par-failure");
-		postInfo->GetNzbInfo()->SetUnpackStatus(NzbInfo::usSkipped);
+		nzbInfo->PrintMessage(Message::mkWarning,
+			"Skipping unpack for %s due to %s", nzbInfo->GetName(),
+			nzbInfo->GetParStatus() == NzbInfo::psManual ? "required par-repair" : "par-failure");
+		nzbInfo->SetUnpackStatus(NzbInfo::usSkipped);
 		unpack = false;
 	}
 
@@ -730,33 +852,15 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post
 
 	nzbInfo->LeavePostProcess();
 
-	if (IsNzbFileCompleted(nzbInfo, true))
+	if (nzbInfo->IsDownloadCompleted(true))
 	{
 		NzbCompleted(downloadQueue, nzbInfo, false);
 	}
 
+	Guard guard(m_waitMutex);
 	m_queuedJobs--;
 }
 
-bool PrePostProcessor::IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars)
-{
-	if (nzbInfo->GetActiveDownloads())
-	{
-		return false;
-	}
-
-	for (FileInfo* fileInfo : nzbInfo->GetFileList())
-	{
-		if ((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) &&
-			!fileInfo->GetDeleted())
-		{
-			return false;
-		}
-	}
-
-	return true;
-}
-
 void PrePostProcessor::UpdatePauseState()
 {
 	bool needPause = false;
@@ -789,16 +893,16 @@ void PrePostProcessor::UpdatePauseState()
 		}
 	}
 
-	if (needPause && !g_Options->GetTempPauseDownload())
+	if (needPause && !g_WorkState->GetTempPauseDownload())
 	{
 		info("Pausing download before post-processing");
 	}
-	else if (!needPause && g_Options->GetTempPauseDownload())
+	else if (!needPause && g_WorkState->GetTempPauseDownload())
 	{
 		info("Unpausing download after post-processing");
 	}
 
-	g_Options->SetTempPauseDownload(needPause);
+	g_WorkState->SetTempPauseDownload(needPause);
 }
 
 bool PrePostProcessor::EditList(DownloadQueue* downloadQueue, IdList* idList,
@@ -837,6 +941,11 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 						postInfo->GetPostThread()->Stop();
 						ok = true;
 					}
+					else if (postInfo->GetNzbInfo()->GetUnpackThread())
+					{
+						((DirectUnpack*)postInfo->GetNzbInfo()->GetUnpackThread())->NzbDeleted(downloadQueue, postInfo->GetNzbInfo());
+						ok = true;
+					}
 					else
 					{
 						error("Internal error in PrePostProcessor::QueueDelete");
@@ -846,7 +955,16 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 				{
 					postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
 						"Deleting queued post-job %s", postInfo->GetNzbInfo()->GetName());
+
 					JobCompleted(downloadQueue, postInfo);
+
+					m_activeJobs.erase(std::remove_if(m_activeJobs.begin(), m_activeJobs.end(),
+						[postInfo](NzbInfo* postJob)
+						{
+							return postInfo == postJob->GetPostInfo();
+						}),
+						m_activeJobs.end());
+
 					ok = true;
 				}
 				break;
@@ -854,5 +972,39 @@ bool PrePostProcessor::PostQueueDelete(DownloadQueue* downloadQueue, IdList* idL
 		}
 	}
 
+	if (ok)
+	{
+		downloadQueue->Save();
+	}
+
 	return ok;
 }
+
+void PrePostProcessor::FileDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileInfo* fileInfo)
+{
+	if (!nzbInfo->GetPostInfo())
+	{
+		g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeFileDownloaded);
+	}
+
+	if (g_Options->GetDirectUnpack() && !g_Options->GetRawArticle() && !g_Options->GetSkipWrite())
+	{
+		bool allowPar;
+		if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsNone &&
+			nzbInfo->GetDirectRenameStatus() != NzbInfo::tsRunning &&
+			DirectUnpack::IsArchiveFilename(fileInfo->GetFilename()) &&
+			CanRunMoreJobs(&allowPar))
+		{
+			NzbParameter* unpackParameter = nzbInfo->GetParameters()->Find("*Unpack:");
+			bool wantUnpack = !(unpackParameter && !strcasecmp(unpackParameter->GetValue(), "no"));
+			if (wantUnpack && nzbInfo->GetFailedArticles() == 0)
+			{
+				DirectUnpack::StartJob(nzbInfo);
+			}
+		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsRunning)
+		{
+			((DirectUnpack*)nzbInfo->GetUnpackThread())->FileDownloaded(downloadQueue, fileInfo);
+		}
+	}
+}

daemon/postprocess/PrePostProcessor.h

@@ -44,6 +44,8 @@ protected:
 private:
 	int m_queuedJobs = 0;
 	RawNzbList m_activeJobs;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	void CheckPostQueue();
 	void CheckRequestPar(DownloadQueue* downloadQueue);
@@ -61,8 +63,8 @@ private:
 	bool PostQueueDelete(DownloadQueue* downloadQueue, IdList* idList);
 	void DownloadQueueUpdate(void* aspect);
 	void DeleteCleanup(NzbInfo* nzbInfo);
-	bool IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars);
 	void WaitJobs();
+	void FileDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileInfo* fileInfo);
 };
 
 extern PrePostProcessor* g_PrePostProcessor;

daemon/postprocess/RarReader.cpp

@@ -173,7 +173,7 @@ bool RarVolume::Skip(DiskFile& file, RarBlock* block, int64 size)
 	uint8 buf[256];
 	while (size > 0)
 	{
-		int64 len = size <= sizeof(buf) ? size : sizeof(buf);
+		int64 len = size <= (int64)sizeof(buf) ? size : (int64)sizeof(buf);
 		if (!Read(file, block, buf, len)) return false;
 		size -= len;
 	}
@@ -268,6 +268,12 @@ RarVolume::RarBlock RarVolume::ReadRar3Block(DiskFile& file)
 	uint16 size = ((uint16)buf[6] << 8) + buf[5];
 
 	uint32 blocksize = size;
+	if (m_encrypted)
+	{
+		// Align to 16 bytes
+		blocksize = (blocksize + ((~blocksize + 1) & (16 - 1)));
+	}
+
 	block.trailsize = blocksize - sizeof(buf);
 
 	uint8 addbuf[4];
@@ -283,8 +289,10 @@ RarVolume::RarBlock RarVolume::ReadRar3Block(DiskFile& file)
 		block.trailsize = blocksize - sizeof(buf) - 4;
 	}
 
+#ifdef DEBUG
 	static int num = 0;
-	debug("%i) %llu, %i, %i, %i, %u, %llu", ++num, (long long)block.crc, (int)block.type, (int)block.flags, (int)size, (int)block.addsize, (long long)block.trailsize);
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
 
 	return block;
 }
@@ -444,8 +452,10 @@ RarVolume::RarBlock RarVolume::ReadRar5Block(DiskFile& file)
 	if ((block.flags & RAR5_BLOCK_DATAAREA) && !ReadV(file, &block, &datasize)) return {0};
 	block.trailsize += datasize;
 
+#ifdef DEBUG
 	static int num = 0;
-	debug("%i) %llu, %i, %i, %i, %u, %llu", ++num, (long long)block.crc, (int)block.type, (int)block.flags, (int)size, (int)block.addsize, (long long)block.trailsize);
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
 
 	return block;
 }
@@ -521,23 +531,25 @@ bool RarVolume::ReadRar5File(DiskFile& file, RarBlock& block, RarFile& innerFile
 		}
 	}
 
-	debug("%llu, %i, %s", (long long)block.trailsize, (int)namelen, (const char*)name);
+	debug("%" PRIu64 ", %" PRIu64 ", %s", block.trailsize, namelen, (const char*)name);
 
 	return true;
 }
 
 void RarVolume::LogDebugInfo()
 {
+#ifdef DEBUG
 	debug("Volume: version:%i, multi:%i, vol-no:%i, new-naming:%i, has-next:%i, encrypted:%i, file-count:%i, [%s]",
 		(int)m_version, (int)m_multiVolume, m_volumeNo, (int)m_newNaming, (int)m_hasNextVolume,
 		(int)m_encrypted, (int)m_files.size(), FileSystem::BaseFileName(m_filename));
 
 	for (RarFile& file : m_files)
 	{
-		debug("  time:%i, size:%lli, attr:%i, split-before:%i, split-after:%i, [%s]",
-			(int)file.m_time, (long long)file.m_size, (int)file.m_attr,
-			(int)file.m_splitBefore, (int)file.m_splitAfter, *file.m_filename);
+		debug("  time:%i, size:%" PRIi64 ", attr:%i, split-before:%i, split-after:%i, [%s]",
+			file.m_time, file.m_size, file.m_attr,
+			file.m_splitBefore, file.m_splitAfter, *file.m_filename);
 	}
+#endif
 }
 
 bool RarVolume::DecryptRar3Prepare(const uint8 salt[8])

daemon/postprocess/RarRenamer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -202,7 +202,8 @@ void RarRenamer::MakeSets()
 	// find first volumes and create initial incomplete sets
 	for (RarVolume& volume : m_volumes)
 	{
-		if (!volume.GetFiles()->empty() && volume.GetVolumeNo() == 0)
+		if (!volume.GetFiles()->empty() && volume.GetVolumeNo() == 0 &&
+			!volume.GetFiles()->front().GetSplitBefore())
 		{
 			m_sets.push_back({&volume});
 		}
@@ -217,6 +218,8 @@ void RarRenamer::MakeSets()
 			while (found)
 			{
 				found = false;
+				std::vector<RarVolume*> candidates;
+
 				RarVolume* lastVolume = set.back();
 				for (RarVolume& volume : *volumes)
 				{
@@ -224,25 +227,53 @@ void RarRenamer::MakeSets()
 						volume.GetVolumeNo() == lastVolume->GetVolumeNo() + 1 &&
 						volume.GetVersion() == lastVolume->GetVersion() &&
 						lastVolume->GetHasNextVolume() &&
-						((volume.GetFiles()->at(0).GetSplitBefore() &&
-						 lastVolume->GetFiles()->at(0).GetSplitAfter() &&
-						 !strcmp(volume.GetFiles()->at(0).GetFilename(), lastVolume->GetFiles()->at(0).GetFilename())) ||
-						 (!volume.GetFiles()->at(0).GetSplitBefore() && !lastVolume->GetFiles()->at(0).GetSplitAfter())))
+						((volume.GetFiles()->front().GetSplitBefore() &&
+						 lastVolume->GetFiles()->back().GetSplitAfter() &&
+						 !strcmp(volume.GetFiles()->front().GetFilename(), lastVolume->GetFiles()->back().GetFilename())) ||
+						 (!volume.GetFiles()->front().GetSplitBefore() && !lastVolume->GetFiles()->back().GetSplitAfter())))
 					{
-						debug("   adding %s", FileSystem::BaseFileName(volume.GetFilename()));
-						set.push_back(&volume);
-						found = true;
-						break;
+						debug("   found candidate %s", FileSystem::BaseFileName(volume.GetFilename()));
+						candidates.push_back(&volume);
 					}
 				}
+
+				RarVolume* nextVolume = nullptr;
+
+				if (candidates.size() > 1)
+				{
+					for (RarVolume* volume : candidates)
+					{
+						if (SameArchiveName(FileSystem::BaseFileName(set[0]->GetFilename()),
+							FileSystem::BaseFileName(volume->GetFilename()), set[0]->GetNewNaming()))
+						{
+							nextVolume = volume;
+							break;
+						}
+					}
+				}
+
+				if (!nextVolume && !candidates.empty())
+				{
+					nextVolume = candidates.front();
+				}
+
+				if (nextVolume)
+				{
+					debug("   adding %s", FileSystem::BaseFileName(nextVolume->GetFilename()));
+					set.push_back(nextVolume);
+					found = true;
+				}
 			}
 
-			bool completed = !set.back()->GetHasNextVolume();
+			RarVolume* lastVolume = set.back();
+			bool completed = !lastVolume->GetHasNextVolume() &&
+				(lastVolume->GetFiles()->empty() || !lastVolume->GetFiles()->back().GetSplitAfter());
 
 			return !completed;
 		}),
 		m_sets.end());
 
+#ifdef DEBUG
 	// debug log
 	for (RarVolumeSet& set : m_sets)
 	{
@@ -252,6 +283,43 @@ void RarRenamer::MakeSets()
 			debug("   %s", FileSystem::BaseFileName(volume->GetFilename()));
 		}
 	}
+#endif
+}
+
+bool RarRenamer::SameArchiveName(const char* filename1, const char* filename2, bool newNaming)
+{
+	if (strlen(filename1) != strlen(filename2))
+	{
+		return false;
+	}
+
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+
+	if (!(ext1 && ext2 && strlen(ext1) == strlen(ext2)))
+	{
+		return false;
+	}
+
+	if (newNaming)
+	{
+		if (ext1 == filename1 || ext2 == filename2)
+		{
+			return false;
+		}
+		BString<1024> name1, name2;
+		name1.Set(filename1, (int)(ext1 - filename1));
+		name2.Set(filename2, (int)(ext2 - filename2));
+		ext1 = strrchr(name1, '.');
+		ext2 = strrchr(name2, '.');
+		return ext1 && ext2 && strlen(ext1) == strlen(ext2) &&
+			!strncmp(ext1, ".part", 5) && !strncmp(ext2, ".part", 5) &&
+			!strncmp(name1, name2, ext1 - name1);
+	}
+	else
+	{
+		return !strncmp(filename1, filename2, ext1 - filename1);
+	}
 }
 
 bool RarRenamer::IsSetProperlyNamed(RarVolumeSet& set)
@@ -280,7 +348,7 @@ bool RarRenamer::IsSetProperlyNamed(RarVolumeSet& set)
 			{
 				setPartLen = partNo.Length();
 			}
-			bool ok = atoi(partNo) == volume->GetVolumeNo() + 1 &&
+			bool ok = (uint32)atoi(partNo) == volume->GetVolumeNo() + 1 &&
 				partNo.Length() == setPartLen &&
 				!strncmp(setBasename, filename, regExPart.GetMatchStart(1));
 			if (!ok)

daemon/postprocess/RarRenamer.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -76,6 +76,7 @@ private:
 	void MakeSets();
 	bool IsSetProperlyNamed(RarVolumeSet& set);
 	RarFile* FindMainFile(RarVolumeSet& set);
+	static bool SameArchiveName(const char* filename1, const char* filename2, bool newNaming);
 };
 
 #endif

daemon/postprocess/Rename.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -137,7 +137,7 @@ void RenameController::ExecRename(const char* destDir, const char* finalDir, con
 		m_rarRenamer.SetInfoName(nzbName);
 		m_rarRenamer.SetIgnoreExt(g_Options->GetUnpackIgnoreExt());
 
-		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password", false);
+		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password");
 		if (parameter)
 		{
 			m_rarRenamer.SetPassword(parameter->GetValue());
@@ -207,13 +207,17 @@ void RenameController::UpdateRarRenameProgress()
 /**
 *  Update file name in the CompletedFiles-list of NZBInfo
 */
-void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFileName)
+void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFilename)
 {
 	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
 	{
-		if (!strcasecmp(completedFile.GetFileName(), oldFilename))
+		if (!strcasecmp(completedFile.GetFilename(), oldFilename))
 		{
-			completedFile.SetFileName(newFileName);
+			if (Util::EmptyStr(completedFile.GetOrigname()))
+			{
+				completedFile.SetOrigname(completedFile.GetFilename());
+			}
+			completedFile.SetFilename(newFilename);
 			break;
 		}
 	}

daemon/postprocess/Rename.h

@@ -78,8 +78,8 @@ private:
 	protected:
 		virtual void UpdateProgress() { m_owner->UpdateRarRenameProgress(); }
 		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
-		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName)
-			{ m_owner->RegisterRenamedFile(oldFilename, newFileName); }
+		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFilename)
+			{ m_owner->RegisterRenamedFile(oldFilename, newFilename); }
 		virtual bool IsStopped() { return m_owner->IsStopped(); };
 	private:
 		RenameController* m_owner;
@@ -92,7 +92,7 @@ private:
 
 	void ExecRename(const char* destDir, const char* finalDir, const char* nzbName);
 	void RenameCompleted();
-	void RegisterRenamedFile(const char* oldFilename, const char* newFileName);
+	void RegisterRenamedFile(const char* oldFilename, const char* newFilename);
 };
 
 #endif

daemon/postprocess/Repair.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@
 #include "DupeCoordinator.h"
 #include "ParParser.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "DiskState.h"
 #include "Log.h"
 #include "FileSystem.h"
@@ -74,7 +75,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 
 	for (CompletedFile& completedFile2 : m_postInfo->GetNzbInfo()->GetCompletedFiles())
 	{
-		if (!strcasecmp(completedFile2.GetFileName(), filename))
+		if (!strcasecmp(completedFile2.GetFilename(), filename))
 		{
 			completedFile = &completedFile2;
 			break;
@@ -85,7 +86,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 		return ParChecker::fsUnknown;
 	}
 
-	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFileName()), completedFile->GetCrc(), (int)completedFile->GetStatus());
+	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFilename()), completedFile->GetCrc(), (int)completedFile->GetStatus());
 
 	*crc = completedFile->GetCrc();
 
@@ -114,6 +115,19 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 		ParChecker::fsUnknown;
 }
 
+const char* RepairController::PostParChecker::FindFileOrigname(const char* filename)
+{
+	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
+	{
+		if (!strcasecmp(completedFile.GetFilename(), filename))
+		{
+			return completedFile.GetOrigname();
+		}
+	}
+
+	return nullptr;
+}
+
 void RepairController::PostParChecker::RequestDupeSources(DupeSourceList* dupeSourceList)
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
@@ -246,7 +260,7 @@ bool RepairController::AddPar(FileInfo* fileInfo, bool deleted)
 	bool sameCollection = fileInfo->GetNzbInfo() == m_parChecker.GetPostInfo()->GetNzbInfo();
 	if (sameCollection && !deleted)
 	{
-		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), (int)PATH_SEPARATOR, fileInfo->GetFilename());
+		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename());
 		m_parChecker.AddParFile(fullFilename);
 	}
 	else
@@ -320,7 +334,7 @@ bool RepairController::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename
 	}
 
 	std::sort(availableBlocks.begin(), availableBlocks.end(),
-		[](BlockInfo& block1, BlockInfo& block2)
+		[](const BlockInfo& block1, const BlockInfo& block2)
 		{
 			return block1.m_blockCount < block2.m_blockCount;
 		});
@@ -355,7 +369,7 @@ bool RepairController::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename
 		{
 			if (blockInfo.m_fileInfo->GetPaused())
 			{
-				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), (int)PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
+				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
 				blockInfo.m_fileInfo->SetPaused(false);
 				blockInfo.m_fileInfo->SetExtraPriority(true);
 			}
@@ -410,7 +424,8 @@ void RepairController::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo,
 
 			if (exactParName)
 			{
-				useFile = ParParser::SameParCollection(fileInfo->GetFilename(), FileSystem::BaseFileName(parFilename));
+				useFile = ParParser::SameParCollection(fileInfo->GetFilename(),
+					FileSystem::BaseFileName(parFilename), fileInfo->GetFilenameConfirmed());
 			}
 			else if (strictParName)
 			{
@@ -518,7 +533,7 @@ void RepairController::UpdateParCheckProgress()
 
 void RepairController::CheckPauseState(PostInfo* postInfo)
 {
-	if (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = postInfo->GetStageTime();
 		time_t startTime = postInfo->GetStartTime();
@@ -527,9 +542,9 @@ void RepairController::CheckPauseState(PostInfo* postInfo)
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
+		while (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 
 			// update time stamps
 

daemon/postprocess/Repair.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -68,6 +68,7 @@ private:
 		virtual void RegisterParredFile(const char* filename);
 		virtual bool IsParredFile(const char* filename);
 		virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments);
+		virtual const char* FindFileOrigname(const char* filename);
 		virtual void RequestDupeSources(DupeSourceList* dupeSourceList);
 		virtual void StatDupeSources(DupeSourceList* dupeSourceList);
 	private:

daemon/postprocess/Unpack.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -50,17 +50,7 @@ void UnpackController::StartJob(PostInfo* postInfo)
 void UnpackController::Run()
 {
 	time_t start = Util::CurrentTime();
-
-	m_cleanedUpDisk = false;
-	m_finalDirCreated = false;
 	m_unpackOk = true;
-	m_unpackStartError = false;
-	m_unpackSpaceError = false;
-	m_unpackDecryptError = false;
-	m_unpackPasswordError = false;
-	m_autoTerminated = false;
-	m_passListTried = false;
-
 	bool unpack;
 
 	{
@@ -70,10 +60,10 @@ void UnpackController::Run()
 		m_finalDir = m_postInfo->GetNzbInfo()->GetFinalDir();
 		m_name = m_postInfo->GetNzbInfo()->GetName();
 
-		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:", false);
+		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:");
 		unpack = !(parameter && !strcasecmp(parameter->GetValue(), "no"));
 
-		parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password", false);
+		parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password");
 		if (parameter)
 		{
 			m_password = parameter->GetValue();
@@ -103,7 +93,7 @@ void UnpackController::Run()
 			m_postInfo->GetLastUnpackStatus() == (int)NzbInfo::usPassword ?
 				 "%s failed: checksum error in the encrypted file. Corrupt file or wrong password." : "%s failed.",
 			*m_infoNameUp);
-		m_postInfo->GetNzbInfo()->SetUnpackStatus((NzbInfo::EUnpackStatus)m_postInfo->GetLastUnpackStatus());
+		m_postInfo->GetNzbInfo()->SetUnpackStatus((NzbInfo::EPostUnpackStatus)m_postInfo->GetLastUnpackStatus());
 	}
 	else if (unpack && hasFiles)
 	{
@@ -113,7 +103,29 @@ void UnpackController::Run()
 
 		if (m_hasRarFiles)
 		{
-			UnpackArchives(upUnrar, false);
+			if (m_hasNotUnpackedRarFiles || m_unpackDirCreated)
+			{
+				if (m_postInfo->GetNzbInfo()->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+				{
+					if (m_unpackDirCreated)
+					{
+						PrintMessage(Message::mkWarning, "Could not find files unpacked by direct unpack, unpacking all files again");
+					}
+					else
+					{
+						PrintMessage(Message::mkInfo, "Found archive files not processed by direct unpack, unpacking all files again");
+					}
+				}
+
+				// Discard info about extracted archives to prevent reusing on next unpack attempt
+				m_postInfo->GetExtractedArchives()->clear();
+
+				UnpackArchives(upUnrar, false);
+			}
+			else
+			{
+				PrintMessage(Message::mkInfo, "Using directly unpacked files");
+			}
 		}
 
 		if (m_hasSevenZipFiles && m_unpackOk)
@@ -179,7 +191,7 @@ void UnpackController::UnpackArchives(EUnpacker unpacker, bool multiVolumes)
 		if (!m_unpackOk && m_hasParFiles && !m_unpackPasswordError &&
 			m_postInfo->GetNzbInfo()->GetParStatus() <= NzbInfo::psSkipped)
 		{
-			// for rar4- or 7z-archives try par-check first, before trying password file
+			debug("For rar4- or 7z-archives try par-check first, before trying password file");
 			return;
 		}
 	}
@@ -206,6 +218,7 @@ void UnpackController::UnpackArchives(EUnpacker unpacker, bool multiVolumes)
 			(m_unpackDecryptError || m_unpackPasswordError) &&
 			infile.ReadLine(password, sizeof(password) - 1))
 		{
+			debug("Password line: %s", password);
 			// trim trailing <CR> and <LF>
 			char* end = password + strlen(password) - 1;
 			while (end >= password && (*end == '\n' || *end == '\r')) *end-- = '\0';
@@ -277,7 +290,8 @@ void UnpackController::ExecuteUnrar(const char* password)
 	}
 
 	params.emplace_back("*.rar");
-	params.push_back(FileSystem::MakeExtendedPath(BString<1024>("%s%c", *m_unpackDir, PATH_SEPARATOR), true));
+	m_unpackExtendedDir = FileSystem::MakeExtendedPath(m_unpackDir, true);
+	params.push_back(*BString<1024>("%s%c", *m_unpackExtendedDir, PATH_SEPARATOR));
 	SetArgs(std::move(params));
 	SetLogPrefix("Unrar");
 	ResetEnv();
@@ -291,7 +305,7 @@ void UnpackController::ExecuteUnrar(const char* password)
 	SetProgressLabel("");
 
 	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
-	m_unpackStartError = exitCode == -1;
+	m_unpackStartError = exitCode == -1 && !m_autoTerminated;
 	m_unpackSpaceError = exitCode == 5;
 	m_unpackPasswordError |= exitCode == 11; // only for rar5-archives
 
@@ -346,7 +360,7 @@ void UnpackController::ExecuteSevenZip(const char* password, bool multiVolumes)
 	SetProgressLabel("");
 
 	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
-	m_unpackStartError = exitCode == -1;
+	m_unpackStartError = exitCode == -1 && !m_autoTerminated;
 
 	if (!m_unpackOk && exitCode > 0)
 	{
@@ -604,6 +618,7 @@ void UnpackController::CreateUnpackDir()
 	const char* destDir = !m_finalDir.Empty() ? *m_finalDir : *m_destDir;
 
 	m_unpackDir.Format("%s%c%s", destDir, PATH_SEPARATOR, "_unpack");
+	m_unpackDirCreated = !FileSystem::DirectoryExists(m_unpackDir);
 
 	detail("Unpacking into %s", *m_unpackDir);
 
@@ -616,12 +631,6 @@ void UnpackController::CreateUnpackDir()
 
 void UnpackController::CheckArchiveFiles()
 {
-	m_hasRarFiles = false;
-	m_hasRenamedArchiveFiles = false;
-	m_hasSevenZipFiles = false;
-	m_hasSevenZipMultiFiles = false;
-	m_hasSplittedFiles = false;
-
 	RegEx regExRar(".*\\.rar$");
 	RegEx regExRarMultiSeq(".*\\.[r-z][0-9][0-9]$");
 	RegEx regExSevenZip(".*\\.7z$");
@@ -641,6 +650,10 @@ void UnpackController::CheckArchiveFiles()
 			if (regExRar.Match(filename))
 			{
 				m_hasRarFiles = true;
+				m_hasNotUnpackedRarFiles |= std::find(
+					m_postInfo->GetExtractedArchives()->begin(),
+					m_postInfo->GetExtractedArchives()->end(),
+					filename) == m_postInfo->GetExtractedArchives()->end();
 			}
 			else if (regExSevenZip.Match(filename))
 			{
@@ -731,7 +744,7 @@ bool UnpackController::Cleanup()
 
 	if (!m_unpackOk && m_finalDirCreated)
 	{
-		FileSystem::RemoveDirectory(m_finalDir);
+		FileSystem::DeleteDirectory(m_finalDir);
 	}
 
 	if (m_unpackOk && ok && g_Options->GetUnpackCleanupDisk())
@@ -839,9 +852,9 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 	// Modify unrar messages for better readability:
 	// remove the destination path part from message "Extracting file.xxx"
 	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: Extracting  ", 19) &&
-		!strncmp(text + 19, m_unpackDir, strlen(m_unpackDir)))
+		!strncmp(text + 19, m_unpackExtendedDir, strlen(m_unpackExtendedDir)))
 	{
-		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackDir) + 1);
+		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackExtendedDir) + 1);
 	}
 
 	m_postInfo->GetNzbInfo()->AddMessage(kind, msgText);
@@ -861,8 +874,10 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 
 	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: Extracting from ", 23))
 	{
+#ifdef DEBUG
 		const char *filename = text + 23;
 		debug("Filename: %s", filename);
+#endif
 		SetProgressLabel(text + 7);
 	}
 
@@ -873,7 +888,8 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 		m_unpackDecryptError = true;
 	}
 
-	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: The specified password is incorrect.", 43))
+	if (m_unpacker == upUnrar && (!strncmp(text, "Unrar: The specified password is incorrect.", 43) ||
+		!strncmp(text, "Unrar: Incorrect password for", 29)))
 	{
 		m_unpackPasswordError = true;
 	}
@@ -916,3 +932,23 @@ void UnpackController::SetProgressLabel(const char* progressLabel)
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	m_postInfo->SetProgressLabel(progressLabel);
 }
+
+bool UnpackController::HasCompletedArchiveFiles(NzbInfo* nzbInfo)
+{
+	RegEx regExRar(".*\\.rar$");
+	RegEx regExSevenZip(".*\\.7z$");
+	RegEx regExSevenZipMulti(".*\\.7z\\.[0-9]+$");
+
+	for (CompletedFile& completedFile: nzbInfo->GetCompletedFiles())
+	{
+		const char* filename = completedFile.GetFilename();
+		if (regExRar.Match(filename) ||
+			regExSevenZip.Match(filename) ||
+			regExSevenZipMulti.Match(filename))
+		{
+			return true;
+		}
+	}
+
+	return false;
+}

daemon/postprocess/Unpack.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -32,6 +32,7 @@ public:
 	virtual void Run();
 	virtual void Stop();
 	static void StartJob(PostInfo* postInfo);
+	static bool HasCompletedArchiveFiles(NzbInfo* nzbInfo);
 
 protected:
 	virtual bool ReadLine(char* buf, int bufSize, FILE* stream);
@@ -65,27 +66,30 @@ private:
 	CString m_destDir;
 	CString m_finalDir;
 	CString m_unpackDir;
+	CString m_unpackExtendedDir;
 	CString m_password;
-	bool m_interDir;
-	bool m_allOkMessageReceived;
-	bool m_noFilesMessageReceived;
-	bool m_hasParFiles;
-	bool m_hasRarFiles;
-	bool m_hasRenamedArchiveFiles;
-	bool m_hasSevenZipFiles;
-	bool m_hasSevenZipMultiFiles;
-	bool m_hasSplittedFiles;
-	bool m_unpackOk;
-	bool m_unpackStartError;
-	bool m_unpackSpaceError;
-	bool m_unpackDecryptError;
-	bool m_unpackPasswordError;
-	bool m_cleanedUpDisk;
-	bool m_autoTerminated;
 	EUnpacker m_unpacker;
-	bool m_finalDirCreated;
+	bool m_interDir = false;
+	bool m_allOkMessageReceived = false;
+	bool m_noFilesMessageReceived = false;
+	bool m_hasParFiles = false;
+	bool m_hasRarFiles = false;
+	bool m_hasNotUnpackedRarFiles = false;
+	bool m_hasRenamedArchiveFiles = false;
+	bool m_hasSevenZipFiles = false;
+	bool m_hasSevenZipMultiFiles = false;
+	bool m_hasSplittedFiles = false;
+	bool m_unpackOk = false;
+	bool m_unpackStartError = false;
+	bool m_unpackSpaceError = false;
+	bool m_unpackDecryptError = false;
+	bool m_unpackPasswordError = false;
+	bool m_cleanedUpDisk = false;
+	bool m_autoTerminated = false;
+	bool m_finalDirCreated = false;
+	bool m_unpackDirCreated = false;
+	bool m_passListTried = false;
 	FileList m_joinedFiles;
-	bool m_passListTried;
 
 	void ExecuteUnpack(EUnpacker unpacker, const char* password, bool multiVolumes);
 	void ExecuteUnrar(const char* password);

daemon/queue/DirectRenamer.cpp

@@ -0,0 +1,570 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "DirectRenamer.h"
+#include "Options.h"
+#include "FileSystem.h"
+#include "ParParser.h"
+
+#ifndef DISABLE_PARCHECK
+#include "par2cmdline.h"
+#include "par2fileformat.h"
+#include "md5.h"
+#endif
+
+class RenameContentAnalyzer : public ArticleContentAnalyzer
+{
+public:
+	virtual void Reset();
+	virtual void Append(const void* buffer, int len);
+	void Finish();
+	const char* GetHash16k() { return m_hash16k; }
+	bool GetParFile() { return m_parFile; }
+	const char* GetParSetId() { return m_parSetId; }
+
+private:
+#ifndef DISABLE_PARCHECK
+	Par2::MD5Context m_md5Context;
+	char m_signature[sizeof(Par2::PACKET_HEADER)];
+#endif
+	int m_dataSize = 0;
+	CString m_hash16k;
+	CString m_parSetId;
+	bool m_parFile = false;
+};
+
+#ifndef DISABLE_PARCHECK
+class DirectParRepairer : public Par2::Par2Repairer
+{
+public:
+	DirectParRepairer() : Par2::Par2Repairer(m_nout, m_nout) {};
+	friend class DirectParLoader;
+
+private:
+	class NullStreamBuf : public std::streambuf {};
+	NullStreamBuf m_nullbuf;
+	std::ostream m_nout{&m_nullbuf};
+};
+
+class DirectParLoader : public Thread
+{
+public:
+	static void StartLoader(DirectRenamer* owner, NzbInfo* nzbInfo);
+	virtual void Run();
+
+private:
+	typedef std::vector<CString> ParFiles;
+
+	DirectRenamer* m_owner;
+	ParFiles m_parFiles;
+	DirectRenamer::FileHashList m_parHashes;
+	int m_nzbId;
+
+	void LoadParFile(const char* parFile);
+};
+
+void DirectParLoader::StartLoader(DirectRenamer* owner, NzbInfo* nzbInfo)
+{
+	nzbInfo->PrintMessage(Message::mkInfo, "Directly checking renamed files for %s", nzbInfo->GetName());
+
+	DirectParLoader* directParLoader = new DirectParLoader();
+	directParLoader->m_owner = owner;
+	directParLoader->m_nzbId = nzbInfo->GetId();
+
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (completedFile.GetParFile())
+		{
+			directParLoader->m_parFiles.emplace_back(BString<1024>("%s%c%s",
+				nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename()));
+		}
+	}
+
+	directParLoader->SetAutoDestroy(true);
+	directParLoader->Start();
+}
+
+void DirectParLoader::Run()
+{
+	debug("Started DirectParLoader");
+
+	for (CString& parFile : m_parFiles)
+	{
+		LoadParFile(parFile);
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (nzbInfo)
+	{
+		// nzb is still in queue
+		m_owner->RenameFiles(downloadQueue, nzbInfo, &m_parHashes);
+	}
+}
+
+void DirectParLoader::LoadParFile(const char* parFile)
+{
+	DirectParRepairer repairer;
+
+	if (!repairer.LoadPacketsFromFile(parFile))
+	{
+		warn("Could not load par2-file %s", parFile);
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (!nzbInfo)
+	{
+		// nzb isn't in queue anymore
+		return;
+	}
+
+	nzbInfo->PrintMessage(Message::mkInfo, "Loaded par2-file %s for direct-rename", FileSystem::BaseFileName(parFile));
+
+	for (std::pair<const Par2::MD5Hash, Par2::Par2RepairerSourceFile*>& entry : repairer.sourcefilemap)
+	{
+		if (IsStopped())
+		{
+			break;
+		}
+
+		Par2::Par2RepairerSourceFile* sourceFile = entry.second;
+		if (!sourceFile || !sourceFile->GetDescriptionPacket())
+		{
+			nzbInfo->PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", FileSystem::BaseFileName(parFile));
+			return;
+		}
+		std::string filename = Par2::DiskFile::TranslateFilename(sourceFile->GetDescriptionPacket()->FileName());
+		std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print();
+
+		debug("file: %s, hash-16k: %s", filename.c_str(), hash.c_str());
+		m_parHashes.emplace_back(filename.c_str(), hash.c_str());
+	}
+}
+#endif
+
+std::unique_ptr<ArticleContentAnalyzer> DirectRenamer::MakeArticleContentAnalyzer()
+{
+	return std::make_unique<RenameContentAnalyzer>();
+}
+
+void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo,
+	ArticleInfo* articleInfo, ArticleContentAnalyzer* articleContentAnalyzer)
+{
+	debug("Applying analyzer data %s for ", fileInfo->GetFilename());
+
+	RenameContentAnalyzer* contentAnalyzer = (RenameContentAnalyzer*)articleContentAnalyzer;
+	contentAnalyzer->Finish();
+
+	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+
+	// we don't support analyzing of files split into articles smaller than 16KB
+	if (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1)
+	{
+		fileInfo->SetHash16k(contentAnalyzer->GetHash16k());
+		debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), fileInfo->GetHash16k());
+	}
+
+	fileInfo->GetNzbInfo()->PrintMessage(Message::mkDetail, "Detected %s %s", (contentAnalyzer->GetParFile() ? "par2-file" : "non-par2-file"), fileInfo->GetFilename());
+
+	if (fileInfo->GetParFile() != contentAnalyzer->GetParFile())
+	{
+		debug("Changing par2-flag for %s", fileInfo->GetFilename());
+		fileInfo->SetParFile(contentAnalyzer->GetParFile());
+
+		int delta = fileInfo->GetParFile() ? 1 : -1;
+
+		nzbInfo->SetParSize(nzbInfo->GetParSize() + fileInfo->GetSize() * delta);
+		nzbInfo->SetParCurrentSuccessSize(nzbInfo->GetParCurrentSuccessSize() + fileInfo->GetSuccessSize() * delta);
+		nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() +
+			fileInfo->GetFailedSize() * delta + fileInfo->GetMissedSize() * delta);
+		nzbInfo->SetParFailedSize(nzbInfo->GetParFailedSize() + fileInfo->GetMissedSize() * delta);
+		nzbInfo->SetRemainingParCount(nzbInfo->GetRemainingParCount() + 1 * delta);
+
+		if (!fileInfo->GetParFile() && fileInfo->GetPaused())
+		{
+			fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Resuming non-par2-file %s", fileInfo->GetFilename());
+			fileInfo->SetPaused(false);
+		}
+
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
+	}
+
+	if (fileInfo->GetParFile())
+	{
+		fileInfo->SetParSetId(contentAnalyzer->GetParSetId());
+		debug("file: %s; setid: %s", fileInfo->GetFilename(), fileInfo->GetParSetId());
+	}
+
+	CheckState(downloadQueue, nzbInfo);
+}
+
+void DirectRenamer::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo)
+{
+	CheckState(downloadQueue, fileInfo->GetNzbInfo());
+}
+
+void DirectRenamer::CheckState(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+#ifndef DISABLE_PARCHECK
+	if (nzbInfo->GetDirectRenameStatus() > NzbInfo::tsRunning)
+	{
+		return;
+	}
+
+	// check if all first articles are successfully downloaded (1)
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (Util::EmptyStr(fileInfo->GetHash16k()) ||
+			(fileInfo->GetParFile() && Util::EmptyStr(fileInfo->GetParSetId())))
+		{
+			return;
+		}
+	}
+
+	// check if all first articles are successfully downloaded (2)
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (Util::EmptyStr(completedFile.GetHash16k()) ||
+			(completedFile.GetParFile() && Util::EmptyStr(completedFile.GetParSetId())))
+		{
+			return;
+		}
+	}
+
+	if (!nzbInfo->GetWaitingPar())
+	{
+		// all first articles downloaded
+		UnpausePars(nzbInfo);
+		nzbInfo->SetWaitingPar(true);
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
+	}
+
+	if (nzbInfo->GetWaitingPar() && !nzbInfo->GetLoadingPar())
+	{
+		// check if all par2-files scheduled for downloading already completed
+		FileList::iterator pos = std::find_if(
+			nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+			[](std::unique_ptr<FileInfo>& fileInfo)
+		{
+			return fileInfo->GetExtraPriority();
+		});
+
+		if (pos == nzbInfo->GetFileList()->end())
+		{
+			// all wanted par2-files are downloaded
+			nzbInfo->SetLoadingPar(true);
+			DirectParLoader::StartLoader(this, nzbInfo);
+			return;
+		}
+	}
+#endif
+}
+
+// Unpause smallest par-files from each par-set
+void DirectRenamer::UnpausePars(NzbInfo* nzbInfo)
+{
+	ParFileList parFiles;
+	CollectPars(nzbInfo, &parFiles);
+
+	std::vector<CString> parsets;
+
+	// sort by size
+	std::sort(parFiles.begin(), parFiles.end(),
+		[nzbInfo](const ParFile& parFile1, const ParFile& parFile2)
+		{
+			FileInfo* fileInfo1 = nzbInfo->GetFileList()->Find(const_cast<ParFile&>(parFile1).GetId());
+			FileInfo* fileInfo2 = nzbInfo->GetFileList()->Find(const_cast<ParFile&>(parFile2).GetId());
+			return (!fileInfo1 && fileInfo2) ||
+				(fileInfo1 && fileInfo2 && fileInfo1->GetSize() < fileInfo2->GetSize());
+		});
+
+	// 1. count already downloaded files
+	for (ParFile& parFile : parFiles)
+	{
+		if (parFile.GetCompleted())
+		{
+			parsets.emplace_back(parFile.GetSetId());
+		}
+	}
+
+	// 2. find smallest par-file from each par-set from not yet completely downloaded files
+	for (ParFile& parFile : parFiles)
+	{
+		std::vector<CString>::iterator pos = std::find(parsets.begin(), parsets.end(), parFile.GetSetId());
+		if (pos == parsets.end())
+		{
+			// this par-set is not yet downloaded
+			parsets.emplace_back(parFile.GetSetId());
+
+			FileInfo* fileInfo = nzbInfo->GetFileList()->Find(parFile.GetId());
+			if (fileInfo)
+			{
+				nzbInfo->PrintMessage(Message::mkInfo, "Increasing priority for par2-file %s", fileInfo->GetFilename());
+				fileInfo->SetPaused(false);
+				fileInfo->SetExtraPriority(true);
+			}
+		}
+	}
+}
+
+void DirectRenamer::CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles)
+{
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (fileInfo->GetParFile())
+		{
+			parFiles->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(), fileInfo->GetParSetId(), false);
+		}
+	}
+
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		if (completedFile.GetParFile())
+		{
+			parFiles->emplace_back(completedFile.GetId(), completedFile.GetFilename(), completedFile.GetParSetId(), true);
+		}
+	}
+}
+
+void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes)
+{
+	int renamedCount = 0;
+
+	bool renamePars = NeedRenamePars(nzbInfo);
+	int vol = 1;
+
+	// rename in-progress files
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		CString newName;
+		if (fileInfo->GetParFile() && renamePars)
+		{
+			newName = BuildNewParName(fileInfo->GetFilename(), nzbInfo->GetDestDir(), fileInfo->GetParSetId(), vol);
+		}
+		else if (!fileInfo->GetParFile())
+		{
+			newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k());
+		}
+
+		if (newName)
+		{
+			bool written = fileInfo->GetOutputFilename() &&
+				!Util::EndsWith(fileInfo->GetOutputFilename(), ".out.tmp", true);
+			if (!written)
+			{
+				nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s",
+					fileInfo->GetFilename(), *newName);
+				if (Util::EmptyStr(fileInfo->GetOrigname()))
+				{
+					fileInfo->SetOrigname(fileInfo->GetFilename());
+				}
+				fileInfo->SetFilename(newName);
+				fileInfo->SetFilenameConfirmed(true);
+				renamedCount++;
+			}
+			else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName))
+			{
+				if (Util::EmptyStr(fileInfo->GetOrigname()))
+				{
+					fileInfo->SetOrigname(fileInfo->GetFilename());
+				}
+				fileInfo->SetFilename(newName);
+				fileInfo->SetFilenameConfirmed(true);
+				renamedCount++;
+			}
+		}
+	}
+
+	// rename completed files
+	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+	{
+		CString newName;
+		if (completedFile.GetParFile() && renamePars)
+		{
+			newName = BuildNewParName(completedFile.GetFilename(), nzbInfo->GetDestDir(), completedFile.GetParSetId(), vol);
+		}
+		else if (!completedFile.GetParFile())
+		{
+			newName = BuildNewRegularName(completedFile.GetFilename(), parHashes, completedFile.GetHash16k());
+		}
+
+		if (newName && RenameCompletedFile(nzbInfo, completedFile.GetFilename(), newName))
+		{
+			if (Util::EmptyStr(completedFile.GetOrigname()))
+			{
+				completedFile.SetOrigname(completedFile.GetFilename());
+			}
+			completedFile.SetFilename(newName);
+			renamedCount++;
+		}
+	}
+
+	if (renamedCount > 0)
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Successfully renamed %i file(s) for %s", renamedCount, nzbInfo->GetName());
+	}
+	else
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "No renamed files found for %s", nzbInfo->GetName());
+	}
+
+	RenameCompleted(downloadQueue, nzbInfo);
+}
+
+CString DirectRenamer::BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k)
+{
+	if (Util::EmptyStr(hash16k))
+	{
+		return nullptr;
+	}
+
+	FileHashList::iterator pos = std::find_if(parHashes->begin(), parHashes->end(),
+		[hash16k](FileHash& parHash)
+	{
+		return !strcmp(parHash.GetHash(), hash16k);
+	});
+
+	if (pos != parHashes->end())
+	{
+		FileHash& parHash = *pos;
+		if (strcasecmp(oldName, parHash.GetFilename()))
+		{
+			return parHash.GetFilename();
+		}
+	}
+
+	return nullptr;
+}
+
+CString DirectRenamer::BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol)
+{
+	BString<1024> newName;
+	BString<1024> destFileName;
+
+	// trying to reuse file suffix
+	const char* suffix = strstr(oldName, ".vol");
+	const char* extension = suffix ? strrchr(suffix, '.') : nullptr;
+	if (suffix && extension && !strcasecmp(extension, ".par2"))
+	{
+		newName.Format("%s%s", setId, suffix);
+		destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName);
+	}
+
+	while (destFileName.Empty() || FileSystem::FileExists(destFileName))
+	{
+		newName.Format("%s.vol%03i+01.PAR2", setId, vol);
+		destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName);
+		vol++;
+	}
+
+	return *newName;
+}
+
+bool DirectRenamer::NeedRenamePars(NzbInfo* nzbInfo)
+{
+	// renaming is needed if par2-files from same par-set have different base names
+	// or if any par2-file has non .par2-extension
+	ParFileList parFiles;
+	CollectPars(nzbInfo, &parFiles);
+
+	for (ParFile& parFile : parFiles)
+	{
+		if (!Util::EndsWith(parFile.GetFilename(), ".par2", false))
+		{
+			return true;
+		}
+
+		for (ParFile& parFile2 : parFiles)
+		{
+			if (&parFile != &parFile2 && !strcmp(parFile.GetSetId(), parFile2.GetSetId()) &&
+				!ParParser::SameParCollection(parFile.GetFilename(), parFile2.GetFilename(), false))
+			{
+					return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+bool DirectRenamer::RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName)
+{
+	BString<1024> oldFullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, oldName);
+	BString<1024> newFullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, newName);
+	nzbInfo->PrintMessage(Message::mkInfo, "Renaming completed file %s to %s", oldName, newName);
+	if (!FileSystem::MoveFile(oldFullFilename, newFullFilename))
+	{
+		nzbInfo->PrintMessage(Message::mkError, "Could not rename completed file %s to %s: %s",
+			*oldFullFilename, *newFullFilename, *FileSystem::GetLastErrorMessage());
+		return false;
+	}
+	return true;
+}
+
+
+void RenameContentAnalyzer::Reset()
+{
+#ifndef DISABLE_PARCHECK
+	m_md5Context.Reset();
+#endif
+	m_dataSize = 0;
+}
+
+void RenameContentAnalyzer::Append(const void* buffer, int len)
+{
+#ifndef DISABLE_PARCHECK
+	if ((size_t)m_dataSize < sizeof(m_signature))
+	{
+		memcpy(m_signature + m_dataSize, buffer, std::min((size_t)len, sizeof(m_signature) - m_dataSize));
+	}
+
+	if ((size_t)m_dataSize >= sizeof(m_signature) && (*(Par2::MAGIC*)m_signature) == Par2::packet_magic)
+	{
+		m_parFile = true;
+		m_parSetId = ((Par2::PACKET_HEADER*)m_signature)->setid.print().c_str();
+	}
+
+	int rem16kSize = std::min(len, 16 * 1024 - m_dataSize);
+	if (rem16kSize > 0)
+	{
+		m_md5Context.Update(buffer, rem16kSize);
+	}
+
+	m_dataSize += len;
+#endif
+}
+
+// Must be called with locked DownloadQueue
+void RenameContentAnalyzer::Finish()
+{
+#ifndef DISABLE_PARCHECK
+	Par2::MD5Hash hash;
+	m_md5Context.Final(hash);
+
+	m_hash16k = hash.print().c_str();
+#endif
+}

daemon/queue/DirectRenamer.h

@@ -0,0 +1,84 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef DIRECTRENAMER_H
+#define DIRECTRENAMER_H
+
+#include "ArticleDownloader.h"
+
+class DirectRenamer
+{
+public:
+	class FileHash
+	{
+	public:
+		FileHash(const char* filename, const char* hash) :
+			m_filename(filename), m_hash(hash) {}
+		const char* GetFilename() { return m_filename; }
+		const char* GetHash() { return m_hash; }
+
+	private:
+		CString m_filename;
+		CString m_hash;
+	};
+
+	typedef std::deque<FileHash> FileHashList;
+
+	class ParFile
+	{
+	public:
+		ParFile(int id, const char* filename, const char* setId, bool completed) :
+			m_id(id), m_filename(filename), m_setId(setId), m_completed(completed) {}
+		int GetId() { return m_id; }
+		const char* GetFilename() { return m_filename; }
+		const char* GetSetId() { return m_setId; }
+		bool GetCompleted() { return m_completed; }
+
+	private:
+		int m_id;
+		CString m_filename;
+		CString m_setId;
+		bool m_completed = false;
+	};
+
+	typedef std::deque<ParFile> ParFileList;
+
+	std::unique_ptr<ArticleContentAnalyzer> MakeArticleContentAnalyzer();
+	void ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo,
+		ArticleInfo* articleInfo, ArticleContentAnalyzer* articleContentAnalyzer);
+	void FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo);
+
+protected:
+	virtual void RenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo) = 0;
+
+private:
+	void CheckState(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void UnpausePars(NzbInfo* nzbInfo);
+	void RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes);
+	bool RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName);
+	bool NeedRenamePars(NzbInfo* nzbInfo);
+	void CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles);
+	CString BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k);
+	CString BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol);
+
+	friend class DirectParLoader;
+};
+
+#endif

daemon/queue/DiskState.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,6 +27,10 @@
 #include "FileSystem.h"
 
 static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version ";
+const int DISKSTATE_QUEUE_VERSION = 62;
+const int DISKSTATE_FILE_VERSION = 6;
+const int DISKSTATE_STATS_VERSION = 3;
+const int DISKSTATE_FEEDS_VERSION = 3;
 
 class StateDiskFile : public DiskFile
 {
@@ -41,12 +45,10 @@ int64 StateDiskFile::PrintLine(const char* format, ...)
 {
 	va_list ap;
 	va_start(ap, format);
-	CString str;
-	str.FormatV(format, ap);
+	BString<1024> str;
+	int len = str.FormatV(format, ap);
 	va_end(ap);
 
-	int len = str.Length();
-
 	// replacing terminating <NULL> with <LF>
 	str[len++] = '\n';
 
@@ -113,8 +115,8 @@ public:
 private:
 	BString<1024> m_destFilename;
 	BString<1024> m_tempFilename;
-	bool m_transactional;
 	int m_formatVersion;
+	bool m_transactional;
 	int m_fileVersion;
 	StateDiskFile m_file;
 
@@ -265,7 +267,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 	bool ok = true;
 
 	{
-		StateFile stateFile("queue", 59, true);
+		StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
 		if (!downloadQueue->GetQueue()->empty())
 		{
 			StateDiskFile* outfile = stateFile.BeginWrite();
@@ -288,7 +290,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 
 	if (saveHistory)
 	{
-		StateFile stateFile("history", 59, true);
+		StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
 		if (!downloadQueue->GetHistory()->empty())
 		{
 			StateDiskFile* outfile = stateFile.BeginWrite();
@@ -309,6 +311,10 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 		}
 	}
 
+	// progress-file isn't needed after saving of full queue data
+	StateFile progressStateFile("progress", DISKSTATE_QUEUE_VERSION, true);
+	progressStateFile.Discard();
+
 	return ok;
 }
 
@@ -320,7 +326,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 	int formatVersion = 0;
 
 	{
-		StateFile stateFile("queue", 59, true);
+		StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
 		if (stateFile.FileExists())
 		{
 			StateDiskFile* infile = stateFile.BeginRead();
@@ -331,7 +337,12 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 
 			formatVersion = stateFile.GetFileVersion();
 
-			if (formatVersion < 47)
+			if (formatVersion <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+			else if (formatVersion < 47)
 			{
 				error("Failed to read queue and history data. Only queue and history from NZBGet v13 or newer can be converted by this NZBGet version. "
 					"Old queue and history data still can be converted using NZBGet v16 as an intermediate version.");
@@ -347,9 +358,8 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 		}
 	}
 
-	if (formatVersion == 0 || formatVersion >= 57)
 	{
-		StateFile stateFile("history", 59, true);
+		StateFile stateFile("progress", DISKSTATE_QUEUE_VERSION, true);
 		if (stateFile.FileExists())
 		{
 			StateDiskFile* infile = stateFile.BeginRead();
@@ -357,10 +367,40 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 			{
 				return false;
 			}
+
+			if (stateFile.GetFileVersion() <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+
+			if (!LoadProgress(downloadQueue->GetQueue(), servers, *infile, stateFile.GetFileVersion())) goto error;
+		}
+	}
+
+	if (formatVersion == 0 || formatVersion >= 57)
+	{
+		StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
+		if (stateFile.FileExists())
+		{
+			StateDiskFile* infile = stateFile.BeginRead();
+			if (!infile)
+			{
+				return false;
+			}
+
+			if (stateFile.GetFileVersion() <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+
 			if (!LoadHistory(downloadQueue->GetHistory(), servers, *infile, stateFile.GetFileVersion())) goto error;
 		}
 	}
 
+	LoadAllFileInfos(downloadQueue);
+
 	CleanupQueueDir(downloadQueue);
 
 	if (!LoadAllFileStates(downloadQueue, servers)) goto error;
@@ -382,6 +422,42 @@ error:
 	return ok;
 }
 
+bool DiskState::SaveDownloadProgress(DownloadQueue* downloadQueue)
+{
+	int count = 0;
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		count += nzbInfo->GetChanged() ? 1 : 0;
+	}
+
+	debug("Saving queue progress to disk");
+
+	bool ok = true;
+
+	{
+		StateFile stateFile("progress", DISKSTATE_QUEUE_VERSION, true);
+		if (count > 0)
+		{
+			StateDiskFile* outfile = stateFile.BeginWrite();
+			if (!outfile)
+			{
+				return false;
+			}
+
+			SaveProgress(downloadQueue->GetQueue(), *outfile, count);
+
+			// now rename to dest file name
+			ok = stateFile.FinishWrite();
+		}
+		else
+		{
+			stateFile.Discard();
+		}
+	}
+
+	return ok;
+}
+
 void DiskState::SaveQueue(NzbList* queue, StateDiskFile& outfile)
 {
 	debug("Saving nzb list to disk");
@@ -414,6 +490,50 @@ error:
 	return false;
 }
 
+void DiskState::SaveProgress(NzbList* queue, StateDiskFile& outfile, int changedCount)
+{
+	debug("Saving nzb progress to disk");
+
+	outfile.PrintLine("%i", changedCount);
+	for (NzbInfo* nzbInfo : queue)
+	{
+		if (nzbInfo->GetChanged())
+		{
+			outfile.PrintLine("%i", nzbInfo->GetId());
+			SaveNzbInfo(nzbInfo, outfile);
+		}
+	}
+}
+
+bool DiskState::LoadProgress(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion)
+{
+	debug("Loading nzb progress from disk");
+
+	// load nzb-infos
+	int size;
+	if (infile.ScanLine("%i", &size) != 1) goto error;
+	for (int i = 0; i < size; i++)
+	{
+		int id;
+		if (infile.ScanLine("%i", &id) != 1) goto error;
+
+		NzbInfo* nzbInfo = queue->Find(id);
+		if (!nzbInfo)
+		{
+			error("NZB with id %i could not be found", id);
+			goto error;
+		}
+
+		if (!LoadNzbInfo(nzbInfo, servers, infile, formatVersion)) goto error;
+	}
+
+	return true;
+
+error:
+	error("Error reading nzb progress from disk");
+	return false;
+}
+
 void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 {
 	outfile.PrintLine("%i", nzbInfo->GetId());
@@ -428,9 +548,10 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i,%i,%i,%i,%i", (int)nzbInfo->GetPriority(),
 		nzbInfo->GetPostInfo() ? (int)nzbInfo->GetPostInfo()->GetStage() + 1 : 0,
 		(int)nzbInfo->GetDeletePaused(), (int)nzbInfo->GetManyDupeFiles(), nzbInfo->GetFeedId());
-	outfile.PrintLine("%i,%i,%i,%i,%i,%i,%i,%i", (int)nzbInfo->GetParStatus(), (int)nzbInfo->GetUnpackStatus(),
-		(int)nzbInfo->GetMoveStatus(), (int)nzbInfo->GetParRenameStatus(), (int)nzbInfo->GetRarRenameStatus(), 
-		(int)nzbInfo->GetDeleteStatus(), (int)nzbInfo->GetMarkStatus(), (int)nzbInfo->GetUrlStatus());
+	outfile.PrintLine("%i,%i,%i,%i,%i,%i,%i,%i,%i", (int)nzbInfo->GetParStatus(), (int)nzbInfo->GetUnpackStatus(),
+		(int)nzbInfo->GetMoveStatus(), (int)nzbInfo->GetParRenameStatus(), (int)nzbInfo->GetRarRenameStatus(),
+		(int)nzbInfo->GetDirectRenameStatus(), (int)nzbInfo->GetDeleteStatus(), (int)nzbInfo->GetMarkStatus(),
+		(int)nzbInfo->GetUrlStatus());
 	outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetUnpackCleanedUpDisk(), (int)nzbInfo->GetHealthPaused(),
 		(int)nzbInfo->GetAddUrlPaused());
 	outfile.PrintLine("%i,%i,%i", nzbInfo->GetFileCount(), nzbInfo->GetParkedFileCount(),
@@ -457,7 +578,7 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i,%i,%i", nzbInfo->GetTotalArticles(), nzbInfo->GetSuccessArticles(), nzbInfo->GetFailedArticles());
 
 	outfile.PrintLine("%s", nzbInfo->GetDupeKey());
-	outfile.PrintLine("%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore());
+	outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore(), (int)nzbInfo->GetDupeHint());
 
 	Util::SplitInt64(nzbInfo->GetDownloadedSize(), &High1, &Low1);
 	outfile.PrintLine("%u,%u,%i,%i,%i,%i,%i", High1, Low1, nzbInfo->GetDownloadSec(), nzbInfo->GetPostTotalSec(),
@@ -466,8 +587,12 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i", (int)nzbInfo->GetCompletedFiles()->size());
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		outfile.PrintLine("%i,%i,%u,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
-			completedFile.GetCrc(), completedFile.GetFileName());
+		outfile.PrintLine("%i,%i,%u,%i,%s,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
+			completedFile.GetCrc(), (int)completedFile.GetParFile(),
+			completedFile.GetHash16k() ? completedFile.GetHash16k() : "",
+			completedFile.GetParSetId() ? completedFile.GetParSetId() : "");
+		outfile.PrintLine("%s", completedFile.GetFilename());
+		outfile.PrintLine("%s", completedFile.GetOrigname() ? completedFile.GetOrigname() : "");
 	}
 
 	outfile.PrintLine("%i", (int)nzbInfo->GetParameters()->size());
@@ -568,17 +693,33 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 	}
 	nzbInfo->SetFeedId(feedId);
 
-	int parStatus, unpackStatus, moveStatus, parRenameStatus, rarRenameStatus, deleteStatus, markStatus, urlStatus;
-	if (formatVersion < 58 && infile.ScanLine("%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
-		&parRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 7) goto error;
-	rarRenameStatus = 0;
-	if (formatVersion >= 58 && infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
-		&parRenameStatus, &rarRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 8) goto error;
+	int parStatus, unpackStatus, moveStatus, parRenameStatus, rarRenameStatus,
+		directRenameStatus, deleteStatus, markStatus, urlStatus;
+	if (formatVersion >= 60)
+	{
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i,%i", &parStatus,
+			&unpackStatus, &moveStatus, &parRenameStatus, &rarRenameStatus, &directRenameStatus,
+			&deleteStatus, &markStatus, &urlStatus) != 9) goto error;
+	}
+	else if (formatVersion >= 58)
+	{
+		directRenameStatus = 0;
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i,%i", &parStatus,
+			&unpackStatus, &moveStatus, &parRenameStatus, &rarRenameStatus, &deleteStatus,
+			&markStatus, &urlStatus) != 8) goto error;
+	}
+	else
+	{
+		rarRenameStatus = directRenameStatus = 0;
+		if (infile.ScanLine("%i,%i,%i,%i,%i,%i,%i", &parStatus, &unpackStatus, &moveStatus,
+			&parRenameStatus, &deleteStatus, &markStatus, &urlStatus) != 7) goto error;
+	}
 	nzbInfo->SetParStatus((NzbInfo::EParStatus)parStatus);
-	nzbInfo->SetUnpackStatus((NzbInfo::EUnpackStatus)unpackStatus);
+	nzbInfo->SetUnpackStatus((NzbInfo::EPostUnpackStatus)unpackStatus);
 	nzbInfo->SetMoveStatus((NzbInfo::EMoveStatus)moveStatus);
-	nzbInfo->SetParRenameStatus((NzbInfo::ERenameStatus)parRenameStatus);
-	nzbInfo->SetRarRenameStatus((NzbInfo::ERenameStatus)rarRenameStatus);
+	nzbInfo->SetParRenameStatus((NzbInfo::EPostRenameStatus)parRenameStatus);
+	nzbInfo->SetRarRenameStatus((NzbInfo::EPostRenameStatus)rarRenameStatus);
+	nzbInfo->SetDirectRenameStatus((NzbInfo::EDirectRenameStatus)directRenameStatus);
 	nzbInfo->SetDeleteStatus((NzbInfo::EDeleteStatus)deleteStatus);
 	nzbInfo->SetMarkStatus((NzbInfo::EMarkStatus)markStatus);
 	if (nzbInfo->GetKind() == NzbInfo::nkNzb ||
@@ -664,10 +805,19 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	nzbInfo->SetDupeKey(buf);
 
-	int dupeMode, dupeScore;
-	if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	int dupeMode, dupeScore, dupeHint;
+	dupeHint = 0; //clang requires initialization in a separate line (due to goto statements)
+	if (formatVersion >= 61)
+	{
+		if (infile.ScanLine("%i,%i,%i", &dupeMode, &dupeScore, &dupeHint) != 3) goto error;
+	}
+	else
+	{
+		if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	}
 	nzbInfo->SetDupeMode((EDupeMode)dupeMode);
 	nzbInfo->SetDupeScore(dupeScore);
+	nzbInfo->SetDupeMode((EDupeMode)dupeHint);
 
 	if (formatVersion >= 48)
 	{
@@ -681,6 +831,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		nzbInfo->SetUnpackSec(unpackSec);
 	}
 
+	nzbInfo->GetCompletedFiles()->clear();
 	if (infile.ScanLine("%i", &fileCount) != 1) goto error;
 	for (int i = 0; i < fileCount; i++)
 	{
@@ -690,10 +841,33 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		char* fileName = buf;
 		int status = 0;
 		uint32 crc = 0;
+		int parFile = 0;
+		char* hash16k = nullptr;
+		char* parSetId = nullptr;
+		char filenameBuf[1024];
+		char origName[1024];
 
 		if (formatVersion >= 49)
 		{
-			if (formatVersion >= 50)
+			if (formatVersion >= 60)
+			{
+				if (sscanf(buf, "%i,%i,%u,%i", &id, &status, &crc, &parFile) != 4) goto error;
+				hash16k = strchr(buf, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k) hash16k = strchr(hash16k+1, ',');
+				if (hash16k)
+				{
+					parSetId = strchr(++hash16k, ',');
+					if (parSetId)
+					{
+						*parSetId++ = '\0';
+						fileName = strchr(parSetId, ',');
+						if (fileName) *fileName = '\0';
+					}
+				}
+			}
+			else if (formatVersion >= 50)
 			{
 				if (sscanf(buf, "%i,%i,%u", &id, &status, &crc) != 3) goto error;
 				fileName = strchr(buf, ',');
@@ -709,11 +883,22 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 			{
 				fileName++;
 			}
+			if (formatVersion >= 62)
+			{
+				if (!infile.ReadLine(filenameBuf, sizeof(filenameBuf))) goto error;
+				fileName = filenameBuf;
+				if (!infile.ReadLine(origName, sizeof(origName))) goto error;
+			}
 		}
 
-		nzbInfo->GetCompletedFiles()->emplace_back(id, fileName, (CompletedFile::EStatus)status, crc);
+		nzbInfo->GetCompletedFiles()->emplace_back(id, fileName,
+			Util::EmptyStr(origName) ? nullptr : origName,
+			(CompletedFile::EStatus)status, crc, (bool)parFile,
+			Util::EmptyStr(hash16k) ? nullptr : hash16k,
+			Util::EmptyStr(parSetId) ? nullptr : parSetId);
 	}
 
+	nzbInfo->GetParameters()->clear();
 	int parameterCount;
 	if (infile.ScanLine("%i", &parameterCount) != 1) goto error;
 	for (int i = 0; i < parameterCount; i++)
@@ -729,6 +914,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		}
 	}
 
+	nzbInfo->GetScriptStatuses()->clear();
 	int scriptCount;
 	if (infile.ScanLine("%i", &scriptCount) != 1) goto error;
 	for (int i = 0; i < scriptCount; i++)
@@ -758,6 +944,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		}
 	}
 
+	nzbInfo->GetFileList()->clear();
 	if (infile.ScanLine("%i", &fileCount) != 1) goto error;
 	for (int i = 0; i < fileCount; i++)
 	{
@@ -775,19 +962,14 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 
 		std::unique_ptr<FileInfo> fileInfo = std::make_unique<FileInfo>();
 		fileInfo->SetId(id);
-
-		bool res = LoadFile(fileInfo.get(), true, false);
-		if (res)
+		fileInfo->SetPaused(paused);
+		if (formatVersion < 56)
 		{
-			fileInfo->SetPaused(paused);
-			if (formatVersion < 56)
-			{
-				fileInfo->SetTime(time);
-			}
-			fileInfo->SetExtraPriority(extraPriority != 0);
-			fileInfo->SetNzbInfo(nzbInfo);
-			nzbInfo->GetFileList()->Add(std::move(fileInfo));
+			fileInfo->SetTime(time);
 		}
+		fileInfo->SetExtraPriority((bool)extraPriority);
+		fileInfo->SetNzbInfo(nzbInfo);
+		nzbInfo->GetFileList()->Add(std::move(fileInfo));
 	}
 
 	return true;
@@ -840,7 +1022,7 @@ bool DiskState::SaveFile(FileInfo* fileInfo)
 	debug("Saving FileInfo %i to disk", fileInfo->GetId());
 
 	BString<100> filename("%i", fileInfo->GetId());
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* outfile = stateFile.BeginWrite();
 	if (!outfile)
@@ -848,15 +1030,16 @@ bool DiskState::SaveFile(FileInfo* fileInfo)
 		return false;
 	}
 
-	return SaveFileInfo(fileInfo, *outfile) && stateFile.FinishWrite();
+	return SaveFileInfo(fileInfo, *outfile, true) && stateFile.FinishWrite();
 }
 
-bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
+bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles)
 {
 	outfile.PrintLine("%s", fileInfo->GetSubject());
 	outfile.PrintLine("%s", fileInfo->GetFilename());
+	outfile.PrintLine("%s", fileInfo->GetOrigname() ? fileInfo->GetOrigname() : "");
 
-	outfile.PrintLine("%i", (int)fileInfo->GetTime());
+	outfile.PrintLine("%i,%i", (int)fileInfo->GetFilenameConfirmed(), (int)fileInfo->GetTime());
 
 	uint32 High, Low;
 	Util::SplitInt64(fileInfo->GetSize(), &High, &Low);
@@ -874,11 +1057,14 @@ bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
 		outfile.PrintLine("%s", *group);
 	}
 
-	outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
-	for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+	if (articles)
 	{
-		outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
-		outfile.PrintLine("%s", articleInfo->GetMessageId());
+		outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
+		for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+		{
+			outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
+			outfile.PrintLine("%s", articleInfo->GetMessageId());
+		}
 	}
 
 	return true;
@@ -894,7 +1080,7 @@ bool DiskState::LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles)
 	debug("Loading FileInfo %i from disk", fileInfo->GetId());
 
 	BString<100> filename("%i", fileInfo->GetId());
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* infile = stateFile.BeginRead();
 	if (!infile)
@@ -915,7 +1101,20 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	if (fileSummary) fileInfo->SetFilename(buf);
 
-	if (formatVersion >= 4)
+	if (formatVersion >= 6)
+	{
+		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+		if (fileSummary) fileInfo->SetOrigname(Util::EmptyStr(buf) ? nullptr : buf);
+	}
+
+	if (formatVersion >= 5)
+	{
+		int time, filenameConfirmed;
+		if (infile.ScanLine("%i,%i", &filenameConfirmed, &time) != 2) goto error;
+		if (fileSummary) fileInfo->SetFilenameConfirmed((bool)filenameConfirmed);
+		if (fileSummary) fileInfo->SetTime((time_t)time);
+	}
+	else if (formatVersion >= 4)
 	{
 		int time;
 		if (infile.ScanLine("%i", &time) != 1) goto error;
@@ -948,10 +1147,9 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
 		if (fileSummary) fileInfo->GetGroups()->push_back(buf);
 	}
 
-	if (infile.ScanLine("%i", &size) != 1) goto error;
-
 	if (articles)
 	{
+		if (infile.ScanLine("%i", &size) != 1) goto error;
 		for (int i = 0; i < size; i++)
 		{
 			int PartNumber, PartSize;
@@ -979,7 +1177,7 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, bool completed)
 	debug("Saving FileState %i to disk", fileInfo->GetId());
 
 	BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* outfile = stateFile.BeginWrite();
 	if (!outfile)
@@ -1001,6 +1199,9 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool c
 	outfile.PrintLine("%u,%u,%u,%u,%u,%u", High1, Low1, High2, Low2, High3, Low3);
 
 	outfile.PrintLine("%s", fileInfo->GetFilename());
+	outfile.PrintLine("%s", fileInfo->GetHash16k() ? fileInfo->GetHash16k() : "");
+	outfile.PrintLine("%s", fileInfo->GetParSetId() ? fileInfo->GetParSetId() : "");
+	outfile.PrintLine("%i", (int)fileInfo->GetParFile());
 
 	SaveServerStats(fileInfo->GetServerStats(), outfile);
 
@@ -1020,7 +1221,7 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, bool complet
 	debug("Loading FileInfo %i from disk", fileInfo->GetId());
 
 	BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
-	StateFile stateFile(filename, 4, false);
+	StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);
 
 	StateDiskFile* infile = stateFile.BeginRead();
 	if (!infile)
@@ -1046,13 +1247,28 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil
 	fileInfo->SetSuccessSize(Util::JoinInt64(High2, Low2));
 	fileInfo->SetFailedSize(Util::JoinInt64(High3, Low3));
 
+	char buf[1024];
+
 	if (formatVersion >= 4)
 	{
-		char buf[1024];
 		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 		fileInfo->SetFilename(buf);
 	}
 
+	if (formatVersion >= 5)
+	{
+		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+		fileInfo->SetHash16k(*buf ? buf : nullptr);
+		if (formatVersion >= 6)
+		{
+			if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+			fileInfo->SetParSetId(*buf ? buf : nullptr);
+		}
+		int parFile = 0;
+		if (infile.ScanLine("%i", &parFile) != 1) goto error;
+		fileInfo->SetParFile((bool)parFile);
+	}
+
 	if (!LoadServerStats(fileInfo->GetServerStats(), servers, infile)) goto error;
 
 	int completedArticles;
@@ -1091,6 +1307,13 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil
 			status = ArticleInfo::aiUndefined;
 		}
 
+		if (status == ArticleInfo::aiFinished && !g_Options->GetDirectWrite() &&
+			!fileInfo->GetForceDirectWrite() && !pa->GetResultFilename())
+		{
+			pa->SetResultFilename(BString<1024>("%s%c%i.%03i", g_Options->GetTempDir(),
+				PATH_SEPARATOR, fileInfo->GetId(), pa->GetPartNumber()));
+		}
+
 		// don't allow all articles be completed or the file will stuck.
 		// such states should never be saved on disk but just in case.
 		if (completedArticles == size - 1 && !completed)
@@ -1330,9 +1553,23 @@ void DiskState::CleanupTempDir(DownloadQueue* downloadQueue)
 	DirBrowser dir(g_Options->GetTempDir());
 	while (const char* filename = dir.Next())
 	{
+		bool garbage = strstr(filename, ".tmp") || strstr(filename, ".dec");
+
 		int id, part;
-		if (strstr(filename, ".tmp") || strstr(filename, ".dec") ||
-			(sscanf(filename, "%i.%i", &id, &part) == 2))
+		if (!garbage && sscanf(filename, "%i.%i", &id, &part) == 2)
+		{
+			garbage = true;
+			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+			{
+				if (nzbInfo->GetFileList()->Find(id))
+				{
+					garbage = false;
+					break;
+				}
+			}
+		}
+
+		if (garbage)
 		{
 			BString<1024> fullFilename("%s%c%s", g_Options->GetTempDir(), PATH_SEPARATOR, filename);
 			FileSystem::DeleteFile(fullFilename);
@@ -1342,6 +1579,51 @@ void DiskState::CleanupTempDir(DownloadQueue* downloadQueue)
 
 void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 {
+	// Prepare sorted id lists for faster search
+
+	std::vector<int> nzbIdList;
+	std::vector<int> fileIdList;
+
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		nzbIdList.push_back(nzbInfo->GetId());
+
+		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		{
+			fileIdList.push_back(fileInfo->GetId());
+		}
+
+		for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+		{
+			fileIdList.push_back(completedFile.GetId());
+		}
+	}
+
+	for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
+	{
+		if (historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			historyInfo->GetKind() == HistoryInfo::hkUrl)
+		{
+			NzbInfo* nzbInfo = historyInfo->GetNzbInfo();
+			nzbIdList.push_back(nzbInfo->GetId());
+
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
+			{
+				fileIdList.push_back(fileInfo->GetId());
+			}
+
+			for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
+			{
+				fileIdList.push_back(completedFile.GetId());
+			}
+		}
+	}
+
+	std::sort(nzbIdList.begin(), nzbIdList.end());
+	std::sort(fileIdList.begin(), fileIdList.end());
+
+	// Do cleanup
+
 	int deletedFiles = 0;
 
 	DirBrowser dir(g_Options->GetQueueDir());
@@ -1354,74 +1636,12 @@ void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 		if ((sscanf(filename, "%i%c", &id, &suffix) == 2 && (suffix == 's' || suffix == 'c')) ||
 			(sscanf(filename, "%i", &id) == 1 && !strchr(filename, '.')))
 		{
-			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
-			{
-				for (FileInfo* fileInfo : nzbInfo->GetFileList())
-				{
-					if (fileInfo->GetId() == id)
-					{
-						goto next;
-					}
-				}
-
-				for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
-				{
-					if (completedFile.GetId() == id)
-					{
-						goto next;
-					}
-				}
-			}
-
-			for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
-			{
-				if (historyInfo->GetKind() == HistoryInfo::hkNzb)
-				{
-					NzbInfo* nzbInfo = historyInfo->GetNzbInfo();
-
-					for (FileInfo* fileInfo : nzbInfo->GetFileList())
-					{
-						if (fileInfo->GetId() == id)
-						{
-							goto next;
-						}
-					}
-
-					for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
-					{
-						if (completedFile.GetId() == id)
-						{
-							goto next;
-						}
-					}
-				}
-			}
-
-			del = true;
+			del = !std::binary_search(fileIdList.begin(), fileIdList.end(), id);
 		}
 
 		if (!del && sscanf(filename, "n%i.log", &id) == 1)
 		{
-			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
-			{
-				if (nzbInfo->GetId() == id)
-				{
-					goto next;
-				}
-			}
-
-			for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
-			{
-				if (historyInfo->GetKind() == HistoryInfo::hkNzb)
-				{
-					if (historyInfo->GetNzbInfo()->GetId() == id)
-					{
-						goto next;
-					}
-				}
-			}
-
-			del = true;
+			del = !std::binary_search(nzbIdList.begin(), nzbIdList.end(), id);
 		}
 
 		if (del)
@@ -1431,8 +1651,6 @@ void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 			FileSystem::DeleteFile(fullFilename);
 			deletedFiles++;
 		}
-
-		next:;
 	}
 
 	if (deletedFiles > 0)
@@ -1450,7 +1668,7 @@ bool DiskState::SaveFeeds(Feeds* feeds, FeedHistory* feedHistory)
 {
 	debug("Saving feeds state to disk");
 
-	StateFile stateFile("feeds", 3, true);
+	StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);
 
 	if (feeds->empty() && feedHistory->empty())
 	{
@@ -1478,7 +1696,7 @@ bool DiskState::LoadFeeds(Feeds* feeds, FeedHistory* feedHistory)
 {
 	debug("Loading feeds state from disk");
 
-	StateFile stateFile("feeds", 3, true);
+	StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);
 
 	if (!stateFile.FileExists())
 	{
@@ -1620,6 +1838,119 @@ void DiskState::CalcFileStats(DownloadQueue* downloadQueue, int formatVersion)
 	}
 }
 
+bool DiskState::SaveAllFileInfos(DownloadQueue* downloadQueue)
+{
+	bool ok = true;
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, true);
+	if (!downloadQueue->GetQueue()->empty())
+	{
+		StateDiskFile* outfile = stateFile.BeginWrite();
+		if (!outfile)
+		{
+			return false;
+		}
+
+		// save file-infos
+
+		int fileCount = 0;
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			fileCount += nzbInfo->GetFileList()->size();
+		}
+		outfile->PrintLine("%i", fileCount);
+
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
+			{
+				outfile->PrintLine("%i", fileInfo->GetId());
+				SaveFileInfo(fileInfo, *outfile, false);
+			}
+		}
+
+		// now rename to dest file name
+		ok = stateFile.FinishWrite();
+	}
+	else
+	{
+		stateFile.Discard();
+	}
+
+	return ok;
+}
+
+bool DiskState::LoadAllFileInfos(DownloadQueue* downloadQueue)
+{
+	if (downloadQueue->GetQueue()->empty())
+	{
+		return true;
+	}
+
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
+	StateDiskFile* infile = nullptr;
+	bool useHibernate = false;
+
+	if (stateFile.FileExists())
+	{
+		infile = stateFile.BeginRead();
+		useHibernate = infile != nullptr;
+		if (useHibernate)
+		{
+			int fileCount = 0;
+			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+			{
+				fileCount += nzbInfo->GetFileList()->size();
+			}
+			int size = 0;
+			useHibernate = infile->ScanLine("%i", &size) == 1 && size == fileCount;
+		}
+		if (!useHibernate)
+		{
+			stateFile.Discard();
+		}
+	}
+
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		RawFileList brokenFileInfos;
+
+		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		{
+			bool res = false;
+			if (useHibernate)
+			{
+				int id = 0;
+				infile->ScanLine("%i", &id);
+				if (id == fileInfo->GetId())
+				{
+					res = LoadFileInfo(fileInfo, *infile, stateFile.GetFileVersion(), true, false);
+				}
+			}
+			if (!res)
+			{
+				res = LoadFile(fileInfo, true, false);
+			}
+			if (!res)
+			{
+				brokenFileInfos.push_back(fileInfo);
+			}
+		}
+
+		for (FileInfo* fileInfo : brokenFileInfos)
+		{
+			nzbInfo->GetFileList()->Remove(fileInfo);
+		}
+	}
+
+	return true;
+}
+
+void DiskState::DiscardQuickFileInfos()
+{
+	StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
+	stateFile.Discard();
+}
+
 bool DiskState::LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers)
 {
 	BString<1024> cacheFlagFilename("%s%c%s", g_Options->GetQueueDir(), PATH_SEPARATOR, "acache");
@@ -1667,7 +1998,7 @@ bool DiskState::SaveStats(Servers* servers, ServerVolumes* serverVolumes)
 {
 	debug("Saving stats to disk");
 
-	StateFile stateFile("stats", 3, true);
+	StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);
 
 	if (servers->empty())
 	{
@@ -1695,7 +2026,7 @@ bool DiskState::LoadStats(Servers* servers, ServerVolumes* serverVolumes, bool*
 {
 	debug("Loading stats from disk");
 
-	StateFile stateFile("stats", 3, true);
+	StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);
 
 	if (!stateFile.FileExists())
 	{

daemon/queue/DiskState.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -36,8 +36,11 @@ public:
 	bool DownloadQueueExists();
 	bool SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory);
 	bool LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers);
+	bool SaveDownloadProgress(DownloadQueue* downloadQueue);
 	bool SaveFile(FileInfo* fileInfo);
 	bool LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles);
+	bool SaveAllFileInfos(DownloadQueue* downloadQueue);
+	void DiscardQuickFileInfos();
 	bool SaveFileState(FileInfo* fileInfo, bool completed);
 	bool LoadFileState(FileInfo* fileInfo, Servers* servers, bool completed);
 	bool LoadArticles(FileInfo* fileInfo);
@@ -55,12 +58,14 @@ public:
 	void LoadNzbMessages(int nzbId, MessageList* messages);
 
 private:
-	bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile);
+	bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles);
 	bool LoadFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, int formatVersion, bool fileSummary, bool articles);
 	bool SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool completed);
 	bool LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFile& infile, int formatVersion, bool completed);
 	void SaveQueue(NzbList* queue, StateDiskFile& outfile);
 	bool LoadQueue(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion);
+	void SaveProgress(NzbList* queue, StateDiskFile& outfile, int changedCount);
+	bool LoadProgress(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion);
 	void SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile);
 	bool LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& infile, int formatVersion);
 	void SaveDupInfo(DupInfo* dupInfo, StateDiskFile& outfile);
@@ -76,6 +81,7 @@ private:
 	bool SaveVolumeStat(ServerVolumes* serverVolumes, StateDiskFile& outfile);
 	bool LoadVolumeStat(Servers* servers, ServerVolumes* serverVolumes, StateDiskFile& infile, int formatVersion);
 	void CalcFileStats(DownloadQueue* downloadQueue, int formatVersion);
+	bool LoadAllFileInfos(DownloadQueue* downloadQueue);
 	bool LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers);
 	void SaveServerStats(ServerStatList* serverStatList, StateDiskFile& outfile);
 	bool LoadServerStats(ServerStatList* serverStatList, Servers* servers, StateDiskFile& infile);

daemon/queue/DownloadInfo.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -40,7 +40,7 @@ void NzbParameterList::SetParameter(const char* name, const char* value)
 	iterator pos = std::find_if(begin(), end(),
 		[name](NzbParameter& parameter)
 		{
-			return !strcmp(parameter.GetName(), name);
+			return !strcasecmp(parameter.GetName(), name);
 		});
 
 	if (emptyVal && pos != end())
@@ -57,12 +57,11 @@ void NzbParameterList::SetParameter(const char* name, const char* value)
 	}
 }
 
-NzbParameter* NzbParameterList::Find(const char* name, bool caseSensitive)
+NzbParameter* NzbParameterList::Find(const char* name)
 {
 	for (NzbParameter& parameter : this)
 	{
-		if ((caseSensitive && !strcmp(parameter.GetName(), name)) ||
-			(!caseSensitive && !strcasecmp(parameter.GetName(), name)))
+		if (!strcasecmp(parameter.GetName(), name))
 		{
 			return &parameter;
 		}
@@ -347,10 +346,12 @@ void NzbInfo::UpdateMinMaxTime()
 	}
 }
 
-void NzbInfo::AddMessage(Message::EKind kind, const char * text)
+void NzbInfo::AddMessage(Message::EKind kind, const char * text, bool print)
 {
-	switch (kind)
+	if (print)
 	{
+		switch (kind)
+		{
 		case Message::mkDetail:
 			detail("%s", text);
 			break;
@@ -370,19 +371,20 @@ void NzbInfo::AddMessage(Message::EKind kind, const char * text)
 		case Message::mkDebug:
 			debug("%s", text);
 			break;
+		}
 	}
 
 	Guard guard(m_logMutex);
 
 	m_messages.emplace_back(++m_idMessageGen, kind, Util::CurrentTime(), text);
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetNzbLog())
+	if (g_Options->GetServerMode() && g_Options->GetNzbLog())
 	{
 		g_DiskState->AppendNzbMessage(m_id, kind, text);
 		m_messageCount++;
 	}
 
-	while (m_messages.size() > (uint32)g_Options->GetLogBufferSize())
+	while (m_messages.size() > (uint32)g_Options->GetLogBuffer())
 	{
 		m_messages.pop_front();
 	}
@@ -470,13 +472,20 @@ void NzbInfo::SetActiveDownloads(int activeDownloads)
 		if (activeDownloads > 0)
 		{
 			m_downloadStartTime = Util::CurrentTime();
+			m_downloadStartSec = m_downloadSec;
 		}
 		else
 		{
-			m_downloadSec += Util::CurrentTime() - m_downloadStartTime;
+			m_downloadSec = m_downloadStartSec + (int)(Util::CurrentTime() - m_downloadStartTime);
 			m_downloadStartTime = 0;
+			m_changed = true;
 		}
 	}
+	else if (activeDownloads > 0)
+	{
+		m_downloadSec = m_downloadStartSec + (int)(Util::CurrentTime() - m_downloadStartTime);
+		m_changed = true;
+	}
 	m_activeDownloads = activeDownloads;
 }
 
@@ -490,6 +499,9 @@ bool NzbInfo::IsDupeSuccess()
 		m_parStatus == NzbInfo::psFailure ||
 		m_unpackStatus == NzbInfo::usFailure ||
 		m_unpackStatus == NzbInfo::usPassword ||
+		m_urlStatus == NzbInfo::lsFailed ||
+		m_urlStatus == NzbInfo::lsScanSkipped ||
+		m_urlStatus == NzbInfo::lsScanFailed ||
 		(m_parStatus == NzbInfo::psSkipped &&
 		 m_unpackStatus == NzbInfo::usSkipped &&
 		 CalcHealth() < CalcCriticalHealth(true)));
@@ -622,6 +634,10 @@ const char* NzbInfo::MakeTextStatus(bool ignoreScriptStatus)
 		{
 			status = "DELETED/DUPE";
 		}
+		else if (m_deleteStatus == NzbInfo::dsGood)
+		{
+			status = "DELETED/GOOD";
+		}
 		else
 		{
 			const char* urlStatusName[] = { "FAILURE/INTERNAL_ERROR", "FAILURE/INTERNAL_ERROR", "FAILURE/INTERNAL_ERROR",
@@ -645,6 +661,7 @@ void NzbInfo::UpdateCurrentStats()
 	m_currentFailedSize = m_failedSize;
 	m_parCurrentSuccessSize = m_parSuccessSize;
 	m_parCurrentFailedSize = m_parFailedSize;
+	m_extraPriority = 0;
 
 	m_currentServerStats.ListOp(&m_serverStats, ServerStatList::soSet);
 
@@ -655,6 +672,7 @@ void NzbInfo::UpdateCurrentStats()
 		m_currentFailedArticles += fileInfo->GetFailedArticles();
 		m_currentSuccessSize += fileInfo->GetSuccessSize();
 		m_currentFailedSize += fileInfo->GetFailedSize();
+		m_extraPriority += fileInfo->GetExtraPriority() ? 1 : 0;
 
 		if (fileInfo->GetPaused())
 		{
@@ -678,6 +696,7 @@ void NzbInfo::UpdateCompletedStats(FileInfo* fileInfo)
 	m_failedSize += fileInfo->GetFailedSize();
 	m_failedArticles += fileInfo->GetFailedArticles();
 	m_successArticles += fileInfo->GetSuccessArticles();
+	m_extraPriority -= fileInfo->GetExtraPriority() ? 1 : 0;
 
 	if (fileInfo->GetParFile())
 	{
@@ -706,6 +725,7 @@ void NzbInfo::UpdateDeletedStats(FileInfo* fileInfo)
 	m_currentSuccessArticles -= fileInfo->GetSuccessArticles();
 	m_currentFailedArticles -= fileInfo->GetFailedArticles() + fileInfo->GetMissedArticles();
 	m_remainingSize -= fileInfo->GetRemainingSize();
+	m_extraPriority -= fileInfo->GetExtraPriority() ? 1 : 0;
 
 	if (fileInfo->GetParFile())
 	{
@@ -725,6 +745,24 @@ void NzbInfo::UpdateDeletedStats(FileInfo* fileInfo)
 	m_currentServerStats.ListOp(fileInfo->GetServerStats(), ServerStatList::soSubtract);
 }
 
+bool NzbInfo::IsDownloadCompleted(bool ignorePausedPars)
+{
+	if (m_activeDownloads)
+	{
+		return false;
+	}
+
+	for (FileInfo* fileInfo : &m_fileList)
+	{
+		if ((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) &&
+			!fileInfo->GetDeleted())
+		{
+			return false;
+		}
+	}
+
+	return true;
+}
 
 void ArticleInfo::AttachSegment(std::unique_ptr<SegmentData> content, int64 offset, int size)
 {
@@ -771,6 +809,15 @@ void FileInfo::SetPaused(bool paused)
 	m_paused = paused;
 }
 
+void FileInfo::SetExtraPriority(bool extraPriority)
+{
+	if (m_extraPriority != extraPriority && m_nzbInfo)
+	{
+		m_nzbInfo->SetExtraPriority(m_nzbInfo->GetExtraPriority() + (extraPriority ? 1 : -1));
+	}
+	m_extraPriority = extraPriority;
+}
+
 void FileInfo::MakeValidFilename()
 {
 	m_filename = FileSystem::MakeValidFilename(m_filename);
@@ -791,9 +838,10 @@ void FileInfo::SetActiveDownloads(int activeDownloads)
 }
 
 
-CompletedFile::CompletedFile(int id, const char* fileName, EStatus status, uint32 crc) :
-	m_id(id), m_fileName(fileName), m_status(status), m_crc(crc)
-
+CompletedFile::CompletedFile(int id, const char* filename, const char* origname, EStatus status,
+	uint32 crc, bool parFile, const char* hash16k, const char* parSetId) :
+	m_id(id), m_filename(filename), m_origname(origname), m_status(status),
+	m_crc(crc), m_parFile(parFile), m_hash16k(hash16k), m_parSetId(parSetId)
 {
 	if (FileInfo::m_idMax < m_id)
 	{

daemon/queue/DownloadInfo.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -140,6 +140,8 @@ public:
 	void SetSubject(const char* subject) { m_subject = subject; }
 	const char* GetFilename() { return m_filename; }
 	void SetFilename(const char* filename) { m_filename = filename; }
+	void SetOrigname(const char* origname) { m_origname = origname; }
+	const char* GetOrigname() { return m_origname; }
 	void MakeValidFilename();
 	bool GetFilenameConfirmed() { return m_filenameConfirmed; }
 	void SetFilenameConfirmed(bool filenameConfirmed) { m_filenameConfirmed = filenameConfirmed; }
@@ -177,7 +179,7 @@ public:
 	bool GetOutputInitialized() { return m_outputInitialized; }
 	void SetOutputInitialized(bool outputInitialized) { m_outputInitialized = outputInitialized; }
 	bool GetExtraPriority() { return m_extraPriority; }
-	void SetExtraPriority(bool extraPriority) { m_extraPriority = extraPriority; }
+	void SetExtraPriority(bool extraPriority);
 	int GetActiveDownloads() { return m_activeDownloads; }
 	void SetActiveDownloads(int activeDownloads);
 	bool GetDupeDeleted() { return m_dupeDeleted; }
@@ -192,6 +194,13 @@ public:
 	void SetPartialState(EPartialState partialState) { m_partialState = partialState; }
 	uint32 GetCrc() { return m_crc; }
 	void SetCrc(uint32 crc) { m_crc = crc; }
+	const char* GetHash16k() { return m_hash16k; }
+	void SetHash16k(const char* hash16k) { m_hash16k = hash16k; }
+	const char* GetParSetId() { return m_parSetId; }
+	void SetParSetId(const char* parSetId) { m_parSetId = parSetId; }
+	bool GetFlushLocked() { return m_flushLocked; }
+	void SetFlushLocked(bool flushLocked) { m_flushLocked = flushLocked; }
+
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 
 private:
@@ -202,6 +211,7 @@ private:
 	ServerStatList m_serverStats;
 	CString m_subject;
 	CString m_filename;
+	CString m_origname;
 	int64 m_size = 0;
 	int64 m_remainingSize = 0;
 	int64 m_successSize = 0;
@@ -228,6 +238,9 @@ private:
 	bool m_forceDirectWrite = false;
 	EPartialState m_partialState = psNone;
 	uint32 m_crc = 0;
+	CString m_hash16k;
+	CString m_parSetId;
+	bool m_flushLocked = false;
 
 	static int m_idGen;
 	static int m_idMax;
@@ -249,18 +262,30 @@ public:
 		cfFailure
 	};
 
-	CompletedFile(int id, const char* fileName, EStatus status, uint32 crc);
+	CompletedFile(int id, const char* filename, const char* oldname, EStatus status,
+		uint32 crc, bool parFile, const char* hash16k, const char* parSetId);
 	int GetId() { return m_id; }
-	void SetFileName(const char* fileName) { m_fileName = fileName; }
-	const char* GetFileName() { return m_fileName; }
+	void SetFilename(const char* filename) { m_filename = filename; }
+	const char* GetFilename() { return m_filename; }
+	void SetOrigname(const char* origname) { m_origname = origname; }
+	const char* GetOrigname() { return m_origname; }
+	bool GetParFile() { return m_parFile; }
 	EStatus GetStatus() { return m_status; }
 	uint32 GetCrc() { return m_crc; }
+	const char* GetHash16k() { return m_hash16k; }
+	void SetHash16k(const char* hash16k) { m_hash16k = hash16k; }
+	const char* GetParSetId() { return m_parSetId; }
+	void SetParSetId(const char* parSetId) { m_parSetId = parSetId; }
 
 private:
 	int m_id;
-	CString m_fileName;
+	CString m_filename;
+	CString m_origname;
 	EStatus m_status;
 	uint32 m_crc;
+	bool m_parFile;
+	CString m_hash16k;
+	CString m_parSetId;
 };
 
 typedef std::deque<CompletedFile> CompletedFileList;
@@ -288,7 +313,7 @@ class NzbParameterList : public NzbParameterListBase
 {
 public:
 	void SetParameter(const char* name, const char* value);
-	NzbParameter* Find(const char* name, bool caseSensitive);
+	NzbParameter* Find(const char* name);
 	void CopyFrom(NzbParameterList* sourceParameters);
 };
 
@@ -332,7 +357,15 @@ enum EDupeMode
 class NzbInfo
 {
 public:
-	enum ERenameStatus
+	enum EDirectRenameStatus
+	{
+		tsNone,
+		tsRunning,
+		tsFailure,
+		tsSuccess
+	};
+
+	enum EPostRenameStatus
 	{
 		rsNone,
 		rsSkipped,
@@ -350,7 +383,15 @@ public:
 		psManual
 	};
 
-	enum EUnpackStatus
+	enum EDirectUnpackStatus
+	{
+		nsNone,
+		nsRunning,
+		nsFailure,
+		nsSuccess
+	};
+
+	enum EPostUnpackStatus
 	{
 		usNone,
 		usSkipped,
@@ -411,6 +452,13 @@ public:
 		nkUrl
 	};
 
+	enum EDupeHint
+	{
+		dhNone,
+		dhRedownloadManual,
+		dhRedownloadAuto
+	};
+
 	int GetId() { return m_id; }
 	void SetId(int id);
 	static void ResetGenId(bool max);
@@ -477,6 +525,9 @@ public:
 	void SetCurrentFailedArticles(int currentFailedArticles) { m_currentFailedArticles = currentFailedArticles; }
 	int GetPriority() { return m_priority; }
 	void SetPriority(int priority) { m_priority = priority; }
+	int GetExtraPriority() { return m_extraPriority; }
+	void SetExtraPriority(int extraPriority) { m_extraPriority = extraPriority; }
+	bool HasExtraPriority() { return m_extraPriority > 0; }
 	bool GetForcePriority() { return m_priority >= FORCE_PRIORITY; }
 	time_t GetMinTime() { return m_minTime; }
 	void SetMinTime(time_t minTime) { m_minTime = minTime; }
@@ -485,14 +536,18 @@ public:
 	void BuildDestDirName();
 	CString BuildFinalDirName();
 	CompletedFileList* GetCompletedFiles() { return &m_completedFiles; }
-	ERenameStatus GetParRenameStatus() { return m_parRenameStatus; }
-	void SetParRenameStatus(ERenameStatus renameStatus) { m_parRenameStatus = renameStatus; }
-	ERenameStatus GetRarRenameStatus() { return m_rarRenameStatus; }
-	void SetRarRenameStatus(ERenameStatus renameStatus) { m_rarRenameStatus = renameStatus; }
+	void SetDirectRenameStatus(EDirectRenameStatus renameStatus) { m_directRenameStatus = renameStatus; }
+	EDirectRenameStatus GetDirectRenameStatus() { return m_directRenameStatus; }
+	EPostRenameStatus GetParRenameStatus() { return m_parRenameStatus; }
+	void SetParRenameStatus(EPostRenameStatus renameStatus) { m_parRenameStatus = renameStatus; }
+	EPostRenameStatus GetRarRenameStatus() { return m_rarRenameStatus; }
+	void SetRarRenameStatus(EPostRenameStatus renameStatus) { m_rarRenameStatus = renameStatus; }
 	EParStatus GetParStatus() { return m_parStatus; }
 	void SetParStatus(EParStatus parStatus) { m_parStatus = parStatus; }
-	EUnpackStatus GetUnpackStatus() { return m_unpackStatus; }
-	void SetUnpackStatus(EUnpackStatus unpackStatus) { m_unpackStatus = unpackStatus; }
+	EDirectUnpackStatus GetDirectUnpackStatus() { return m_directUnpackStatus; }
+	void SetDirectUnpackStatus(EDirectUnpackStatus directUnpackStatus) { m_directUnpackStatus = directUnpackStatus; }
+	EPostUnpackStatus GetUnpackStatus() { return m_unpackStatus; }
+	void SetUnpackStatus(EPostUnpackStatus unpackStatus) { m_unpackStatus = unpackStatus; }
 	ECleanupStatus GetCleanupStatus() { return m_cleanupStatus; }
 	void SetCleanupStatus(ECleanupStatus cleanupStatus) { m_cleanupStatus = cleanupStatus; }
 	EMoveStatus GetMoveStatus() { return m_moveStatus; }
@@ -538,6 +593,8 @@ public:
 	void SetDupeScore(int dupeScore) { m_dupeScore = dupeScore; }
 	EDupeMode GetDupeMode() { return m_dupeMode; }
 	void SetDupeMode(EDupeMode dupeMode) { m_dupeMode = dupeMode; }
+	EDupeHint GetDupeHint() { return m_dupeHint; }
+	void SetDupeHint(EDupeHint dupeHint) { m_dupeHint = dupeHint; }
 	uint32 GetFullContentHash() { return m_fullContentHash; }
 	void SetFullContentHash(uint32 fullContentHash) { m_fullContentHash = fullContentHash; }
 	uint32 GetFilteredContentHash() { return m_filteredContentHash; }
@@ -556,6 +613,8 @@ public:
 	void SetUnpackSec(int unpackSec) { m_unpackSec = unpackSec; }
 	time_t GetDownloadStartTime() { return m_downloadStartTime; }
 	void SetDownloadStartTime(time_t downloadStartTime) { m_downloadStartTime = downloadStartTime; }
+	bool GetChanged() { return m_changed; }
+	void SetChanged(bool changed) { m_changed = changed; }
 	void SetReprocess(bool reprocess) { m_reprocess = reprocess; }
 	bool GetReprocess() { return m_reprocess; }
 	time_t GetQueueScriptTime() { return m_queueScriptTime; }
@@ -571,15 +630,24 @@ public:
 	void LeavePostProcess();
 	bool IsDupeSuccess();
 	const char* MakeTextStatus(bool ignoreScriptStatus);
-	void AddMessage(Message::EKind kind, const char* text);
+	void AddMessage(Message::EKind kind, const char* text, bool print = true);
 	void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
 	int GetMessageCount() { return m_messageCount; }
 	void SetMessageCount(int messageCount) { m_messageCount = messageCount; }
 	int GetCachedMessageCount() { return m_cachedMessageCount; }
 	GuardedMessageList GuardCachedMessages() { return GuardedMessageList(&m_messages, &m_logMutex); }
+	bool GetAllFirst() { return m_allFirst; }
+	void SetAllFirst(bool allFirst) { m_allFirst = allFirst; }
+	bool GetWaitingPar() { return m_waitingPar; }
+	void SetWaitingPar(bool waitingPar) { m_waitingPar = waitingPar; }
+	bool GetLoadingPar() { return m_loadingPar; }
+	void SetLoadingPar(bool loadingPar) { m_loadingPar = loadingPar; }
+	Thread* GetUnpackThread() { return m_unpackThread; }
+	void SetUnpackThread(Thread* unpackThread) { m_unpackThread = unpackThread; }
 	void UpdateCurrentStats();
 	void UpdateCompletedStats(FileInfo* fileInfo);
 	void UpdateDeletedStats(FileInfo* fileInfo);
+	bool IsDownloadCompleted(bool ignorePausedPars);
 
 	static const int FORCE_PRIORITY = 900;
 
@@ -617,11 +685,14 @@ private:
 	time_t m_minTime = 0;
 	time_t m_maxTime = 0;
 	int m_priority = 0;
+	int m_extraPriority = 0;
 	CompletedFileList m_completedFiles;
-	ERenameStatus m_parRenameStatus = rsNone;
-	ERenameStatus m_rarRenameStatus = rsNone;
+	EDirectRenameStatus m_directRenameStatus = tsNone;
+	EPostRenameStatus m_parRenameStatus = rsNone;
+	EPostRenameStatus m_rarRenameStatus = rsNone;
 	EParStatus m_parStatus = psNone;
-	EUnpackStatus m_unpackStatus = usNone;
+	EDirectUnpackStatus m_directUnpackStatus = nsNone;
+	EPostUnpackStatus m_unpackStatus = usNone;
 	ECleanupStatus m_cleanupStatus = csNone;
 	EMoveStatus m_moveStatus = msNone;
 	EDeleteStatus m_deleteStatus = dsNone;
@@ -642,6 +713,7 @@ private:
 	CString m_dupeKey = "";
 	int m_dupeScore = 0;
 	EDupeMode m_dupeMode = dmScore;
+	EDupeHint m_dupeHint = dhNone;
 	uint32 m_fullContentHash = 0;
 	uint32 m_filteredContentHash = 0;
 	FileList m_fileList;
@@ -655,17 +727,23 @@ private:
 	std::unique_ptr<PostInfo> m_postInfo;
 	int64 m_downloadedSize = 0;
 	time_t m_downloadStartTime = 0;
+	int m_downloadStartSec = 0;
 	int m_downloadSec = 0;
 	int m_postTotalSec = 0;
 	int m_parSec = 0;
 	int m_repairSec = 0;
 	int m_unpackSec = 0;
 	bool m_reprocess = false;
+	bool m_changed = false;
 	time_t m_queueScriptTime = 0;
 	bool m_parFull = false;
 	int m_messageCount = 0;
 	int m_cachedMessageCount = 0;
 	int m_feedId = 0;
+	bool m_allFirst = false;
+	bool m_waitingPar = false;
+	bool m_loadingPar = false;
+	Thread* m_unpackThread = nullptr;
 
 	static int m_idGen;
 	static int m_idMax;
@@ -698,6 +776,7 @@ public:
 	};
 
 	typedef std::vector<CString> ParredFiles;
+	typedef std::vector<CString> ExtractedArchives;
 
 	NzbInfo* GetNzbInfo() { return m_nzbInfo; }
 	void SetNzbInfo(NzbInfo* nzbInfo) { m_nzbInfo = nzbInfo; }
@@ -736,6 +815,7 @@ public:
 	Thread* GetPostThread() { return m_postThread; }
 	void SetPostThread(Thread* postThread) { m_postThread = postThread; }
 	ParredFiles* GetParredFiles() { return &m_parredFiles; }
+	ExtractedArchives* GetExtractedArchives() { return &m_extractedArchives; }
 
 private:
 	NzbInfo* m_nzbInfo = nullptr;
@@ -757,6 +837,7 @@ private:
 	time_t m_stageTime = 0;
 	Thread* m_postThread = nullptr;
 	ParredFiles m_parredFiles;
+	ExtractedArchives m_extractedArchives;
 };
 
 typedef std::vector<int> IdList;
@@ -820,7 +901,7 @@ public:
 	};
 
 	HistoryInfo(std::unique_ptr<NzbInfo> nzbInfo) : m_info(nzbInfo.release()),
-		m_kind(nzbInfo->GetKind() == NzbInfo::nkNzb ? hkNzb : hkUrl) {}
+		m_kind(GetNzbInfo()->GetKind() == NzbInfo::nkNzb ? hkNzb : hkUrl) {}
 	HistoryInfo(std::unique_ptr<DupInfo> dupInfo) : m_info(dupInfo.release()), m_kind(hkDup) {}
 	~HistoryInfo();
 	EKind GetKind() { return m_kind; }
@@ -833,8 +914,8 @@ public:
 	const char* GetName();
 
 private:
-	EKind m_kind;
 	void* m_info;
+	EKind m_kind;
 	time_t m_time = 0;
 };
 
@@ -850,9 +931,16 @@ public:
 		eaNzbFound,
 		eaNzbAdded,
 		eaNzbDeleted,
+		eaNzbNamed,
+		eaNzbReturned,
 		eaFileCompleted,
 		eaFileDeleted,
-		eaUrlCompleted
+		eaUrlFound,
+		eaUrlAdded,
+		eaUrlDeleted,
+		eaUrlCompleted,
+		eaUrlFailed,
+		eaUrlReturned
 	};
 
 	struct Aspect
@@ -872,7 +960,7 @@ public:
 		eaFileResume, // resume (unpause) files
 		eaFileDelete, // delete files
 		eaFilePauseAllPars, // pause only (all) pars (does not affect other files)
-		eaFilePauseExtraPars, // pause only (almost all) pars, except main par-file (does not affect other files)
+		eaFilePauseExtraPars, // pause (almost all) pars, except main par-file (does not affect other files)
 		eaFileReorder, // set file order
 		eaFileSplit, // split - create new group from selected files
 		eaGroupMoveOffset, // move group to offset relative to the current position in download-queue
@@ -898,6 +986,7 @@ public:
 		eaGroupSetDupeScore, // set duplicate score
 		eaGroupSetDupeMode, // set duplicate mode
 		eaGroupSort, // sort groups
+		eaGroupSortFiles, // sort files for optimal download order
 		eaPostDelete, // cancel post-processing
 		eaHistoryDelete, // hide history-item
 		eaHistoryFinalDelete, // delete history-item
@@ -932,6 +1021,7 @@ public:
 	virtual bool EditList(IdList* idList, NameList* nameList, EMatchMode matchMode, EEditAction action, const char* args) = 0;
 	virtual void HistoryChanged() = 0;
 	virtual void Save() = 0;
+	virtual void SaveChanged() = 0;
 	void CalcRemainingSize(int64* remaining, int64* remainingForced);
 
 protected:

daemon/queue/DupeCoordinator.cpp

@@ -31,8 +31,8 @@ bool DupeCoordinator::SameNameOrKey(const char* name1, const char* dupeKey1,
 	const char* name2, const char* dupeKey2)
 {
 	bool hasDupeKeys = !Util::EmptyStr(dupeKey1) && !Util::EmptyStr(dupeKey2);
-	return (hasDupeKeys && !strcmp(dupeKey1, dupeKey2)) ||
-		(!hasDupeKeys && !strcmp(name1, name2));
+	return (hasDupeKeys && !strcasecmp(dupeKey1, dupeKey2)) ||
+		(!hasDupeKeys && !strcasecmp(name1, name2));
 }
 
 /**
@@ -202,12 +202,19 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		}
 	}
 
+	if (!sameContent && nzbInfo->GetDupeHint() != NzbInfo::dhNone)
+	{
+		// dupe check when "download again" URLs: checking same content only
+		return;
+	}
+
 	if (!sameContent && !good && nzbInfo->GetDupeMode() == dmScore)
 	{
 		// nzb-files having success-duplicates in recent history (with different content) are added to history for backup
 		for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
 		{
-			if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+			if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+				 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 				historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 				SameNameOrKey(historyInfo->GetNzbInfo()->GetName(), historyInfo->GetNzbInfo()->GetDupeKey(),
 					nzbInfo->GetName(), nzbInfo->GetDupeKey()) &&
@@ -237,7 +244,7 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 				sameContent ? "exactly same content" : good ? "good status" : "success status");
 		}
 
-		if (nzbInfo->GetFeedId())
+		if (nzbInfo->GetFeedId() && nzbInfo->GetDupeHint() == NzbInfo::dhNone)
 		{
 			warn("%s", *message);
 			// Flag saying QueueCoordinator to skip nzb-file
@@ -263,7 +270,9 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		{
 			NzbInfo* queuedNzbInfo = (*it++).get();
 			if (queuedNzbInfo != nzbInfo &&
-				queuedNzbInfo->GetKind() == NzbInfo::nkNzb &&
+				queuedNzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+				(queuedNzbInfo->GetKind() == NzbInfo::nkNzb ||
+				 (queuedNzbInfo->GetKind() == NzbInfo::nkUrl && nzbInfo->GetKind() == NzbInfo::nkUrl)) &&
 				queuedNzbInfo->GetDupeMode() != dmForce &&
 				SameNameOrKey(queuedNzbInfo->GetName(), queuedNzbInfo->GetDupeKey(),
 					nzbInfo->GetName(), nzbInfo->GetDupeKey()))
@@ -286,9 +295,13 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 					// the existing queue item is moved to history as dupe-backup
 					info("Moving collection %s with lower duplicate score to history", queuedNzbInfo->GetName());
 					queuedNzbInfo->SetDeleteStatus(NzbInfo::dsDupe);
+					int oldSize = downloadQueue->GetQueue()->size();
 					downloadQueue->EditEntry(queuedNzbInfo->GetId(),
 						DownloadQueue::eaGroupDelete, nullptr);
+					int newSize = downloadQueue->GetQueue()->size();
+					index += oldSize == newSize ? 1 : 0;
 					it = downloadQueue->GetQueue()->begin() + index;
+					index--;
 				}
 			}
 		}
@@ -377,7 +390,8 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI
 	HistoryInfo* historyDupe = nullptr;
 	for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
 	{
-		if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+		if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 			historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 			historyInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsDupe &&
 			historyInfo->GetNzbInfo()->CalcHealth() >= historyInfo->GetNzbInfo()->CalcCriticalHealth(true) &&
@@ -395,6 +409,7 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI
 	if (historyDupe)
 	{
 		info("Found duplicate %s for %s", historyDupe->GetNzbInfo()->GetName(), nzbName);
+		historyDupe->GetNzbInfo()->SetDupeHint(NzbInfo::dhRedownloadAuto);
 		g_HistoryCoordinator->Redownload(downloadQueue, historyDupe);
 	}
 }
@@ -465,7 +480,8 @@ void DupeCoordinator::HistoryCleanup(DownloadQueue* downloadQueue, HistoryInfo*
 	{
 		HistoryInfo* historyInfo = (*it).get();
 
-		if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+		if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 			historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 			historyInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsDupe &&
 			historyInfo != markHistoryInfo &&

daemon/queue/HistoryCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -87,7 +87,7 @@ void HistoryCoordinator::ServiceWork()
 
 void HistoryCoordinator::DeleteDiskFiles(NzbInfo* nzbInfo)
 {
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		// delete parked files
 		g_DiskState->DiscardFiles(nzbInfo);
@@ -130,8 +130,9 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 	for (FileInfo* fileInfo : nzbInfo->GetFileList())
 	{
 		nzbInfo->UpdateCompletedStats(fileInfo);
-		nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(),
-			fileInfo->GetFilename(), CompletedFile::cfNone, 0);
+		nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(),
+			fileInfo->GetOrigname(), CompletedFile::cfNone, 0, fileInfo->GetParFile(),
+			fileInfo->GetHash16k(), fileInfo->GetParSetId());
 	}
 
 	// Cleaning up parked files if par-check was successful or unpack was successful or
@@ -169,7 +170,7 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 			(completedFile.GetStatus() == CompletedFile::cfPartial &&
 			 &completedFile == &*nzbInfo->GetCompletedFiles()->rbegin()))
 		{
-			nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFileName());
+			nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFilename());
 			nzbInfo->SetParkedFileCount(nzbInfo->GetParkedFileCount() + 1);
 		}
 	}
@@ -178,6 +179,13 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 	nzbInfo->SetRemainingParCount(0);
 	nzbInfo->SetParking(false);
 
+	if (nzbInfo->GetDirectRenameStatus() == NzbInfo::tsRunning)
+	{
+		nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure);
+	}
+
+	nzbInfo->SetDupeHint(NzbInfo::dhNone);
+
 	nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName());
 }
 
@@ -379,6 +387,7 @@ void HistoryCoordinator::MoveToQueue(DownloadQueue* downloadQueue, HistoryList::
 	if (!nzbInfo->GetUnpackCleanedUpDisk())
 	{
 		nzbInfo->SetUnpackStatus(NzbInfo::usNone);
+		nzbInfo->SetDirectUnpackStatus(NzbInfo::nsNone);
 		nzbInfo->SetCleanupStatus(NzbInfo::csNone);
 		nzbInfo->SetParRenameStatus(NzbInfo::rsNone);
 		nzbInfo->SetRarRenameStatus(NzbInfo::rsNone);
@@ -415,6 +424,8 @@ void HistoryCoordinator::MoveToQueue(DownloadQueue* downloadQueue, HistoryList::
 		// start postprocessing
 		debug("Restarting postprocessing for %s", *nicename);
 		g_PrePostProcessor->NzbDownloaded(downloadQueue, nzbInfo);
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaNzbReturned, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
 	}
 }
 
@@ -427,8 +438,13 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 		historyInfo->DiscardNzbInfo();
 		nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 		nzbInfo->SetDeleteStatus(NzbInfo::dsNone);
+		nzbInfo->SetDupeHint(nzbInfo->GetDupeHint() == NzbInfo::dhNone ? NzbInfo::dhRedownloadManual : nzbInfo->GetDupeHint());
 		downloadQueue->GetQueue()->Add(std::unique_ptr<NzbInfo>(nzbInfo), true);
 		downloadQueue->GetHistory()->erase(itHistory);
+
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaUrlReturned, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
+
 		return;
 	}
 
@@ -493,6 +509,8 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	nzbInfo->SetParStatus(NzbInfo::psNone);
 	nzbInfo->SetParRenameStatus(NzbInfo::rsNone);
 	nzbInfo->SetRarRenameStatus(NzbInfo::rsNone);
+	nzbInfo->SetDirectRenameStatus(NzbInfo::tsNone);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsNone);
 	nzbInfo->SetDownloadedSize(0);
 	nzbInfo->SetDownloadSec(0);
 	nzbInfo->SetPostTotalSec(0);
@@ -500,6 +518,9 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	nzbInfo->SetRepairSec(0);
 	nzbInfo->SetUnpackSec(0);
 	nzbInfo->SetExtraParBlocks(0);
+	nzbInfo->SetAllFirst(false);
+	nzbInfo->SetWaitingPar(false);
+	nzbInfo->SetLoadingPar(false);
 	nzbInfo->GetCompletedFiles()->clear();
 	nzbInfo->GetServerStats()->clear();
 	nzbInfo->GetCurrentServerStats()->clear();
@@ -511,6 +532,9 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	MoveToQueue(downloadQueue, itHistory, historyInfo, false);
 
 	g_PrePostProcessor->NzbAdded(downloadQueue, nzbInfo);
+
+	DownloadQueue::Aspect aspect = {DownloadQueue::eaNzbReturned, downloadQueue, nzbInfo, nullptr};
+	downloadQueue->Notify(&aspect);
 }
 
 void HistoryCoordinator::HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory,
@@ -587,7 +611,7 @@ void HistoryCoordinator::HistoryRetry(DownloadQueue* downloadQueue, HistoryList:
 				  g_DiskState->LoadFileState(fileInfo.get(), g_ServerPool->GetServers(), true) &&
 				  (resetFailed || fileInfo->GetRemainingSize() > 0))))
 			{
-				fileInfo->SetFilename(completedFile.GetFileName());
+				fileInfo->SetFilename(completedFile.GetFilename());
 				fileInfo->SetNzbInfo(nzbInfo);
 
 				BString<1024> outputFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename());

daemon/queue/HistoryCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ public:
 	void Redownload(DownloadQueue* downloadQueue, HistoryInfo* historyInfo);
 
 protected:
-	virtual int ServiceInterval() { return 600000; }
+	virtual int ServiceInterval() { return 60 * 60; }
 	virtual void ServiceWork();
 
 private:

daemon/queue/NzbFile.cpp

@@ -117,6 +117,12 @@ void NzbFile::ParseSubject(FileInfo* fileInfo, bool TryQuotes)
 {
 	// Example subject: some garbage "title" yEnc (10/99)
 
+	if (!fileInfo->GetSubject())
+	{
+		// Malformed file element without subject. We generate subject using internal element id.
+		fileInfo->SetSubject(CString::FormatStr("%d", fileInfo->GetId()));
+	}
+
 	// strip the "yEnc (10/99)"-suffix
 	BString<1024> subject = fileInfo->GetSubject();
 	char* end = subject + strlen(subject) - 1;
@@ -374,7 +380,7 @@ void NzbFile::ProcessFiles()
 
 	CalcHashes();
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		for (FileInfo* fileInfo : m_nzbInfo->GetFileList())
 		{
@@ -595,6 +601,10 @@ bool NzbFile::ParseNzb(IUnknown* nzb)
 
 bool NzbFile::Parse()
 {
+#ifdef DISABLE_LIBXML2
+	error("Could not parse rss feed, program was compiled without libxml2 support");
+	return false;
+#else
 	xmlSAXHandler SAX_handler = {0};
 	SAX_handler.startElement = reinterpret_cast<startElementSAXFunc>(SAX_StartElement);
 	SAX_handler.endElement = reinterpret_cast<endElementSAXFunc>(SAX_EndElement);
@@ -623,6 +633,7 @@ bool NzbFile::Parse()
 	ProcessFiles();
 
 	return true;
+#endif
 }
 
 void NzbFile::Parse_StartElement(const char *name, const char **atts)
@@ -805,7 +816,11 @@ void NzbFile::SAX_characters(NzbFile* file, const char * xmlstr, int len)
 
 void* NzbFile::SAX_getEntity(NzbFile* file, const char * name)
 {
+#ifdef DISABLE_LIBXML2
+	void* e = nullptr;
+#else
 	xmlEntityPtr e = xmlGetPredefinedEntity((xmlChar* )name);
+#endif
 	if (!e)
 	{
 		file->m_nzbInfo->AddMessage(Message::mkWarning, "entity not found");

daemon/queue/QueueCoordinator.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "nzbget.h"
 #include "QueueCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "ArticleDownloader.h"
 #include "ArticleWriter.h"
@@ -58,21 +59,39 @@ void QueueCoordinator::CoordinatorDownloadQueue::Save()
 		return;
 	}
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		g_DiskState->SaveDownloadQueue(this, m_historyChanged);
+		m_stateChanged = true;
+	}
+
+	for (NzbInfo* nzbInfo : GetQueue())
+	{
+		nzbInfo->SetChanged(false);
 	}
 
 	m_wantSave = false;
 	m_historyChanged = false;
+
+	// queue has changed, time to wake up if in standby
+	m_owner->WakeUp();
+}
+
+void QueueCoordinator::CoordinatorDownloadQueue::SaveChanged()
+{
+	if (g_Options->GetServerMode())
+	{
+		g_DiskState->SaveDownloadProgress(this);
+		m_stateChanged = true;
+	}
 }
 
 QueueCoordinator::QueueCoordinator()
 {
 	debug("Creating QueueCoordinator");
 
-	m_downloadQueue.m_owner = this;
 	CoordinatorDownloadQueue::Init(&m_downloadQueue);
+	g_WorkState->Attach(this);
 }
 
 QueueCoordinator::~QueueCoordinator()
@@ -98,11 +117,11 @@ void QueueCoordinator::Load()
 	bool perfectServerMatch = true;
 	bool queueLoaded = false;
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue())
+	if (g_Options->GetServerMode())
 	{
 		statLoaded = g_StatMeter->Load(&perfectServerMatch);
 
-		if (g_Options->GetReloadQueue() && g_DiskState->DownloadQueueExists())
+		if (g_DiskState->DownloadQueueExists())
 		{
 			queueLoaded = g_DiskState->LoadDownloadQueue(downloadQueue, g_ServerPool->GetServers());
 		}
@@ -129,7 +148,7 @@ void QueueCoordinator::Load()
 		downloadQueue->Save();
 
 		// re-save file states into diskstate to update server ids
-		if (g_Options->GetServerMode() && g_Options->GetSaveQueue())
+		if (g_Options->GetServerMode())
 		{
 			for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 			{
@@ -172,8 +191,9 @@ void QueueCoordinator::Run()
 	AdjustDownloadsLimit();
 	bool wasStandBy = true;
 	bool articeDownloadsRunning = false;
-	int resetCounter = 0;
+	time_t lastReset = 0;
 	g_StatMeter->IntervalCheck();
+	int waitInterval = 100;
 
 	while (!IsStopped())
 	{
@@ -194,7 +214,7 @@ void QueueCoordinator::Run()
 				downloadsChecked = true;
 				m_hasMoreJobs = hasMoreArticles || articeDownloadsRunning;
 				if (hasMoreArticles && !IsStopped() && (int)m_activeDownloads.size() < m_downloadsLimit &&
-					(!g_Options->GetTempPauseDownload() || fileInfo->GetExtraPriority()))
+					(!g_WorkState->GetTempPauseDownload() || fileInfo->GetExtraPriority()))
 				{
 					StartArticleDownload(fileInfo, articleInfo, connection);
 					articeDownloadsRunning = true;
@@ -222,46 +242,65 @@ void QueueCoordinator::Run()
 		if (standBy != wasStandBy)
 		{
 			g_StatMeter->EnterLeaveStandBy(standBy);
+			g_WorkState->SetDownloading(!standBy);
 			wasStandBy = standBy;
 			if (standBy)
 			{
-				SavePartialState();
+				SaveAllPartialState();
 			}
 		}
 
 		// sleep longer in StandBy
-		int sleepInterval = downloadStarted ? 0 : standBy ? 100 : 5;
-		usleep(sleepInterval * 1000);
-
-		if (!standBy)
+		if (standBy)
 		{
+			Guard guard(m_waitMutex);
+			// sleeping max. 2 seconds; can't sleep much longer because we can't rely on
+			// notifications from 'WorkState' and we also have periodical work to do here
+			waitInterval = std::min(waitInterval * 2, 2000);
+			m_waitCond.WaitFor(m_waitMutex, waitInterval, [&]{ return m_hasMoreJobs || IsStopped(); });
+		}
+		else
+		{
+			int sleepInterval = downloadStarted ? 0 : 5;
+			Util::Sleep(sleepInterval);
 			g_StatMeter->AddSpeedReading(0);
+			waitInterval = 100;
 		}
 
-		Util::SetStandByMode(standBy);
-
-		resetCounter += sleepInterval;
-		if (resetCounter >= 1000)
+		if (lastReset != Util::CurrentTime())
 		{
 			// this code should not be called too often, once per second is OK
 			g_ServerPool->CloseUnusedConnections();
 			ResetHangingDownloads();
 			if (!standBy)
 			{
-				SavePartialState();
+				SaveAllPartialState();
 			}
-			resetCounter = 0;
 			g_StatMeter->IntervalCheck();
+			g_Log->IntervalCheck();
 			AdjustDownloadsLimit();
+			Util::SetStandByMode(standBy);
+			lastReset = Util::CurrentTime();
 		}
 	}
 
 	WaitJobs();
-	SavePartialState();
+	SaveAllPartialState();
+	SaveQueueIfChanged();
+	SaveAllFileState();
 
 	debug("Exiting QueueCoordinator-loop");
 }
 
+void QueueCoordinator::WakeUp()
+{
+	debug("Waking up QueueCoordinator");
+	// Resume Run()
+	Guard guard(m_waitMutex);
+	m_hasMoreJobs = true;
+	m_waitCond.NotifyAll();
+}
+
 void QueueCoordinator::WaitJobs()
 {
 	// waiting for downloads
@@ -276,7 +315,7 @@ void QueueCoordinator::WaitJobs()
 				break;
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 
@@ -327,7 +366,7 @@ NzbInfo* QueueCoordinator::AddNzbFileToQueue(std::unique_ptr<NzbInfo> nzbInfo, N
 		for (FileInfo* fileInfo: nzbInfo->GetFileList())
 		{
 			allPaused &= fileInfo->GetPaused();
-			if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+			if (g_Options->GetServerMode())
 			{
 				g_DiskState->DiscardFile(fileInfo->GetId(), true, false, false);
 			}
@@ -394,11 +433,6 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 {
 	debug("CheckDupeFileInfos");
 
-	if (!g_Options->GetDupeCheck() || nzbInfo->GetDupeMode() == dmForce)
-	{
-		return;
-	}
-
 	RawFileList dupeList;
 
 	int index1 = 0;
@@ -415,9 +449,19 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 				(fileInfo->GetSize() < fileInfo2->GetSize() ||
 				 (fileInfo->GetSize() == fileInfo2->GetSize() && index2 < index1)))
 			{
-				warn("File \"%s\" appears twice in collection, adding only the biggest file", fileInfo->GetFilename());
-				dupe = true;
-				break;
+				// If more than two files have same filename we don't filter them out since that
+				// naming might be intentional and correct filenames must be read from article bodies.
+				int dupeCount = (int)std::count_if(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+					[fileInfo2](std::unique_ptr<FileInfo>& fileInfo3)
+					{
+						return !strcmp(fileInfo3->GetFilename(), fileInfo2->GetFilename());
+					});
+				if (dupeCount == 2)
+				{
+					warn("File \"%s\" appears twice in collection, adding only the biggest file", fileInfo->GetFilename());
+					dupe = true;
+					break;
+				}
 			}
 		}
 		if (dupe)
@@ -431,7 +475,7 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 	{
 		nzbInfo->UpdateDeletedStats(fileInfo);
 		nzbInfo->GetFileList()->Remove(fileInfo);
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		if (g_Options->GetServerMode())
 		{
 			g_DiskState->DiscardFile(fileInfo->GetId(), true, false, false);
 		}
@@ -443,12 +487,18 @@ void QueueCoordinator::Stop()
 	Thread::Stop();
 
 	debug("Stopping ArticleDownloads");
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
-	for (ArticleDownloader* articleDownloader : m_activeDownloads)
 	{
-		articleDownloader->Stop();
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		for (ArticleDownloader* articleDownloader : m_activeDownloads)
+		{
+			articleDownloader->Stop();
+		}
 	}
 	debug("ArticleDownloads are notified");
+
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
 }
 
 /*
@@ -460,8 +510,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 	// if the file doesn't have any articles left for download, we store that fact and search again,
 	// ignoring all files which were previously marked as not having any articles.
 
-	// special case: if the file has ExtraPriority-flag set, it has the highest priority and the
-	// Paused-flag is ignored.
+	// special case: if the file has ExtraPriority-flag set, it has the highest priority.
 
 	//debug("QueueCoordinator::GetNextArticle()");
 
@@ -476,20 +525,34 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 
 		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 		{
-			for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
+			bool nzbHigherPriority = fileInfo &&
+				((nzbInfo->HasExtraPriority() == fileInfo->GetNzbInfo()->HasExtraPriority() &&
+					nzbInfo->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
+					(nzbInfo->HasExtraPriority() > fileInfo->GetNzbInfo()->HasExtraPriority()));
+
+			bool nzbPaused = nzbInfo->GetFileList()->size() - nzbInfo->GetPausedFileCount() <= 0;
+
+			if ((!fileInfo || nzbHigherPriority) && !nzbPaused &&
+				(!(g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) || nzbInfo->GetForcePriority()))
 			{
-				if ((checkedFiles.empty() ||
-					 std::find(checkedFiles.begin(), checkedFiles.end(), fileInfo1) == checkedFiles.end()) &&
-					!fileInfo1->GetPaused() && !fileInfo1->GetDeleted() &&
-					(g_Options->GetPropagationDelay() == 0 ||
-					 (int)fileInfo1->GetTime() < (int)curDate - g_Options->GetPropagationDelay()) &&
-					(!(g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) || nzbInfo->GetForcePriority()) &&
-					(!fileInfo ||
-					 (fileInfo1->GetExtraPriority() == fileInfo->GetExtraPriority() &&
-					  fileInfo1->GetNzbInfo()->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
-					 (fileInfo1->GetExtraPriority() > fileInfo->GetExtraPriority())))
+				for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
 				{
-					fileInfo = fileInfo1;
+					bool alreadyChecked = !checkedFiles.empty() &&
+						std::find(checkedFiles.begin(), checkedFiles.end(), fileInfo1) != checkedFiles.end();
+
+					bool propagationWait = g_Options->GetPropagationDelay() > 0 &&
+						(int)fileInfo1->GetTime() + g_Options->GetPropagationDelay() >= (int)curDate;
+
+					bool higherPriority = fileInfo &&
+						((fileInfo1->GetExtraPriority() == fileInfo->GetExtraPriority() &&
+							fileInfo1->GetNzbInfo()->GetPriority() > fileInfo->GetNzbInfo()->GetPriority()) ||
+							(fileInfo1->GetExtraPriority() > fileInfo->GetExtraPriority()));
+
+					if (!alreadyChecked && !propagationWait && !fileInfo1->GetPaused() && 
+						!fileInfo1->GetDeleted() && (!fileInfo || higherPriority))
+					{
+						fileInfo = fileInfo1;
+					}
 				}
 			}
 		}
@@ -500,7 +563,15 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 			break;
 		}
 
-		if (fileInfo->GetArticles()->empty() && g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		if (g_Options->GetDirectRename() &&
+			fileInfo->GetNzbInfo()->GetDirectRenameStatus() <= NzbInfo::tsRunning &&
+			!fileInfo->GetNzbInfo()->GetAllFirst() &&
+			GetNextFirstArticle(fileInfo->GetNzbInfo(), fileInfo, articleInfo))
+		{
+			return true;
+		}
+
+		if (fileInfo->GetArticles()->empty() && g_Options->GetServerMode())
 		{
 			g_DiskState->LoadArticles(fileInfo);
 			LoadPartialState(fileInfo);
@@ -512,8 +583,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 			if (article->GetStatus() == ArticleInfo::aiUndefined)
 			{
 				articleInfo = article;
-				ok = true;
-				break;
+				return true;
 			}
 		}
 
@@ -525,7 +595,39 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 		}
 	}
 
-	return ok;
+	return false;
+}
+
+bool QueueCoordinator::GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo, ArticleInfo* &articleInfo)
+{
+	// find a file not renamed yet
+	for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
+	{
+		if (!fileInfo1->GetFilenameConfirmed())
+		{
+			if (fileInfo1->GetArticles()->empty() && g_Options->GetServerMode())
+			{
+				g_DiskState->LoadArticles(fileInfo1);
+				LoadPartialState(fileInfo1);
+			}
+			if (!fileInfo1->GetArticles()->empty())
+			{
+				ArticleInfo* article = fileInfo1->GetArticles()->at(0).get();
+				if (article->GetStatus() == ArticleInfo::aiUndefined)
+				{
+					fileInfo = fileInfo1;
+					articleInfo = article;
+					nzbInfo->SetDirectRenameStatus(NzbInfo::tsRunning);
+					return true;
+				}
+			}
+		}
+	}
+
+	// no more files for renaming remained
+	nzbInfo->SetAllFirst(true);
+
+	return false;
 }
 
 void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* articleInfo, NntpConnection* connection)
@@ -539,7 +641,12 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art
 	articleDownloader->SetArticleInfo(articleInfo);
 	articleDownloader->SetConnection(connection);
 
-	BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size());
+	if (articleInfo->GetPartNumber() == 1 && g_Options->GetDirectRename() && !g_Options->GetRawArticle())
+	{
+		articleDownloader->SetContentAnalyzer(m_directRenamer.MakeArticleContentAnalyzer());
+	}
+
+	BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size());
 	articleDownloader->SetInfoName(infoName);
 
 	articleInfo->SetStatus(ArticleInfo::aiRunning);
@@ -550,11 +657,18 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art
 	articleDownloader->Start();
 }
 
-void QueueCoordinator::Update(Subject* Caller, void* Aspect)
+void QueueCoordinator::Update(Subject* caller, void* aspect)
 {
+	if (caller == g_WorkState)
+	{
+		debug("Notification from WorkState received");
+		WakeUp();
+		return;
+	}
+
 	debug("Notification from ArticleDownloader received");
 
-	ArticleDownloader* articleDownloader = (ArticleDownloader*)Caller;
+	ArticleDownloader* articleDownloader = (ArticleDownloader*)caller;
 	if ((articleDownloader->GetStatus() == ArticleDownloader::adFinished) ||
 		(articleDownloader->GetStatus() == ArticleDownloader::adFailed) ||
 		(articleDownloader->GetStatus() == ArticleDownloader::adRetry))
@@ -568,12 +682,14 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 	debug("Article downloaded");
 
 	FileInfo* fileInfo = articleDownloader->GetFileInfo();
-	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
-	ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
-	bool retry = false;
-	bool fileCompleted = false;
+	bool completeFileParts = false;
 
 	{
+		NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+		ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
+		bool retry = false;
+		bool fileCompleted = false;
+
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
 		if (articleDownloader->GetStatus() == ArticleDownloader::adFinished)
@@ -598,6 +714,10 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 		{
 			articleInfo->SetStatus(ArticleInfo::aiUndefined);
 			retry = true;
+			if (articleInfo->GetPartNumber() == 1)
+			{
+				nzbInfo->SetAllFirst(false);
+			}
 		}
 
 		if (!retry)
@@ -619,13 +739,22 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 			articleDownloader->GetStatus() == ArticleDownloader::adFinished &&
 			articleDownloader->GetArticleFilename())
 		{
-			fileInfo->SetFilename(articleDownloader->GetArticleFilename());
-			fileInfo->MakeValidFilename();
+			// in "FileNaming=auto"-mode prefer filename from nzb-file to filename read from article
+			// if the name from article seems to be obfuscated
+			bool useFilenameFromArticle = g_Options->GetFileNaming() == Options::nfArticle ||
+				(g_Options->GetFileNaming() == Options::nfAuto &&
+				 !Util::AlphaNum(articleDownloader->GetArticleFilename()) &&
+				 !nzbInfo->GetManyDupeFiles());
+			if (useFilenameFromArticle)
+			{
+				fileInfo->SetFilename(articleDownloader->GetArticleFilename());
+				fileInfo->MakeValidFilename();
+			}
 			fileInfo->SetFilenameConfirmed(true);
 			if (g_Options->GetDupeCheck() &&
 				nzbInfo->GetDupeMode() != dmForce &&
 				!nzbInfo->GetManyDupeFiles() &&
-				FileSystem::FileExists(nzbInfo->GetDestDir(), fileInfo->GetFilename()))
+				FileSystem::FileExists(BString<1024>("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename())))
 			{
 				warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", fileInfo->GetFilename());
 				fileCompleted = false;
@@ -634,6 +763,11 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 			}
 		}
 
+		if (articleDownloader->GetContentAnalyzer() && articleDownloader->GetStatus() == ArticleDownloader::adFinished)
+		{
+			m_directRenamer.ArticleDownloaded(downloadQueue, fileInfo, articleInfo, articleDownloader->GetContentAnalyzer());
+		}
+
 		nzbInfo->SetDownloadedSize(nzbInfo->GetDownloadedSize() + articleDownloader->GetDownloadedSize());
 
 		CheckHealth(downloadQueue, fileInfo);
@@ -642,35 +776,45 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
 		{
 			fileCompleted = true;
 		}
+
+		completeFileParts = fileCompleted && (!fileInfo->GetDeleted() || nzbInfo->GetParking());
+
+		if (!completeFileParts)
+		{
+			DeleteDownloader(downloadQueue, articleDownloader, false);
+		}
 	}
 
-	bool deleteFileObj = false;
-
-	if (fileCompleted && (!fileInfo->GetDeleted() || nzbInfo->GetParking()))
+	if (completeFileParts)
 	{
 		// all jobs done
 		articleDownloader->CompleteFileParts();
 		fileInfo->SetPartialChanged(false);
-		deleteFileObj = true;
-	}
 
-	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		DeleteDownloader(downloadQueue, articleDownloader, true);
+	}
+}
 
-		bool hasOtherDownloaders = fileInfo->GetActiveDownloads() > 1;
-		deleteFileObj |= fileInfo->GetDeleted() && !hasOtherDownloaders;
+void QueueCoordinator::DeleteDownloader(DownloadQueue* downloadQueue,
+	ArticleDownloader* articleDownloader, bool fileCompleted)
+{
+	FileInfo* fileInfo = articleDownloader->GetFileInfo();
+	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
+	bool hasOtherDownloaders = fileInfo->GetActiveDownloads() > 1;
+	bool deleteFileObj = fileCompleted || (fileInfo->GetDeleted() && !hasOtherDownloaders);
 
-		// remove downloader from downloader list
-		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), articleDownloader));
+	// remove downloader from downloader list
+	m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), articleDownloader));
 
-		fileInfo->SetActiveDownloads(fileInfo->GetActiveDownloads() - 1);
-		nzbInfo->SetActiveDownloads(nzbInfo->GetActiveDownloads() - 1);
+	fileInfo->SetActiveDownloads(fileInfo->GetActiveDownloads() - 1);
+	nzbInfo->SetActiveDownloads(nzbInfo->GetActiveDownloads() - 1);
 
-		if (deleteFileObj)
-		{
-			DeleteFileInfo(downloadQueue, fileInfo, fileCompleted);
-			downloadQueue->Save();
-		}
+	if (deleteFileObj)
+	{
+		DeleteFileInfo(downloadQueue, fileInfo, fileCompleted);
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
 	}
 }
 
@@ -678,7 +822,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 {
 	while (g_ArticleCache->FileBusy(fileInfo))
 	{
-		usleep(5*1000);
+		Util::Sleep(5);
 	}
 
 	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
@@ -702,7 +846,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 		fileInfo->GetSuccessArticles() > 0 || fileInfo->GetFailedArticles() > 0 ? CompletedFile::cfPartial :
 		CompletedFile::cfNone;
 
-	if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+	if (g_Options->GetServerMode())
 	{
 		g_DiskState->DiscardFile(fileInfo->GetId(), fileStatus == CompletedFile::cfSuccess || (fileDeleted && !parking), true, false);
 		if (fileStatus == CompletedFile::cfPartial && (completed || parking))
@@ -722,8 +866,20 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 			fileInfo->GetId(),
 			completed && fileInfo->GetOutputFilename() ?
 			FileSystem::BaseFileName(fileInfo->GetOutputFilename()) : fileInfo->GetFilename(),
-			fileStatus,
-			fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0);
+			fileInfo->GetOrigname(), fileStatus,
+			fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0,
+			fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId());
+	}
+
+	if (g_Options->GetDirectRename())
+	{
+		m_directRenamer.FileDownloaded(downloadQueue, fileInfo);
+	}
+
+	if (nzbInfo->GetDirectRenameStatus() == NzbInfo::tsRunning &&
+		!nzbInfo->GetDeleting() && nzbInfo->IsDownloadCompleted(true))
+	{
+		DiscardDirectRename(downloadQueue, nzbInfo);
 	}
 
 	std::unique_ptr<FileInfo> srcFileInfo = nzbInfo->GetFileList()->Remove(fileInfo);
@@ -756,9 +912,29 @@ void QueueCoordinator::DiscardTempFiles(FileInfo* fileInfo)
 	}
 }
 
-void QueueCoordinator::SavePartialState()
+void QueueCoordinator::SaveQueueIfChanged()
 {
-	if (!(g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetContinuePartial()))
+	if (!g_Options->GetServerMode())
+	{
+		return;
+	}
+
+	bool hasChanges = false;
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		hasChanges |= nzbInfo->GetChanged();
+	}
+
+	if (hasChanges)
+	{
+		downloadQueue->Save();
+	}
+}
+
+void QueueCoordinator::SaveAllPartialState()
+{
+	if (!g_Options->GetServerMode() || !g_Options->GetContinuePartial())
 	{
 		return;
 	}
@@ -768,19 +944,26 @@ void QueueCoordinator::SavePartialState()
 	{
 		for (FileInfo* fileInfo : nzbInfo->GetFileList())
 		{
-			if (fileInfo->GetPartialChanged())
-			{
-				debug("Saving partial state for %s", fileInfo->GetFilename());
-				if (fileInfo->GetPartialState() == FileInfo::psCompleted)
-				{
-					g_DiskState->DiscardFile(fileInfo->GetId(), false, false, true);
-				}
-				g_DiskState->SaveFileState(fileInfo, false);
-				fileInfo->SetPartialChanged(false);
-				fileInfo->SetPartialState(FileInfo::psPartial);
-			}
+			SavePartialState(fileInfo);
 		}
 	}
+
+	downloadQueue->SaveChanged();
+}
+
+void QueueCoordinator::SavePartialState(FileInfo* fileInfo)
+{
+	if (fileInfo->GetPartialChanged())
+	{
+		debug("Saving partial state for %s", fileInfo->GetFilename());
+		if (fileInfo->GetPartialState() == FileInfo::psCompleted)
+		{
+			g_DiskState->DiscardFile(fileInfo->GetId(), false, false, true);
+		}
+		g_DiskState->SaveFileState(fileInfo, false);
+		fileInfo->SetPartialChanged(false);
+		fileInfo->SetPartialState(FileInfo::psPartial);
+	}
 }
 
 void QueueCoordinator::LoadPartialState(FileInfo* fileInfo)
@@ -802,6 +985,15 @@ void QueueCoordinator::LoadPartialState(FileInfo* fileInfo)
 	}
 }
 
+void QueueCoordinator::SaveAllFileState()
+{
+	if (g_Options->GetServerMode() && m_downloadQueue.m_stateChanged)
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		g_DiskState->SaveAllFileInfos(downloadQueue);
+	}
+}
+
 void QueueCoordinator::CheckHealth(DownloadQueue* downloadQueue, FileInfo* fileInfo)
 {
 	if (g_Options->GetHealthCheck() == Options::hcNone ||
@@ -844,9 +1036,9 @@ void QueueCoordinator::LogDebugInfo()
 	downloadQueue->CalcRemainingSize(&remaining, &remainingForced);
 	info("     Remaining: %.1f MB, Forced: %.1f MB", remaining / 1024.0 / 1024.0, remainingForced / 1024.0 / 1024.0);
 	info("     Download: %s, Post-process: %s, Scan: %s",
-		 (g_Options->GetPauseDownload() ? "paused" : g_Options->GetTempPauseDownload() ? "temp-paused" : "active"),
-		 (g_Options->GetPausePostProcess() ? "paused" : "active"),
-		 (g_Options->GetPauseScan() ? "paused" : "active"));
+		 (g_WorkState->GetPauseDownload() ? "paused" : g_WorkState->GetTempPauseDownload() ? "temp-paused" : "active"),
+		 (g_WorkState->GetPausePostProcess() ? "paused" : "active"),
+		 (g_WorkState->GetPauseScan() ? "paused" : "active"));
 
 	info("   ---------- QueueCoordinator");
 	info("    Active Downloads: %i, Limit: %i", (int)m_activeDownloads.size(), m_downloadsLimit);
@@ -858,7 +1050,7 @@ void QueueCoordinator::LogDebugInfo()
 
 void QueueCoordinator::ResetHangingDownloads()
 {
-	if (g_Options->GetTerminateTimeout() == 0 && g_Options->GetArticleTimeout() == 0)
+	if (g_Options->GetArticleTimeout() == 0)
 	{
 		return;
 	}
@@ -866,46 +1058,16 @@ void QueueCoordinator::ResetHangingDownloads()
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[tm](ArticleDownloader* articleDownloader)
+	for (ArticleDownloader* articleDownloader : m_activeDownloads)
+	{
+		if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetArticleTimeout() + 1 &&
+			articleDownloader->GetStatus() == ArticleDownloader::adRunning)
 		{
-			if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetArticleTimeout() + 1 &&
-				articleDownloader->GetStatus() == ArticleDownloader::adRunning)
-			{
-				error("Cancelling hanging download %s @ %s", articleDownloader->GetInfoName(),
-					articleDownloader->GetConnectionName());
-				articleDownloader->Stop();
-			}
-
-			if (tm - articleDownloader->GetLastUpdateTime() > g_Options->GetTerminateTimeout() &&
-				articleDownloader->GetStatus() == ArticleDownloader::adRunning)
-			{
-				ArticleInfo* articleInfo = articleDownloader->GetArticleInfo();
-				debug("Terminating hanging download %s", articleDownloader->GetInfoName());
-				if (articleDownloader->Terminate())
-				{
-					error("Terminated hanging download %s @ %s", articleDownloader->GetInfoName(),
-						articleDownloader->GetConnectionName());
-					articleInfo->SetStatus(ArticleInfo::aiUndefined);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s @ %s", articleDownloader->GetInfoName(),
-						articleDownloader->GetConnectionName());
-				}
-
-				articleDownloader->GetFileInfo()->SetActiveDownloads(articleDownloader->GetFileInfo()->GetActiveDownloads() - 1);
-				articleDownloader->GetFileInfo()->GetNzbInfo()->SetActiveDownloads(articleDownloader->GetFileInfo()->GetNzbInfo()->GetActiveDownloads() - 1);
-				articleDownloader->GetFileInfo()->GetNzbInfo()->SetDownloadedSize(articleDownloader->GetFileInfo()->GetNzbInfo()->GetDownloadedSize() + articleDownloader->GetDownloadedSize());
-
-				// it's not safe to destroy pArticleDownloader, because the state of object is unknown
-				delete articleDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging download %s @ %s", articleDownloader->GetInfoName(),
+				articleDownloader->GetConnectionName());
+			articleDownloader->Stop();
+		}
+	}
 }
 
 /*
@@ -1183,3 +1345,144 @@ bool QueueCoordinator::SplitQueueEntries(DownloadQueue* downloadQueue, RawFileLi
 
 	return true;
 }
+
+void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (g_Options->GetServerMode() && !fileInfo->GetArticles()->empty())
+		{
+			// save new file name into disk state file
+			g_DiskState->SaveFile(fileInfo);
+		}
+	}
+
+	DiscardDirectRename(downloadQueue, nzbInfo);
+
+	nzbInfo->SetDirectRenameStatus(NzbInfo::tsSuccess);
+
+	if (g_Options->GetParCheck() != Options::pcForce)
+	{
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupResume, nullptr);
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupPauseAllPars, nullptr);
+	}
+
+	if (g_Options->GetReorderFiles())
+	{
+		nzbInfo->PrintMessage(Message::mkInfo, "Reordering files for %s", nzbInfo->GetName());
+		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr);
+	}
+
+	nzbInfo->SetChanged(true);
+	downloadQueue->SaveChanged();
+
+	DownloadQueue::Aspect namedAspect = { DownloadQueue::eaNzbNamed, downloadQueue, nzbInfo, nullptr };
+	downloadQueue->Notify(&namedAspect);
+}
+
+void QueueCoordinator::DiscardDirectRename(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	int64 discardedSize = 0;
+	int discardedCount = 0;
+
+	for (FileInfo* fileInfo : nzbInfo->GetFileList())
+	{
+		if (fileInfo->GetParFile() && fileInfo->GetCompletedArticles() == 1 &&
+			fileInfo->GetActiveDownloads() == 0)
+		{
+			bool locked = false;
+			{
+				Guard contentGuard = g_ArticleCache->GuardContent();
+				locked = fileInfo->GetFlushLocked();
+				if (!locked)
+				{
+					fileInfo->SetFlushLocked(true);
+				}
+			}
+
+			if (!locked)
+			{
+				// discard downloaded articles from partially downloaded par-files
+				discardedSize += fileInfo->GetSuccessSize();
+				discardedCount++;
+
+				DiscardDownloadedArticles(nzbInfo, fileInfo);
+			}
+
+			if (!locked)
+			{
+				Guard contentGuard = g_ArticleCache->GuardContent();
+				fileInfo->SetFlushLocked(false);
+			}
+		}
+
+		if (g_Options->GetServerMode() &&
+			!fileInfo->GetArticles()->empty() && g_Options->GetContinuePartial() &&
+			fileInfo->GetActiveDownloads() == 0 && fileInfo->GetCachedArticles() == 0)
+		{
+			// discard article infos to free up memory if possible
+			debug("Discarding article infos for %s/%s", nzbInfo->GetName(), fileInfo->GetFilename());
+			fileInfo->SetPartialChanged(true);
+			SavePartialState(fileInfo);
+			fileInfo->GetArticles()->clear();
+		}
+	}
+
+	if (discardedSize > 0)
+	{
+		nzbInfo->PrintMessage(Message::mkDetail, "Discarded %s from %i files used for direct renaming",
+			*Util::FormatSize(discardedSize), discardedCount);
+	}
+}
+
+void QueueCoordinator::DiscardDownloadedArticles(NzbInfo* nzbInfo, FileInfo* fileInfo)
+{
+	nzbInfo->SetRemainingSize(nzbInfo->GetRemainingSize() + fileInfo->GetSuccessSize() + fileInfo->GetFailedSize());
+	if (fileInfo->GetPaused())
+	{
+		nzbInfo->SetPausedSize(nzbInfo->GetPausedSize() + fileInfo->GetSuccessSize() + fileInfo->GetFailedSize());
+	}
+	nzbInfo->GetCurrentServerStats()->ListOp(fileInfo->GetServerStats(), ServerStatList::soSubtract);
+	fileInfo->GetServerStats()->clear();
+
+	nzbInfo->SetCurrentSuccessArticles(nzbInfo->GetCurrentSuccessArticles() - fileInfo->GetSuccessArticles());
+	nzbInfo->SetCurrentSuccessSize(nzbInfo->GetCurrentSuccessSize() - fileInfo->GetSuccessSize());
+	nzbInfo->SetParCurrentSuccessSize(nzbInfo->GetParCurrentSuccessSize() - fileInfo->GetSuccessSize());
+	fileInfo->SetSuccessSize(0);
+	fileInfo->SetSuccessArticles(0);
+
+	nzbInfo->SetCurrentFailedArticles(nzbInfo->GetCurrentFailedArticles() - fileInfo->GetFailedArticles());
+	nzbInfo->SetCurrentFailedSize(nzbInfo->GetCurrentFailedSize() - fileInfo->GetFailedSize());
+	nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() - fileInfo->GetFailedSize());
+	fileInfo->SetFailedSize(0);
+	fileInfo->SetFailedArticles(0);
+
+	fileInfo->SetCompletedArticles(0);
+	fileInfo->SetRemainingSize(fileInfo->GetSize() - fileInfo->GetMissedSize());
+
+	// discard temporary files
+	DiscardTempFiles(fileInfo);
+	g_DiskState->DiscardFile(fileInfo->GetId(), false, true, false);
+
+	fileInfo->SetOutputFilename(nullptr);
+	fileInfo->SetOutputInitialized(false);
+	fileInfo->SetCachedArticles(0);
+	fileInfo->SetPartialChanged(false);
+	fileInfo->SetPartialState(FileInfo::psNone);
+
+	if (g_Options->GetServerMode())
+	{
+		// free up memory used by articles if possible
+		fileInfo->GetArticles()->clear();
+	}
+	else
+	{
+		// reset article states if discarding isn't possible
+		for (ArticleInfo* articleInfo : fileInfo->GetArticles())
+		{
+			articleInfo->SetStatus(ArticleInfo::aiUndefined);
+			articleInfo->SetResultFilename(nullptr);
+			articleInfo->DiscardSegment();
+		}
+	}
+}

daemon/queue/QueueCoordinator.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -30,6 +30,7 @@
 #include "Observer.h"
 #include "QueueEditor.h"
 #include "NntpConnection.h"
+#include "DirectRenamer.h"
 
 class QueueCoordinator : public Thread, public Observer, public Debuggable
 {
@@ -40,7 +41,7 @@ public:
 	virtual ~QueueCoordinator();
 	virtual void Run();
 	virtual void Stop();
-	void Update(Subject* Caller, void* Aspect);
+	void Update(Subject* caller, void* aspect);
 
 	// editing queue
 	NzbInfo* AddNzbFileToQueue(std::unique_ptr<NzbInfo> nzbInfo, NzbInfo* urlInfo, bool addFirst);
@@ -60,37 +61,63 @@ private:
 	class CoordinatorDownloadQueue : public DownloadQueue
 	{
 	public:
+		CoordinatorDownloadQueue(QueueCoordinator* owner) : m_owner(owner) {}
 		virtual bool EditEntry(int ID, EEditAction action, const char* args);
 		virtual bool EditList(IdList* idList, NameList* nameList, EMatchMode matchMode,
 			EEditAction action, const char* args);
 		virtual void HistoryChanged() { m_historyChanged = true; }
 		virtual void Save();
+		virtual void SaveChanged();
 	private:
 		QueueCoordinator* m_owner;
 		bool m_massEdit = false;
 		bool m_wantSave = false;
 		bool m_historyChanged = false;
+		bool m_stateChanged = false;
 		friend class QueueCoordinator;
 	};
 
-	CoordinatorDownloadQueue m_downloadQueue;
+	class CoordinatorDirectRenamer : public DirectRenamer
+	{
+	public:
+		CoordinatorDirectRenamer(QueueCoordinator* owner) : m_owner(owner) {}
+	protected:
+		virtual void RenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+			{ m_owner->DirectRenameCompleted(downloadQueue, nzbInfo); }
+	private:
+		QueueCoordinator* m_owner;
+	};
+
+	CoordinatorDownloadQueue m_downloadQueue{this};
 	ActiveDownloads m_activeDownloads;
 	QueueEditor m_queueEditor;
+	CoordinatorDirectRenamer m_directRenamer{this};
 	bool m_hasMoreJobs = true;
 	int m_downloadsLimit;
 	int m_serverConfigGeneration = 0;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	bool GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
+	bool GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
 	void StartArticleDownload(FileInfo* fileInfo, ArticleInfo* articleInfo, NntpConnection* connection);
 	void ArticleCompleted(ArticleDownloader* articleDownloader);
+	void DeleteDownloader(DownloadQueue* downloadQueue, ArticleDownloader* articleDownloader, bool fileCompleted);
 	void DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fileInfo, bool completed);
+	void DirectRenameCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void DiscardDirectRename(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void DiscardDownloadedArticles(NzbInfo* nzbInfo, FileInfo* fileInfo);
 	void CheckHealth(DownloadQueue* downloadQueue, FileInfo* fileInfo);
 	void ResetHangingDownloads();
 	void AdjustDownloadsLimit();
 	void Load();
-	void SavePartialState();
+	void SaveQueueIfChanged();
+	void SaveAllPartialState();
+	void SavePartialState(FileInfo* fileInfo);
 	void LoadPartialState(FileInfo* fileInfo);
+	void SaveAllFileState();
 	void WaitJobs();
+	void WakeUp();
 };
 
 extern QueueCoordinator* g_QueueCoordinator;

daemon/queue/QueueEditor.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -29,6 +29,7 @@
 #include "PrePostProcessor.h"
 #include "HistoryCoordinator.h"
 #include "UrlCoordinator.h"
+#include "ParParser.h"
 
 const int MAX_ID = 1000000000;
 
@@ -436,7 +437,11 @@ bool QueueEditor::InternEditList(ItemList* itemList,
 						{
 							EditGroup(item.m_nzbInfo, action, args);
 						}
+						break;
 
+					case DownloadQueue::eaGroupSortFiles:
+						SortGroupFiles(item.m_nzbInfo);
+						break;
 
 					default:
 						// suppress compiler warning "enumeration not handled in switch"
@@ -645,7 +650,7 @@ void QueueEditor::PrepareList(ItemList* itemList, IdList* idList,
 bool QueueEditor::BuildIdListFromNameList(IdList* idList, NameList* nameList, DownloadQueue::EMatchMode matchMode, DownloadQueue::EEditAction action)
 {
 #ifndef HAVE_REGEX_H
-	if (matchMode == mmRegEx)
+	if (matchMode == DownloadQueue::mmRegEx)
 	{
 		return false;
 	}
@@ -835,10 +840,7 @@ void QueueEditor::PausePars(RawFileList* fileList, bool extraParsOnly)
 
 	for (FileInfo* fileInfo : fileList)
 	{
-		BString<1024> loFileName = fileInfo->GetFilename();
-		for (char* p = loFileName; *p; p++) *p = tolower(*p); // convert string to lowercase
-
-		if (strstr(loFileName, ".par2"))
+		if (fileInfo->GetParFile())
 		{
 			if (!extraParsOnly)
 			{
@@ -846,6 +848,8 @@ void QueueEditor::PausePars(RawFileList* fileList, bool extraParsOnly)
 			}
 			else
 			{
+				BString<1024> loFileName = fileInfo->GetFilename();
+				for (char* p = loFileName; *p; p++) *p = tolower(*p); // convert string to lowercase
 				if (strstr(loFileName, ".vol"))
 				{
 					Vols.push_back(fileInfo);
@@ -1259,6 +1263,54 @@ void QueueEditor::SetNzbDupeParam(NzbInfo* nzbInfo, DownloadQueue::EEditAction a
 	}
 }
 
+void QueueEditor::SortGroupFiles(NzbInfo* nzbInfo)
+{
+	debug("QueueEditor: sorting inner files for '%s'", nzbInfo->GetName());
+
+	std::sort(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+		[](const std::unique_ptr<FileInfo>& fileInfo1, const std::unique_ptr<FileInfo>& fileInfo2)
+		{
+			if (!fileInfo1->GetParFile() && !fileInfo2->GetParFile())
+			{
+				// ".rar"-files are ordered before ".r01", etc. files
+				int len1 = strlen(fileInfo1->GetFilename());
+				int len2 = strlen(fileInfo2->GetFilename());
+				const char* ext1 = strrchr(fileInfo1->GetFilename(), '.');
+				const char* ext2 = strrchr(fileInfo2->GetFilename(), '.');
+				int ext1len = ext1 ? strlen(ext1) : 0;
+				int ext2len = ext2 ? strlen(ext2) : 0;
+				bool sameBaseName = len1 == len2 && ext1len == 4 && ext2len == 4 &&
+					!strncmp(fileInfo1->GetFilename(), fileInfo2->GetFilename(), len1 - 4);
+				if (sameBaseName && !strcmp(ext1, ".rar") && ext2[1] == 'r' &&
+					isdigit(ext2[2]) && isdigit(ext2[3]))
+				{
+					return true;
+				}
+				else if (sameBaseName && !strcmp(ext2, ".rar") && ext1[1] == 'r' &&
+					isdigit(ext1[2]) && isdigit(ext1[3]))
+				{
+					return false;
+				}
+			}
+			else if (fileInfo1->GetParFile() && fileInfo2->GetParFile() &&
+				ParParser::SameParCollection(fileInfo1->GetFilename(), fileInfo2->GetFilename(),
+					fileInfo1->GetFilenameConfirmed() && fileInfo2->GetFilenameConfirmed()))
+			{
+				return fileInfo1->GetSize() < fileInfo2->GetSize();
+			}
+			else if (!fileInfo1->GetParFile() && fileInfo2->GetParFile())
+			{
+				return true;
+			}
+			else if (fileInfo1->GetParFile() && !fileInfo2->GetParFile())
+			{
+				return false;
+			}
+
+			return strcmp(fileInfo1->GetFilename(), fileInfo2->GetFilename()) < 0;
+		});
+}
+
 bool QueueEditor::DeleteUrl(NzbInfo* nzbInfo, DownloadQueue::EEditAction action)
 {
 	return g_UrlCoordinator->DeleteQueueEntry(m_downloadQueue, nzbInfo, action == DownloadQueue::eaGroupFinalDelete);

daemon/queue/QueueEditor.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,9 +33,9 @@ private:
 	class EditItem
 	{
 	public:
-		int m_offset;
 		FileInfo* m_fileInfo;
 		NzbInfo* m_nzbInfo;
+		int m_offset;
 
 		EditItem(FileInfo* fileInfo, NzbInfo* nzbInfo, int offset) :
 			m_fileInfo(fileInfo), m_nzbInfo(nzbInfo), m_offset(offset) {}
@@ -68,6 +68,7 @@ private:
 	void DeleteEntry(FileInfo* fileInfo);
 	void MoveEntry(FileInfo* fileInfo, int offset);
 	void MoveGroup(NzbInfo* nzbInfo, int offset);
+	void SortGroupFiles(NzbInfo* nzbInfo);
 	bool ItemListContainsItem(ItemList* itemList, int id);
 
 	friend class GroupSorter;

daemon/queue/Scanner.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Scanner.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "QueueCoordinator.h"
 #include "HistoryCoordinator.h"
@@ -28,6 +29,8 @@
 #include "Util.h"
 #include "FileSystem.h"
 
+int Scanner::m_idGen = 0;
+
 Scanner::QueueData::QueueData(const char* filename, const char* nzbName, const char* category,
 	int priority, const char* dupeKey, int dupeScore, EDupeMode dupeMode,
 	NzbParameterList* parameters, bool addTop, bool addPaused, NzbInfo* urlInfo,
@@ -71,60 +74,72 @@ void Scanner::QueueData::SetNzbId(int nzbId)
 
 void Scanner::InitOptions()
 {
-	m_nzbDirInterval = g_Options->GetNzbDirInterval() * 1000;
+	m_nzbDirInterval = 1;
 	m_scanScript = ScanScriptController::HasScripts();
 }
 
+int Scanner::ServiceInterval()
+{
+	return m_requestedNzbDirScan ? Service::Now :
+		g_Options->GetNzbDirInterval() <= 0 ? Service::Sleep :
+		// g_Options->GetPauseScan() ? Service::Sleep :   // for that to work we need to react on changing of pause-state
+		m_nzbDirInterval;
+}
+
 void Scanner::ServiceWork()
 {
+	debug("Scanner service work");
+
 	if (!DownloadQueue::IsLoaded())
 	{
 		return;
 	}
 
+	m_nzbDirInterval = g_Options->GetNzbDirInterval();
+
+	if (g_WorkState->GetPauseScan() && !m_requestedNzbDirScan)
+	{
+		return;
+	}
+
+	debug("Scanner service work: doing work");
+
 	Guard guard(m_scanMutex);
 
-	if (m_requestedNzbDirScan ||
-		(!g_Options->GetPauseScan() && g_Options->GetNzbDirInterval() > 0 &&
-		 m_nzbDirInterval >= g_Options->GetNzbDirInterval() * 1000))
+	// check nzbdir every g_pOptions->GetNzbDirInterval() seconds or if requested
+	bool checkStat = !m_requestedNzbDirScan;
+	m_requestedNzbDirScan = false;
+	m_scanning = true;
+	CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
+	if (!checkStat && m_scanScript)
 	{
-		// check nzbdir every g_pOptions->GetNzbDirInterval() seconds or if requested
-		bool checkStat = !m_requestedNzbDirScan;
-		m_requestedNzbDirScan = false;
-		m_scanning = true;
+		// if immediate scan requested, we need second scan to process files extracted by scan-scripts
 		CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
-		if (!checkStat && m_scanScript)
-		{
-			// if immediate scan requested, we need second scan to process files extracted by scan-scripts
-			CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
-		}
-		m_scanning = false;
-		m_nzbDirInterval = 0;
-
-		// if NzbDirFileAge is less than NzbDirInterval (that can happen if NzbDirInterval
-		// is set for rare scans like once per hour) we make 4 scans:
-		//   - one additional scan is neccessary to check sizes of detected files;
-		//   - another scan is required to check files which were extracted by scan-scripts;
-		//   - third scan is needed to check sizes of extracted files.
-		if (g_Options->GetNzbDirInterval() > 0 && g_Options->GetNzbDirFileAge() < g_Options->GetNzbDirInterval())
-		{
-			int maxPass = m_scanScript ? 3 : 1;
-			if (m_pass < maxPass)
-			{
-				// scheduling another scan of incoming directory in NzbDirFileAge seconds.
-				m_nzbDirInterval = (g_Options->GetNzbDirInterval() - g_Options->GetNzbDirFileAge()) * 1000;
-				m_pass++;
-			}
-			else
-			{
-				m_pass = 0;
-			}
-		}
-
-		DropOldFiles();
-		m_queueList.clear();
 	}
-	m_nzbDirInterval += 200;
+	m_scanning = false;
+
+	// if NzbDirFileAge is less than NzbDirInterval (that can happen if NzbDirInterval
+	// is set for rare scans like once per hour) we make 4 scans:
+	//   - one additional scan is neccessary to check sizes of detected files;
+	//   - another scan is required to check files which were extracted by scan-scripts;
+	//   - third scan is needed to check sizes of extracted files.
+	if (g_Options->GetNzbDirInterval() > 0 && g_Options->GetNzbDirFileAge() < g_Options->GetNzbDirInterval())
+	{
+		int maxPass = m_scanScript ? 3 : 1;
+		if (m_pass < maxPass)
+		{
+			// scheduling another scan of incoming directory in NzbDirFileAge seconds.
+			m_nzbDirInterval = g_Options->GetNzbDirFileAge();
+			m_pass++;
+		}
+		else
+		{
+			m_pass = 0;
+		}
+	}
+
+	DropOldFiles();
+	m_queueList.clear();
 }
 
 /**
@@ -369,7 +384,10 @@ void Scanner::InitPPParameters(const char* category, NzbParameterList* parameter
 		}
 	}
 
-	parameters->SetParameter("*Unpack:", unpack ? "yes" : "no");
+	if (!parameters->Find("*Unpack:"))
+	{
+		parameters->SetParameter("*Unpack:", unpack ? "yes" : "no");
+	}
 
 	if (!Util::EmptyStr(extensions))
 	{
@@ -379,10 +397,12 @@ void Scanner::InitPPParameters(const char* category, NzbParameterList* parameter
 		{
 			for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
 			{
+				BString<1024> paramName("%s:", scriptName);
 				if ((script.GetPostScript() || script.GetQueueScript()) &&
+					!parameters->Find(paramName) &&
 					FileSystem::SameFilename(scriptName, script.GetName()))
 				{
-					parameters->SetParameter(BString<1024>("%s:", scriptName), "yes");
+					parameters->SetParameter(paramName, "yes");
 				}
 			}
 		}
@@ -431,6 +451,7 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch
 		nzbInfo->SetUrl(urlInfo->GetUrl());
 		nzbInfo->SetUrlStatus(urlInfo->GetUrlStatus());
 		nzbInfo->SetFeedId(urlInfo->GetFeedId());
+		nzbInfo->SetDupeHint(urlInfo->GetDupeHint());
 	}
 
 	if (nzbFile.GetPassword())
@@ -451,7 +472,14 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch
 	{
 		addedNzb = g_QueueCoordinator->AddNzbFileToQueue(std::move(nzbInfo), std::move(urlInfo), addTop);
 	}
-	else if (!urlInfo)
+	else if (urlInfo)
+	{
+		for (Message& message : nzbInfo->GuardCachedMessages())
+		{
+			urlInfo->AddMessage(message.GetKind(), message.GetText(), false);
+		}
+	}
+	else
 	{
 		nzbInfo->SetDeleteStatus(NzbInfo::dsScan);
 		addedNzb = g_QueueCoordinator->AddNzbFileToQueue(std::move(nzbInfo), std::move(urlInfo), addTop);
@@ -471,16 +499,17 @@ void Scanner::ScanNzbDir(bool syncMode)
 		Guard guard(m_scanMutex);
 		m_scanning = true;
 		m_requestedNzbDirScan = true;
+		WakeUp();
 	}
 
 	while (syncMode && (m_scanning || m_requestedNzbDirScan))
 	{
-		usleep(100 * 1000);
+		Util::Sleep(100);
 	}
 }
 
 Scanner::EAddStatus Scanner::AddExternalFile(const char* nzbName, const char* category,
-	int priority, const char* dupeKey, int dupeScore,  EDupeMode dupeMode,
+	int priority, const char* dupeKey, int dupeScore, EDupeMode dupeMode,
 	NzbParameterList* parameters, bool addTop, bool addPaused, NzbInfo* urlInfo,
 	const char* fileName, const char* buffer, int bufSize, int* nzbId)
 {
@@ -493,8 +522,8 @@ Scanner::EAddStatus Scanner::AddExternalFile(const char* nzbName, const char* ca
 	}
 	else
 	{
-		int num = 1;
-		while (num == 1 || FileSystem::FileExists(tempFileName))
+		int num = ++m_idGen;
+		while (tempFileName.Empty() || FileSystem::FileExists(tempFileName))
 		{
 			tempFileName.Format("%s%cnzb-%i.tmp", g_Options->GetTempDir(), PATH_SEPARATOR, num);
 			num++;

daemon/queue/Scanner.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -46,7 +46,7 @@ public:
 	void InitPPParameters(const char* category, NzbParameterList* parameters, bool reset);
 
 protected:
-	virtual int ServiceInterval() { return 200; }
+	virtual int ServiceInterval();
 	virtual void ServiceWork();
 
 private:
@@ -114,6 +114,7 @@ private:
 	QueueList m_queueList;
 	bool m_scanning = false;
 	Mutex m_scanMutex;
+	static int m_idGen;
 
 	void CheckIncomingNzbs(const char* directory, const char* category, bool checkStat);
 	bool AddFileToQueue(const char* filename, const char* nzbName, const char* category,

daemon/queue/UrlCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "UrlCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "WebDownloader.h"
 #include "Util.h"
 #include "FileSystem.h"
@@ -60,6 +61,12 @@ void UrlDownloader::ProcessHeader(const char* line)
 	}
 }
 
+UrlCoordinator::UrlCoordinator()
+{
+	m_downloadQueueObserver.m_owner = this;
+	DownloadQueue::Guard()->Attach(&m_downloadQueueObserver);
+}
+
 UrlCoordinator::~UrlCoordinator()
 {
 	debug("Destroying UrlCoordinator");
@@ -79,41 +86,45 @@ void UrlCoordinator::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
-	int resetCounter = 0;
-
 	while (!IsStopped())
 	{
+		time_t lastReset = 0;
 		bool downloadStarted = false;
-		if (!g_Options->GetPauseDownload() || g_Options->GetUrlForce())
+
 		{
-			// start download for next URL
+			NzbInfo* nzbInfo = nullptr;
 			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 			if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
 			{
-				NzbInfo* nzbInfo = GetNextUrl(downloadQueue);
-				bool hasMoreUrls = nzbInfo != nullptr;
-				bool urlDownloadsRunning = !m_activeDownloads.empty();
-				m_hasMoreJobs = hasMoreUrls || urlDownloadsRunning;
-				if (hasMoreUrls && !IsStopped())
+				nzbInfo = GetNextUrl(downloadQueue);
+				if (nzbInfo && (!g_WorkState->GetPauseDownload() || g_Options->GetUrlForce()))
 				{
 					StartUrlDownload(nzbInfo);
 					downloadStarted = true;
 				}
 			}
+			m_hasMoreJobs = !m_activeDownloads.empty() || nzbInfo;
 		}
 
-		int sleepInterval = downloadStarted ? 0 : 100;
-		usleep(sleepInterval * 1000);
-
-		resetCounter += sleepInterval;
-		if (resetCounter >= 1000)
+		if (lastReset != Util::CurrentTime())
 		{
 			// this code should not be called too often, once per second is OK
 			ResetHangingDownloads();
-			resetCounter = 0;
+			lastReset = Util::CurrentTime();
+		}
+
+		if (!m_hasMoreJobs && !IsStopped())
+		{
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&] { return m_hasMoreJobs || IsStopped(); });
+		}
+		else
+		{
+			int sleepInterval = downloadStarted ? 0 : 100;
+			Util::Sleep(sleepInterval);
 		}
 	}
 
@@ -136,7 +147,7 @@ void UrlCoordinator::WaitJobs()
 				break;
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 
@@ -148,18 +159,38 @@ void UrlCoordinator::Stop()
 	Thread::Stop();
 
 	debug("Stopping UrlDownloads");
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
-	for (UrlDownloader* urlDownloader : m_activeDownloads)
 	{
-		urlDownloader->Stop();
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		for (UrlDownloader* urlDownloader : m_activeDownloads)
+		{
+			urlDownloader->Stop();
+		}
 	}
 	debug("UrlDownloads are notified");
+
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
+}
+
+void UrlCoordinator::DownloadQueueUpdate(Subject* caller, void* aspect)
+{
+	debug("Notification from download queue received");
+
+	DownloadQueue::Aspect* queueAspect = (DownloadQueue::Aspect*)aspect;
+	if (queueAspect->action == DownloadQueue::eaUrlAdded ||
+		queueAspect->action == DownloadQueue::eaUrlReturned)
+	{
+		// Resume Run()
+		Guard guard(m_waitMutex);
+		m_hasMoreJobs = true;
+		m_waitCond.NotifyAll();
+	}
 }
 
 void UrlCoordinator::ResetHangingDownloads()
 {
-	const int timeout = g_Options->GetTerminateTimeout();
-	if (timeout == 0)
+	if (g_Options->GetUrlTimeout() == 0)
 	{
 		return;
 	}
@@ -167,32 +198,15 @@ void UrlCoordinator::ResetHangingDownloads()
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[timeout, tm](UrlDownloader* urlDownloader)
+	for (UrlDownloader* urlDownloader: m_activeDownloads)
+	{
+		if (tm - urlDownloader->GetLastUpdateTime() > g_Options->GetUrlTimeout() + 10 &&
+			urlDownloader->GetStatus() == UrlDownloader::adRunning)
 		{
-			if (tm - urlDownloader->GetLastUpdateTime() > timeout &&
-				urlDownloader->GetStatus() == UrlDownloader::adRunning)
-			{
-				NzbInfo* nzbInfo = urlDownloader->GetNzbInfo();
-				debug("Terminating hanging download %s", urlDownloader->GetInfoName());
-				if (urlDownloader->Terminate())
-				{
-					error("Terminated hanging download %s", urlDownloader->GetInfoName());
-					nzbInfo->SetUrlStatus(NzbInfo::lsNone);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s", urlDownloader->GetInfoName());
-				}
-
-				// it's not safe to destroy urlDownloader, because the state of object is unknown
-				delete urlDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging url download %s", urlDownloader->GetInfoName());
+			urlDownloader->Stop();
+		}
+	}
 }
 
 void UrlCoordinator::LogDebugInfo()
@@ -212,8 +226,6 @@ void UrlCoordinator::LogDebugInfo()
  */
 NzbInfo* UrlCoordinator::GetNextUrl(DownloadQueue* downloadQueue)
 {
-	bool pauseDownload = g_Options->GetPauseDownload();
-
 	NzbInfo* nzbInfo = nullptr;
 
 	for (NzbInfo* nzbInfo1 : downloadQueue->GetQueue())
@@ -221,7 +233,6 @@ NzbInfo* UrlCoordinator::GetNextUrl(DownloadQueue* downloadQueue)
 		if (nzbInfo1->GetKind() == NzbInfo::nkUrl &&
 			nzbInfo1->GetUrlStatus() == NzbInfo::lsNone &&
 			nzbInfo1->GetDeleteStatus() == NzbInfo::dsNone &&
-			(!pauseDownload || g_Options->GetUrlForce()) &&
 			(!nzbInfo || nzbInfo1->GetPriority() > nzbInfo->GetPriority()))
 		{
 			nzbInfo = nzbInfo1;
@@ -295,13 +306,11 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 		// remove downloader from downloader list
 		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), urlDownloader));
 
-		nzbInfo->SetActiveDownloads(0);
-
 		retry = urlDownloader->GetStatus() == WebDownloader::adRetry && !nzbInfo->GetDeleting();
 
 		if (nzbInfo->GetDeleting())
 		{
-			nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
+			nzbInfo->SetDeleteStatus(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ? NzbInfo::dsManual : nzbInfo->GetDeleteStatus());
 			nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 			nzbInfo->SetDeleting(false);
 		}
@@ -327,6 +336,7 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 	if (retry)
 	{
+		nzbInfo->SetActiveDownloads(0);
 		return;
 	}
 
@@ -342,7 +352,7 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 		if (addStatus == Scanner::asSuccess)
 		{
-			// if scanner has successfully added nzb-file to queue, our pNZBInfo is
+			// if scanner has successfully added nzb-file to queue, our nzbInfo is
 			// already removed from queue and destroyed
 			return;
 		}
@@ -354,31 +364,13 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 	g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeUrlCompleted);
 
-	std::unique_ptr<NzbInfo> oldNzbInfo;
-
 	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
-		// delete URL from queue
-		oldNzbInfo = downloadQueue->GetQueue()->Remove(nzbInfo);
+		nzbInfo->SetActiveDownloads(0);
 
-		// add failed URL to history
-		if (g_Options->GetKeepHistory() > 0 &&
-			nzbInfo->GetUrlStatus() != NzbInfo::lsFinished &&
-			!nzbInfo->GetAvoidHistory())
-		{
-			std::unique_ptr<HistoryInfo> historyInfo = std::make_unique<HistoryInfo>(std::move(oldNzbInfo));
-			historyInfo->SetTime(Util::CurrentTime());
-			downloadQueue->GetHistory()->Add(std::move(historyInfo), true);
-			downloadQueue->HistoryChanged();
-		}
-
-		downloadQueue->Save();
-	}
-
-	if (oldNzbInfo)
-	{
-		g_DiskState->DiscardFiles(oldNzbInfo.get());
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaUrlFailed, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
 	}
 }
 
@@ -398,26 +390,40 @@ bool UrlCoordinator::DeleteQueueEntry(DownloadQueue* downloadQueue, NzbInfo* nzb
 				return true;
 			}
 		}
+
+		return false;
 	}
 
 	info("Deleting URL %s", nzbInfo->GetName());
 
-	nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
+	nzbInfo->SetDeleteStatus(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ? NzbInfo::dsManual : nzbInfo->GetDeleteStatus());
 	nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 
-	std::unique_ptr<NzbInfo> oldNzbInfo = downloadQueue->GetQueue()->Remove(nzbInfo);
-
-	if (g_Options->GetKeepHistory() > 0 && !avoidHistory)
-	{
-		std::unique_ptr<HistoryInfo> historyInfo = std::make_unique<HistoryInfo>(std::move(oldNzbInfo));
-		historyInfo->SetTime(Util::CurrentTime());
-		downloadQueue->GetHistory()->Add(std::move(historyInfo), true);
-		downloadQueue->HistoryChanged();
-	}
-	else
-	{
-		g_DiskState->DiscardFiles(oldNzbInfo.get());
-	}
+	DownloadQueue::Aspect deletedAspect = {DownloadQueue::eaUrlDeleted, downloadQueue, nzbInfo, nullptr};
+	downloadQueue->Notify(&deletedAspect);
 
 	return true;
 }
+
+void UrlCoordinator::AddUrlToQueue(std::unique_ptr<NzbInfo> nzbInfo, bool addFirst)
+{
+	debug("Adding URL to queue");
+										
+	NzbInfo* addedNzb = nzbInfo.get();
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	DownloadQueue::Aspect foundAspect = {DownloadQueue::eaUrlFound, downloadQueue, addedNzb, nullptr};
+	downloadQueue->Notify(&foundAspect);
+
+	if (addedNzb->GetDeleteStatus() != NzbInfo::dsManual)
+	{
+		downloadQueue->GetQueue()->Add(std::move(nzbInfo), addFirst);
+
+		DownloadQueue::Aspect addedAspect = {DownloadQueue::eaUrlAdded, downloadQueue, addedNzb, nullptr};
+		downloadQueue->Notify(&addedAspect);
+	}
+
+	downloadQueue->Save();
+}
+

daemon/queue/UrlCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,12 +33,14 @@ class UrlDownloader;
 class UrlCoordinator : public Thread, public Observer, public Debuggable
 {
 public:
+	UrlCoordinator();
 	virtual ~UrlCoordinator();
 	virtual void Run();
 	virtual void Stop();
 	void Update(Subject* caller, void* aspect);
 
 	// Editing the queue
+	void AddUrlToQueue(std::unique_ptr<NzbInfo> nzbInfo, bool addFirst);
 	bool HasMoreJobs() { return m_hasMoreJobs; }
 	bool DeleteQueueEntry(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool avoidHistory);
 
@@ -48,15 +50,26 @@ protected:
 private:
 	typedef std::list<UrlDownloader*> ActiveDownloads;
 
+	class DownloadQueueObserver: public Observer
+	{
+	public:
+		UrlCoordinator* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->DownloadQueueUpdate(caller, aspect); }
+	};
+
 	ActiveDownloads m_activeDownloads;
 	bool m_hasMoreJobs = true;
 	bool m_force;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
+	DownloadQueueObserver m_downloadQueueObserver;
 
 	NzbInfo* GetNextUrl(DownloadQueue* downloadQueue);
 	void StartUrlDownload(NzbInfo* nzbInfo);
 	void UrlCompleted(UrlDownloader* urlDownloader);
 	void ResetHangingDownloads();
 	void WaitJobs();
+	void DownloadQueueUpdate(Subject* caller, void* aspect);
 };
 
 extern UrlCoordinator* g_UrlCoordinator;