#784: removed expired root certificate

from ca root certificate store: certificate “DST Root CA X3” used by
Lets Encrypt

.gitignore

@@ -26,11 +26,14 @@
 # GNU Autotools
 .deps/
 config.h
+config.h.in~
 config.log
 config.status
 Makefile
 stamp-h1
 autom4te.cache/
+.dirstamp
+*.o-*
 
 # Visual Studio User-specific files
 *.suo
@@ -57,7 +60,15 @@ ipch/
 *.svclog
 *.scc
 *.sln
+.vscode/
+
+# macOS
+.DS_Store
 
 # NZBGet specific
 nzbget
 code_revision.cpp
+*.temp
+*.pyc
+pytest.ini
+.cache

.lgtm.yml

@@ -0,0 +1,17 @@
+# Configuration file for integration with http://lgtm.com
+
+path_classifiers:
+  library:
+    # exclude these directories from default alerts report:
+    - lib
+    - webui/lib
+
+extraction:
+  cpp:
+    configure:
+      command:
+        # compile with tests to activate scanning of C++ sources for tests
+        - ./configure --enable-tests
+
+queries:
+  - exclude: js/incomplete-sanitization  # this one gives false positives only and nothing useful

.travis.yml

@@ -0,0 +1,67 @@
+sudo: required
+dist: trusty
+language: cpp
+
+matrix:
+  include:
+
+    - compiler: gcc
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-5
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-5
+
+    - compiler: gcc
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-4.9
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.9
+
+    - compiler: gcc
+      addons:
+        apt:
+          packages:
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=g++-4.8
+        - CXXFLAGS="-std=c++11 -O2 -s"
+        - CONFIGUREOPTS="--disable-cpp-check"
+
+    - compiler: clang
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-precise-3.6
+          packages:
+            - clang-3.6
+            - unrar
+            - p7zip-full
+            - par2
+      env:
+        - COMPILER=clang++-3.6
+
+install:
+  - sudo pip install -U pytest
+
+script:
+  - $COMPILER --version
+  - CXX=$COMPILER ./configure $CONFIGUREOPTS --enable-tests && make
+  - ./nzbget --tests
+  - cd tests/functional && pytest -v

COPYING

@@ -1,12 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -15,7 +15,7 @@ software--to make sure the software is free for all its users.  This
 General Public License applies to most of the Free Software
 Foundation's software and to any other program whose authors commit to
 using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
+the GNU Lesser General Public License instead.)  You can apply it to
 your programs, too.
 
   When we speak of free software, we are referring to freedom, not
@@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
+
+                    GNU GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License applies to any program or other work which contains
@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
     License.  (Exception: if the Program itself is interactive but
     does not normally print such an announcement, your work based on
     the Program is not required to print an announcement.)
-
+
 These requirements apply to the modified work as a whole.  If
 identifiable sections of that work are not derived from the Program,
 and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
 access to copy the source code from the same place counts as
 distribution of the source code, even though third parties are not
 compelled to copy the source along with the object code.
-
+
   4. You may not copy, modify, sublicense, or distribute the Program
 except as expressly provided under this License.  Any attempt
 otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
 
 This section is intended to make thoroughly clear what is believed to
 be a consequence of the rest of this License.
-
+
   8. If the distribution and/or use of the Program is restricted in
 certain countries either by patents or by copyrighted interfaces, the
 original copyright holder who places the Program under this License
@@ -255,7 +255,7 @@ make exceptions for this.  Our decision will be guided by the two goals
 of preserving the free status of all derivatives of our free software and
 of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
 
   If you develop a new program, and you want it to be of the greatest
 possible use to the public, the best way to achieve this is to make it
@@ -303,10 +303,9 @@ the "copyright" line and a pointer to where the full notice is found.
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -336,5 +335,5 @@ necessary.  Here is a sample; alter the names:
 This General Public License does not permit incorporating your program into
 proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

ChangeLog

@@ -1,7 +1,561 @@
-nzbget-17.0:
+nzbget-21.2-testing:
   - please see repository change log at
     https://github.com/nzbget/nzbget/commits/develop
 
+nzbget-21.1:
+  - fixed crash on systems with 64-bit time;
+  - corrected icon in Windows "uninstall program" list;
+  - allow special characters in URL for username and password;
+  - improved reporting for binding errors on Windows;
+  - fixed unicode space characters in javascript files, which could cause issues
+    with nginx proxy;
+  - fixed negative values for "FileSizeLo" in json-rpc;
+  - corrected url detection in rpc-method "append";
+  - added support for new error messages in unrar 5.80;
+  - now always using snapshots when reading directory contents:
+    - in previous versions snapshots were used on macOS only;
+    - now they are used on all OSes;
+    - this solves issue with leftovers during directory cleanup, which could
+      happen on certain OSes when working with network drives;
+  - fixed file allocating on file systems where sparse files are not supported:
+    - the issue could happen when InterDir was located on a network drive;
+  - fixed crash caused by malformed nzb files;
+  - fixed GROUP command in nserv;
+  - updated url of the global certificate storage file in the build scripts;
+  - fixed: file selector in WebKit based browsers doesn't allow to choose the
+    same file again;
+  - removed outdated links from web interface;
+  - fixed PC sleep mode not working (Windows only);
+  - set "SameSite" attribute for cookies;
+  - corrected typo in about dialog of web interface;
+  - updated license text: changed address of Free Software Foundation and minor
+    formatting changes.
+
+nzbget-21.0:
+  - reworked duplicate handling to support URLs, especially when using RSS
+    feeds:
+      - queue items added via URLs (to be fetched by nzbget) are no longer
+        immediately fetched;
+      - instead url-items are handled by duplicate check similar to nzb-items
+        and may be placed into history as duplicate backups;
+      - if an url-item needs to be downloaded as backup for a failed other item
+        the nzb-file is fetched via provided URL;
+      - this greatly reduces the number of nzbs fetched from indexers when using
+        RSS feeds and duplicate handling;
+  - improved support for Android devices:
+    - now providing a separate installer package for Android;
+    - the package contains binaries built using Android NDK;
+    - this improves compatibility with Android, in particular with Android 8,
+      where general Linux installer version of NZBGet didn't work anymore due
+      to security changes in Android;
+    - android installer app is updated to use the new android installer package
+      instead of general Linux package;
+  - thoroughly optimised the program to reduce power consumption in idle state:
+    - number of CPU wake ups in idle state has been reduced from hundreds times
+      per second to about only one per second;
+  - optimisations for large queues with thousands of items:
+    - speed up saving of queue state and reduced number of queue state savings;
+    - improved queue state format to reduce amount of state data saved during
+      downloading;
+    - in tests download speed for very large queue (16000 items) has been
+      increased from 45 MB/s to 300 MB/s (comparing to 400 MB/s with small
+      queue); 
+  - added native support for aarch64 architecture (ARM 64 Bit CPU) in Linux and
+    Android installers;
+  - force par-check for nzbs without archives;
+  - added functional tests for unpack CRC error;
+  - click on nzbget logo in web-interface now switches to downloads tab instead
+    of showing "About dialog" which has been moved into settings;
+  - improved handling of files splitted via par2;
+  - added python 3 compatibility to EMail.py script;
+  - added python 3 compatibility to Logger.py script;
+  - proper UTF-8 encoding of email content in EMail.py script;
+  - improved error reporting for queue disk state corruption;
+  - updated unrar to 5.7 and 7-zip to 19.0;
+  - Windows installer now includes unrar in 32 bit and 64 bit variants;
+  - allowing wildcards in option AuthorizedIP;
+  - removed suggestion of RC4 cipher;
+  - better description of option UMask;
+  - integrated LGTM code analyser tool into project;
+  - fixed: failed downloads not having any par2- or archive- files were moved to
+    DestDir instead of remaining in InterDir;
+  - fixed crash when using FIPS version of OpenSSL;
+  - fixed compatibility issue with OpenSSL compiled without compression support;
+  - fixed deprecated OpenSSL calls;
+  - fixed potential crash in built-in web-server;
+  - fixed: statistics for session download time and speed may be way off on high
+    load;
+  - fixed many compilation warnings in GCC;
+  - fixed: macOS menubar widget could not connect if password contained special
+    characters;
+  - fixed: remote clients not displaying current download speed;
+  - fixed: remote server could crash when feed with invalid api request;
+  - fixed trimming of relative paths in config.
+
+nzbget-20.0:
+  - massive performance optimisations in downloader:
+        - improved yEnc decoder;
+        - improved CRC32 calculation;
+        - processing data in one pass;
+        - SIMD implementation of decoder and CRC functions on x86 and ARM CPUs;
+          SIMD code relies on node-yencode library by Anime Tosho
+          (https://github.com/animetosho/node-yencode);
+        - overall performance improvement up to +500% on x86 and +250% on ARM
+          (better speed or less CPU usage);
+  - using glibc instead of uClibc in universal installer builds for Linux:
+        - this significantly improves performance;
+        - compatibility with Android and other systems is hopefully also improved;
+        - in universal installer glibc is used on x86 and ARM;
+        - uClibc is still used on MIPS and PPC;
+  - performance optimisations in web-interface:
+        - reduced number of requests when loading webui by combining all
+          javascript-files into one and all css-files into one;
+        - reduced load time by calling multiple API-methods simultaneously;
+        - extensive use of browser caching for static files significantly
+          reduces the amount of data transferred on webui initialisation;
+        - extensive use of browser caching for API requests reduces the amount
+          of data transferred during webui status updates, especially when
+          nzbget is in idle state and there are no changes in download queue or
+          history;
+        - avoid work in browser on status updates if there are no changes in
+          download queue or history;
+        - support for keep alive connections significantly reduces overhead for
+          establishing connections on webui status updates, especially when
+          connecting to nzbget via TLS/SSL (avoiding TLS handshakes);
+  - a number of performance optimisations for large download queue with
+    thousands of items:
+        - much faster loading of queue from disk greatly improves program start
+          up time;
+        - improved queue management for faster download speed;
+  - now offering 64 bit binaries for Windows:
+        - installer includes 32 and 64 bit nzbget binaries;
+        - when updating from older versions the 64 bit binary is installed
+          automatically, although into the old location to keep all your
+          shortcuts intact;
+        - using word "windows" instead of "win32" in the setup file name;
+  - automatic update check:
+        - new option "UpdateCheck" to check for stable or testing versions (or
+          disable);
+        - when a new version is found a notification is shown;
+        - the update check is enabled by default for stable versions;
+  - significantly improved logging performance, especially in debug builds;
+  - par-check prints par2 creator application to help identify creator app
+    issues;
+  - added support for Unix domain sockets (POSIX only);
+  - better error handling when fetching rss feeds;
+  - updated POSIX build files to newer autotools version:
+        - compatibility with newer autotools;
+        - compatibility with newer platforms such as aarch64;
+  - better username/password validation when testing connection on settings
+    page;
+  - improved rar-renamer to better handle certain cases;
+  - new option "SkipWrite" for easier speed tests;
+  - support for redirect codes 303, 307 and 308 in web-client for fetching of
+    rss feeds and nzb-files;
+  - installer for FreeBSD is now built using Clang instead of GCC; this fixes
+    incompatibility with FreeBSD 11;
+  - universal Linux installer can now be used on FreeBSD (via Linux
+    compatibility mode);
+  - updated unrar to v5.50;
+  - more robust news server connection test;
+  - enhancements in NServ:
+        - memory cache to reduce disk load during speed tests - new command line
+          switch "-m";
+        - speed control - new command line switches "-w" and "-r";
+        - show IP address of incoming connection;
+  - changed default location of log-file;
+  - better handling of broken connections;
+  - removed obsolete or less useful options "SaveQueue", "ReloadQueue",
+    "TerminateTimeout", "AccurateRate", "BrokenLog";
+  - renamed option "LogBufferSize" to "LogBuffer";
+  - passwords of failed login attempts are no longer printed to log to improve
+    security;
+  - cookies in web interface are now saved with "httpOnly" attribute to improve
+    security;
+  - titles and duplicate keys in duplicate check are now case insensitive;
+  - added LibreSSL support;
+  - web interface now has a better icon for favorites or home screen of mobile
+    devices;
+  - improved duplicate detection for obfuscated downloads having files with
+    same subjects;
+  - direct rename and direct unpack are now active by default on new
+    installations, except for slow Linux systems;
+  - added advice for letsencrypt in option descriptions;
+  - fixed incorrect renaming in rar-renamer which could cause some downloads to
+    fail;
+  - fixed race condition in queue script coordinator which could cause crashes;
+  - fixed: post-processing parameters were sometimes case sensitive causing
+    issues;
+  - fixed DNS resolving issues on Android;
+  - fixed: backup servers not used on certain article decoding errors;
+  - fixed: when direct rename was active certain downloads with damaged
+    par2-files become paused at near completion and required manual resuming;
+  - fixed: crash when flushing article cache after direct rename;
+  - fixed: deleting active par-job may crash the program;
+  - fixed: functional tests may fail on Windows due to locked files;
+  - fixed: unpack using password file doesn't work on Windows;
+  - fixed: compiler error when building using GnuTLS;
+  - fixed: Linux installer failure on android emulator;
+  - fixed: options formatted as password fields when they shouldn't;
+  - fixed: slightly off article statistics after direct rename;
+  - fixed: NServ terminated if client interrupted connection;
+  - fixed: example pp-scripts may not work properly if nzbget password or
+    username contained special characters;
+  - fix in functional tests to not rely on sizes of externally generated files;
+  - fixed: option AuthorizedIP did not work with IPv6 addresses;
+  - fixed crash on certain malformed articles;
+  - fixed crash which could happen on Windows when reloading or terminating the
+    program;
+  - fixed logging of IPv6 addresses.
+
+nzbget-19.1:
+  - proper handling of changing category (and destination path) during direct
+    unpack; direct unpack now gracefully aborts with cleanup; the files will
+    be unpacked during post-processing stage;
+  - fixed: password protected downloads of certain kind may sometimes end up
+    with no files if direct unpack was active;
+  - fixed: rar-renamer mistakenly renamed some encrypted rar3 files causing
+    unnecessary repair;
+  - fixed: rar-renamer could not process some encrypted rar3-archives.
+
+nzbget-19.0:
+  - unpack during downloading:
+        - downloaded files can now be unpacked as soon as every archive part is
+          downloaded;
+        - new option "DirectUnpack" to activate direct unpacking;
+        - direct unpack works even with obfuscated downloads; option
+         "DirectRename" (see below) must be active for that;
+        - option "ReorderFiles" (see below) should be also active for optimal
+          file download order;
+        - direct unpack works for rar-archives; 7-zip archives and simply
+          splitted files are processed by default unpack module;
+        - direct unpack obviously works only for healthy download; if download
+          is damaged the direct unpack cancels and the download is unpacked
+          during post-processing stage after files are repaired;
+        - direct unpack reduces the time needed to complete download and
+      post-processing;
+        - it also allows to start watching of video files during download
+          (requires compatible video player software);
+  - renaming of obfuscated file names during downloading:
+        - correct file names for obfuscated downloads are now determined during
+          download stage (instead of post-processing stage);
+        - downloaded files are saved into disk directly with correct names;
+        - direct renaming uses par2-files to restore correct file names;
+        - new option "DirectRename" to activate direct renaming;
+        - new queue-event NZB_NAMED, sent after the inner files are renamed;
+  - automatic reordering of files:
+        - inner files within nzb reordered to ensure download of files in
+          archive parts order;
+        - the files are reordered when nzb is added to queue;
+        - if direct renaming is active (option "DirectRename") the files are
+      reordered again after the correct names becomes known;
+        - new option "ReorderFiles";
+        - new command "GroupSortFiles" in api-method "editqueue";
+        - new subcommand "SF" of remote command "-E/--edit";
+  - new option "FileNaming" to control how to name obfuscated files (before they
+    get renamed by par-rename, rar-rename or direct-rename);
+  - TLS certificate verification:
+        - when connecting to a news server (for downloading) or a web server
+         (for fetching of rss feeds and nzb-files) the authenticity of the
+          server is validated using server security certificate. If the check
+          fails that means the connection cannot be trusted and must be closed
+          with an error message explaining the security issue;
+        - new options "CertCheck" and "CertStore";
+        - official NZBGet packages come with activated certificate check;
+        - when updating from an older NZBGet version the option CertCheck will
+          be automatically activated when the settings is saved (switch to
+          Settings page in web-interface and click "Save all changed");
+  - authentication via form in web-interface as alternative to HTTP
+    authentication:
+        - that must help with password tools having issues with HTTP
+          authentication dialog;
+        - new option "FormAuth";
+  - drop-downs (context menus) for priority, category and status columns:
+        - quicker changing of priority and category;
+        - easier access to actions via drop-down (context menu) in status
+          column;
+  - extensions scripts can now be executed from settings page:
+        - script authors define custom buttons;
+        - when clicked the script is executed in a special mode and obtain extra
+          parameters;
+        - example script "Email.py" extended with button "Send test e-mail";
+  - on Windows NZBGet can now associate itself with nzb-files:
+        - use option in Windows installer to register NZBGet for nzb-files;
+  - unrar shipped within Linux package is now compiled with "fallocate" option
+    to improve compatibility with media players when watching videos during
+    downloading and unpacking;
+  - support for HTTP-header "X-Forwarded-For" in IP-logging;
+  - improvements in RSS feed view in phone mode;
+  - set name, password and dupe info when adding via URL by click on a button
+    near URL field in web-interface;
+  - backup-badge for items in history similar to downloads tab;
+  - show backup icon in history in phone theme;
+  - added support for ECC certificates in built-in web-server;
+  - save changes before performing actions in history dialog;
+  - proper exit code on client command success or failure.
+  - added host name to all error messages regarding connection issues;
+  - new button "Volume Statistics" in section "News Servers" of settings page;
+    shows the same volume data as in global statistics dialog;
+  - new option "ServerX.Notes" for user comments on news servers;
+  - new parameters for api-method "servervolumes" as a performance optimization
+    measure to reduce amount of transferred data;
+  - new option to force connection to news servers via ipv4 or ipv6;
+  - removed unnecessary requests to news servers;
+  - updated unrar to v5.40;
+  - clear script execution log before executing script;
+  - added support for crash dumps on Windows:
+        - renamed option "DumpCore" to "CrashDump";
+        - new option "CrashTrace" to make it possible to disable default
+          printing off call stack in order to produce more relevant crash dumps;
+  - fixed: startup scheduler tasks can be executed again;
+  - fixed: "fatal" messages when compiling from sources.
+  - fixed: per-nzb download statistics could be wrong if the program was
+    reloaded during downloading.
+  - fixed crash which may happen between post-processing steps;
+  - fixed: asterix (*) was sometimes passed as parameter to extension scripts
+    (Windows only);
+  - fixed potential crash during update via web-interface.
+
+nzbget-18.1:
+  - fixed: crash during download caused by a race condition;
+  - fixed: sleep mode did not work on Windows;
+  - fixed: queue was not saved after deleting of queued post-jobs;
+  - fixed: possible crash at the end of post-processing;
+  - fixed: "undefined" in reorder extension scripts.
+
+nzbget-18.0:
+  - automatic deobfuscation of rar-archives without par-files:
+        - obfuscated downloads not having par-files can now be successfully
+          unpacked;
+        - also helps with downloads where rar-files were obfuscated before
+          creating par-files;
+        - new options "RarRename" and "UnpackIgnoreExt";
+  - multi post-processing:
+        - in addition to classic post-processing strategy where items are
+          processed one after another it is now possible to post-process
+          multiple items at the same time;
+        - new option "PostStrategy" to choose from four: sequential, balanced,
+          aggressive, rocket;
+        - in "balanced" strategy downloads needing repair do not block other
+          items which are processed sequentially but simultaneously with
+          repairing item;
+        - in "aggressive" mode up to three items are post-processed at the same
+          time and in "rocket" mode up to six items (including up to two repair
+          tasks);
+  - unified extension scripts settings:
+        - options "PostScript", "QueueScript", "ScanScript" and "FeedScript"
+          were replaced with one option "Extensions";
+        - users don't need to know the technical details os extension scripts as
+          all scripts are now can be selected at one place;
+        - easier activation of complex extension scripts which previously needed
+          to be selected in multiple options for their proper work;
+  - reordering download queue with drag and drop in web-interface:
+        - new actions "GroupMoveBefore" and "GroupMoveAfter" in API-method
+          "editqueue";
+  - priorities are now displayed as a column instead of badge; that makes it
+    possible to manually sort on priority;
+  - removed vertical lines in tables; looks better in combination with new
+    priority column;
+  - keyboard shortcuts in web-interface;
+  - improved UI to prevent accidental deletion of many items:
+        - visual indication of records selected on other pages;
+        - extra warning when deleting many records from history;
+  - additional options in "custom pause dialog";
+  - better handing of damaged par2-files in par-renamer:
+        - if par-renamer can't load a (damaged) par2-file then another par2-file
+          is downloaded and par-renamer tries again;
+        - reverted non-strict par2-filename matching to handle article subjects
+          with non-parseable filenames;
+  - better handling of obfuscated par-files;
+  - splitted option "Retries" into "ArticleRetries" and "UrlRetries"; option
+    "RetryInterval" into "ArticleInterval" and "UrlInterval";
+  - scheduler tasks can be started at program launch:
+        - use asterisk as TaskX.Time;
+  - graceful termination of scheduler scripts:
+        - scripts receive signal SIGINT (CTRL+BREAK on Windows) before
+          termination;
+  - added support for nZEDb attributes in rss feeds;
+  - better cleanup handling: if parameter "unpack" is disabled for an nzb-file
+    the cleanup isn't performed for it;
+  - fields containing passwords are now displayed as protected fields;
+  - showing password-badge for nzbs with passwords;
+  - allow control of what tab is shown when opening web-interface:
+    add "#downloads", "#history", "#messages" or "#settings" to the URL,
+    for example "http://localhost:6789/#history" or
+    "http://localhost:6789/index.html#history";
+  - functional testing to ensure program quality:
+        - implemented built-in simple nntp server to be used for functional
+          testing;
+        - created a number of tests;
+        - new features come with additional tests;
+  - improved API-method "append" in combination with duplicate check; method
+    returns nzb-id also for items added straight to history;
+  - removed parameter "offset" from api-method "editqueue":
+        - when needed the "offset" is now passed within parameter "Args" as
+          string;
+        - old method signature is supported for compatibility;
+  - improved error reporting on feed parse errors;
+  - highlighting selected rows with alternative colors;
+  - improved selecting of par2-file for repair;
+  - splitted config section "Download Queue" and moved many options into new
+    section "Connection";
+  - disabled SSLv3 in built-in web-server;
+  - multiple recipients in the example pp-script "EMail.py";
+  - added compatibility with openssl 1.1.0;
+  - fixed TLS handshake error when using GnuTLS;
+  - fixed: sorting of selected items may give wrong results;
+  - fixed: search box filter in feed view were not reset.
+
+nzbget-17.1:
+  - adjustments and fixes for "Retry failed articles" function, better handling
+    of certain corner cases;
+  - partial compatibility with gcc 4.8;
+  - removed unnecessary debug logging to javascript console;
+  - improved error reporting on certain file operations;
+  - corrected option description;
+  - corrected text in history delete confirmation dialog;
+  - fixed performance issue on certain Windows systems;
+  - fixed: root drive paths on Windows could not be used (for example
+    "NzbDir=N:\");
+  - fixed hanging after marking as BAD from queue script;
+  - fixed: old nzbget.exe was deleted even when installing into a new directory
+    (Windows only);
+  - fixed: compilation error if configured with unit tests but without par-module;
+  - fixed crash on malformed articles;
+  - fixed javascript error on Chrome for Linux;
+  - fixed compilation error if configured without TLS.
+
+nzbget-17.0:
+  - reworked the full source code base to utilize modern C++ features:
+        - with the main motivation to make the code nicer and more fun to work
+          with;
+        - to compile NZBGet from source a compiler supporting C++14 is required;
+        - most users don't have to compile on their own and can use official
+          installers offered on download page;
+        - for a detailed list of internal changes see Milestone "Modern C++" on
+          GitHub;
+  - now offering an official installer for FreeBSD:
+        - automatic installation;
+        - built-in update via web-interface;
+        - currently supporting only x86_64 CPU architecture;
+  - full support for Unicode and extra long file paths (more than 260 characters)
+    on Windows including:
+        - downloading;
+        - par-verification and -repair;
+        - par-renaming (deobfuscation);
+        - unpacking;
+        - post-processing;
+  - added download volume quota management:
+        - new options "MonthlyQuota", "QuotaStartDay", "DailyQuota";
+        - downloading is suspended when the quota is reached and automatically
+          resumed when the next billing period starts (month or day);
+        - new fields in RPC-method "status": MonthSizeLo, MonthSizeHi,
+          MonthSizeMB, DaySizeLo, DaySizeHi, DaySizeMB, QuotaReached.
+          MonthSizes are related to current billing month taking option
+          "QuotaStartDay" into account;
+        - download volume for "this month" shown in web-interface in statistics
+          dialog shows data for current billing month (taking option
+          "QuotaStartDay" into account);
+        - remaining time is shown in orange when the quota is reached;
+        - dialog "statistics and status" may show extra row "Download quota:
+          reached";
+  - new function "Retry failed articles" in history:
+        - failed downloads can be now tried again but in contrast to command
+          "Download again" only failed articles are downloaded whereas
+          successfully downloaded pieces are reused;
+        - new command "HistoryRetryFailed" of RPC-method "editqueue";
+        - new subcommand "F" of command line switch "-E/--edit" for history;
+  - reworked options to delete already downloaded files when deleting downloads
+    from queue:
+        - removed option "DeleteCleanupDisk";
+        - in the "Delete downloads confirmation dialog" allowing user to decide
+          if the already downloaded files must be deleted or not;
+        - option "HealthCheck" extended with new possible value "Park"; Health
+          check now offers:
+            - "Delete" - to move download into history and delete already
+               downloaded files;
+            - "Park" - to move download into history and keep already downloaded
+               files;
+        - remote command "GroupDelete" now always delete already downloaded files;
+        - new remote command "GroupParkDelete" keeps already downloaded files;
+        - new subcommand "DP" of console command "--edit/-E" to delete download
+          from queue and keep already downloaded files;
+  - new queue script event "NZB_MARKED"; new env var "NZBNA_MARKSTATUS" is
+    passed to queue scripts:
+  - new option "ShellOverride" allows to configure path to python, bash, etc.;
+    useful on systems with non-standard paths; eliminating the need to change
+    shebang for every script; also makes it possible to put scripts on non-exec
+    file systems;
+  - reduced disk fragmentation in direct write mode on Windows; this improves
+    unpack speed;
+  - news servers can now be configured as optional; marking server as optional
+    tells NZBGet to ignore this server if a connection to this server cannot be
+    established;
+  - added support for tvdbid and tvmazeid in rss feeds;
+  - added button to save nzb-log into a file directly from web-ui;
+  - queue-scripts can now change destination after download is completed and
+    before unpack;
+  - queue-scripts save messages into nzb-log;
+  - showing number of selected items in confirmation box when deleting or
+    performing other actions on multiple items in web-interface;
+  - built-in web-server can now use certificate chain files through option
+    "SecureCert", when compiled using OpenSSL;
+  - added support for SNI in TLS/SSL;
+  - better error reporting when using GnuTLS;
+  - allowing character "=" in dupe-badges;
+  - par-check doesn't ignore files from option "ExtCleanupDisk" anymore; only
+    files listed in option "ParIgnoreExt" are ignored;
+  - low-level messages from par2-module are now added to log (as DETAIL);
+  - option "ScriptDir" now accepts multiple directories;
+  - path to original queued nzb-file is now passed to scripts;
+  - hidden files and directories are now ignored by the scanner of incoming nzb
+    directory;
+  - separated disk state files for queue and history for better performance;
+  - improved replacing of invalid characters in file names in certain cases;
+  - automatically removing orphaned diskstate files from QueueDir;
+  - added support for file names with reserved words on Windows;
+  - added several settings to change behavior of web-interface, new section
+    "WEB-INTERFACE" on settings page;
+  - showing various status info in browser window title:
+        - number of downloads, current speed, remaining time, pause state;
+        - new option "WindowTitle";
+  - improved tray icon (Windows) to look better on a dark background;
+  - feed scripts must now return exit codes; this is to prevent queueing of
+    all files from the feed if the feed script crashes;
+  - improved error reporting on DNS lookup errors;
+  - fixed: splitted files were not always joined;
+  - fixed: wrong encoding in file names of downloaded files;
+  - fixed: queue-scripts not called for failed URLs if the scripts were set in
+    category’s option "PostScript";
+  - fixed: crash when executing command "--printconfig";
+  - fixed: error messages when trying to delete intermediate directory on Windows;
+  - fixed: web-ui didn't load in Chrome on iOS;
+  - improved POSIX configure script - now using pkg-config for all required
+    libraries;
+  - improved Windows installer - scripts are now installed into a subdirectory
+    of default "MainDir" (C:\ProgramData\NZBGet\scripts) instead of program's
+    directory;
+  - updated and corrected option descriptions.
+
+nzbget-16.4:
+  - fixed resource (socket) leak which may cause "too many open files"
+    errors with a possible crash.
+
+nzbget-16.3:
+  - fixed: certain downloads may fail due to a bug in the workaround
+    implemented in v16.2.
+
+nzbget-16.2:
+  - implemented workaround to deal with malformed news server responses,
+    which may still contain useful data.
+
+nzbget-16.1:
+  - fixed issues with reverse proxies;
+  - fixed unpack failure on older AMD CPUs, when installed via
+    universal Linux installer;
+  - fixed: when the program was started from setup the default directories
+    were created with wrong permission (Windows only).
+
 nzbget-16.0:
   - moved project hosting to GitHub:
         - moved source code repository from subversion to git;

Makefile.am

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,6 +28,8 @@ nzbget_SOURCES = \
 	daemon/connect/WebDownloader.h \
 	daemon/extension/FeedScript.cpp \
 	daemon/extension/FeedScript.h \
+	daemon/extension/CommandScript.cpp \
+	daemon/extension/CommandScript.h \
 	daemon/extension/NzbScript.cpp \
 	daemon/extension/NzbScript.h \
 	daemon/extension/PostScript.cpp \
@@ -56,8 +58,8 @@ nzbget_SOURCES = \
 	daemon/frontend/LoggableFrontend.h \
 	daemon/frontend/NCursesFrontend.cpp \
 	daemon/frontend/NCursesFrontend.h \
-    daemon/main/CommandLineParser.cpp \
-    daemon/main/CommandLineParser.h \
+	daemon/main/CommandLineParser.cpp \
+	daemon/main/CommandLineParser.h \
 	daemon/main/DiskService.cpp \
 	daemon/main/DiskService.h \
 	daemon/main/Maintenance.cpp \
@@ -66,6 +68,8 @@ nzbget_SOURCES = \
 	daemon/main/nzbget.h \
 	daemon/main/Options.cpp \
 	daemon/main/Options.h \
+	daemon/main/WorkState.cpp \
+	daemon/main/WorkState.h \
 	daemon/main/Scheduler.cpp \
 	daemon/main/Scheduler.h \
 	daemon/main/StackTrace.cpp \
@@ -90,16 +94,26 @@ nzbget_SOURCES = \
 	daemon/postprocess/DupeMatcher.h \
 	daemon/postprocess/ParChecker.cpp \
 	daemon/postprocess/ParChecker.h \
-	daemon/postprocess/ParCoordinator.cpp \
-	daemon/postprocess/ParCoordinator.h \
 	daemon/postprocess/ParParser.cpp \
 	daemon/postprocess/ParParser.h \
 	daemon/postprocess/ParRenamer.cpp \
 	daemon/postprocess/ParRenamer.h \
 	daemon/postprocess/PrePostProcessor.cpp \
 	daemon/postprocess/PrePostProcessor.h \
+	daemon/postprocess/RarRenamer.cpp \
+	daemon/postprocess/RarRenamer.h \
+	daemon/postprocess/RarReader.cpp \
+	daemon/postprocess/RarReader.h \
+	daemon/postprocess/Rename.cpp \
+	daemon/postprocess/Rename.h \
+	daemon/postprocess/Repair.cpp \
+	daemon/postprocess/Repair.h \
 	daemon/postprocess/Unpack.cpp \
 	daemon/postprocess/Unpack.h \
+	daemon/postprocess/DirectUnpack.cpp \
+	daemon/postprocess/DirectUnpack.h \
+	daemon/queue/DirectRenamer.cpp \
+	daemon/queue/DirectRenamer.h \
 	daemon/queue/DiskState.cpp \
 	daemon/queue/DiskState.h \
 	daemon/queue/DownloadInfo.cpp \
@@ -146,6 +160,16 @@ nzbget_SOURCES = \
 	daemon/util/FileSystem.h \
 	daemon/util/Util.cpp \
 	daemon/util/Util.h \
+	daemon/nserv/NServMain.h \
+	daemon/nserv/NServMain.cpp \
+	daemon/nserv/NServFrontend.h \
+	daemon/nserv/NServFrontend.cpp \
+	daemon/nserv/NntpServer.h \
+	daemon/nserv/NntpServer.cpp \
+	daemon/nserv/NzbGenerator.h \
+	daemon/nserv/NzbGenerator.cpp \
+	daemon/nserv/YEncoder.h \
+	daemon/nserv/YEncoder.cpp \
 	code_revision.cpp
 
 if WITH_PAR2
@@ -174,8 +198,6 @@ nzbget_SOURCES += \
 	lib/par2/md5.cpp \
 	lib/par2/md5.h \
 	lib/par2/par2cmdline.h \
-	lib/par2/par2creatorsourcefile.cpp \
-	lib/par2/par2creatorsourcefile.h \
 	lib/par2/par2fileformat.cpp \
 	lib/par2/par2fileformat.h \
 	lib/par2/par2repairer.cpp \
@@ -194,6 +216,25 @@ nzbget_SOURCES += \
 	lib/par2/verificationpacket.h
 endif
 
+# Simd decoder and Crc32
+nzbget_SOURCES += \
+	lib/yencode/YEncode.h \
+	lib/yencode/SimdInit.cpp \
+	lib/yencode/SimdDecoder.cpp \
+	lib/yencode/ScalarDecoder.cpp \
+	lib/yencode/Sse2Decoder.cpp \
+	lib/yencode/Ssse3Decoder.cpp \
+	lib/yencode/PclmulCrc.cpp \
+	lib/yencode/NeonDecoder.cpp \
+	lib/yencode/AcleCrc.cpp \
+	lib/yencode/SliceCrc.cpp
+
+lib/yencode/Sse2Decoder.$(OBJEXT) : CXXFLAGS+=$(SSE2_CXXFLAGS)
+lib/yencode/Ssse3Decoder.$(OBJEXT) : CXXFLAGS+=$(SSSE3_CXXFLAGS)
+lib/yencode/PclmulCrc.$(OBJEXT) : CXXFLAGS+=$(PCLMUL_CXXFLAGS)
+lib/yencode/NeonDecoder.$(OBJEXT) : CXXFLAGS+=$(NEON_CXXFLAGS)
+lib/yencode/AcleCrc.$(OBJEXT) : CXXFLAGS+=$(ACLECRC_CXXFLAGS)
+
 AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/connect \
 	-I$(srcdir)/daemon/extension \
@@ -205,7 +246,9 @@ AM_CPPFLAGS = \
 	-I$(srcdir)/daemon/queue \
 	-I$(srcdir)/daemon/remote \
 	-I$(srcdir)/daemon/util \
-	-I$(srcdir)/lib/par2
+	-I$(srcdir)/daemon/nserv \
+	-I$(srcdir)/lib/par2 \
+	-I$(srcdir)/lib/yencode
 
 if WITH_TESTS
 nzbget_SOURCES += \
@@ -217,13 +260,22 @@ nzbget_SOURCES += \
 	tests/main/CommandLineParserTest.cpp \
 	tests/main/OptionsTest.cpp \
 	tests/feed/FeedFilterTest.cpp \
-	tests/postprocess/ParCheckerTest.cpp \
-	tests/postprocess/ParRenamerTest.cpp \
 	tests/postprocess/DupeMatcherTest.cpp \
+	tests/postprocess/RarRenamerTest.cpp \
+	tests/postprocess/RarReaderTest.cpp \
+	tests/postprocess/DirectUnpackTest.cpp \
 	tests/queue/NzbFileTest.cpp \
+	tests/nntp/ServerPoolTest.cpp \
+	tests/util/FileSystemTest.cpp \
 	tests/util/NStringTest.cpp \
 	tests/util/UtilTest.cpp
 
+if WITH_PAR2
+nzbget_SOURCES += \
+	tests/postprocess/ParCheckerTest.cpp \
+	tests/postprocess/ParRenamerTest.cpp
+endif
+
 AM_CPPFLAGS += \
 	-I$(srcdir)/lib/catch \
 	-I$(srcdir)/tests/suite
@@ -233,7 +285,8 @@ EXTRA_DIST = \
 	$(windows_FILES) \
 	$(osx_FILES) \
 	$(linux_FILES) \
-	$(testdata_FILES)
+	$(testdata_FILES) \
+	$(par2doc_FILES)
 
 windows_FILES = \
 	daemon/windows/StdAfx.cpp \
@@ -252,9 +305,9 @@ windows_FILES = \
 	windows/resources/trayicon_idle.ico \
 	windows/resources/trayicon_paused.ico \
 	windows/resources/trayicon_working.ico \
-	windows/setup/nzbget-setup.nsi \
-	windows/setup/install.bmp \
-	windows/setup/uninstall.bmp
+	windows/resources/install.bmp \
+	windows/resources/uninstall.bmp \
+	windows/nzbget-setup.nsi
 
 osx_FILES = \
 	osx/App_Prefix.pch \
@@ -294,16 +347,18 @@ linux_FILES = \
 	linux/build-info.txt \
 	linux/build-nzbget \
 	linux/build-unpack \
+	linux/build-toolchain-android \
 	linux/build-toolchain-freebsd
 
-
 doc_FILES = \
-	lib/par2/AUTHORS \
-	lib/par2/README \
 	README \
 	ChangeLog \
 	COPYING
 
+par2doc_FILES = \
+	lib/par2/AUTHORS \
+	lib/par2/README
+
 exampleconf_FILES = \
 	nzbget.conf
 
@@ -337,7 +392,9 @@ webui_FILES = \
 	webui/img/favicon.ico \
 	webui/img/download-anim-green-2x.png \
 	webui/img/download-anim-orange-2x.png \
-	webui/img/transmit-reload-2x.gif
+	webui/img/transmit-reload-2x.gif \
+	webui/img/favicon-256x256-opaque.png \
+	webui/img/favicon-256x256.png
 
 scripts_FILES = \
 	scripts/EMail.py \
@@ -358,7 +415,36 @@ testdata_FILES = \
 	tests/testdata/parchecker/testfile.par2 \
 	tests/testdata/parchecker/testfile.vol00+1.PAR2 \
 	tests/testdata/parchecker/testfile.vol01+2.PAR2 \
-	tests/testdata/parchecker/testfile.vol03+3.PAR2
+	tests/testdata/parchecker/testfile.vol03+3.PAR2 \
+	tests/testdata/parchecker2/crc.txt \
+	tests/testdata/parchecker2/testfile.7z.001 \
+	tests/testdata/parchecker2/testfile.7z.002 \
+	tests/testdata/parchecker2/testfile.7z.003 \
+	tests/testdata/parchecker2/testfile.7z.par2 \
+	tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \
+	tests/testdata/rarrenamer/testfile3.part01.rar \
+	tests/testdata/rarrenamer/testfile3.part02.rar \
+	tests/testdata/rarrenamer/testfile3.part03.rar \
+	tests/testdata/rarrenamer/testfile5.part01.rar \
+	tests/testdata/rarrenamer/testfile5.part02.rar \
+	tests/testdata/rarrenamer/testfile5.part03.rar \
+	tests/testdata/rarrenamer/testfile3oldnam.rar \
+	tests/testdata/rarrenamer/testfile3oldnam.r00 \
+	tests/testdata/rarrenamer/testfile3oldnam.r01 \
+	tests/testdata/rarrenamer/testfile3encdata.part01.rar \
+	tests/testdata/rarrenamer/testfile3encdata.part02.rar \
+	tests/testdata/rarrenamer/testfile3encdata.part03.rar \
+	tests/testdata/rarrenamer/testfile3encnam.part01.rar \
+	tests/testdata/rarrenamer/testfile3encnam.part02.rar \
+	tests/testdata/rarrenamer/testfile3encnam.part03.rar \
+	tests/testdata/rarrenamer/testfile5encdata.part01.rar \
+	tests/testdata/rarrenamer/testfile5encdata.part02.rar \
+	tests/testdata/rarrenamer/testfile5encdata.part03.rar \
+	tests/testdata/rarrenamer/testfile5encnam.part01.rar \
+	tests/testdata/rarrenamer/testfile5encnam.part02.rar \
+	tests/testdata/rarrenamer/testfile5encnam.part03.rar
 
 # Install
 dist_doc_DATA = $(doc_FILES)
@@ -409,21 +495,21 @@ uninstall-conf:
 #    we create new file "code_revision.c" with empty revision number.
 code_revision.cpp: FORCE
 	@ if test -d ./.git ; then \
-		B="$(shell git branch | sed -n -e 's/^\* \(.*\)/\1/p')"; \
-		M="$(shell git status --porcelain)" ; \
+		B=`git branch | sed -n -e 's/^\* \(.*\)/\1/p'`; \
+		M=`git status --porcelain` ; \
 		if test "$$M" != "" ; then \
 			M="M" ; \
 		fi ; \
 		if test "$$B" = "master" ; then \
 			V="$$M" ; \
 		elif test "$$B" = "develop" ; then \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M" ; \
 		else \
-			V="$(shell git rev-list HEAD | wc -l | xargs)" ; \
+			V=`git rev-list HEAD | wc -l | xargs` ; \
 			V="$${V}$$M ($$B)" ; \
 		fi ; \
-		H="$(shell test -f ./code_revision.cpp && head -n 1 code_revision.cpp)"; \
+		H=`test -f ./code_revision.cpp && head -n 1 code_revision.cpp`; \
 		if test "/* $$V */" != "$$H" ; then \
 			( \
 			echo "/* $$V */" ;\

README.md

@@ -1,15 +1,19 @@
 # NZBGet #
+[![License](https://img.shields.io/badge/license-GPL-blue.svg)](http://www.gnu.org/licenses/)
+[![Build Status](https://img.shields.io/travis/nzbget/nzbget/develop.svg)](https://travis-ci.org/nzbget/nzbget)
+[![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/nzbget/nzbget.svg?label=code%20quality:%20c%2b%2b)](https://lgtm.com/projects/g/nzbget/nzbget/context:cpp)
+[![Code Quality: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/nzbget/nzbget.svg?label=code%20quality:%20js)](https://lgtm.com/projects/g/nzbget/nzbget/context:javascript)
+[![Total Alerts](https://img.shields.io/lgtm/alerts/g/nzbget/nzbget.svg)](https://lgtm.com/projects/g/nzbget/nzbget/alerts)
+
+[![Total downloads](https://img.shields.io/github/downloads/nzbget/nzbget/total.svg)](https://github.com/nzbget/nzbget/releases)
+[![Downloads (latest release)](https://img.shields.io/github/downloads/nzbget/nzbget/latest/total.svg?label=latest%20release)](https://github.com/nzbget/nzbget/releases/latest)
+
 NZBGet is a binary downloader, which downloads files from Usenet
 based on information given in nzb-files. 
 
-NZBGet is written in C++ and is known for its extraordinary performance and efficiency.
+NZBGet is written in C++ and is known for its performance and efficiency.
 
-NZBGet can be run at almost every platform - classic PCs, NAS, media players, SAT-receivers, WLAN-routers, etc.
-The download area provides precompiled binaries
-for Windows, Mac OS X and Linux (compatible with many CPUs and platform variants). For other platforms
-the program can be compiled from sources.
-
-- [Home page (nzbget.net)](http://nzbget.net) - for first time visitors, learn more about NZBGet;
-- [Downloads](http://nzbget.net/download) - get the binaries and sources;
-- [Documentation](https://github.com/nzbget/nzbget/wiki) - installation manuals, HOW-TOs, API;
-- [Forum](http://forum.nzbget.net) - get support, share your ideas, scripts, add-ons.
+NZBGet can run on almost any device - classic PC, NAS, media player, SAT-receiver, WLAN-router, etc.
+The download area provides precompiled binaries for Windows, macOS, Linux (compatible with
+many CPUs and platform variants), FreeBSD and Android. For other platforms
+the program can be compiled from sources.

config.h.in

@@ -3,16 +3,15 @@
 /* Define to 1 to include debug-code */
 #undef DEBUG
 
-/* Define to 1 if deleting of files during reading of directory is not
-   properly supported by OS */
-#undef DIRBROWSER_SNAPSHOT
-
 /* Define to 1 to not use curses */
 #undef DISABLE_CURSES
 
 /* Define to 1 to disable gzip-support */
 #undef DISABLE_GZIP
 
+/* Define to 1 to not use libxml2, only for development purposes */
+#undef DISABLE_LIBXML2
+
 /* Define to 1 to disable par-verification and repair */
 #undef DISABLE_PARCHECK
 
@@ -83,6 +82,9 @@
 /* Define to 1 to use GnuTLS library for TLS/SSL-support. */
 #undef HAVE_LIBGNUTLS
 
+/* Define to 1 if lockf is supported */
+#undef HAVE_LOCKF
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
@@ -92,9 +94,15 @@
 /* Define to 1 if you have the <ncurses/ncurses.h> header file. */
 #undef HAVE_NCURSES_NCURSES_H
 
-/* Define to 1 to use OpenSSL library for TLS/SSL-support. */
+/* Define to 1 to use Nettle library for decryption. */
+#undef HAVE_NETTLE
+
+/* Define to 1 to use OpenSSL library for TLS/SSL-support and decryption. */
 #undef HAVE_OPENSSL
 
+/* Define to 1 if pthread_cancel is supported */
+#undef HAVE_PTHREAD_CANCEL
+
 /* Define to 1 if you have the <regex.h> header file. */
 #undef HAVE_REGEX_H
 
@@ -131,6 +139,9 @@
 /* Define to 1 if variadic macros are supported */
 #undef HAVE_VARIADIC_MACROS
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#undef HAVE_X509_CHECK_HOST
+
 /* Define to 1 to exclude debug-code */
 #undef NDEBUG
 
@@ -149,6 +160,9 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME
 
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
 
@@ -164,9 +178,10 @@
 /* Version number of package */
 #undef VERSION
 
-/* Define to 1 if your processor stores words with the most significant byte
-   first (like Motorola and SPARC, unlike Intel and VAX). */
-#undef WORDS_BIGENDIAN
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
 
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS

configure.ac

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2021 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,22 +20,16 @@
 #                                               -*- Autoconf -*-
 # Process this file with autoconf to produce a configure script.
 
-AC_PREREQ(2.59)
-AC_INIT(nzbget, 17.0-testing, hugbug@users.sourceforge.net)
+AC_PREREQ(2.65)
+AC_INIT(nzbget, 21.2-testing, hugbug@users.sourceforge.net)
 AC_CONFIG_AUX_DIR(posix)
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE([foreign])
+AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_CONFIG_SRCDIR([daemon/main/nzbget.cpp])
 AC_CONFIG_HEADERS([config.h])
 AM_MAINTAINER_MODE
 
-
-dnl
-dnl Set default library path, if not specified in environment variable "LIBPREF".
-dnl
-if test "$LIBPREF" = ""; then
-	LIBPREF="/usr"
-fi
+m4_include([posix/ax_cxx_compile_stdcxx.m4])
 
 
 dnl
@@ -71,8 +65,7 @@ fi
 dnl
 dnl Checks for header files.
 dnl
-AC_CHECK_HEADERS(sys/prctl.h)
-AC_CHECK_HEADERS(regex.h)
+AC_CHECK_HEADERS(sys/prctl.h regex.h endian.h getopt.h)
 
 
 dnl
@@ -84,6 +77,19 @@ AC_SEARCH_LIBS([inet_addr], [nsl])
 AC_SEARCH_LIBS([hstrerror], [resolv])
 
 
+dnl
+dnl Android NDK restrictions
+dnl
+AC_CHECK_FUNC(lockf,
+	[AC_CHECK_DECL(lockf,
+		[AC_DEFINE([HAVE_LOCKF], 1, [Define to 1 if lockf is supported])],,
+		[#include <unistd.h>])])
+AC_CHECK_FUNC(pthread_cancel,
+	[AC_CHECK_DECL(pthread_cancel,
+		[AC_DEFINE([HAVE_PTHREAD_CANCEL], 1, [Define to 1 if pthread_cancel is supported])],,
+		[#include <pthread.h>])])
+
+
 dnl
 dnl Getopt
 dnl
@@ -152,7 +158,7 @@ if test "$FOUND" = "no"; then
 		[	char* szHost; struct hostent hinfobuf; char* strbuf; int h_errnop;
 			struct hostent* hinfo = gethostbyname_r(szHost, &hinfobuf, strbuf, 1024, &h_errnop); ],
 		AC_MSG_RESULT([[yes, and it takes 5 arguments]])
-		FOUND="yes"                      
+		FOUND="yes"
 		AC_DEFINE([HAVE_GETHOSTBYNAME_R_5], 1, [Define to 1 if gethostbyname_r takes 5 arguments]),
 		FOUND="no")
 	
@@ -202,32 +208,20 @@ AC_TRY_COMPILE([
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (size_t*)NULL)],[
-    AC_MSG_RESULT(size_t)
-    SOCKLEN_T=size_t],[
-    AC_TRY_COMPILE([
+	AC_MSG_RESULT(size_t)
+	SOCKLEN_T=size_t],[
+	AC_TRY_COMPILE([
 #include <stddef.h>
 #include <sys/types.h>
 #include <sys/socket.h>],[
 (void)getsockopt (1, 1, 1, NULL, (int*)NULL)],[
-      AC_MSG_RESULT(int)
-      SOCKLEN_T=int],[
-      AC_MSG_WARN(could not determine)
-      SOCKLEN_T=int])])])
+	AC_MSG_RESULT(int)
+	SOCKLEN_T=int],[
+	AC_MSG_WARN(could not determine)
+	SOCKLEN_T=int])])])
 AC_DEFINE_UNQUOTED(SOCKLEN_T, $SOCKLEN_T, [Determine what socket length (socklen_t) data type is])
 
 
-dnl
-dnl Dir-browser's snapshot
-dnl
-AC_MSG_CHECKING(whether dir-browser snapshot workaround is needed)
-if test "$target_vendor" == "apple"; then
-	AC_MSG_RESULT([[yes]])
-	AC_DEFINE([DIRBROWSER_SNAPSHOT], 1, [Define to 1 if deleting of files during reading of directory is not properly supported by OS])
-else
-	AC_MSG_RESULT([[no]])
-fi
-
-
 dnl
 dnl check cpu cores via sysconf
 dnl
@@ -244,26 +238,36 @@ AC_TRY_COMPILE(
 dnl 
 dnl checks for libxml2 includes and libraries.
 dnl
-AC_ARG_WITH(libxml2_includes,
-	[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
-	[CPPFLAGS="${CPPFLAGS} -I${withval}"]
-	[INCVAL="yes"],
-	[INCVAL="no"])
-AC_ARG_WITH(libxml2_libraries,
-	[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
-	[LDFLAGS="${LDFLAGS} -L${withval}"]
-	[LIBVAL="yes"],
-	[LIBVAL="no"])
-if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
-	PKG_CHECK_MODULES(libxml2, libxml-2.0,
-		[LIBS="${LIBS} $libxml2_LIBS"]
-		[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+AC_MSG_CHECKING(whether to use libxml2)
+AC_ARG_ENABLE(libxml2,
+	[AS_HELP_STRING([--disable-libxml2], [do not use libxml2 (removes dependency from libxml2-library, only for development purposes)])],
+	[USELIBXML2=$enableval],
+	[USELIBXML2=yes] )
+AC_MSG_RESULT($USELIBXML2)
+if test "$USELIBXML2" = "yes"; then
+	AC_ARG_WITH(libxml2_includes,
+		[AS_HELP_STRING([--with-libxml2-includes=DIR], [libxml2 include directory])],
+		[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+		[INCVAL="yes"],
+		[INCVAL="no"])
+	AC_ARG_WITH(libxml2_libraries,
+		[AS_HELP_STRING([--with-libxml2-libraries=DIR], [libxml2 library directory])],
+		[LDFLAGS="${LDFLAGS} -L${withval}"]
+		[LIBVAL="yes"],
+		[LIBVAL="no"])
+	if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+		PKG_CHECK_MODULES(libxml2, libxml-2.0,
+			[LIBS="${LIBS} $libxml2_LIBS"]
+			[CPPFLAGS="${CPPFLAGS} $libxml2_CFLAGS"],
+			AC_MSG_ERROR("libxml2 library not found"))
+	fi
+	AC_CHECK_HEADER(libxml/tree.h,,
+		AC_MSG_ERROR("libxml2 header files not found"))
+	AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
 		AC_MSG_ERROR("libxml2 library not found"))
+else
+	AC_DEFINE([DISABLE_LIBXML2],1,[Define to 1 to not use libxml2, only for development purposes])
 fi
-AC_CHECK_HEADER(libxml/tree.h,,
-	AC_MSG_ERROR("libxml2 header files not found"))
-AC_SEARCH_LIBS([xmlNewNode], [xml2], ,
-	AC_MSG_ERROR("libxml2 library not found"))
 
 
 dnl
@@ -276,17 +280,23 @@ AC_ARG_ENABLE(curses,
 	[USECURSES=yes] )
 AC_MSG_RESULT($USECURSES)
 if test "$USECURSES" = "yes"; then
-	INCVAL="${LIBPREF}/include"
-	LIBVAL="${LIBPREF}/lib"
 	AC_ARG_WITH(libcurses_includes,
 		[AS_HELP_STRING([--with-libcurses-includes=DIR], [libcurses include directory])],
-		[INCVAL="$withval"])
-	CPPFLAGS="${CPPFLAGS} -I${INCVAL}"
+		[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+		[INCVAL="yes"],
+		[INCVAL="no"])
 	AC_ARG_WITH(libcurses_libraries,
 		[AS_HELP_STRING([--with-libcurses-libraries=DIR], [libcurses library directory])],
-		[LIBVAL="$withval"])
-	LDFLAGS="${LDFLAGS} -L${LIBVAL}"
-	
+		[LDFLAGS="${LDFLAGS} -L${withval}"]
+		[LIBVAL="yes"],
+		[LIBVAL="no"])
+	if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+		PKG_CHECK_MODULES(ncurses, ncurses,
+			[LIBS="${LIBS} $ncurses_LIBS"]
+			[CPPFLAGS="${CPPFLAGS} $ncurses_CFLAGS"],
+			AC_MSG_ERROR("ncurses library not found"))
+	fi
+
 	AC_CHECK_HEADER(ncurses.h,
 		FOUND=yes
 		AC_DEFINE([HAVE_NCURSES_H],1,[Define to 1 if you have the <ncurses.h> header file.]),
@@ -308,6 +318,8 @@ if test "$USECURSES" = "yes"; then
 	fi
 	AC_SEARCH_LIBS([refresh], [ncurses curses],, 
 		AC_ERROR([Couldn't find curses library]))
+	AC_SEARCH_LIBS([nodelay], [ncurses curses tinfo],, 
+		AC_ERROR([Couldn't find curses library]))
 else
 	AC_DEFINE([DISABLE_CURSES],1,[Define to 1 to not use curses])
 fi
@@ -325,11 +337,8 @@ AC_MSG_RESULT($ENABLEPARCHECK)
 if test "$ENABLEPARCHECK" = "yes"; then
 	dnl PAR2 checks. 
 	dnl 
-	dnl Checks for header files.
-	AC_CHECK_HEADERS([endian.h] [getopt.h])
 	dnl Checks for typedefs, structures, and compiler characteristics.
 	AC_TYPE_SIZE_T
-	AC_C_BIGENDIAN
 	AC_FUNC_FSEEKO
 	dnl Checks for library functions.
 	AC_CHECK_FUNCS([stricmp])
@@ -372,8 +381,7 @@ if test "$USETLS" = "yes"; then
 		if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
 			PKG_CHECK_MODULES([openssl], [openssl],
 				[LIBS="${LIBS} $openssl_LIBS"]
-				[CPPFLAGS="${CPPFLAGS} $openssl_CFLAGS"],
-				FOUND=no)
+				[CPPFLAGS="${CPPFLAGS} $openssl_CFLAGS"])
 		fi
 
 		AC_CHECK_HEADER(openssl/ssl.h,
@@ -384,8 +392,8 @@ if test "$USETLS" = "yes"; then
 			AC_MSG_ERROR([Couldn't find OpenSSL headers (ssl.h)])
 		fi
 		if test "$FOUND" = "yes"; then
-			AC_SEARCH_LIBS([CRYPTO_set_locking_callback], [crypto],
-				AC_SEARCH_LIBS([SSL_library_init], [ssl],
+			AC_SEARCH_LIBS([ASN1_OBJECT_free], [crypto],
+				AC_SEARCH_LIBS([SSL_CTX_new], [ssl],
 					FOUND=yes,
 					FOUND=no),
 				FOUND=no)
@@ -394,22 +402,29 @@ if test "$USETLS" = "yes"; then
 			fi
 			if test "$FOUND" = "yes"; then
 				TLSLIB="OpenSSL"
-				AC_DEFINE([HAVE_OPENSSL],1,[Define to 1 to use OpenSSL library for TLS/SSL-support.])
+				AC_DEFINE([HAVE_OPENSSL],1,[Define to 1 to use OpenSSL library for TLS/SSL-support and decryption.])
+				AC_SEARCH_LIBS([X509_check_host], [crypto],
+					AC_DEFINE([HAVE_X509_CHECK_HOST],1,[Define to 1 if OpenSSL supports function "X509_check_host".]))
 			fi
 		fi
 	fi
 
 	if test "$TLSLIB" = "GnuTLS" -o "$TLSLIB" = ""; then
-		INCVAL="${LIBPREF}/include"
-		LIBVAL="${LIBPREF}/lib"
 		AC_ARG_WITH(libgnutls_includes,
 			[AS_HELP_STRING([--with-libgnutls-includes=DIR], [GnuTLS include directory])],
-			[INCVAL="$withval"])
-		CPPFLAGS="${CPPFLAGS} -I${INCVAL}"
+			[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+			[INCVAL="yes"],
+			[INCVAL="no"])
 		AC_ARG_WITH(libgnutls_libraries,
 			[AS_HELP_STRING([--with-libgnutls-libraries=DIR], [GnuTLS library directory])],
-			[LIBVAL="$withval"])
-		LDFLAGS="${LDFLAGS} -L${LIBVAL}"
+			[LDFLAGS="${LDFLAGS} -L${withval}"]
+			[LIBVAL="yes"],
+			[LIBVAL="no"])
+		if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+			PKG_CHECK_MODULES([gnutls], [gnutls],
+				[LIBS="${LIBS} $gnutls_LIBS"]
+				[CPPFLAGS="${CPPFLAGS} $gnutls_CFLAGS"])
+		fi
 
 		AC_CHECK_HEADER(gnutls/gnutls.h,
 			FOUND=yes
@@ -451,6 +466,39 @@ if test "$USETLS" = "yes"; then
 				AC_DEFINE([HAVE_LIBGNUTLS],1,[Define to 1 to use GnuTLS library for TLS/SSL-support.])
 			fi
 		fi
+
+		if test "$TLSLIB" = "GnuTLS"; then
+			AC_ARG_WITH(libnettle_includes,
+				[AS_HELP_STRING([--with-libnettle-includes=DIR], [Nettle include directory])],
+				[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+				[INCVAL="yes"],
+				[INCVAL="no"])
+			AC_ARG_WITH(libnettle_libraries,
+				[AS_HELP_STRING([--with-libnettle-libraries=DIR], [Nettle library directory])],
+				[LDFLAGS="${LDFLAGS} -L${withval}"]
+				[LIBVAL="yes"],
+				[LIBVAL="no"])
+			if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+				PKG_CHECK_MODULES([nettle], [nettle],
+					[LIBS="${LIBS} $nettle_LIBS"]
+					[CPPFLAGS="${CPPFLAGS} $nettle_CFLAGS"])
+			fi
+			AC_CHECK_HEADER(nettle/sha.h,
+				FOUND=yes,
+				FOUND=no)
+			if test "$FOUND" = "no"; then
+				AC_MSG_ERROR([Couldn't find Nettle headers (sha.h)])
+			fi
+			AC_SEARCH_LIBS([nettle_pbkdf2_hmac_sha256], [nettle],
+				FOUND=yes,
+				FOUND=no)
+			if test "$FOUND" = "no"; then
+				AC_MSG_ERROR([Couldn't find Nettle library, required when using GnuTLS])
+			fi
+			if test "$FOUND" = "yes"; then
+				AC_DEFINE([HAVE_NETTLE],1,[Define to 1 to use Nettle library for decryption.])
+			fi
+		fi
 	fi
 
 	if test "$TLSLIB" = ""; then
@@ -475,16 +523,21 @@ AC_ARG_ENABLE(gzip,
 	[USEZLIB=yes] )
 AC_MSG_RESULT($USEZLIB)
 if test "$USEZLIB" = "yes"; then
-	INCVAL="${LIBPREF}/include"
-	LIBVAL="${LIBPREF}/lib"
 	AC_ARG_WITH(zlib_includes,
 		[AS_HELP_STRING([--with-zlib-includes=DIR], [zlib include directory])],
-		[INCVAL="$withval"])
-	CPPFLAGS="${CPPFLAGS} -I${INCVAL}"
+		[CPPFLAGS="${CPPFLAGS} -I${withval}"]
+		[INCVAL="yes"],
+		[INCVAL="no"])
 	AC_ARG_WITH(zlib_libraries,
 		[AS_HELP_STRING([--with-zlib-libraries=DIR], [zlib library directory])],
-		[LIBVAL="$withval"])
-	LDFLAGS="${LDFLAGS} -L${LIBVAL}"
+		[LDFLAGS="${LDFLAGS} -L${withval}"]
+		[LIBVAL="yes"],
+		[LIBVAL="no"])
+	if test "$INCVAL" = "no" -o "$LIBVAL" = "no"; then
+		PKG_CHECK_MODULES([zlib], [zlib],
+			[LIBS="${LIBS} $zlib_LIBS"]
+			[CPPFLAGS="${CPPFLAGS} $zlib_CFLAGS"])
+	fi
 	
 	AC_CHECK_HEADER(zlib.h,,
 		AC_MSG_ERROR("zlib header files not found"))
@@ -495,6 +548,36 @@ else
 fi
 
 
+dnl
+dnl Determine if CPU supports SIMD instructions
+dnl
+AC_MSG_CHECKING(whether to use SIMD-optimized routines)
+USE_SIMD=no
+case $host_cpu in
+	i?86|x86_64)
+		SSE2_CXXFLAGS="-msse2"
+		SSSE3_CXXFLAGS="-mssse3"
+		PCLMUL_CXXFLAGS="-msse4.1 -mpclmul"
+		USE_SIMD=yes
+		;;
+	arm*)
+		NEON_CXXFLAGS="-mfpu=neon"
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+	aarch64)
+		ACLECRC_CXXFLAGS="-march=armv8-a+crc -fpermissive"
+		USE_SIMD=yes
+		;;
+esac
+AC_MSG_RESULT($USE_SIMD)
+AC_SUBST([SSE2_CXXFLAGS])
+AC_SUBST([SSSE3_CXXFLAGS])
+AC_SUBST([PCLMUL_CXXFLAGS])
+AC_SUBST([NEON_CXXFLAGS])
+AC_SUBST([ACLECRC_CXXFLAGS])
+
+
 dnl 
 dnl Some Linux systems require an empty signal handler for SIGCHLD
 dnl in order for exit codes to be correctly delivered to parent process.
@@ -553,11 +636,10 @@ dnl
 dnl variadic macros
 dnl 
 AC_MSG_CHECKING(for variadic macros)
-AC_COMPILE_IFELSE([
-	#define macro(...)   macrofunc(__VA_ARGS__)
-	int macrofunc(int a, int b) { return a + b; }
-	int test() { return macro(1, 2); }
-	],
+AC_TRY_COMPILE(
+	[ #define macro(...)   macrofunc(__VA_ARGS__) ]
+	[ int macrofunc(int a, int b) { return a + b; } ],
+	[ int a=macro(1, 2); ],
 	AC_MSG_RESULT([yes])
 	AC_DEFINE([HAVE_VARIADIC_MACROS], 1, Define to 1 if variadic macros are supported),
 	AC_MSG_RESULT([no]))

daemon/connect/Connection.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -41,13 +41,22 @@ public:
 		csConnected,
 		csDisconnected,
 		csListening,
-		csCancelled
+		csCancelled,
+		csBroken
+	};
+
+	enum EIPVersion
+	{
+		ipAuto,
+		ipV4,
+		ipV6
 	};
 
 	Connection(const char* host, int port, bool tls);
 	Connection(SOCKET socket, bool tls);
 	virtual ~Connection();
 	static void Init();
+	static void Final();
 	virtual bool Connect();
 	virtual bool Disconnect();
 	bool Bind();
@@ -65,12 +74,14 @@ public:
 	const char* GetCipher() { return m_cipher; }
 	void SetCipher(const char* cipher) { m_cipher = cipher; }
 	void SetTimeout(int timeout) { m_timeout = timeout; }
+	void SetIPVersion(EIPVersion ipVersion) { m_ipVersion = ipVersion; }
 	EStatus GetStatus() { return m_status; }
 	void SetSuppressErrors(bool suppressErrors);
 	bool GetSuppressErrors() { return m_suppressErrors; }
 	const char* GetRemoteAddr();
 	bool GetGracefull() { return m_gracefull; }
 	void SetGracefull(bool gracefull) { m_gracefull = gracefull; }
+	void SetForceClose(bool forceClose) { m_forceClose = forceClose; }
 #ifndef DISABLE_TLS
 	bool StartTls(bool isClient, const char* certFile, const char* keyFile);
 #endif
@@ -80,6 +91,7 @@ protected:
 	CString m_host;
 	int m_port;
 	bool m_tls;
+	EIPVersion m_ipVersion = ipAuto;
 	SOCKET m_socket = INVALID_SOCKET;
 	CString m_cipher;
 	CharBuffer m_readBuf;
@@ -90,8 +102,8 @@ protected:
 	bool m_suppressErrors = true;
 	BString<100> m_remoteAddr;
 	int m_totalBytesRead = 0;
-	bool m_broken = false;
 	bool m_gracefull = false;
+	bool m_forceClose = false;
 
 	struct SockAddr
 	{
@@ -106,9 +118,9 @@ protected:
 	class ConTlsSocket: public TlsSocket
 	{
 	public:
-		ConTlsSocket(SOCKET socket, bool isClient, const char* certFile,
-			const char* keyFile, const char* cipher, Connection* owner) :
-			TlsSocket(socket, isClient, certFile, keyFile, cipher), m_owner(owner) {}
+		ConTlsSocket(SOCKET socket, bool isClient, const char* host,
+			const char* certFile, const char* keyFile, const char* cipher, Connection* owner) :
+			TlsSocket(socket, isClient, host, certFile, keyFile, cipher), m_owner(owner) {}
 	protected:
 		virtual void PrintError(const char* errMsg) { m_owner->PrintError(errMsg); }
 	private:
@@ -124,12 +136,13 @@ protected:
 #endif
 #endif
 
-	void ReportError(const char* msgPrefix, const char* msgArg, bool PrintErrCode, int herrno = 0,
-		const char* herrMsg = nullptr);
+	void ReportError(const char* msgPrefix, const char* msgArg, bool printErrCode, int errCode = 0,
+		const char* errMsg = nullptr);
 	virtual void PrintError(const char* errMsg);
+	int GetLastNetworkError();
 	bool DoConnect();
 	bool DoDisconnect();
-	bool InitSocketOpts();
+	bool InitSocketOpts(SOCKET socket);
 	bool ConnectWithTimeout(void* address, int address_len);
 #ifndef HAVE_GETADDRINFO
 	in_addr_t ResolveHostAddr(const char* host);
@@ -139,10 +152,6 @@ protected:
 	int send(SOCKET s, const char* buf, int len, int flags);
 	void CloseTls();
 #endif
-
-private:
-	static void Final();
-	friend class ConnectionFinalizer;
 };
 
 #endif

daemon/connect/TlsSocket.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,17 +25,10 @@
 #include "TlsSocket.h"
 #include "Thread.h"
 #include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
 
-class TlsSocketFinalizer
-{
-public:
-	~TlsSocketFinalizer()
-	{
-		TlsSocket::Final();
-	}
-};
-
-std::unique_ptr<TlsSocketFinalizer> m_tlsSocketFinalizer;
+CString TlsSocket::m_certStore;
 
 #ifdef HAVE_LIBGNUTLS
 #ifdef NEED_GCRYPT_LOCKING
@@ -89,6 +82,12 @@ static struct gcry_thread_cbs gcry_threads_Mutex =
 
 #ifdef HAVE_OPENSSL
 
+#ifndef CRYPTO_set_locking_callback
+#define NEED_CRYPTO_LOCKING
+#endif
+
+#ifdef NEED_CRYPTO_LOCKING
+
 /**
  * Mutexes for OpenSSL
  */
@@ -108,17 +107,6 @@ static void openssl_locking(int mode, int n, const char* file, int line)
 	}
 }
 
-/*
-static uint32 openssl_thread_id(void)
-{
-#ifdef WIN32
-	return (uint32)GetCurrentThreadId();
-#else
-	return (uint32)pthread_self();
-#endif
-}
-*/
-
 static struct CRYPTO_dynlock_value* openssl_dynlock_create(const char *file, int line)
 {
 	return (CRYPTO_dynlock_value*)new Mutex();
@@ -143,6 +131,7 @@ static void openssl_dynlock_lock(int mode, struct CRYPTO_dynlock_value *l, const
 	}
 }
 
+#endif /* NEED_CRYPTO_LOCKING */
 #endif /* HAVE_OPENSSL */
 
 
@@ -172,24 +161,24 @@ void TlsSocket::Init()
 #endif /* HAVE_LIBGNUTLS */
 
 #ifdef HAVE_OPENSSL
+
+#ifdef NEED_CRYPTO_LOCKING
 	for (int i = 0, num = CRYPTO_num_locks(); i < num; i++)
 	{
 		g_OpenSSLMutexes.emplace_back(std::make_unique<Mutex>());
 	}
 
+	CRYPTO_set_locking_callback(openssl_locking);
+	CRYPTO_set_dynlock_create_callback(openssl_dynlock_create);
+	CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy);
+	CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock);
+#endif /* NEED_CRYPTO_LOCKING */
+
 	SSL_load_error_strings();
 	SSL_library_init();
 	OpenSSL_add_all_algorithms();
 
-	CRYPTO_set_locking_callback(openssl_locking);
-	//CRYPTO_set_id_callback(openssl_thread_id);
-	CRYPTO_set_dynlock_create_callback(openssl_dynlock_create);
-	CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy);
-	CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock);
-
 #endif /* HAVE_OPENSSL */
-
-	m_tlsSocketFinalizer = std::make_unique<TlsSocketFinalizer>();
 }
 
 void TlsSocket::Final()
@@ -197,18 +186,49 @@ void TlsSocket::Final()
 #ifdef HAVE_LIBGNUTLS
 	gnutls_global_deinit();
 #endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+#ifndef LIBRESSL_VERSION_NUMBER
+	FIPS_mode_set(0);
+#endif
+#ifdef NEED_CRYPTO_LOCKING
+	CRYPTO_set_locking_callback(nullptr);
+	CRYPTO_set_id_callback(nullptr);
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	ERR_remove_state(0);
+#endif
+#if	OPENSSL_VERSION_NUMBER >= 0x10002000L && ! defined (LIBRESSL_VERSION_NUMBER)
+	SSL_COMP_free_compression_methods();
+#endif
+	//ENGINE_cleanup();
+	CONF_modules_free();
+	CONF_modules_unload(1);
+#ifndef OPENSSL_NO_COMP
+	COMP_zlib_cleanup();
+#endif
+	ERR_free_strings();
+	EVP_cleanup();
+	CRYPTO_cleanup_all_ex_data();
+#endif /* HAVE_OPENSSL */
 }
 
 TlsSocket::~TlsSocket()
 {
 	Close();
+
+#ifdef HAVE_OPENSSL
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	ERR_remove_state(0);
+#endif
+#endif
 }
 
-void TlsSocket::ReportError(const char* errMsg)
+void TlsSocket::ReportError(const char* errMsg, bool suppressable)
 {
 #ifdef HAVE_LIBGNUTLS
 	const char* errstr = gnutls_strerror(m_retCode);
-	if (m_suppressErrors)
+	if (suppressable && m_suppressErrors)
 	{
 		debug("%s: %s", errMsg, errstr);
 	}
@@ -219,16 +239,14 @@ void TlsSocket::ReportError(const char* errMsg)
 #endif /* HAVE_LIBGNUTLS */
 
 #ifdef HAVE_OPENSSL
-	int errcode;
+	int errcode = ERR_get_error();
 	do
 	{
-		errcode = ERR_get_error();
-
 		char errstr[1024];
 		ERR_error_string_n(errcode, errstr, sizeof(errstr));
 		errstr[1024-1] = '\0';
 
-		if (m_suppressErrors)
+		if (suppressable && m_suppressErrors)
 		{
 			debug("%s: %s", errMsg, errstr);
 		}
@@ -240,6 +258,8 @@ void TlsSocket::ReportError(const char* errMsg)
 		{
 			PrintError(errMsg);
 		}
+
+		errcode = ERR_get_error();
 	} while (errcode);
 #endif /* HAVE_OPENSSL */
 }
@@ -256,7 +276,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_certificate_allocate_credentials(&cred);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -268,7 +288,7 @@ bool TlsSocket::Start()
 			m_certFile, m_keyFile, GNUTLS_X509_FMT_PEM);
 		if (m_retCode != 0)
 		{
-			ReportError("Could not load certificate or key file");
+			ReportError("Could not load certificate or key file", false);
 			Close();
 			return false;
 		}
@@ -278,7 +298,7 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_init(&sess, m_isClient ? GNUTLS_CLIENT : GNUTLS_SERVER);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
@@ -287,21 +307,33 @@ bool TlsSocket::Start()
 
 	m_initialized = true;
 
-	const char* priority = !m_cipher.Empty() ? m_cipher.Str() : "NORMAL";
+	const char* priority = !m_cipher.Empty() ? m_cipher.Str() :
+		(m_certFile && m_keyFile ? "NORMAL:!VERS-SSL3.0" : "NORMAL");
 
 	m_retCode = gnutls_priority_set_direct((gnutls_session_t)m_session, priority, nullptr);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not select cipher for TLS session");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
 
+	if (m_host)
+	{
+		m_retCode = gnutls_server_name_set((gnutls_session_t)m_session, GNUTLS_NAME_DNS, m_host, m_host.Length());
+		if (m_retCode != 0)
+		{
+			ReportError("Could not set hostname for TLS");
+			Close();
+			return false;
+		}
+	}
+
 	m_retCode = gnutls_credentials_set((gnutls_session_t)m_session, GNUTLS_CRD_CERTIFICATE,
 		(gnutls_certificate_credentials_t*)m_context);
 	if (m_retCode != 0)
 	{
-		ReportError("Could not initialize TLS session");
+		ReportError("Could not initialize TLS session", false);
 		Close();
 		return false;
 	}
@@ -311,7 +343,13 @@ bool TlsSocket::Start()
 	m_retCode = gnutls_handshake((gnutls_session_t)m_session);
 	if (m_retCode != 0)
 	{
-		ReportError("TLS handshake failed");
+		ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -325,7 +363,7 @@ bool TlsSocket::Start()
 
 	if (!m_context)
 	{
-		ReportError("Could not create TLS context");
+		ReportError("Could not create TLS context", false);
 		return false;
 	}
 
@@ -333,34 +371,77 @@ bool TlsSocket::Start()
 	{
 		if (SSL_CTX_use_certificate_chain_file((SSL_CTX*)m_context, m_certFile) != 1)
 		{
-			ReportError("Could not load certificate file");
+			ReportError("Could not load certificate file", false);
 			Close();
 			return false;
 		}
 		if (SSL_CTX_use_PrivateKey_file((SSL_CTX*)m_context, m_keyFile, SSL_FILETYPE_PEM) != 1)
 		{
-			ReportError("Could not load key file");
+			ReportError("Could not load key file", false);
 			Close();
 			return false;
 		}
+		if (!SSL_CTX_set_options((SSL_CTX*)m_context, SSL_OP_NO_SSLv3))
+		{
+			ReportError("Could not select minimum protocol version for TLS", false);
+			Close();
+			return false;
+		}
+
+		// For ECC certificates
+		EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
+		if (!ecdh)
+		{
+			ReportError("Could not generate ecdh parameters for TLS", false);
+			Close();
+			return false;
+		}
+		if (!SSL_CTX_set_tmp_ecdh((SSL_CTX*)m_context, ecdh))
+		{
+			ReportError("Could not set ecdh parameters for TLS", false);
+			EC_KEY_free(ecdh);
+			Close();
+			return false;
+		}
+		EC_KEY_free(ecdh);
+	}
+
+	if (m_isClient && !m_certStore.Empty())
+	{
+		// Enable certificate validation
+		if (SSL_CTX_load_verify_locations((SSL_CTX*)m_context, m_certStore, nullptr) != 1)
+		{
+			ReportError("Could not set certificate store location", false);
+			Close();
+			return false;
+		}
+
+		SSL_CTX_set_verify((SSL_CTX*)m_context, SSL_VERIFY_PEER, nullptr);
 	}
 
 	m_session = SSL_new((SSL_CTX*)m_context);
 	if (!m_session)
 	{
-		ReportError("Could not create TLS session");
+		ReportError("Could not create TLS session", false);
 		Close();
 		return false;
 	}
 
 	if (!m_cipher.Empty() && !SSL_set_cipher_list((SSL*)m_session, m_cipher))
 	{
-		ReportError("Could not select cipher for TLS");
+		ReportError("Could not select cipher for TLS", false);
 		Close();
 		return false;
 	}
 
-	if (!SSL_set_fd((SSL*)m_session, m_socket))
+	if (m_isClient && m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host))
+	{
+		ReportError("Could not set host name for TLS");
+		Close();
+		return false;
+	}
+
+	if (!SSL_set_fd((SSL*)m_session, (int)m_socket))
 	{
 		ReportError("Could not set the file descriptor for TLS");
 		Close();
@@ -370,7 +451,23 @@ bool TlsSocket::Start()
 	int error_code = m_isClient ? SSL_connect((SSL*)m_session) : SSL_accept((SSL*)m_session);
 	if (error_code < 1)
 	{
-		ReportError("TLS handshake failed");
+		long verifyRes = SSL_get_verify_result((SSL*)m_session);
+		if (verifyRes != X509_V_OK)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification",
+				*m_host, X509_verify_cert_error_string(verifyRes)));
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS handshake failed for %s", *m_host));
+		}
+		Close();
+		return false;
+	}
+
+	if (m_isClient && !m_certStore.Empty() && !ValidateCert())
+	{
 		Close();
 		return false;
 	}
@@ -380,6 +477,128 @@ bool TlsSocket::Start()
 #endif /* HAVE_OPENSSL */
 }
 
+bool TlsSocket::ValidateCert()
+{
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER >= 0x030104
+#if	GNUTLS_VERSION_NUMBER >= 0x030306
+	if (FileSystem::DirectoryExists(m_certStore))
+	{
+		if (gnutls_certificate_set_x509_trust_dir((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+	else
+#endif
+	{
+		if (gnutls_certificate_set_x509_trust_file((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0)
+		{
+			ReportError("Could not set certificate store location");
+			return false;
+		}
+	}
+
+	unsigned int status = 0;
+	if (gnutls_certificate_verify_peers3((gnutls_session_t)m_session, m_host, &status) != 0 ||
+		gnutls_certificate_type_get((gnutls_session_t)m_session) != GNUTLS_CRT_X509)
+	{
+		ReportError("Could not verify TLS certificate");
+		return false;
+	}
+
+	if (status != 0)
+	{
+		if (status & GNUTLS_CERT_UNEXPECTED_OWNER)
+		{
+			// Extracting hostname from the certificate
+			unsigned int cert_list_size = 0;
+			const gnutls_datum_t* cert_list = gnutls_certificate_get_peers((gnutls_session_t)m_session, &cert_list_size);
+			if (cert_list_size > 0)
+			{
+				gnutls_x509_crt_t cert;
+				gnutls_x509_crt_init(&cert);
+				gnutls_x509_crt_import(cert, &cert_list[0], GNUTLS_X509_FMT_DER);
+				char dn[256];
+				size_t size = sizeof(dn);
+				if (gnutls_x509_crt_get_dn_by_oid(cert, GNUTLS_OID_X520_COMMON_NAME, 0, 0, dn, &size) == 0)
+				{
+					PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+						" For more info visit http://nzbget.net/certificate-verification", *m_host, dn));
+					gnutls_x509_crt_deinit(cert);
+					return false;
+				}
+				gnutls_x509_crt_deinit(cert);
+			}
+		}
+
+		gnutls_datum_t msgdata;
+		if (gnutls_certificate_verification_status_print(status, GNUTLS_CRT_X509, &msgdata, 0) == 0)
+		{
+			PrintError(BString<1024>("TLS certificate verification failed for %s: %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host, msgdata.data));
+			gnutls_free(&msgdata);
+		}
+		else
+		{
+			ReportError(BString<1024>("TLS certificate verification failed for %s."
+				" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		}
+		return false;
+	}
+#endif
+	return true;
+#endif /* HAVE_LIBGNUTLS */
+
+#ifdef HAVE_OPENSSL
+	// verify a server certificate was presented during the negotiation
+	X509* cert = SSL_get_peer_certificate((SSL*)m_session);
+	if (!cert)
+	{
+		PrintError(BString<1024>("TLS certificate verification failed for %s: no certificate provided by server."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host));
+		return false;
+	}
+
+#ifdef HAVE_X509_CHECK_HOST
+	// hostname verification
+	if (!m_host.Empty() && X509_check_host(cert, m_host, m_host.Length(), 0, nullptr) != 1)
+	{
+		const unsigned char* certHost = nullptr;
+        // Find the position of the CN field in the Subject field of the certificate
+        int common_name_loc = X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1);
+        if (common_name_loc >= 0)
+		{
+			// Extract the CN field
+			X509_NAME_ENTRY* common_name_entry = X509_NAME_get_entry(X509_get_subject_name(cert), common_name_loc);
+			if (common_name_entry != nullptr)
+			{
+				// Convert the CN field to a C string
+				ASN1_STRING* common_name_asn1 = X509_NAME_ENTRY_get_data(common_name_entry);
+				if (common_name_asn1 != nullptr)
+				{
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+					certHost = ASN1_STRING_get0_data(common_name_asn1);
+#else
+					certHost = ASN1_STRING_data(common_name_asn1);
+#endif
+				}
+			}
+        }
+
+		PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)."
+			" For more info visit http://nzbget.net/certificate-verification", *m_host, certHost));
+		X509_free(cert);
+		return false;
+	}
+#endif
+
+	X509_free(cert);
+	return true;
+#endif /* HAVE_OPENSSL */
+}
+
 void TlsSocket::Close()
 {
 	if (m_session)

daemon/connect/TlsSocket.h

@@ -27,12 +27,14 @@
 class TlsSocket
 {
 public:
-	TlsSocket(SOCKET socket, bool isClient, const char* certFile,
-		const char* keyFile, const char* cipher) :
-		m_socket(socket), m_isClient(isClient), m_certFile(certFile),
-		m_keyFile(keyFile), m_cipher(cipher) {}
+	TlsSocket(SOCKET socket, bool isClient, const char* host,
+		const char* certFile, const char* keyFile, const char* cipher) :
+		m_socket(socket), m_isClient(isClient), m_host(host),
+		m_certFile(certFile), m_keyFile(keyFile), m_cipher(cipher) {}
 	virtual ~TlsSocket();
 	static void Init();
+	static void InitOptions(const char* certStore) { m_certStore = certStore; }
+	static void Final();
 	bool Start();
 	void Close();
 	int Send(const char* buffer, int size);
@@ -41,26 +43,26 @@ public:
 
 protected:
 	virtual void PrintError(const char* errMsg);
-	
+
 private:
+	SOCKET m_socket;
 	bool m_isClient;
+	CString m_host;
 	CString m_certFile;
 	CString m_keyFile;
 	CString m_cipher;
-	SOCKET m_socket;
 	bool m_suppressErrors = false;
 	bool m_initialized = false;
 	bool m_connected = false;
 	int m_retCode;
+	static CString m_certStore;
 
 	// using "void*" to prevent the including of GnuTLS/OpenSSL header files into TlsSocket.h
 	void* m_context = nullptr;
 	void* m_session = nullptr;
 
-	void ReportError(const char* errMsg);
-
-	static void Final();
-	friend class TlsSocketFinalizer;
+	void ReportError(const char* errMsg, bool suppressable = true);
+	bool ValidateCert();
 };
 
 #endif

daemon/connect/WebDownloader.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "WebDownloader.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Util.h"
 #include "FileSystem.h"
 
@@ -54,7 +55,7 @@ void WebDownloader::Run()
 
 	SetStatus(adRunning);
 
-	int remainedDownloadRetries = g_Options->GetRetries() > 0 ? g_Options->GetRetries() : 1;
+	int remainedDownloadRetries = g_Options->GetUrlRetries() > 0 ? g_Options->GetUrlRetries() : 1;
 	int remainedConnectRetries = remainedDownloadRetries > 10 ? remainedDownloadRetries : 10;
 	if (!m_retry)
 	{
@@ -72,19 +73,19 @@ void WebDownloader::Run()
 
 		if ((((Status == adFailed) && (remainedDownloadRetries > 1)) ||
 			((Status == adConnectError) && (remainedConnectRetries > 1)))
-			&& !IsStopped() && !(!m_force && g_Options->GetPauseDownload()))
+			&& !IsStopped() && !(!m_force && g_WorkState->GetPauseDownload()))
 		{
-			detail("Waiting %i sec to retry", g_Options->GetRetryInterval());
+			detail("Waiting %i sec to retry", g_Options->GetUrlInterval());
 			int msec = 0;
-			while (!IsStopped() && (msec < g_Options->GetRetryInterval() * 1000) &&
-				!(!m_force && g_Options->GetPauseDownload()))
+			while (!IsStopped() && (msec < g_Options->GetUrlInterval() * 1000) &&
+				!(!m_force && g_WorkState->GetPauseDownload()))
 			{
-				usleep(100 * 1000);
+				Util::Sleep(100);
 				msec += 100;
 			}
 		}
 
-		if (IsStopped() || (!m_force && g_Options->GetPauseDownload()))
+		if (IsStopped() || (!m_force && g_WorkState->GetPauseDownload()))
 		{
 			Status = adRetry;
 			break;
@@ -441,7 +442,9 @@ WebDownloader::EStatus WebDownloader::CheckResponse(const char* response)
 		warn("URL %s failed: %s", *m_infoName, hTTPResponse);
 		return adNotFound;
 	}
-	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3))
+	else if (!strncmp(hTTPResponse, "301", 3) || !strncmp(hTTPResponse, "302", 3) ||
+		 !strncmp(hTTPResponse, "303", 3) || !strncmp(hTTPResponse, "307", 3) ||
+		 !strncmp(hTTPResponse, "308", 3))
 	{
 		m_redirecting = true;
 		return adRunning;
@@ -648,21 +651,6 @@ void WebDownloader::Stop()
 	debug("WebDownloader stopped successfully");
 }
 
-bool WebDownloader::Terminate()
-{
-	std::unique_ptr<Connection> connection = std::move(m_connection);
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		connection.reset();
-	}
-	return terminated;
-}
-
 void WebDownloader::FreeConnection()
 {
 	if (m_connection)

daemon/connect/WebDownloader.h

@@ -50,7 +50,6 @@ public:
 	virtual void Stop();
 	EStatus Download();
 	EStatus DownloadWithRedirects(int maxRedirects);
-	bool Terminate();
 	void SetInfoName(const char* infoName) { m_infoName = infoName; }
 	const char* GetInfoName() { return m_infoName; }
 	void SetUrl(const char* url);

daemon/extension/CommandScript.cpp

@@ -0,0 +1,131 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "CommandScript.h"
+#include "Options.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+
+static const int COMMANDPROCESS_SUCCESS = 93;
+static const int COMMANDPROCESS_ERROR = 94;
+
+bool CommandScriptController::StartScript(const char* scriptName, const char* command,
+	std::unique_ptr<Options::OptEntries> modifiedOptions)
+{
+	CommandScriptController* scriptController = new CommandScriptController();
+	scriptController->m_script = scriptName;
+	scriptController->m_command = command;
+	scriptController->m_logId = g_CommandScriptLog->Reset();
+	scriptController->m_modifiedOptions = std::move(modifiedOptions);
+
+	scriptController->SetAutoDestroy(true);
+
+	scriptController->Start();
+
+	for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
+	{
+		if (FileSystem::SameFilename(scriptName, script.GetName()))
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+void CommandScriptController::Run()
+{
+	ExecuteScriptList(m_script);
+}
+
+void CommandScriptController::ExecuteScript(ScriptConfig::Script* script)
+{
+	PrintMessage(Message::mkInfo, "Executing script %s with command %s", script->GetName(), *m_command);
+
+	SetArgs({script->GetLocation()});
+
+	BString<1024> infoName("script %s with command %s", script->GetName(), *m_command);
+	SetInfoName(infoName);
+
+	SetLogPrefix(script->GetDisplayName());
+	PrepareParams(script->GetName());
+
+	int exitCode = Execute();
+
+	infoName[0] = 'S'; // uppercase
+	SetLogPrefix(nullptr);
+
+	switch (exitCode)
+	{
+		case COMMANDPROCESS_SUCCESS:
+			PrintMessage(Message::mkInfo, "%s successful", *infoName);
+			break;
+
+		case COMMANDPROCESS_ERROR:
+		case -1: // Execute() returns -1 if the process could not be started (file not found or other problem)
+			PrintMessage(Message::mkError, "%s failed", *infoName);
+			break;
+
+		default:
+			PrintMessage(Message::mkError, "%s failed (terminated with unknown status)", *infoName);
+			break;
+	}
+}
+
+void CommandScriptController::PrepareParams(const char* scriptName)
+{
+	ResetEnv();
+
+	SetEnvVar("NZBCP_COMMAND", m_command);
+
+	PrepareEnvScript(nullptr, scriptName);
+}
+
+const char* CommandScriptController::GetOptValue(const char* name, const char* value)
+{
+	Options::OptEntry* entry = m_modifiedOptions->FindOption(name);
+	return entry ? entry->GetValue() : value;
+}
+
+void CommandScriptController::AddMessage(Message::EKind kind, const char * text)
+{
+	NzbScriptController::AddMessage(kind, text);
+	g_CommandScriptLog->AddMessage(m_logId, kind, text);
+}
+
+
+int CommandScriptLog::Reset()
+{
+	Guard guard(m_logMutex);
+	m_messages.clear();
+	return ++m_idScriptGen;
+}
+
+void CommandScriptLog::AddMessage(int scriptId, Message::EKind kind, const char * text)
+{
+	Guard guard(m_logMutex);
+
+	// save only messages from the last started script
+	if (scriptId == m_idScriptGen)
+	{
+		m_messages.emplace_back(++m_idMessageGen, kind, Util::CurrentTime(), text);
+	}
+}

daemon/extension/CommandScript.h

@@ -0,0 +1,63 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef COMMANDSCRIPT_H
+#define COMMANDSCRIPT_H
+
+#include "NzbScript.h"
+#include "Log.h"
+
+class CommandScriptController : public Thread, public NzbScriptController
+{
+public:
+	virtual void Run();
+	static bool StartScript(const char* scriptName, const char* command, std::unique_ptr<Options::OptEntries> modifiedOptions);
+
+protected:
+	virtual void ExecuteScript(ScriptConfig::Script* script);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+	virtual const char* GetOptValue(const char* name, const char* value);
+
+private:
+	CString m_script;
+	CString m_command;
+	int m_logId;
+	std::unique_ptr<Options::OptEntries> m_modifiedOptions;
+
+	void PrepareParams(const char* scriptName);
+};
+
+class CommandScriptLog
+{
+public:
+	GuardedMessageList GuardMessages() { return GuardedMessageList(&m_messages, &m_logMutex); }
+	int Reset();
+	void AddMessage(int scriptId, Message::EKind kind, const char* text);
+
+private:
+	MessageList m_messages;
+	Mutex m_logMutex;
+	int m_idMessageGen;
+	int m_idScriptGen;
+};
+
+extern CommandScriptLog* g_CommandScriptLog;
+
+#endif

daemon/extension/FeedScript.cpp

@@ -24,13 +24,20 @@
 #include "Log.h"
 #include "Util.h"
 
-void FeedScriptController::ExecuteScripts(const char* feedScript, const char* feedFile, int feedId)
+static const int FEED_SUCCESS = 93;
+
+void FeedScriptController::ExecuteScripts(const char* feedScript, const char* feedFile, int feedId, bool* success)
 {
 	FeedScriptController scriptController;
 	scriptController.m_feedFile = feedFile;
 	scriptController.m_feedId = feedId;
 
 	scriptController.ExecuteScriptList(feedScript);
+
+	if (success)
+	{
+		*success = scriptController.m_success;
+	}
 }
 
 void FeedScriptController::ExecuteScript(ScriptConfig::Script* script)
@@ -42,13 +49,22 @@ void FeedScriptController::ExecuteScript(ScriptConfig::Script* script)
 
 	PrintMessage(Message::mkInfo, "Executing feed-script %s for Feed%i", script->GetName(), m_feedId);
 
-	SetScript(script->GetLocation());
+	SetArgs({script->GetLocation()});
+
+	BString<1024> infoName("feed-script %s for Feed%i", script->GetName(), m_feedId);
+	SetInfoName(infoName);
 
-	SetInfoName(BString<1024>("feed-script %s for Feed%i", script->GetName(), m_feedId));
 	SetLogPrefix(script->GetDisplayName());
 	PrepareParams(script->GetName());
 
-	Execute();
+	int exitCode = Execute();
+
+	if (exitCode != FEED_SUCCESS)
+	{
+		infoName[0] = 'F'; // uppercase
+		PrintMessage(Message::mkError, "%s failed", *infoName);
+		m_success = false;
+	}
 
 	SetLogPrefix(nullptr);
 }

daemon/extension/FeedScript.h

@@ -26,7 +26,7 @@
 class FeedScriptController : public NzbScriptController
 {
 public:
-	static void ExecuteScripts(const char* feedScript, const char* feedFile, int feedId);
+	static void ExecuteScripts(const char* feedScript, const char* feedFile, int feedId, bool* success);
 
 protected:
 	virtual void ExecuteScript(ScriptConfig::Script* script);
@@ -34,6 +34,7 @@ protected:
 private:
 	const char* m_feedFile;
 	int m_feedId;
+	bool m_success = true;
 
 	void PrepareParams(const char* scriptName);
 };

daemon/extension/NzbScript.cpp

@@ -77,7 +77,6 @@ void NzbScriptController::ExecuteScriptList(const char* scriptList)
 			{
 				if (FileSystem::SameFilename(scriptName, script.GetName()))
 				{
-					Reset();
 					ExecuteScript(&script);
 					break;
 				}

daemon/extension/PostScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@
 #include "Log.h"
 #include "Util.h"
 #include "Options.h"
+#include "WorkState.h"
 
 static const int POSTPROCESS_PARCHECK = 92;
 static const int POSTPROCESS_SUCCESS = 93;
@@ -85,7 +86,7 @@ void PostScriptController::ExecuteScript(ScriptConfig::Script* script)
 		m_postInfo->SetProgressLabel(progressLabel);
 	}
 
-	SetScript(script->GetLocation());
+	SetArgs({script->GetLocation()});
 
 	BString<1024> infoName("post-process-script %s for %s", script->GetName(), m_postInfo->GetNzbInfo()->GetName());
 	SetInfoName(infoName);
@@ -100,7 +101,7 @@ void PostScriptController::ExecuteScript(ScriptConfig::Script* script)
 	infoName[0] = 'P'; // uppercase
 
 	SetLogPrefix(nullptr);
-	ScriptStatus::EStatus status = AnalyseExitCode(exitCode);
+	ScriptStatus::EStatus status = AnalyseExitCode(exitCode, infoName);
 
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
@@ -118,6 +119,7 @@ void PostScriptController::PrepareParams(const char* scriptName)
 	SetEnvVar("NZBPP_NZBNAME", m_postInfo->GetNzbInfo()->GetName());
 	SetEnvVar("NZBPP_DIRECTORY", m_postInfo->GetNzbInfo()->GetDestDir());
 	SetEnvVar("NZBPP_NZBFILENAME", m_postInfo->GetNzbInfo()->GetFilename());
+	SetEnvVar("NZBPP_QUEUEDFILE", m_postInfo->GetNzbInfo()->GetQueuedFilename());
 	SetEnvVar("NZBPP_URL", m_postInfo->GetNzbInfo()->GetUrl());
 	SetEnvVar("NZBPP_FINALDIR", m_postInfo->GetNzbInfo()->GetFinalDir());
 	SetEnvVar("NZBPP_CATEGORY", m_postInfo->GetNzbInfo()->GetCategory());
@@ -175,7 +177,7 @@ void PostScriptController::PrepareParams(const char* scriptName)
 	PrepareEnvScript(m_postInfo->GetNzbInfo()->GetParameters(), scriptName);
 }
 
-ScriptStatus::EStatus PostScriptController::AnalyseExitCode(int exitCode)
+ScriptStatus::EStatus PostScriptController::AnalyseExitCode(int exitCode, const char* upInfoName)
 {
 	// The ScriptStatus is accumulated for all scripts:
 	// If any script has failed the status is "failure", etc.
@@ -183,28 +185,28 @@ ScriptStatus::EStatus PostScriptController::AnalyseExitCode(int exitCode)
 	switch (exitCode)
 	{
 		case POSTPROCESS_SUCCESS:
-			PrintMessage(Message::mkInfo, "%s successful", GetInfoName());
+			PrintMessage(Message::mkInfo, "%s successful", upInfoName);
 			return ScriptStatus::srSuccess;
 
 		case POSTPROCESS_ERROR:
 		case -1: // Execute() returns -1 if the process could not be started (file not found or other problem)
-			PrintMessage(Message::mkError, "%s failed", GetInfoName());
+			PrintMessage(Message::mkError, "%s failed", upInfoName);
 			return ScriptStatus::srFailure;
 
 		case POSTPROCESS_NONE:
-			PrintMessage(Message::mkInfo, "%s skipped", GetInfoName());
+			PrintMessage(Message::mkInfo, "%s skipped", upInfoName);
 			return ScriptStatus::srNone;
 
 #ifndef DISABLE_PARCHECK
 		case POSTPROCESS_PARCHECK:
 			if (m_postInfo->GetNzbInfo()->GetParStatus() > NzbInfo::psSkipped)
 			{
-				PrintMessage(Message::mkError, "%s requested par-check/repair, but the collection was already checked", GetInfoName());
+				PrintMessage(Message::mkError, "%s requested par-check/repair, but the collection was already checked", upInfoName);
 				return ScriptStatus::srFailure;
 			}
 			else
 			{
-				PrintMessage(Message::mkInfo, "%s requested par-check/repair", GetInfoName());
+				PrintMessage(Message::mkInfo, "%s requested par-check/repair", upInfoName);
 				m_postInfo->SetRequestParCheck(true);
 				m_postInfo->SetForceRepair(true);
 				return ScriptStatus::srSuccess;
@@ -213,7 +215,7 @@ ScriptStatus::EStatus PostScriptController::AnalyseExitCode(int exitCode)
 #endif
 
 		default:
-			PrintMessage(Message::mkError, "%s failed (terminated with unknown status)", GetInfoName());
+			PrintMessage(Message::mkError, "%s failed (terminated with unknown status)", upInfoName);
 			return ScriptStatus::srFailure;
 	}
 }
@@ -271,16 +273,16 @@ void PostScriptController::AddMessage(Message::EKind kind, const char* text)
 		m_postInfo->SetProgressLabel(text);
 	}
 
-	if (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = m_postInfo->GetStageTime();
 		time_t startTime = m_postInfo->GetStartTime();
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
+		while (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 
 			// update time stamps
 

daemon/extension/PostScript.h

@@ -41,7 +41,7 @@ private:
 	ScriptConfig::Script* m_script;
 
 	void PrepareParams(const char* scriptName);
-	ScriptStatus::EStatus AnalyseExitCode(int exitCode);
+	ScriptStatus::EStatus AnalyseExitCode(int exitCode, const char* upInfoName);
 };
 
 #endif

daemon/extension/QueueScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,7 +27,14 @@
 #include "Util.h"
 #include "FileSystem.h"
 
-static const char* QUEUE_EVENT_NAMES[] = { "FILE_DOWNLOADED", "URL_COMPLETED", "NZB_ADDED", "NZB_DOWNLOADED", "NZB_DELETED" };
+static const char* QUEUE_EVENT_NAMES[] = {
+	"FILE_DOWNLOADED",
+	"URL_COMPLETED",
+	"NZB_MARKED",
+	"NZB_ADDED",
+	"NZB_NAMED",
+	"NZB_DOWNLOADED",
+	"NZB_DELETED" };
 
 class QueueScriptController : public Thread, public NzbScriptController
 {
@@ -45,6 +52,7 @@ private:
 	CString m_url;
 	CString m_category;
 	CString m_destDir;
+	CString m_queuedFilename;
 	int m_id;
 	int m_priority;
 	CString m_dupeKey;
@@ -57,6 +65,7 @@ private:
 	bool m_markBad;
 	NzbInfo::EDeleteStatus m_deleteStatus;
 	NzbInfo::EUrlStatus m_urlStatus;
+	NzbInfo::EMarkStatus m_markStatus;
 
 	void PrepareParams(const char* scriptName);
 };
@@ -71,6 +80,7 @@ void QueueScriptController::StartScript(NzbInfo* nzbInfo, ScriptConfig::Script*
 	scriptController->m_url = nzbInfo->GetUrl();
 	scriptController->m_category = nzbInfo->GetCategory();
 	scriptController->m_destDir = nzbInfo->GetDestDir();
+	scriptController->m_queuedFilename = nzbInfo->GetQueuedFilename();
 	scriptController->m_id = nzbInfo->GetId();
 	scriptController->m_priority = nzbInfo->GetPriority();
 	scriptController->m_dupeKey = nzbInfo->GetDupeKey();
@@ -83,6 +93,7 @@ void QueueScriptController::StartScript(NzbInfo* nzbInfo, ScriptConfig::Script*
 	scriptController->m_markBad = false;
 	scriptController->m_deleteStatus = nzbInfo->GetDeleteStatus();
 	scriptController->m_urlStatus = nzbInfo->GetUrlStatus();
+	scriptController->m_markStatus = nzbInfo->GetMarkStatus();
 	scriptController->SetAutoDestroy(true);
 
 	scriptController->Start();
@@ -100,9 +111,9 @@ void QueueScriptController::Run()
 		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_id);
 		if (nzbInfo)
 		{
-			PrintMessage(Message::mkWarning, "Cancelling download and deleting %s", *m_nzbName);
+			nzbInfo->PrintMessage(Message::mkWarning, "Cancelling download and deleting %s", *m_nzbName);
 			nzbInfo->SetDeleteStatus(NzbInfo::dsBad);
-			downloadQueue->EditEntry(m_id, DownloadQueue::eaGroupDelete, 0, nullptr);
+			downloadQueue->EditEntry(m_id, DownloadQueue::eaGroupDelete, nullptr);
 		}
 	}
 
@@ -114,7 +125,7 @@ void QueueScriptController::ExecuteScript(ScriptConfig::Script* script)
 	PrintMessage(m_event == QueueScriptCoordinator::qeFileDownloaded ? Message::mkDetail : Message::mkInfo,
 		"Executing queue-script %s for %s", script->GetName(), FileSystem::BaseFileName(m_nzbName));
 
-	SetScript(script->GetLocation());
+	SetArgs({script->GetLocation()});
 
 	BString<1024> infoName("queue-script %s for %s", script->GetName(), FileSystem::BaseFileName(m_nzbName));
 	SetInfoName(infoName);
@@ -136,6 +147,7 @@ void QueueScriptController::PrepareParams(const char* scriptName)
 	SetIntEnvVar("NZBNA_NZBID", m_id);
 	SetEnvVar("NZBNA_FILENAME", m_nzbFilename);
 	SetEnvVar("NZBNA_DIRECTORY", m_destDir);
+	SetEnvVar("NZBNA_QUEUEDFILE", m_queuedFilename);
 	SetEnvVar("NZBNA_URL", m_url);
 	SetEnvVar("NZBNA_CATEGORY", m_category);
 	SetIntEnvVar("NZBNA_PRIORITY", m_priority);
@@ -155,6 +167,9 @@ void QueueScriptController::PrepareParams(const char* scriptName)
 	const char* urlStatusName[] = { "NONE", "UNKNOWN", "SUCCESS", "FAILURE", "UNKNOWN", "SCAN_SKIPPED", "SCAN_FAILURE" };
 	SetEnvVar("NZBNA_URLSTATUS", urlStatusName[m_urlStatus]);
 
+	const char* markStatusName[] = { "NONE", "BAD", "GOOD", "SUCCESS" };
+	SetEnvVar("NZBNA_MARKSTATUS", markStatusName[m_markStatus]);
+
 	PrepareEnvScript(&m_parameters, scriptName);
 }
 
@@ -201,9 +216,7 @@ void QueueScriptController::AddMessage(Message::EKind kind, const char* text)
 			NzbInfo* nzbInfo = QueueScriptCoordinator::FindNzbInfo(downloadQueue, m_id);
 			if (nzbInfo)
 			{
-				SetLogPrefix(nullptr);
-				PrintMessage(Message::mkWarning, "Marking %s as bad", *m_nzbName);
-				SetLogPrefix(m_script->GetDisplayName());
+				nzbInfo->PrintMessage(Message::mkWarning, "Marking %s as bad", *m_nzbName);
 				nzbInfo->SetMarkStatus(NzbInfo::ksBad);
 			}
 		}
@@ -324,21 +337,7 @@ bool QueueScriptCoordinator::UsableScript(ScriptConfig::Script& script, NzbInfo*
 		return false;
 	}
 
-	// check queue-scripts
-	const char* queueScript = g_Options->GetQueueScript();
-	if (!Util::EmptyStr(queueScript))
-	{
-		Tokenizer tok(queueScript, ",;");
-		while (const char* scriptName = tok.Next())
-		{
-			if (FileSystem::SameFilename(scriptName, script.GetName()))
-			{
-				return true;
-			}
-		}
-	}
-
-	// check post-processing-scripts assigned for that nzb
+	// check extension scripts assigned for that nzb
 	for (NzbParameter& parameter : nzbInfo->GetParameters())
 	{
 		const char* varname = parameter.GetName();
@@ -356,17 +355,17 @@ bool QueueScriptCoordinator::UsableScript(ScriptConfig::Script& script, NzbInfo*
 		}
 	}
 
-	// for URL-events the post-processing scripts are not assigned yet;
-	// instead we take the default post-processing scripts for the category (or global)
+	// for URL-events the extension scripts are not assigned yet;
+	// instead we take the default extension scripts for the category (or global)
 	if (event == qeUrlCompleted)
 	{
-		const char* postScript = g_Options->GetPostScript();
+		const char* postScript = g_Options->GetExtensions();
 		if (!Util::EmptyStr(nzbInfo->GetCategory()))
 		{
 			Options::Category* categoryObj = g_Options->FindCategory(nzbInfo->GetCategory(), false);
-			if (categoryObj && !Util::EmptyStr(categoryObj->GetPostScript()))
+			if (categoryObj && !Util::EmptyStr(categoryObj->GetExtensions()))
 			{
-				postScript = categoryObj->GetPostScript();
+				postScript = categoryObj->GetExtensions();
 			}
 		}
 
@@ -410,13 +409,12 @@ void QueueScriptCoordinator::CheckQueue()
 		return;
 	}
 
-	m_curItem.reset();
-
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 	Guard guard(m_queueMutex);
 
+	m_curItem.reset();
 	NzbInfo* curNzbInfo = nullptr;
-	Queue::iterator itCurItem = m_queue.end();
+	Queue::iterator itCurItem;
 
 	for (Queue::iterator it = m_queue.begin(); it != m_queue.end(); )
 	{
@@ -425,11 +423,20 @@ void QueueScriptCoordinator::CheckQueue()
 		NzbInfo* nzbInfo = FindNzbInfo(downloadQueue, queueItem->GetNzbId());
 
 		// in a case this nzb must not be processed further - delete queue script from queue
-		if (!nzbInfo ||
-			(nzbInfo->GetDeleteStatus() != NzbInfo::dsNone && queueItem->GetEvent() != qeNzbDeleted) ||
-			nzbInfo->GetMarkStatus() == NzbInfo::ksBad)
+		EEvent event = queueItem->GetEvent();
+		bool ignoreEvent = !nzbInfo ||
+			(nzbInfo->GetDeleteStatus() != NzbInfo::dsNone && event != qeNzbDeleted && event != qeNzbMarked) ||
+			(nzbInfo->GetMarkStatus() == NzbInfo::ksBad && event != qeNzbMarked);
+
+		if (ignoreEvent)
 		{
 			it = m_queue.erase(it);
+			if (curNzbInfo)
+			{
+				// process from the beginning, while "erase" invalidated "itCurItem"
+				curNzbInfo = nullptr;
+				it = m_queue.begin();
+			}
 			continue;
 		}
 
@@ -442,7 +449,7 @@ void QueueScriptCoordinator::CheckQueue()
 		it++;
 	}
 
-	if (itCurItem != m_queue.end())
+	if (curNzbInfo)
 	{
 		m_curItem = std::move(*itCurItem);
 		m_queue.erase(itCurItem);

daemon/extension/QueueScript.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -31,7 +31,9 @@ public:
 	{
 		qeFileDownloaded, // lowest priority
 		qeUrlCompleted,
+		qeNzbMarked,
 		qeNzbAdded,
+		qeNzbNamed,
 		qeNzbDownloaded,
 		qeNzbDeleted // highest priority
 	};

daemon/extension/ScanScript.cpp

@@ -25,6 +25,22 @@
 #include "Log.h"
 #include "FileSystem.h"
 
+class ScanScriptCheck : public NzbScriptController
+{
+protected:
+	virtual void ExecuteScript(ScriptConfig::Script* script) { has |= script->GetScanScript(); }
+	bool has = false;
+	friend class ScanScriptController;
+};
+
+
+bool ScanScriptController::HasScripts()
+{
+	ScanScriptCheck check;
+	check.ExecuteScriptList(g_Options->GetExtensions());
+	return check.has;
+}
+
 void ScanScriptController::ExecuteScripts(const char* nzbFilename,
 	const char* url, const char* directory, CString* nzbName, CString* category,
 	int* priority, NzbParameterList* parameters, bool* addTop, bool* addPaused,
@@ -45,7 +61,18 @@ void ScanScriptController::ExecuteScripts(const char* nzbFilename,
 	scriptController.m_dupeMode = dupeMode;
 	scriptController.m_prefixLen = 0;
 
-	scriptController.ExecuteScriptList(g_Options->GetScanScript());
+	const char* extensions = g_Options->GetExtensions();
+
+	if (!Util::EmptyStr(*category))
+	{
+		Options::Category* categoryObj = g_Options->FindCategory(*category, false);
+		if (categoryObj && !Util::EmptyStr(categoryObj->GetExtensions()))
+		{
+			extensions = categoryObj->GetExtensions();
+		}
+	}
+
+	scriptController.ExecuteScriptList(extensions);
 }
 
 void ScanScriptController::ExecuteScript(ScriptConfig::Script* script)
@@ -57,7 +84,7 @@ void ScanScriptController::ExecuteScript(ScriptConfig::Script* script)
 
 	PrintMessage(Message::mkInfo, "Executing scan-script %s for %s", script->GetName(), FileSystem::BaseFileName(m_nzbFilename));
 
-	SetScript(script->GetLocation());
+	SetArgs({script->GetLocation()});
 
 	BString<1024> infoName("scan-script %s for %s", script->GetName(), FileSystem::BaseFileName(m_nzbFilename));
 	SetInfoName(infoName);

daemon/extension/ScanScript.h

@@ -30,6 +30,7 @@ public:
 		const char* directory, CString* nzbName, CString* category, int* priority,
 		NzbParameterList* parameters, bool* addTop, bool* addPaused,
 		CString* dupeKey, int* dupeScore, EDupeMode* dupeMode);
+	static bool HasScripts();
 
 protected:
 	virtual void ExecuteScript(ScriptConfig::Script* script);

daemon/extension/SchedulerScript.cpp

@@ -42,7 +42,6 @@ void SchedulerScriptController::StartScript(const char* param, bool externalProc
 
 	if (externalProcess)
 	{
-		scriptController->SetScript(argv[0]);
 		scriptController->SetArgs(std::move(argv));
 	}
 
@@ -70,11 +69,17 @@ void SchedulerScriptController::ExecuteScript(ScriptConfig::Script* script)
 		return;
 	}
 
-	PrintMessage(Message::mkInfo, "Executing scheduler-script %s for Task%i", script->GetName(), m_taskId);
+	BString<1024> taskName(" for Task%i", m_taskId);
+	if (m_taskId == 0)
+	{
+		taskName = "";
+	}
 
-	SetScript(script->GetLocation());
+	PrintMessage(Message::mkInfo, "Executing scheduler-script %s%s", script->GetName(), *taskName);
 
-	BString<1024> infoName("scheduler-script %s for Task%i", script->GetName(), m_taskId);
+	SetArgs({script->GetLocation()});
+
+	BString<1024> infoName("scheduler-script %s%s", script->GetName(), *taskName);
 	SetInfoName(infoName);
 
 	SetLogPrefix(script->GetDisplayName());

daemon/extension/ScriptConfig.cpp

@@ -33,25 +33,14 @@ static const char* SCHEDULER_SCRIPT_SIGNATURE = "SCHEDULER";
 static const char* FEED_SCRIPT_SIGNATURE = "FEED";
 static const char* END_SCRIPT_SIGNATURE = " SCRIPT";
 static const char* QUEUE_EVENTS_SIGNATURE = "### QUEUE EVENTS:";
-
-
-ScriptConfig::Script::Script(const char* name, const char* location)
-{
-	m_name = name;
-	m_location = location;
-	m_displayName = name;
-	m_postScript = false;
-	m_scanScript = false;
-	m_queueScript = false;
-	m_schedulerScript = false;
-	m_feedScript = false;
-}
-
+static const char* TASK_TIME_SIGNATURE = "### TASK TIME:";
+static const char* DEFINITION_SIGNATURE = "###";
 
 void ScriptConfig::InitOptions()
 {
 	InitScripts();
 	InitConfigTemplates();
+	CreateTasks();
 }
 
 bool ScriptConfig::LoadConfig(Options::OptEntries* optEntries)
@@ -81,7 +70,7 @@ bool ScriptConfig::LoadConfig(Options::OptEntries* optEntries)
 
 		CString optname;
 		CString optvalue;
-		if (g_Options->SplitOptionString(buf, optname, optvalue))
+		if (Options::SplitOptionString(buf, optname, optvalue))
 		{
 			optEntries->emplace_back(optname, optvalue);
 		}
@@ -89,6 +78,8 @@ bool ScriptConfig::LoadConfig(Options::OptEntries* optEntries)
 
 	infile.Close();
 
+	Options::ConvertOldOptions(optEntries);
+
 	return true;
 }
 
@@ -180,7 +171,7 @@ bool ScriptConfig::LoadConfigTemplates(ConfigTemplates* configTemplates)
 	LoadScripts(&scriptList);
 
 	const int beginSignatureLen = strlen(BEGIN_SCRIPT_SIGNATURE);
-	const int queueEventsSignatureLen = strlen(QUEUE_EVENTS_SIGNATURE);
+	const int definitionSignatureLen = strlen(DEFINITION_SIGNATURE);
 
 	for (Script& script : scriptList)
 	{
@@ -194,6 +185,7 @@ bool ScriptConfig::LoadConfigTemplates(ConfigTemplates* configTemplates)
 		StringBuilder templ;
 		char buf[1024];
 		bool inConfig = false;
+		bool inHeader = false;
 
 		while (infile.ReadLine(buf, sizeof(buf) - 1))
 		{
@@ -210,12 +202,13 @@ bool ScriptConfig::LoadConfigTemplates(ConfigTemplates* configTemplates)
 					break;
 				}
 				inConfig = true;
+				inHeader = true;
 				continue;
 			}
 
-			bool skip = !strncmp(buf, QUEUE_EVENTS_SIGNATURE, queueEventsSignatureLen);
+			inHeader &= !strncmp(buf, DEFINITION_SIGNATURE, definitionSignatureLen);
 
-			if (inConfig && !skip)
+			if (inConfig && !inHeader)
 			{
 				templ.Append(buf);
 			}
@@ -287,11 +280,6 @@ void ScriptConfig::LoadScripts(Scripts* scripts)
 
 void ScriptConfig::LoadScriptDir(Scripts* scripts, const char* directory, bool isSubDir)
 {
-	CharBuffer buffer(1024*10 + 1);
-
-	const int beginSignatureLen = strlen(BEGIN_SCRIPT_SIGNATURE);
-	const int queueEventsSignatureLen = strlen(QUEUE_EVENTS_SIGNATURE);
-
 	DirBrowser dir(directory);
 	while (const char* filename = dir.Next())
 	{
@@ -307,54 +295,10 @@ void ScriptConfig::LoadScriptDir(Scripts* scripts, const char* directory, bool i
 					continue;
 				}
 
-				// check if the file contains pp-script-signature
-				DiskFile infile;
-				if (infile.Open(fullFilename, DiskFile::omRead))
+				Script script(scriptName, fullFilename);
+				if (LoadScriptFile(&script))
 				{
-					// read first 10KB of the file and look for signature
-					int readBytes = (int)infile.Read(buffer, buffer.Size() - 1);
-					infile.Close();
-					buffer[readBytes] = '\0';
-
-					// split buffer into lines
-					Tokenizer tok(buffer, "\n\r", true);
-					while (char* line = tok.Next())
-					{
-						if (!strncmp(line, BEGIN_SCRIPT_SIGNATURE, beginSignatureLen) &&
-							strstr(line, END_SCRIPT_SIGNATURE))
-						{
-							bool postScript = strstr(line, POST_SCRIPT_SIGNATURE);
-							bool scanScript = strstr(line, SCAN_SCRIPT_SIGNATURE);
-							bool queueScript = strstr(line, QUEUE_SCRIPT_SIGNATURE);
-							bool schedulerScript = strstr(line, SCHEDULER_SCRIPT_SIGNATURE);
-							bool feedScript = strstr(line, FEED_SCRIPT_SIGNATURE);
-							if (postScript || scanScript || queueScript || schedulerScript || feedScript)
-							{
-								char* queueEvents = nullptr;
-								if (queueScript)
-								{
-									while (char* line = tok.Next())
-									{
-										if (!strncmp(line, QUEUE_EVENTS_SIGNATURE, queueEventsSignatureLen))
-										{
-											queueEvents = line + queueEventsSignatureLen;
-											break;
-										}
-									}
-								}
-
-								scripts->emplace_back(scriptName, fullFilename);
-								Script& script = scripts->back();
-								script.SetPostScript(postScript);
-								script.SetScanScript(scanScript);
-								script.SetQueueScript(queueScript);
-								script.SetSchedulerScript(schedulerScript);
-								script.SetFeedScript(feedScript);
-								script.SetQueueEvents(queueEvents);
-								break;
-							}
-						}
-					}
+					scripts->push_back(std::move(script));
 				}
 			}
 			else if (!isSubDir)
@@ -365,6 +309,109 @@ void ScriptConfig::LoadScriptDir(Scripts* scripts, const char* directory, bool i
 	}
 }
 
+bool ScriptConfig::LoadScriptFile(Script* script)
+{
+	DiskFile infile;
+	if (!infile.Open(script->GetLocation(), DiskFile::omRead))
+	{
+		return false;
+	}
+
+	CharBuffer buffer(1024 * 10 + 1);
+
+	const int beginSignatureLen = strlen(BEGIN_SCRIPT_SIGNATURE);
+	const int queueEventsSignatureLen = strlen(QUEUE_EVENTS_SIGNATURE);
+	const int taskTimeSignatureLen = strlen(TASK_TIME_SIGNATURE);
+	const int definitionSignatureLen = strlen(DEFINITION_SIGNATURE);
+
+	// check if the file contains pp-script-signature
+	// read first 10KB of the file and look for signature
+	int readBytes = (int)infile.Read(buffer, buffer.Size() - 1);
+	infile.Close();
+	buffer[readBytes] = '\0';
+
+	bool postScript = false;
+	bool scanScript = false;
+	bool queueScript = false;
+	bool schedulerScript = false;
+	bool feedScript = false;
+	char* queueEvents = nullptr;
+	char* taskTime = nullptr;
+
+	bool inConfig = false;
+	bool afterConfig = false;
+
+	// Declarations "QUEUE EVENT:" and "TASK TIME:" can be placed:
+	// - in script definition body (between opening and closing script signatures);
+	// - immediately before script definition (before opening script signature);
+	// - immediately after script definition (after closing script signature).
+	// The last two pissibilities are provided to increase compatibility of scripts with older
+	// nzbget versions which do not expect the extra declarations in the script defintion body.
+
+	Tokenizer tok(buffer, "\n\r", true);
+	while (char* line = tok.Next())
+	{
+		if (!strncmp(line, QUEUE_EVENTS_SIGNATURE, queueEventsSignatureLen))
+		{
+			queueEvents = line + queueEventsSignatureLen;
+		}
+		else if (!strncmp(line, TASK_TIME_SIGNATURE, taskTimeSignatureLen))
+		{
+			taskTime = line + taskTimeSignatureLen;
+		}
+
+		bool header = !strncmp(line, DEFINITION_SIGNATURE, definitionSignatureLen);
+		if (!header && !inConfig)
+		{
+			queueEvents = nullptr;
+			taskTime = nullptr;
+		}
+
+		if (!header && afterConfig)
+		{
+			break;
+		}
+
+		if (!strncmp(line, BEGIN_SCRIPT_SIGNATURE, beginSignatureLen) && strstr(line, END_SCRIPT_SIGNATURE))
+		{
+			if (!inConfig)
+			{
+				inConfig = true;
+				postScript = strstr(line, POST_SCRIPT_SIGNATURE);
+				scanScript = strstr(line, SCAN_SCRIPT_SIGNATURE);
+				queueScript = strstr(line, QUEUE_SCRIPT_SIGNATURE);
+				schedulerScript = strstr(line, SCHEDULER_SCRIPT_SIGNATURE);
+				feedScript = strstr(line, FEED_SCRIPT_SIGNATURE);
+			}
+			else
+			{
+				afterConfig = true;
+			}
+		}
+	}
+
+	if (!(postScript || scanScript || queueScript || schedulerScript || feedScript))
+	{
+		return false;
+	}
+
+	// trim decorations
+	char* p;
+	while (queueEvents && *queueEvents && *(p = queueEvents + strlen(queueEvents) - 1) == '#') *p = '\0';
+	if (queueEvents) queueEvents = Util::Trim(queueEvents);
+	while (taskTime && *taskTime && *(p = taskTime + strlen(taskTime) - 1) == '#') *p = '\0';
+	if (taskTime) taskTime = Util::Trim(taskTime);
+
+	script->SetPostScript(postScript);
+	script->SetScanScript(scanScript);
+	script->SetQueueScript(queueScript);
+	script->SetSchedulerScript(schedulerScript);
+	script->SetFeedScript(feedScript);
+	script->SetQueueEvents(queueEvents);
+	script->SetTaskTime(taskTime);
+
+	return true;
+}
 
 BString<1024> ScriptConfig::BuildScriptName(const char* directory, const char* filename, bool isSubDir)
 {
@@ -431,3 +478,23 @@ void ScriptConfig::BuildScriptDisplayNames(Scripts* scripts)
 		script.SetDisplayName(displayName);
 	}
 }
+
+void ScriptConfig::CreateTasks()
+{
+	for (Script& script : m_scripts)
+	{
+		if (script.GetSchedulerScript() && !Util::EmptyStr(script.GetTaskTime()))
+		{
+			Tokenizer tok(g_Options->GetExtensions(), ",;");
+			while (const char* scriptName = tok.Next())
+			{
+				if (FileSystem::SameFilename(scriptName, script.GetName()))
+				{
+					g_Options->CreateSchedulerTask(0, script.GetTaskTime(), 
+						nullptr, Options::scScript, script.GetName());
+					break;
+				}
+			}
+		}
+	}
+}

daemon/extension/ScriptConfig.h

@@ -31,7 +31,8 @@ public:
 	class Script
 	{
 	public:
-		Script(const char* name, const char* location);
+		Script(const char* name, const char* location) :
+			m_name(name), m_location(location), m_displayName(name) {};
 		Script(Script&&) = default;
 		const char* GetName() { return m_name; }
 		const char* GetLocation() { return m_location; }
@@ -49,17 +50,20 @@ public:
 		void SetFeedScript(bool feedScript) { m_feedScript = feedScript; }
 		void SetQueueEvents(const char* queueEvents) { m_queueEvents = queueEvents; }
 		const char* GetQueueEvents() { return m_queueEvents; }
+		void SetTaskTime(const char* taskTime) { m_taskTime = taskTime; }
+		const char* GetTaskTime() { return m_taskTime; }
 
 	private:
 		CString m_name;
 		CString m_location;
 		CString m_displayName;
-		bool m_postScript;
-		bool m_scanScript;
-		bool m_queueScript;
-		bool m_schedulerScript;
-		bool m_feedScript;
+		bool m_postScript = false;
+		bool m_scanScript = false;
+		bool m_queueScript = false;
+		bool m_schedulerScript = false;
+		bool m_feedScript = false;
 		CString m_queueEvents;
+		CString m_taskTime;
 	};
 
 	typedef std::list<Script> Scripts;
@@ -75,8 +79,6 @@ public:
 	private:
 		Script m_script;
 		CString m_template;
-
-		friend class Options;
 	};
 
 	typedef std::deque<ConfigTemplate> ConfigTemplates;
@@ -94,9 +96,11 @@ private:
 
 	void InitScripts();
 	void InitConfigTemplates();
+	void CreateTasks();
 	void LoadScriptDir(Scripts* scripts, const char* directory, bool isSubDir);
 	void BuildScriptDisplayNames(Scripts* scripts);
 	void LoadScripts(Scripts* scripts);
+	bool LoadScriptFile(Script* script);
 	BString<1024>BuildScriptName(const char* directory, const char* filename, bool isSubDir);
 	bool ScriptExists(Scripts* scripts, const char* scriptName);
 };

daemon/feed/FeedCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "FeedCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "WebDownloader.h"
 #include "Util.h"
 #include "FileSystem.h"
@@ -29,6 +30,7 @@
 #include "FeedScript.h"
 #include "DiskState.h"
 #include "DupeCoordinator.h"
+#include "UrlCoordinator.h"
 
 std::unique_ptr<RegEx>& FeedCoordinator::FilterHelper::GetRegEx(int id)
 {
@@ -65,6 +67,9 @@ FeedCoordinator::FeedCoordinator()
 
 	m_downloadQueueObserver.m_owner = this;
 	DownloadQueue::Guard()->Attach(&m_downloadQueueObserver);
+
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
 }
 
 FeedCoordinator::~FeedCoordinator()
@@ -84,68 +89,77 @@ void FeedCoordinator::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
-	if (g_Options->GetServerMode() && g_Options->GetSaveQueue() && g_Options->GetReloadQueue())
+	if (g_Options->GetServerMode())
 	{
 		Guard guard(m_downloadsMutex);
 		g_DiskState->LoadFeeds(&m_feeds, &m_feedHistory);
 	}
 
-	int sleepInterval = 100;
-	int updateCounter = 0;
-	int cleanupCounter = 60000;
-
+	time_t lastCleanup = 0;
 	while (!IsStopped())
 	{
-		usleep(sleepInterval * 1000);
-
-		updateCounter += sleepInterval;
-		if (updateCounter >= 1000)
+		// this code should not be called too often, once per second is OK
+		if (!g_WorkState->GetPauseDownload() || m_force || g_Options->GetUrlForce())
 		{
-			// this code should not be called too often, once per second is OK
+			Guard guard(m_downloadsMutex);
 
-			if (!g_Options->GetPauseDownload() || m_force || g_Options->GetUrlForce())
+			time_t current = Util::CurrentTime();
+			if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
 			{
-				Guard guard(m_downloadsMutex);
-
-				time_t current = Util::CurrentTime();
-				if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
+				m_force = false;
+				// check feed list and update feeds
+				for (FeedInfo* feedInfo : &m_feeds)
 				{
-					m_force = false;
-					// check feed list and update feeds
-					for (FeedInfo* feedInfo : &m_feeds)
+					if (((feedInfo->GetInterval() > 0 &&
+						 (feedInfo->GetNextUpdate() == 0 ||
+						  current >= feedInfo->GetNextUpdate() ||
+						  current < feedInfo->GetNextUpdate() - feedInfo->GetInterval() * 60)) ||
+						feedInfo->GetFetch()) &&
+						feedInfo->GetStatus() != FeedInfo::fsRunning)
 					{
-						if (((feedInfo->GetInterval() > 0 &&
-							(current - feedInfo->GetLastUpdate() >= feedInfo->GetInterval() * 60 ||
-							 current < feedInfo->GetLastUpdate())) ||
-							feedInfo->GetFetch()) &&
-							feedInfo->GetStatus() != FeedInfo::fsRunning)
-						{
-							StartFeedDownload(feedInfo, feedInfo->GetFetch());
-						}
-						else if (feedInfo->GetFetch())
-						{
-							m_force = true;
-						}
+						StartFeedDownload(feedInfo, feedInfo->GetFetch());
+					}
+					else if (feedInfo->GetFetch())
+					{
+						m_force = true;
 					}
 				}
 			}
-
-			CheckSaveFeeds();
-			ResetHangingDownloads();
-			updateCounter = 0;
 		}
 
-		cleanupCounter += sleepInterval;
-		if (cleanupCounter >= 60000)
+		CheckSaveFeeds();
+		ResetHangingDownloads();
+
+		if (std::abs(Util::CurrentTime() - lastCleanup) >= 60)
 		{
 			// clean up feed history once a minute
 			CleanupHistory();
 			CleanupCache();
 			CheckSaveFeeds();
-			cleanupCounter = 0;
+			lastCleanup = Util::CurrentTime();
+		}
+
+		Guard guard(m_downloadsMutex);
+		if (m_force)
+		{
+			// don't sleep too long if there active feeds scheduled for redownload
+			m_waitCond.WaitFor(m_downloadsMutex, 1000, [&]{ return IsStopped(); });
+		}
+		else
+		{
+			// no active jobs, we can sleep longer:
+			//  - if option "UrlForce" is active or if the feed list is empty we need to wake up
+			//    only when a new feed preview is requested. We could wait indefinitely for that
+			//    but we need to do some job every now and then and therefore we sleep only 60 seconds.
+			//  - if option "UrlForce" is disabled we need also to wake up when state "DownloadPaused"
+			//    is changed. We detect this via notification from 'WorkState'. However such
+			//    notifications are not 100% reliable due to possible race conditions. Therefore
+			//    we sleep for max. 5 seconds.
+			int waitInterval = g_Options->GetUrlForce() || m_feeds.empty() ? 60000 : 5000;
+			m_waitCond.WaitFor(m_downloadsMutex, waitInterval, [&]{ return m_force || IsStopped(); });
 		}
 	}
 
@@ -159,7 +173,7 @@ void FeedCoordinator::Run()
 			completed = m_activeDownloads.size() == 0;
 		}
 		CheckSaveFeeds();
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 	debug("FeedCoordinator: Downloads are completed");
@@ -178,12 +192,20 @@ void FeedCoordinator::Stop()
 		feedDownloader->Stop();
 	}
 	debug("UrlDownloads are notified");
+
+	// Resume Run() to exit it
+	m_waitCond.NotifyAll();
+}
+
+void FeedCoordinator::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_force = true;
+	m_waitCond.NotifyAll();
 }
 
 void FeedCoordinator::ResetHangingDownloads()
 {
-	const int timeout = g_Options->GetTerminateTimeout();
-	if (timeout == 0)
+	if (g_Options->GetUrlTimeout() == 0)
 	{
 		return;
 	}
@@ -191,31 +213,15 @@ void FeedCoordinator::ResetHangingDownloads()
 	Guard guard(m_downloadsMutex);
 	time_t tm = Util::CurrentTime();
 
-	m_activeDownloads.erase(std::remove_if(m_activeDownloads.begin(), m_activeDownloads.end(),
-		[timeout, tm](FeedDownloader* feedDownloader)
+	for (FeedDownloader* feedDownloader: m_activeDownloads)
+	{
+		if (tm - feedDownloader->GetLastUpdateTime() > g_Options->GetUrlTimeout() + 10 &&
+			feedDownloader->GetStatus() == FeedDownloader::adRunning)
 		{
-			if (tm - feedDownloader->GetLastUpdateTime() > timeout &&
-				feedDownloader->GetStatus() == FeedDownloader::adRunning)
-			{
-				debug("Terminating hanging download %s", feedDownloader->GetInfoName());
-				if (feedDownloader->Terminate())
-				{
-					error("Terminated hanging download %s", feedDownloader->GetInfoName());
-					feedDownloader->GetFeedInfo()->SetStatus(FeedInfo::fsUndefined);
-				}
-				else
-				{
-					error("Could not terminate hanging download %s", feedDownloader->GetInfoName());
-				}
-
-				// it's not safe to destroy feedDownloader, because the state of object is unknown
-				delete feedDownloader;
-
-				return true;
-			}
-			return false;
-		}),
-		m_activeDownloads.end());
+			error("Cancelling hanging feed download %s", feedDownloader->GetInfoName());
+			feedDownloader->Stop();
+		}
+	}
 }
 
 void FeedCoordinator::LogDebugInfo()
@@ -239,14 +245,7 @@ void FeedCoordinator::StartFeedDownload(FeedInfo* feedInfo, bool force)
 	feedDownloader->Attach(this);
 	feedDownloader->SetFeedInfo(feedInfo);
 	feedDownloader->SetUrl(feedInfo->GetUrl());
-	if (strlen(feedInfo->GetName()) > 0)
-	{
-		feedDownloader->SetInfoName(feedInfo->GetName());
-	}
-	else
-	{
-		feedDownloader->SetInfoName(NzbInfo::MakeNiceUrlName(feedInfo->GetUrl(), ""));
-	}
+	feedDownloader->SetInfoName(feedInfo->GetName());
 	feedDownloader->SetForce(force || g_Options->GetUrlForce());
 
 	BString<1024> outFilename;
@@ -298,23 +297,29 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), feedDownloader));
 	}
 
+	SchedulerNextUpdate(feedInfo, statusOK);
+
 	if (statusOK)
 	{
 		if (!feedInfo->GetPreview())
 		{
+			bool scriptSuccess = true;
 			FeedScriptController::ExecuteScripts(
-				!Util::EmptyStr(feedInfo->GetFeedScript()) ? feedInfo->GetFeedScript(): g_Options->GetFeedScript(),
-				feedInfo->GetOutputFilename(), feedInfo->GetId());
+				!Util::EmptyStr(feedInfo->GetExtensions()) ? feedInfo->GetExtensions(): g_Options->GetExtensions(),
+				feedInfo->GetOutputFilename(), feedInfo->GetId(), &scriptSuccess);
+			if (!scriptSuccess)
+			{
+				feedInfo->SetStatus(FeedInfo::fsFailed);
+				return;
+			}
 
-			std::unique_ptr<FeedFile> feedFile = std::make_unique<FeedFile>(feedInfo->GetOutputFilename());
-			bool parsed = feedFile->Parse();
-			FileSystem::DeleteFile(feedInfo->GetOutputFilename());
+			std::unique_ptr<FeedFile> feedFile = parseFeed(feedInfo);
 
 			std::vector<std::unique_ptr<NzbInfo>> addedNzbs;
 
 			{
 				Guard guard(m_downloadsMutex);
-				if (parsed)
+				if (feedFile)
 				{
 					std::unique_ptr<FeedItemList> feedItems = feedFile->DetachFeedItems();
 					addedNzbs = ProcessFeed(feedInfo, feedItems.get());
@@ -325,12 +330,10 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 				m_save = true;
 			}
 
-			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 			for (std::unique_ptr<NzbInfo>& nzbInfo : addedNzbs)
 			{
-				downloadQueue->GetQueue()->Add(std::move(nzbInfo));
+				g_UrlCoordinator->AddUrlToQueue(std::move(nzbInfo), false);
 			}
-			downloadQueue->Save();
 		}
 		feedInfo->SetStatus(FeedInfo::fsFinished);
 	}
@@ -340,6 +343,30 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 	}
 }
 
+void FeedCoordinator::SchedulerNextUpdate(FeedInfo* feedInfo, bool success)
+{
+	time_t current = Util::CurrentTime();
+	int interval;
+
+	if (success)
+	{
+		interval = feedInfo->GetInterval() * 60;
+		feedInfo->SetLastInterval(0);
+	}
+	else
+	{
+		// On failure schedule next update sooner:
+		// starting with 1 minute and increasing, but not greater than FeedX.Interval
+		interval = feedInfo->GetLastInterval() * 2;
+		interval = std::max(interval, 60);
+		interval = std::min(interval, feedInfo->GetInterval() * 60);
+		feedInfo->SetLastInterval(interval);
+	}
+
+	detail("Scheduling update for feed %s in %i minute(s)", feedInfo->GetName(), interval / 60);
+	feedInfo->SetNextUpdate(current + interval);
+}
+
 void FeedCoordinator::FilterFeed(FeedInfo* feedInfo, FeedItemList* feedItems)
 {
 	debug("Filtering feed %s", feedInfo->GetName());
@@ -439,8 +466,7 @@ std::unique_ptr<NzbInfo> FeedCoordinator::CreateNzbInfo(FeedInfo* feedInfo, Feed
 	if (!nzbName.Empty())
 	{
 		BString<1024> nzbName2("%s.nzb", *nzbName);
-		FileSystem::MakeValidFilename(nzbName2, '_', false);
-		nzbInfo->SetFilename(nzbName2);
+		nzbInfo->SetFilename(FileSystem::MakeValidFilename(nzbName2));
 	}
 
 	nzbInfo->SetCategory(feedItemInfo.GetAddCategory());
@@ -449,6 +475,9 @@ std::unique_ptr<NzbInfo> FeedCoordinator::CreateNzbInfo(FeedInfo* feedInfo, Feed
 	nzbInfo->SetDupeKey(feedItemInfo.GetDupeKey());
 	nzbInfo->SetDupeScore(feedItemInfo.GetDupeScore());
 	nzbInfo->SetDupeMode(feedItemInfo.GetDupeMode());
+	nzbInfo->SetSize(feedItemInfo.GetSize());
+	nzbInfo->SetMinTime(feedItemInfo.GetTime());
+	nzbInfo->SetMaxTime(feedItemInfo.GetTime());
 
 	return nzbInfo;
 }
@@ -464,7 +493,7 @@ std::shared_ptr<FeedItemList> FeedCoordinator::ViewFeed(int id)
 
 	return PreviewFeed(feedInfo->GetId(), feedInfo->GetName(), feedInfo->GetUrl(), feedInfo->GetFilter(),
 		feedInfo->GetBacklog(), feedInfo->GetPauseNzb(), feedInfo->GetCategory(),
-		feedInfo->GetPriority(), feedInfo->GetInterval(), feedInfo->GetFeedScript(), 0, nullptr);
+		feedInfo->GetPriority(), feedInfo->GetInterval(), feedInfo->GetExtensions(), 0, nullptr);
 }
 
 std::shared_ptr<FeedItemList> FeedCoordinator::PreviewFeed(int id,
@@ -514,30 +543,30 @@ std::shared_ptr<FeedItemList> FeedCoordinator::PreviewFeed(int id,
 			}
 
 			StartFeedDownload(feedInfo.get(), true);
+
+			m_force = true;
+			m_waitCond.NotifyAll();
 		}
 
 		// wait until the download in a separate thread completes
 		while (feedInfo->GetStatus() == FeedInfo::fsRunning)
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 		}
 
 		// now can process the feed
 
-		std::unique_ptr<FeedFile> feedFile;
-
-		if (feedInfo->GetStatus() == FeedInfo::fsFinished)
+		if (feedInfo->GetStatus() != FeedInfo::fsFinished)
 		{
-			FeedScriptController::ExecuteScripts(
-				!Util::EmptyStr(feedInfo->GetFeedScript()) ? feedInfo->GetFeedScript(): g_Options->GetFeedScript(),
-				feedInfo->GetOutputFilename(), feedInfo->GetId());
-			feedFile = std::make_unique<FeedFile>(feedInfo->GetOutputFilename());
+			return nullptr;
 		}
 
-		bool parsed = feedFile && feedFile->Parse();
-		FileSystem::DeleteFile(feedInfo->GetOutputFilename());
+		FeedScriptController::ExecuteScripts(
+			!Util::EmptyStr(feedInfo->GetExtensions()) ? feedInfo->GetExtensions(): g_Options->GetExtensions(),
+			feedInfo->GetOutputFilename(), feedInfo->GetId(), nullptr);
 
-		if (!parsed)
+		std::unique_ptr<FeedFile> feedFile = parseFeed(feedInfo.get());
+		if (!feedFile)
 		{
 			return nullptr;
 		}
@@ -581,6 +610,23 @@ void FeedCoordinator::FetchFeed(int id)
 			m_force = true;
 		}
 	}
+
+	m_waitCond.NotifyAll();
+}
+
+std::unique_ptr<FeedFile> FeedCoordinator::parseFeed(FeedInfo* feedInfo)
+{
+	std::unique_ptr<FeedFile> feedFile = std::make_unique<FeedFile>(feedInfo->GetOutputFilename(), feedInfo->GetName());
+	if (feedFile->Parse())
+	{
+		FileSystem::DeleteFile(feedInfo->GetOutputFilename());
+	}
+	else
+	{
+		error("Feed file %s kept for troubleshooting (will be deleted on next successful feed fetch)", feedInfo->GetOutputFilename());
+		feedFile.reset();
+	}
+	return feedFile;
 }
 
 void FeedCoordinator::DownloadQueueUpdate(Subject* caller, void* aspect)
@@ -612,11 +658,11 @@ bool FeedCoordinator::HasActiveDownloads()
 
 void FeedCoordinator::CheckSaveFeeds()
 {
-	debug("CheckSaveFeeds");
 	Guard guard(m_downloadsMutex);
 	if (m_save)
 	{
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
+		debug("CheckSaveFeeds: save");
+		if (g_Options->GetServerMode())
 		{
 			g_DiskState->SaveFeeds(&m_feeds, &m_feedHistory);
 		}

daemon/feed/FeedCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@
 #include "Thread.h"
 #include "WebDownloader.h"
 #include "DownloadInfo.h"
+#include "FeedFile.h"
 #include "FeedInfo.h"
 #include "Observer.h"
 #include "Util.h"
@@ -66,6 +67,13 @@ private:
 		virtual void Update(Subject* caller, void* aspect) { m_owner->DownloadQueueUpdate(caller, aspect); }
 	};
 
+	class WorkStateObserver: public Observer
+	{
+	public:
+		FeedCoordinator* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	class FeedCacheItem
 	{
 	public:
@@ -105,9 +113,12 @@ private:
 	FeedHistory m_feedHistory;
 	Mutex m_downloadsMutex;
 	DownloadQueueObserver m_downloadQueueObserver;
+	WorkStateObserver m_workStateObserver;
 	bool m_force = false;
 	bool m_save = false;
 	FeedCache m_feedCache;
+	ConditionVar m_waitCond;
+	bool m_wokenUp = false;
 
 	void StartFeedDownload(FeedInfo* feedInfo, bool force);
 	void FeedCompleted(FeedDownloader* feedDownloader);
@@ -119,6 +130,9 @@ private:
 	void CleanupHistory();
 	void CleanupCache();
 	void CheckSaveFeeds();
+	std::unique_ptr<FeedFile> parseFeed(FeedInfo* feedInfo);
+	void SchedulerNextUpdate(FeedInfo* feedInfo, bool success);
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 extern FeedCoordinator* g_FeedCoordinator;

daemon/feed/FeedFile.cpp

@@ -25,8 +25,8 @@
 #include "Options.h"
 #include "Util.h"
 
-FeedFile::FeedFile(const char* fileName) :
-	m_fileName(fileName)
+FeedFile::FeedFile(const char* fileName, const char* infoName) :
+	m_fileName(fileName), m_infoName(infoName)
 {
 	debug("Creating FeedFile");
 
@@ -113,7 +113,7 @@ bool FeedFile::Parse()
 	{
 		_bstr_t r(doc->GetparseError()->reason);
 		const char* errMsg = r;
-		error("Error parsing rss feed: %s", errMsg);
+		error("Error parsing rss feed %s: %s", *m_infoName, errMsg);
 		return false;
 	}
 
@@ -248,7 +248,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 		//<newznab:attr name="size" value="5423523453534" />
 		if (feedItemInfo.GetSize() == 0)
 		{
-			tag = node->selectSingleNode("newznab:attr[@name='size']");
+			tag = node->selectSingleNode("newznab:attr[@name='size'] | nZEDb:attr[@name='size']");
 			if (tag)
 			{
 				attr = tag->Getattributes()->getNamedItem("value");
@@ -262,7 +262,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 		}
 
 		//<newznab:attr name="imdb" value="1588173"/>
-		tag = node->selectSingleNode("newznab:attr[@name='imdb']");
+		tag = node->selectSingleNode("newznab:attr[@name='imdb'] | nZEDb:attr[@name='imdb']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -275,7 +275,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 		}
 
 		//<newznab:attr name="rageid" value="33877"/>
-		tag = node->selectSingleNode("newznab:attr[@name='rageid']");
+		tag = node->selectSingleNode("newznab:attr[@name='rageid'] | nZEDb:attr[@name='rageid']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -288,7 +288,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 		}
 
 		//<newznab:attr name="tdvdbid" value="33877"/>
-		tag = node->selectSingleNode("newznab:attr[@name='tvdbid']");
+		tag = node->selectSingleNode("newznab:attr[@name='tvdbid'] | nZEDb:attr[@name='tvdbid']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -301,7 +301,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 		}
 
 		//<newznab:attr name="tvmazeid" value="33877"/>
-		tag = node->selectSingleNode("newznab:attr[@name='tvmazeid']");
+		tag = node->selectSingleNode("newznab:attr[@name='tvmazeid'] | nZEDb:attr[@name='tvmazeid']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -315,7 +315,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 
 		//<newznab:attr name="episode" value="E09"/>
 		//<newznab:attr name="episode" value="9"/>
-		tag = node->selectSingleNode("newznab:attr[@name='episode']");
+		tag = node->selectSingleNode("newznab:attr[@name='episode'] | nZEDb:attr[@name='episode']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -328,7 +328,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 
 		//<newznab:attr name="season" value="S03"/>
 		//<newznab:attr name="season" value="3"/>
-		tag = node->selectSingleNode("newznab:attr[@name='season']");
+		tag = node->selectSingleNode("newznab:attr[@name='season'] | nZEDb:attr[@name='season']");
 		if (tag)
 		{
 			attr = tag->Getattributes()->getNamedItem("value");
@@ -339,7 +339,7 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 			}
 		}
 
-		MSXML::IXMLDOMNodeListPtr itemList = node->selectNodes("newznab:attr");
+		MSXML::IXMLDOMNodeListPtr itemList = node->selectNodes("newznab:attr | nZEDb:attr");
 		for (int i = 0; i < itemList->Getlength(); i++)
 		{
 			MSXML::IXMLDOMNodePtr node = itemList->Getitem(i);
@@ -360,6 +360,10 @@ bool FeedFile::ParseFeed(IUnknown* nzb)
 
 bool FeedFile::Parse()
 {
+#ifdef DISABLE_LIBXML2
+	error("Could not parse rss feed, program was compiled without libxml2 support");
+	return false;
+#else
 	xmlSAXHandler SAX_handler = {0};
 	SAX_handler.startElement = reinterpret_cast<startElementSAXFunc>(SAX_StartElement);
 	SAX_handler.endElement = reinterpret_cast<endElementSAXFunc>(SAX_EndElement);
@@ -373,11 +377,12 @@ bool FeedFile::Parse()
 
 	if (ret != 0)
 	{
-		error("Failed to parse rss feed");
+		error("Failed to parse rss feed %s", *m_infoName);
 		return false;
 	}
 
 	return true;
+#endif
 }
 
 void FeedFile::Parse_StartElement(const char *name, const char **atts)
@@ -407,7 +412,8 @@ void FeedFile::Parse_StartElement(const char *name, const char **atts)
 			}
 		}
 	}
-	else if (m_feedItemInfo && !strcmp("newznab:attr", name) &&
+	else if (m_feedItemInfo && 
+		(!strcmp("newznab:attr", name) || !strcmp("nZEDb:attr", name)) &&
 		atts[0] && atts[1] && atts[2] && atts[3] &&
 		!strcmp("name", atts[0]) && !strcmp("value", atts[2]))
 	{
@@ -565,7 +571,11 @@ void FeedFile::SAX_characters(FeedFile* file, const char * xmlstr, int len)
 
 void* FeedFile::SAX_getEntity(FeedFile* file, const char * name)
 {
+#ifdef DISABLE_LIBXML2
+	void* e = nullptr;
+#else
 	xmlEntityPtr e = xmlGetPredefinedEntity((xmlChar* )name);
+#endif
 	if (!e)
 	{
 		warn("entity not found");
@@ -592,6 +602,6 @@ void FeedFile::SAX_error(FeedFile* file, const char *msg, ...)
 
 	// remove trailing CRLF
 	for (char* pend = errMsg + strlen(errMsg) - 1; pend >= errMsg && (*pend == '\n' || *pend == '\r' || *pend == ' '); pend--) *pend = '\0';
-	error("Error parsing rss feed: %s", errMsg);
+	error("Error parsing rss feed %s: %s", *file->m_infoName, errMsg);
 }
 #endif

daemon/feed/FeedFile.h

@@ -27,7 +27,7 @@
 class FeedFile
 {
 public:
-	FeedFile(const char* fileName);
+	FeedFile(const char* fileName, const char* infoName);
 	bool Parse();
 	std::unique_ptr<FeedItemList> DetachFeedItems() { return std::move(m_feedItems); }
 
@@ -36,6 +36,7 @@ public:
 private:
 	std::unique_ptr<FeedItemList> m_feedItems;
 	CString m_fileName;
+	CString m_infoName;
 
 	void ParseSubject(FeedItemInfo& feedItemInfo);
 #ifdef WIN32

daemon/feed/FeedFilter.cpp

@@ -51,7 +51,7 @@ bool FeedFilter::Term::MatchValue(const char* strValue, int64 intValue)
 
 	if (m_command < fcEqual && !strValue)
 	{
-		intBuf.Format("%lld", intValue);
+		intBuf.Format("%" PRId64, intValue);
 		strValue = intBuf;
 	}
 
@@ -897,7 +897,7 @@ void FeedFilter::Rule::ExpandRefValues(FeedItemInfo& feedItemInfo, CString* dest
 			break; // error
 		}
 
-		curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+		curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 	}
 
 	*destStr = std::move(curvalue);

daemon/feed/FeedInfo.cpp

@@ -23,16 +23,20 @@
 #include "Util.h"
 
 FeedInfo::FeedInfo(int id, const char* name, const char* url, bool backlog, int interval,
-		const char* filter, bool pauseNzb, const char* category, int priority, const char* feedScript) :
+		const char* filter, bool pauseNzb, const char* category, int priority, const char* extensions) :
 	m_backlog(backlog), m_interval(interval), m_pauseNzb(pauseNzb), m_priority(priority)
 {
 	m_id = id;
 	m_name = name ? name : "";
+	if (m_name.Length() == 0)
+	{
+		m_name.Format("Feed%i", m_id);
+	}
 	m_url = url ? url : "";
 	m_filter = filter ? filter : "";
 	m_filterHash = Util::HashBJ96(m_filter, strlen(m_filter), 0);
 	m_category = category ? category : "";
-	m_feedScript = feedScript ? feedScript : "";
+	m_extensions = extensions ? extensions : "";
 }
 
 

daemon/feed/FeedInfo.h

@@ -38,7 +38,7 @@ public:
 
 	FeedInfo(int id, const char* name, const char* url, bool backlog, int interval,
 		const char* filter, bool pauseNzb, const char* category, int priority,
-		const char* feedScript);
+		const char* extensions);
 	int GetId() { return m_id; }
 	const char* GetName() { return m_name; }
 	const char* GetUrl() { return m_url; }
@@ -48,9 +48,13 @@ public:
 	bool GetPauseNzb() { return m_pauseNzb; }
 	const char* GetCategory() { return m_category; }
 	int GetPriority() { return m_priority; }
-	const char* GetFeedScript() { return m_feedScript; }
+	const char* GetExtensions() { return m_extensions; }
 	time_t GetLastUpdate() { return m_lastUpdate; }
 	void SetLastUpdate(time_t lastUpdate) { m_lastUpdate = lastUpdate; }
+	time_t GetNextUpdate() { return m_nextUpdate; }
+	void SetNextUpdate(time_t nextUpdate) { m_nextUpdate = nextUpdate; }
+	int GetLastInterval() { return m_lastInterval; }
+	void SetLastInterval(int lastInterval) { m_lastInterval = lastInterval; }
 	bool GetPreview() { return m_preview; }
 	void SetPreview(bool preview) { m_preview = preview; }
 	EStatus GetStatus() { return m_status; }
@@ -68,20 +72,22 @@ private:
 	int m_id;
 	CString m_name;
 	CString m_url;
+	bool m_backlog;
 	int m_interval;
 	CString m_filter;
 	uint32 m_filterHash;
 	bool m_pauseNzb;
 	CString m_category;
-	CString m_feedScript;
+	CString m_extensions;
 	int m_priority;
 	time_t m_lastUpdate = 0;
+	time_t m_nextUpdate = 0;
+	int m_lastInterval = 0;
 	bool m_preview = false;
 	EStatus m_status = fsUndefined;
 	CString m_outputFilename;
 	bool m_fetch = false;
 	bool m_force = false;
-	bool m_backlog;
 };
 
 typedef std::deque<std::unique_ptr<FeedInfo>> Feeds;

daemon/frontend/Frontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "nzbget.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Frontend.h"
 #include "Log.h"
 #include "Connection.h"
@@ -33,9 +34,24 @@ Frontend::Frontend()
 {
 	debug("Creating Frontend");
 
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
+
 	m_updateInterval = g_Options->GetUpdateInterval();
 }
 
+void Frontend::Stop()
+{
+	Thread::Stop();
+
+	m_waitCond.NotifyAll();
+}
+
+void Frontend::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_waitCond.NotifyAll();
+}
+
 bool Frontend::PrepareData()
 {
 	if (IsRemoteMode())
@@ -57,8 +73,8 @@ bool Frontend::PrepareData()
 		if (m_summary)
 		{
 			m_currentDownloadSpeed = g_StatMeter->CalcCurrentDownloadSpeed();
-			m_pauseDownload = g_Options->GetPauseDownload();
-			m_downloadLimit = g_Options->GetDownloadRate();
+			m_pauseDownload = g_WorkState->GetPauseDownload();
+			m_downloadLimit = g_WorkState->GetSpeedLimit();
 			m_threadCount = Thread::GetThreadCount();
 			g_StatMeter->CalcTotalStat(&m_upTimeSec, &m_dnTimeSec, &m_allBytes, &m_standBy);
 
@@ -108,8 +124,8 @@ void Frontend::ServerPauseUnpause(bool pause)
 	}
 	else
 	{
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(pause);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(pause);
 	}
 }
 
@@ -121,7 +137,7 @@ void Frontend::ServerSetDownloadRate(int rate)
 	}
 	else
 	{
-		g_Options->SetDownloadRate(rate);
+		g_WorkState->SetSpeedLimit(rate);
 	}
 }
 
@@ -133,7 +149,7 @@ bool Frontend::ServerEditQueue(DownloadQueue::EEditAction action, int offset, in
 	}
 	else
 	{
-		return DownloadQueue::Guard()->EditEntry(id, action, offset, nullptr);
+		return DownloadQueue::Guard()->EditEntry(id, action, CString::FormatStr("%i", offset));
 	}
 }
 
@@ -307,3 +323,16 @@ bool Frontend::RequestEditQueue(DownloadQueue::EEditAction action, int offset, i
 	IdList ids = { id };
 	return client.RequestServerEditQueue(action, offset, nullptr, &ids, nullptr, rmId);
 }
+
+void Frontend::Wait(int milliseconds)
+{
+	if (g_WorkState->GetPauseFrontend())
+	{
+		Guard guard(m_waitMutex);
+		m_waitCond.WaitFor(m_waitMutex, 2000);
+	}
+	else
+	{
+		Util::Sleep(milliseconds);
+	}
+}

daemon/frontend/Frontend.h

@@ -27,6 +27,7 @@
 #include "DownloadInfo.h"
 #include "MessageBase.h"
 #include "QueueEditor.h"
+#include "Observer.h"
 
 class Frontend : public Thread
 {
@@ -51,7 +52,10 @@ protected:
 	int m_dnTimeSec = 0;
 	int64 m_allBytes = 0;
 	bool m_standBy = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
+	virtual void Stop();
 	bool PrepareData();
 	void FreeData();
 	GuardedMessageList GuardMessages();
@@ -63,12 +67,22 @@ protected:
 	bool RequestSetDownloadRate(int rate);
 	bool ServerEditQueue(DownloadQueue::EEditAction action, int offset, int entry);
 	bool RequestEditQueue(DownloadQueue::EEditAction action, int offset, int id);
+	void Wait(int milliseconds);
 
 private:
+	class WorkStateObserver : public Observer
+	{
+	public:
+		Frontend* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	MessageList m_remoteMessages;
+	WorkStateObserver m_workStateObserver;
 
 	bool RequestMessages();
 	bool RequestFileList();
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 #endif

daemon/frontend/LoggableFrontend.cpp

@@ -20,6 +20,7 @@
 
 
 #include "nzbget.h"
+#include "Util.h"
 #include "LoggableFrontend.h"
 #include "Log.h"
 
@@ -30,7 +31,7 @@ void LoggableFrontend::Run()
 	while (!IsStopped())
 	{
 		Update();
-		usleep(m_updateInterval * 1000);
+		Wait(m_updateInterval);
 	}
 	// Printing the last messages
 	Update();

daemon/frontend/NCursesFrontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -213,8 +213,10 @@ void NCursesFrontend::Run()
 			m_dataUpdatePos = m_updateInterval;
 		}
 
-		usleep(10 * 1000);
-		m_dataUpdatePos -= 10;
+		// update more often (sleep shorter) if need faster reaction on user input
+		int sleepInterval = m_inputMode == normal ? 100 : 10;
+		Wait(sleepInterval);
+		m_dataUpdatePos -= sleepInterval;
 	}
 
 	FreeData();
@@ -338,7 +340,8 @@ int NCursesFrontend::CalcQueueSize()
 	}
 	else
 	{
-		for (NzbInfo* nzbInfo : DownloadQueue::Guard()->GetQueue())
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 		{
 			queueSize += nzbInfo->GetFileList()->size();
 		}
@@ -660,7 +663,8 @@ void NCursesFrontend::PrintFileQueue()
 	int pausedFiles = 0;
 	int fileNum = 0;
 
-	for (NzbInfo* nzbInfo : DownloadQueue::Guard()->GetQueue())
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 	{
 		for (FileInfo* fileInfo : nzbInfo->GetFileList())
 		{
@@ -736,7 +740,7 @@ void NCursesFrontend::PrintFilename(FileInfo * fileInfo, int row, bool selected)
 	BString<1024> nzbNiceName;
 	if (m_showNzbname)
 	{
-		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR);
+		nzbNiceName.Format("%s%c", fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR);
 	}
 
 	BString<1024> text("%s%i%s%s%s %s%s (%s%s)%s", Brace1, fileInfo->GetId(),
@@ -1014,7 +1018,8 @@ bool NCursesFrontend::EditQueue(DownloadQueue::EEditAction action, int offset)
 	else
 	{
 		int fileNum = 0;
-		for (NzbInfo* nzbInfo : DownloadQueue::Guard()->GetQueue())
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 		{
 			for (FileInfo* fileInfo : nzbInfo->GetFileList())
 			{

daemon/main/CommandLineParser.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -442,6 +442,10 @@ void CommandLineParser::InitCommandLine(int argc, const char* const_argv[])
 					{
 						m_editQueueAction = DownloadQueue::eaHistoryRedownload;
 					}
+					else if (!strcasecmp(optarg, "F"))
+					{
+						m_editQueueAction = DownloadQueue::eaHistoryRetryFailed;
+					}
 					else if (!strcasecmp(optarg, "O"))
 					{
 						m_editQueueAction = DownloadQueue::eaHistorySetParameter;
@@ -509,6 +513,14 @@ void CommandLineParser::InitCommandLine(int argc, const char* const_argv[])
 					{
 						m_editQueueAction = group ? DownloadQueue::eaGroupDelete : DownloadQueue::eaFileDelete;
 					}
+					else if (!strcasecmp(optarg, "DP"))
+					{
+						m_editQueueAction = DownloadQueue::eaGroupParkDelete;
+					}
+					else if (!strcasecmp(optarg, "SF"))
+					{
+						m_editQueueAction = DownloadQueue::eaGroupSortFiles;
+					}
 					else if (!strcasecmp(optarg, "C") || !strcasecmp(optarg, "K") || !strcasecmp(optarg, "CP"))
 					{
 						// switch "K" is provided for compatibility with v. 0.8.0 and can be removed in future versions
@@ -733,11 +745,11 @@ void CommandLineParser::PrintUsage(const char* com)
 		"                 S          Print only server status\n"
 		"    <RegEx>                 Regular expression (only with options \"FR\", \"GR\")\n"
 		"                            using POSIX Extended Regular Expression Syntax\n"
-		"  -P, --pause   [D|O|S]  Pause server\n"
+		"  -P, --pause   [D|O|S]     Pause server\n"
 		"                 D          Pause download queue (default)\n"
 		"                 O          Pause post-processor queue\n"
 		"                 S          Pause scan of incoming nzb-directory\n"
-		"  -U, --unpause [D|O|S]  Unpause server\n"
+		"  -U, --unpause [D|O|S]     Unpause server\n"
 		"                 D          Unpause download queue (default)\n"
 		"                 O          Unpause post-processor queue\n"
 		"                 S          Unpause scan of incoming nzb-directory\n"
@@ -764,17 +776,18 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       T                    Move to top of queue\n"
 		"       B                    Move to bottom of queue\n"
 		"       D                    Delete\n"
-		"    - for files (F) and groups (G):\n"
 		"       P                    Pause\n"
 		"       U                    Resume (unpause)\n"
 		"    - for groups (G):\n"
 		"       A                    Pause all pars\n"
 		"       R                    Pause extra pars\n"
+		"       DP                   Delete but keep downloaded files\n"
 		"       I <priority>         Set priority (signed integer)\n"
 		"       C <name>             Set category\n"
 		"       CP <name>            Set category and apply post-process parameters\n"
 		"       N <name>             Rename\n"
 		"       M                    Merge\n"
+		"       SF                   Sort inner files for optimal order\n"
 		"       S <name>             Split - create new group from selected files\n"
 		"       O <name>=<value>     Set post-process parameter\n"
 		"    - for post-jobs (O):\n"
@@ -784,6 +797,7 @@ void CommandLineParser::PrintUsage(const char* com)
 		"       P                    Post-process again\n"
 		"       R                    Download remaining files\n"
 		"       A                    Download again\n"
+		"       F                    Retry download of failed articles\n"
 		"       O <name>=<value>     Set post-process parameter\n"
 		"       B                    Mark as bad\n"
 		"       G                    Mark as good\n"
@@ -880,7 +894,7 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			if (p)
 			{
 				BString<100> buf;
-				buf.Set(optarg, p - optarg);
+				buf.Set(optarg, (int)(p - optarg));
 				editQueueIdFrom = atoi(buf);
 				editQueueIdTo = atoi(p + 1);
 				if (editQueueIdFrom <= 0 || editQueueIdTo <= 0)
@@ -901,25 +915,18 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			}
 
 			int editQueueIdCount = 0;
-			if (editQueueIdTo != 0)
+			if (editQueueIdFrom < editQueueIdTo)
 			{
-				if (editQueueIdFrom < editQueueIdTo)
-				{
-					editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
-				}
-				else
-				{
-					editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
-				}
+				editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
 			}
 			else
 			{
-				editQueueIdCount = 1;
+				editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
 			}
 
 			for (int i = 0; i < editQueueIdCount; i++)
 			{
-				if (editQueueIdFrom < editQueueIdTo || editQueueIdTo == 0)
+				if (editQueueIdFrom < editQueueIdTo)
 				{
 					m_editQueueIdList.push_back(editQueueIdFrom + i);
 				}

daemon/main/CommandLineParser.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by

daemon/main/DiskService.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,23 +21,39 @@
 #include "nzbget.h"
 #include "DiskService.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "StatMeter.h"
 #include "Log.h"
 #include "Util.h"
 #include "FileSystem.h"
 
+DiskService::DiskService()
+{
+	g_WorkState->Attach(this);
+}
+
+void DiskService::Update(Subject* caller, void* aspect)
+{
+	WakeUp();
+}
+
+int DiskService::ServiceInterval()
+{
+	return m_waitingRequiredDir ? 1 :
+		g_Options->GetDiskSpace() <= 0 ? Service::Sleep :
+		// notifications from 'WorkState' are not 100% reliable due to race conditions
+		!g_WorkState->GetDownloading() ? 10 :
+		1;
+}
+
 void DiskService::ServiceWork()
 {
-	m_interval++;
-	if (m_interval == 5)
+	debug("Disk service work");
+
+	if (g_Options->GetDiskSpace() > 0 && g_WorkState->GetDownloading())
 	{
-		if (!g_Options->GetPauseDownload() &&
-			g_Options->GetDiskSpace() > 0 && !g_StatMeter->GetStandBy())
-		{
-			// check free disk space every 1 second
-			CheckDiskSpace();
-		}
-		m_interval = 0;
+		// check free disk space every 1 second
+		CheckDiskSpace();
 	}
 
 	if (m_waitingRequiredDir)
@@ -48,11 +64,13 @@ void DiskService::ServiceWork()
 
 void DiskService::CheckDiskSpace()
 {
+	debug("Disk service work: check disk space");
+
 	int64 freeSpace = FileSystem::FreeDiskSize(g_Options->GetDestDir());
 	if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 	{
 		warn("Low disk space on %s. Pausing download", g_Options->GetDestDir());
-		g_Options->SetPauseDownload(true);
+		g_WorkState->SetPauseDownload(true);
 	}
 
 	if (!Util::EmptyStr(g_Options->GetInterDir()))
@@ -61,13 +79,15 @@ void DiskService::CheckDiskSpace()
 		if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 		{
 			warn("Low disk space on %s. Pausing download", g_Options->GetInterDir());
-			g_Options->SetPauseDownload(true);
+			g_WorkState->SetPauseDownload(true);
 		}
 	}
 }
 
 void DiskService::CheckRequiredDir()
 {
+	debug("Disk service work: check required dir");
+
 	if (!Util::EmptyStr(g_Options->GetRequiredDir()))
 	{
 		bool allExist = true;
@@ -97,7 +117,7 @@ void DiskService::CheckRequiredDir()
 		info("All required directories available");
 	}
 
-	g_Options->SetTempPauseDownload(false);
-	g_Options->SetTempPausePostprocess(false);
+	g_WorkState->SetTempPauseDownload(false);
+	g_WorkState->SetTempPausePostprocess(false);
 	m_waitingRequiredDir = false;
 }

daemon/main/DiskService.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,15 +22,19 @@
 #define DISKSERVICE_H
 
 #include "Service.h"
+#include "Observer.h"
 
-class DiskService : public Service
+class DiskService : public Service, public Observer
 {
+public:
+	DiskService();
+
 protected:
-	virtual int ServiceInterval() { return 200; }
+	virtual int ServiceInterval();
 	virtual void ServiceWork();
+	virtual void Update(Subject* caller, void* aspect);
 
 private:
-	int m_interval = 0;
 	bool m_waitingRequiredDir = true;
 	bool m_waitingReported = false;
 

daemon/main/Maintenance.cpp

@@ -68,7 +68,7 @@ Maintenance::~Maintenance()
 	{
 		while (m_updateScriptController)
 		{
-			usleep(20*1000);
+			Util::Sleep(20);
 		}
 	}
 }
@@ -122,7 +122,7 @@ bool Maintenance::StartUpdate(EBranch branch)
 	m_messages.clear();
 
 	m_updateScriptController = new UpdateScriptController();
-	m_updateScriptController->SetScript(m_updateScript);
+	m_updateScriptController->SetArgs({*m_updateScript});
 	m_updateScriptController->SetBranch(branch);
 	m_updateScriptController->SetAutoDestroy(true);
 
@@ -179,7 +179,7 @@ bool Maintenance::ReadPackageInfoStr(const char* key, CString& value)
 		return false;
 	}
 
-	int len = pend - p;
+	size_t len = pend - p;
 	if (len >= sizeof(fileName))
 	{
 		error("Could not parse file %s", *fileName);
@@ -274,7 +274,7 @@ void UpdateInfoScriptController::ExecuteScript(const char* script, CString& upda
 	detail("Executing update-info-script %s", FileSystem::BaseFileName(script));
 
 	UpdateInfoScriptController scriptController;
-	scriptController.SetScript(script);
+	scriptController.SetArgs({script});
 
 	BString<1024> infoName("update-info-script %s", FileSystem::BaseFileName(script));
 	scriptController.SetInfoName(infoName);
@@ -373,7 +373,7 @@ bool Signature::ReadSignature()
 			{
 				hexSig[sigLen - 2] = '\0'; // trim trailing ",
 			}
-			for (; *hexSig && *(hexSig+1);)
+			while (*hexSig && *(hexSig+1) && output != m_signature + sizeof(m_signature))
 			{
 				uchar c1 = *hexSig++;
 				uchar c2 = *hexSig++;

daemon/main/Options.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -59,25 +59,28 @@ static const char* OPTION_RESTRICTEDUSERNAME	= "RestrictedUsername";
 static const char* OPTION_RESTRICTEDPASSWORD	= "RestrictedPassword";
 static const char* OPTION_ADDUSERNAME			= "AddUsername";
 static const char* OPTION_ADDPASSWORD			= "AddPassword";
+static const char* OPTION_FORMAUTH				= "FormAuth";
 static const char* OPTION_SECURECONTROL			= "SecureControl";
 static const char* OPTION_SECUREPORT			= "SecurePort";
 static const char* OPTION_SECURECERT			= "SecureCert";
 static const char* OPTION_SECUREKEY				= "SecureKey";
+static const char* OPTION_CERTSTORE				= "CertStore";
+static const char* OPTION_CERTCHECK				= "CertCheck";
 static const char* OPTION_AUTHORIZEDIP			= "AuthorizedIP";
 static const char* OPTION_ARTICLETIMEOUT		= "ArticleTimeout";
 static const char* OPTION_URLTIMEOUT			= "UrlTimeout";
-static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_REMOTETIMEOUT			= "RemoteTimeout";
 static const char* OPTION_FLUSHQUEUE			= "FlushQueue";
-static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
-static const char* OPTION_BROKENLOG				= "BrokenLog";
 static const char* OPTION_NZBLOG				= "NzbLog";
-static const char* OPTION_DECODE				= "Decode";
-static const char* OPTION_RETRIES				= "Retries";
-static const char* OPTION_RETRYINTERVAL			= "RetryInterval";
-static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
+static const char* OPTION_RAWARTICLE			= "RawArticle";
+static const char* OPTION_SKIPWRITE				= "SkipWrite";
+static const char* OPTION_ARTICLERETRIES		= "ArticleRetries";
+static const char* OPTION_ARTICLEINTERVAL		= "ArticleInterval";
+static const char* OPTION_URLRETRIES			= "UrlRetries";
+static const char* OPTION_URLINTERVAL			= "UrlInterval";
 static const char* OPTION_CONTINUEPARTIAL		= "ContinuePartial";
 static const char* OPTION_URLCONNECTIONS		= "UrlConnections";
-static const char* OPTION_LOGBUFFERSIZE			= "LogBufferSize";
+static const char* OPTION_LOGBUFFER				= "LogBuffer";
 static const char* OPTION_INFOTARGET			= "InfoTarget";
 static const char* OPTION_WARNINGTARGET			= "WarningTarget";
 static const char* OPTION_ERRORTARGET			= "ErrorTarget";
@@ -87,13 +90,14 @@ static const char* OPTION_PARCHECK				= "ParCheck";
 static const char* OPTION_PARREPAIR				= "ParRepair";
 static const char* OPTION_PARSCAN				= "ParScan";
 static const char* OPTION_PARQUICK				= "ParQuick";
+static const char* OPTION_POSTSTRATEGY			= "PostStrategy";
+static const char* OPTION_FILENAMING			= "FileNaming";
 static const char* OPTION_PARRENAME				= "ParRename";
 static const char* OPTION_PARBUFFER				= "ParBuffer";
 static const char* OPTION_PARTHREADS			= "ParThreads";
+static const char* OPTION_RARRENAME				= "RarRename";
 static const char* OPTION_HEALTHCHECK			= "HealthCheck";
-static const char* OPTION_SCANSCRIPT			= "ScanScript";
-static const char* OPTION_QUEUESCRIPT			= "QueueScript";
-static const char* OPTION_FEEDSCRIPT			= "FeedScript";
+static const char* OPTION_DIRECTRENAME			= "DirectRename";
 static const char* OPTION_UMASK					= "UMask";
 static const char* OPTION_UPDATEINTERVAL		= "UpdateInterval";
 static const char* OPTION_CURSESNZBNAME			= "CursesNzbName";
@@ -104,32 +108,38 @@ static const char* OPTION_DIRECTWRITE			= "DirectWrite";
 static const char* OPTION_WRITEBUFFER			= "WriteBuffer";
 static const char* OPTION_NZBDIRINTERVAL		= "NzbDirInterval";
 static const char* OPTION_NZBDIRFILEAGE			= "NzbDirFileAge";
-static const char* OPTION_PARCLEANUPQUEUE		= "ParCleanupQueue";
 static const char* OPTION_DISKSPACE				= "DiskSpace";
-static const char* OPTION_DUMPCORE				= "DumpCore";
+static const char* OPTION_CRASHTRACE			= "CrashTrace";
+static const char* OPTION_CRASHDUMP				= "CrashDump";
 static const char* OPTION_PARPAUSEQUEUE			= "ParPauseQueue";
 static const char* OPTION_SCRIPTPAUSEQUEUE		= "ScriptPauseQueue";
 static const char* OPTION_NZBCLEANUPDISK		= "NzbCleanupDisk";
-static const char* OPTION_DELETECLEANUPDISK		= "DeleteCleanupDisk";
 static const char* OPTION_PARTIMELIMIT			= "ParTimeLimit";
 static const char* OPTION_KEEPHISTORY			= "KeepHistory";
-static const char* OPTION_ACCURATERATE			= "AccurateRate";
 static const char* OPTION_UNPACK				= "Unpack";
+static const char* OPTION_DIRECTUNPACK			= "DirectUnpack";
 static const char* OPTION_UNPACKCLEANUPDISK		= "UnpackCleanupDisk";
 static const char* OPTION_UNRARCMD				= "UnrarCmd";
 static const char* OPTION_SEVENZIPCMD			= "SevenZipCmd";
 static const char* OPTION_UNPACKPASSFILE		= "UnpackPassFile";
 static const char* OPTION_UNPACKPAUSEQUEUE		= "UnpackPauseQueue";
 static const char* OPTION_SCRIPTORDER			= "ScriptOrder";
-static const char* OPTION_POSTSCRIPT			= "PostScript";
+static const char* OPTION_EXTENSIONS			= "Extensions";
 static const char* OPTION_EXTCLEANUPDISK		= "ExtCleanupDisk";
 static const char* OPTION_PARIGNOREEXT			= "ParIgnoreExt";
+static const char* OPTION_UNPACKIGNOREEXT		= "UnpackIgnoreExt";
 static const char* OPTION_FEEDHISTORY			= "FeedHistory";
 static const char* OPTION_URLFORCE				= "UrlForce";
 static const char* OPTION_TIMECORRECTION		= "TimeCorrection";
 static const char* OPTION_PROPAGATIONDELAY		= "PropagationDelay";
 static const char* OPTION_ARTICLECACHE			= "ArticleCache";
 static const char* OPTION_EVENTINTERVAL			= "EventInterval";
+static const char* OPTION_SHELLOVERRIDE			= "ShellOverride";
+static const char* OPTION_MONTHLYQUOTA			= "MonthlyQuota";
+static const char* OPTION_QUOTASTARTDAY			= "QuotaStartDay";
+static const char* OPTION_DAILYQUOTA			= "DailyQuota";
+static const char* OPTION_REORDERFILES			= "ReorderFiles";
+static const char* OPTION_UPDATECHECK			= "UpdateCheck";
 
 // obsolete options
 static const char* OPTION_POSTLOGKIND			= "PostLogKind";
@@ -150,6 +160,19 @@ static const char* OPTION_NZBPROCESS			= "NZBProcess";
 static const char* OPTION_NZBADDEDPROCESS		= "NZBAddedProcess";
 static const char* OPTION_CREATELOG				= "CreateLog";
 static const char* OPTION_RESETLOG				= "ResetLog";
+static const char* OPTION_PARCLEANUPQUEUE		= "ParCleanupQueue";
+static const char* OPTION_DELETECLEANUPDISK		= "DeleteCleanupDisk";
+static const char* OPTION_HISTORYCLEANUPDISK	= "HistoryCleanupDisk";
+static const char* OPTION_SCANSCRIPT			= "ScanScript";
+static const char* OPTION_QUEUESCRIPT			= "QueueScript";
+static const char* OPTION_FEEDSCRIPT			= "FeedScript";
+static const char* OPTION_DECODE				= "Decode";
+static const char* OPTION_SAVEQUEUE				= "SaveQueue";
+static const char* OPTION_RELOADQUEUE			= "ReloadQueue";
+static const char* OPTION_TERMINATETIMEOUT		= "TerminateTimeout";
+static const char* OPTION_ACCURATERATE			= "AccurateRate";
+static const char* OPTION_CREATEBROKENLOG		= "CreateBrokenLog";
+static const char* OPTION_BROKENLOG				= "BrokenLog";
 
 const char* BoolNames[] = { "yes", "no", "true", "false", "1", "0", "on", "off", "enable", "disable", "enabled", "disabled" };
 const int BoolValues[] = { 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
@@ -183,10 +206,13 @@ bool Options::OptEntry::Restricted()
 
 	bool restricted = !strcasecmp(m_name, OPTION_CONTROLIP) ||
 		!strcasecmp(m_name, OPTION_CONTROLPORT) ||
+		!strcasecmp(m_name, OPTION_FORMAUTH) ||
 		!strcasecmp(m_name, OPTION_SECURECONTROL) ||
 		!strcasecmp(m_name, OPTION_SECUREPORT) ||
 		!strcasecmp(m_name, OPTION_SECURECERT) ||
 		!strcasecmp(m_name, OPTION_SECUREKEY) ||
+		!strcasecmp(m_name, OPTION_CERTSTORE) ||
+		!strcasecmp(m_name, OPTION_CERTCHECK) ||
 		!strcasecmp(m_name, OPTION_AUTHORIZEDIP) ||
 		!strcasecmp(m_name, OPTION_DAEMONUSERNAME) ||
 		!strcasecmp(m_name, OPTION_UMASK) ||
@@ -320,6 +346,7 @@ void Options::Init(const char* exeName, const char* configFilename, bool noConfi
 		return;
 	}
 
+	ConvertOldOptions(&m_optEntries);
 	InitOptions();
 	CheckOptions();
 
@@ -394,7 +421,7 @@ void Options::InitDefaults()
 	SetOption(OPTION_QUEUEDIR, "${MainDir}/queue");
 	SetOption(OPTION_NZBDIR, "${MainDir}/nzb");
 	SetOption(OPTION_LOCKFILE, "${MainDir}/nzbget.lock");
-	SetOption(OPTION_LOGFILE, "${DestDir}/nzbget.log");
+	SetOption(OPTION_LOGFILE, "${MainDir}/nzbget.log");
 	SetOption(OPTION_SCRIPTDIR, "${MainDir}/scripts");
 	SetOption(OPTION_REQUIREDDIR, "");
 	SetOption(OPTION_WRITELOG, "append");
@@ -411,25 +438,28 @@ void Options::InitDefaults()
 	SetOption(OPTION_ADDUSERNAME, "");
 	SetOption(OPTION_ADDPASSWORD, "");
 	SetOption(OPTION_CONTROLPORT, "6789");
+	SetOption(OPTION_FORMAUTH, "no");
 	SetOption(OPTION_SECURECONTROL, "no");
 	SetOption(OPTION_SECUREPORT, "6791");
 	SetOption(OPTION_SECURECERT, "");
 	SetOption(OPTION_SECUREKEY, "");
+	SetOption(OPTION_CERTSTORE, "");
+	SetOption(OPTION_CERTCHECK, "no");
 	SetOption(OPTION_AUTHORIZEDIP, "");
 	SetOption(OPTION_ARTICLETIMEOUT, "60");
 	SetOption(OPTION_URLTIMEOUT, "60");
-	SetOption(OPTION_SAVEQUEUE, "yes");
+	SetOption(OPTION_REMOTETIMEOUT, "90");
 	SetOption(OPTION_FLUSHQUEUE, "yes");
-	SetOption(OPTION_RELOADQUEUE, "yes");
-	SetOption(OPTION_BROKENLOG, "yes");
 	SetOption(OPTION_NZBLOG, "yes");
-	SetOption(OPTION_DECODE, "yes");
-	SetOption(OPTION_RETRIES, "3");
-	SetOption(OPTION_RETRYINTERVAL, "10");
-	SetOption(OPTION_TERMINATETIMEOUT, "600");
+	SetOption(OPTION_RAWARTICLE, "no");
+	SetOption(OPTION_SKIPWRITE, "no");
+	SetOption(OPTION_ARTICLERETRIES, "3");
+	SetOption(OPTION_ARTICLEINTERVAL, "10");
+	SetOption(OPTION_URLRETRIES, "3");
+	SetOption(OPTION_URLINTERVAL, "10");
 	SetOption(OPTION_CONTINUEPARTIAL, "no");
 	SetOption(OPTION_URLCONNECTIONS, "4");
-	SetOption(OPTION_LOGBUFFERSIZE, "1000");
+	SetOption(OPTION_LOGBUFFER, "1000");
 	SetOption(OPTION_INFOTARGET, "both");
 	SetOption(OPTION_WARNINGTARGET, "both");
 	SetOption(OPTION_ERRORTARGET, "both");
@@ -439,15 +469,16 @@ void Options::InitDefaults()
 	SetOption(OPTION_PARREPAIR, "yes");
 	SetOption(OPTION_PARSCAN, "extended");
 	SetOption(OPTION_PARQUICK, "yes");
+	SetOption(OPTION_POSTSTRATEGY, "sequential");
+	SetOption(OPTION_FILENAMING, "article");
 	SetOption(OPTION_PARRENAME, "yes");
 	SetOption(OPTION_PARBUFFER, "16");
 	SetOption(OPTION_PARTHREADS, "1");
+	SetOption(OPTION_RARRENAME, "yes");
 	SetOption(OPTION_HEALTHCHECK, "none");
+	SetOption(OPTION_DIRECTRENAME, "no");
 	SetOption(OPTION_SCRIPTORDER, "");
-	SetOption(OPTION_POSTSCRIPT, "");
-	SetOption(OPTION_SCANSCRIPT, "");
-	SetOption(OPTION_QUEUESCRIPT, "");
-	SetOption(OPTION_FEEDSCRIPT, "");
+	SetOption(OPTION_EXTENSIONS, "");
 	SetOption(OPTION_DAEMONUSERNAME, "root");
 	SetOption(OPTION_UMASK, "1000");
 	SetOption(OPTION_UPDATEINTERVAL, "200");
@@ -459,17 +490,16 @@ void Options::InitDefaults()
 	SetOption(OPTION_WRITEBUFFER, "0");
 	SetOption(OPTION_NZBDIRINTERVAL, "5");
 	SetOption(OPTION_NZBDIRFILEAGE, "60");
-	SetOption(OPTION_PARCLEANUPQUEUE, "yes");
 	SetOption(OPTION_DISKSPACE, "250");
-	SetOption(OPTION_DUMPCORE, "no");
+	SetOption(OPTION_CRASHTRACE, "no");
+	SetOption(OPTION_CRASHDUMP, "no");
 	SetOption(OPTION_PARPAUSEQUEUE, "no");
 	SetOption(OPTION_SCRIPTPAUSEQUEUE, "no");
 	SetOption(OPTION_NZBCLEANUPDISK, "no");
-	SetOption(OPTION_DELETECLEANUPDISK, "no");
 	SetOption(OPTION_PARTIMELIMIT, "0");
 	SetOption(OPTION_KEEPHISTORY, "7");
-	SetOption(OPTION_ACCURATERATE, "no");
 	SetOption(OPTION_UNPACK, "no");
+	SetOption(OPTION_DIRECTUNPACK, "no");
 	SetOption(OPTION_UNPACKCLEANUPDISK, "no");
 #ifdef WIN32
 	SetOption(OPTION_UNRARCMD, "unrar.exe");
@@ -482,12 +512,19 @@ void Options::InitDefaults()
 	SetOption(OPTION_UNPACKPAUSEQUEUE, "no");
 	SetOption(OPTION_EXTCLEANUPDISK, "");
 	SetOption(OPTION_PARIGNOREEXT, "");
+	SetOption(OPTION_UNPACKIGNOREEXT, "");
 	SetOption(OPTION_FEEDHISTORY, "7");
 	SetOption(OPTION_URLFORCE, "yes");
 	SetOption(OPTION_TIMECORRECTION, "0");
 	SetOption(OPTION_PROPAGATIONDELAY, "0");
 	SetOption(OPTION_ARTICLECACHE, "0");
 	SetOption(OPTION_EVENTINTERVAL, "0");
+	SetOption(OPTION_SHELLOVERRIDE, "");
+	SetOption(OPTION_MONTHLYQUOTA, "0");
+	SetOption(OPTION_QUOTASTARTDAY, "1");
+	SetOption(OPTION_DAILYQUOTA, "0");
+	SetOption(OPTION_REORDERFILES, "no");
+	SetOption(OPTION_UPDATECHECK, "none");
 }
 
 void Options::InitOptFile()
@@ -605,8 +642,6 @@ void Options::CheckDir(CString& dir, const char* optionName,
 
 		FileSystem::NormalizePathSeparators((char*)usedir2);
 		dir = usedir2;
-
-		usedir2[usedir2.Length() - 1] = '\0';
 		SetOption(optionName, usedir2);
 	}
 
@@ -634,10 +669,7 @@ void Options::InitOptions()
 
 	m_configTemplate		= GetOption(OPTION_CONFIGTEMPLATE);
 	m_scriptOrder			= GetOption(OPTION_SCRIPTORDER);
-	m_postScript			= GetOption(OPTION_POSTSCRIPT);
-	m_scanScript			= GetOption(OPTION_SCANSCRIPT);
-	m_queueScript			= GetOption(OPTION_QUEUESCRIPT);
-	m_feedScript			= GetOption(OPTION_FEEDSCRIPT);
+	m_extensions			= GetOption(OPTION_EXTENSIONS);
 	m_controlIp				= GetOption(OPTION_CONTROLIP);
 	m_controlUsername		= GetOption(OPTION_CONTROLUSERNAME);
 	m_controlPassword		= GetOption(OPTION_CONTROLPASSWORD);
@@ -647,6 +679,7 @@ void Options::InitOptions()
 	m_addPassword			= GetOption(OPTION_ADDPASSWORD);
 	m_secureCert			= GetOption(OPTION_SECURECERT);
 	m_secureKey				= GetOption(OPTION_SECUREKEY);
+	m_certStore				= GetOption(OPTION_CERTSTORE);
 	m_authorizedIp			= GetOption(OPTION_AUTHORIZEDIP);
 	m_lockFile				= GetOption(OPTION_LOCKFILE);
 	m_daemonUsername		= GetOption(OPTION_DAEMONUSERNAME);
@@ -656,17 +689,21 @@ void Options::InitOptions()
 	m_unpackPassFile		= GetOption(OPTION_UNPACKPASSFILE);
 	m_extCleanupDisk		= GetOption(OPTION_EXTCLEANUPDISK);
 	m_parIgnoreExt			= GetOption(OPTION_PARIGNOREEXT);
+	m_unpackIgnoreExt		= GetOption(OPTION_UNPACKIGNOREEXT);
+	m_shellOverride			= GetOption(OPTION_SHELLOVERRIDE);
 
 	m_downloadRate			= ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024;
 	m_articleTimeout		= ParseIntValue(OPTION_ARTICLETIMEOUT, 10);
 	m_urlTimeout			= ParseIntValue(OPTION_URLTIMEOUT, 10);
-	m_terminateTimeout		= ParseIntValue(OPTION_TERMINATETIMEOUT, 10);
-	m_retries				= ParseIntValue(OPTION_RETRIES, 10);
-	m_retryInterval			= ParseIntValue(OPTION_RETRYINTERVAL, 10);
+	m_remoteTimeout			= ParseIntValue(OPTION_REMOTETIMEOUT, 10);
+	m_articleRetries		= ParseIntValue(OPTION_ARTICLERETRIES, 10);
+	m_articleInterval		= ParseIntValue(OPTION_ARTICLEINTERVAL, 10);
+	m_urlRetries			= ParseIntValue(OPTION_URLRETRIES, 10);
+	m_urlInterval			= ParseIntValue(OPTION_URLINTERVAL, 10);
 	m_controlPort			= ParseIntValue(OPTION_CONTROLPORT, 10);
 	m_securePort			= ParseIntValue(OPTION_SECUREPORT, 10);
 	m_urlConnections		= ParseIntValue(OPTION_URLCONNECTIONS, 10);
-	m_logBufferSize			= ParseIntValue(OPTION_LOGBUFFERSIZE, 10);
+	m_logBuffer				= ParseIntValue(OPTION_LOGBUFFER, 10);
 	m_rotateLog				= ParseIntValue(OPTION_ROTATELOG, 10);
 	m_umask					= ParseIntValue(OPTION_UMASK, 8);
 	m_updateInterval		= ParseIntValue(OPTION_UPDATEINTERVAL, 10);
@@ -688,36 +725,41 @@ void Options::InitOptions()
 	m_eventInterval			= ParseIntValue(OPTION_EVENTINTERVAL, 10);
 	m_parBuffer				= ParseIntValue(OPTION_PARBUFFER, 10);
 	m_parThreads			= ParseIntValue(OPTION_PARTHREADS, 10);
+	m_monthlyQuota			= ParseIntValue(OPTION_MONTHLYQUOTA, 10);
+	m_quotaStartDay			= ParseIntValue(OPTION_QUOTASTARTDAY, 10);
+	m_dailyQuota			= ParseIntValue(OPTION_DAILYQUOTA, 10);
 
-	m_brokenLog				= (bool)ParseEnumValue(OPTION_BROKENLOG, BoolCount, BoolNames, BoolValues);
 	m_nzbLog				= (bool)ParseEnumValue(OPTION_NZBLOG, BoolCount, BoolNames, BoolValues);
 	m_appendCategoryDir		= (bool)ParseEnumValue(OPTION_APPENDCATEGORYDIR, BoolCount, BoolNames, BoolValues);
 	m_continuePartial		= (bool)ParseEnumValue(OPTION_CONTINUEPARTIAL, BoolCount, BoolNames, BoolValues);
-	m_saveQueue				= (bool)ParseEnumValue(OPTION_SAVEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_flushQueue			= (bool)ParseEnumValue(OPTION_FLUSHQUEUE, BoolCount, BoolNames, BoolValues);
 	m_dupeCheck				= (bool)ParseEnumValue(OPTION_DUPECHECK, BoolCount, BoolNames, BoolValues);
 	m_parRepair				= (bool)ParseEnumValue(OPTION_PARREPAIR, BoolCount, BoolNames, BoolValues);
 	m_parQuick				= (bool)ParseEnumValue(OPTION_PARQUICK, BoolCount, BoolNames, BoolValues);
 	m_parRename				= (bool)ParseEnumValue(OPTION_PARRENAME, BoolCount, BoolNames, BoolValues);
-	m_reloadQueue			= (bool)ParseEnumValue(OPTION_RELOADQUEUE, BoolCount, BoolNames, BoolValues);
+	m_rarRename				= (bool)ParseEnumValue(OPTION_RARRENAME, BoolCount, BoolNames, BoolValues);
+	m_directRename			= (bool)ParseEnumValue(OPTION_DIRECTRENAME, BoolCount, BoolNames, BoolValues);
 	m_cursesNzbName			= (bool)ParseEnumValue(OPTION_CURSESNZBNAME, BoolCount, BoolNames, BoolValues);
 	m_cursesTime			= (bool)ParseEnumValue(OPTION_CURSESTIME, BoolCount, BoolNames, BoolValues);
 	m_cursesGroup			= (bool)ParseEnumValue(OPTION_CURSESGROUP, BoolCount, BoolNames, BoolValues);
 	m_crcCheck				= (bool)ParseEnumValue(OPTION_CRCCHECK, BoolCount, BoolNames, BoolValues);
 	m_directWrite			= (bool)ParseEnumValue(OPTION_DIRECTWRITE, BoolCount, BoolNames, BoolValues);
-	m_parCleanupQueue		= (bool)ParseEnumValue(OPTION_PARCLEANUPQUEUE, BoolCount, BoolNames, BoolValues);
-	m_decode				= (bool)ParseEnumValue(OPTION_DECODE, BoolCount, BoolNames, BoolValues);
-	m_dumpCore				= (bool)ParseEnumValue(OPTION_DUMPCORE, BoolCount, BoolNames, BoolValues);
+	m_rawArticle			= (bool)ParseEnumValue(OPTION_RAWARTICLE, BoolCount, BoolNames, BoolValues);
+	m_skipWrite				= (bool)ParseEnumValue(OPTION_SKIPWRITE, BoolCount, BoolNames, BoolValues);
+	m_crashTrace			= (bool)ParseEnumValue(OPTION_CRASHTRACE, BoolCount, BoolNames, BoolValues);
+	m_crashDump				= (bool)ParseEnumValue(OPTION_CRASHDUMP, BoolCount, BoolNames, BoolValues);
 	m_parPauseQueue			= (bool)ParseEnumValue(OPTION_PARPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_scriptPauseQueue		= (bool)ParseEnumValue(OPTION_SCRIPTPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_nzbCleanupDisk		= (bool)ParseEnumValue(OPTION_NZBCLEANUPDISK, BoolCount, BoolNames, BoolValues);
-	m_deleteCleanupDisk		= (bool)ParseEnumValue(OPTION_DELETECLEANUPDISK, BoolCount, BoolNames, BoolValues);
-	m_accurateRate			= (bool)ParseEnumValue(OPTION_ACCURATERATE, BoolCount, BoolNames, BoolValues);
+	m_formAuth				= (bool)ParseEnumValue(OPTION_FORMAUTH, BoolCount, BoolNames, BoolValues);
 	m_secureControl			= (bool)ParseEnumValue(OPTION_SECURECONTROL, BoolCount, BoolNames, BoolValues);
 	m_unpack				= (bool)ParseEnumValue(OPTION_UNPACK, BoolCount, BoolNames, BoolValues);
+	m_directUnpack			= (bool)ParseEnumValue(OPTION_DIRECTUNPACK, BoolCount, BoolNames, BoolValues);
 	m_unpackCleanupDisk		= (bool)ParseEnumValue(OPTION_UNPACKCLEANUPDISK, BoolCount, BoolNames, BoolValues);
 	m_unpackPauseQueue		= (bool)ParseEnumValue(OPTION_UNPACKPAUSEQUEUE, BoolCount, BoolNames, BoolValues);
 	m_urlForce				= (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues);
+	m_certCheck				= (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues);
+	m_reorderFiles			= (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues);
 
 	const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" };
 	const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses };
@@ -734,9 +776,19 @@ void Options::InitOptions()
 	const int ParScanCount = 4;
 	m_parScan = (EParScan)ParseEnumValue(OPTION_PARSCAN, ParScanCount, ParScanNames, ParScanValues);
 
-	const char* HealthCheckNames[] = { "pause", "delete", "none" };
-	const int HealthCheckValues[] = { hcPause, hcDelete, hcNone };
-	const int HealthCheckCount = 3;
+	const char* PostStrategyNames[] = { "sequential", "balanced", "aggressive", "rocket" };
+	const int PostStrategyValues[] = { ppSequential, ppBalanced, ppAggressive, ppRocket };
+	const int PostStrategyCount = 4;
+	m_postStrategy = (EPostStrategy)ParseEnumValue(OPTION_POSTSTRATEGY, PostStrategyCount, PostStrategyNames, PostStrategyValues);
+
+	const char* FileNamingNames[] = { "auto", "article", "nzb" };
+	const int FileNamingValues[] = { nfAuto, nfArticle, nfNzb };
+	const int FileNamingCount = 4;
+	m_fileNaming = (EFileNaming)ParseEnumValue(OPTION_FILENAMING, FileNamingCount, FileNamingNames, FileNamingValues);
+
+	const char* HealthCheckNames[] = { "pause", "delete", "park", "none" };
+	const int HealthCheckValues[] = { hcPause, hcDelete, hcPark, hcNone };
+	const int HealthCheckCount = 4;
 	m_healthCheck = (EHealthCheck)ParseEnumValue(OPTION_HEALTHCHECK, HealthCheckCount, HealthCheckNames, HealthCheckValues);
 
 	const char* TargetNames[] = { "screen", "log", "both", "none" };
@@ -863,7 +915,7 @@ void Options::SetOption(const char* optname, const char* value)
 			const char* varvalue = GetOption(variable);
 			if (varvalue)
 			{
-				curvalue.Replace(dollar - curvalue, 2 + varlen + 1, varvalue);
+				curvalue.Replace((int)(dollar - curvalue), 2 + varlen + 1, varvalue);
 			}
 			else
 			{
@@ -933,6 +985,13 @@ void Options::InitServers()
 			joinGroup = (bool)ParseEnumValue(BString<100>("Server%i.JoinGroup", n), BoolCount, BoolNames, BoolValues);
 		}
 
+		const char* noptional = GetOption(BString<100>("Server%i.Optional", n));
+		bool optional = false;
+		if (noptional)
+		{
+			optional = (bool)ParseEnumValue(BString<100>("Server%i.Optional", n), BoolCount, BoolNames, BoolValues);
+		}
+
 		const char* ntls = GetOption(BString<100>("Server%i.Encryption", n));
 		bool tls = false;
 		if (ntls)
@@ -941,18 +1000,29 @@ void Options::InitServers()
 #ifdef DISABLE_TLS
 			if (tls)
 			{
-				ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", optname);
+				ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support",
+					*BString<100>("Server%i.Encryption", n));
 				tls = false;
 			}
 #endif
 			m_tls |= tls;
 		}
 
+		const char* nipversion = GetOption(BString<100>("Server%i.IpVersion", n));
+		int ipversion = 0;
+		if (nipversion)
+		{
+			const char* IpVersionNames[] = {"auto", "ipv4", "ipv6"};
+			const int IpVersionValues[] = {0, 4, 6};
+			const int IpVersionCount = 3;
+			ipversion = ParseEnumValue(BString<100>("Server%i.IpVersion", n), IpVersionCount, IpVersionNames, IpVersionValues);
+		}
+
 		const char* ncipher = GetOption(BString<100>("Server%i.Cipher", n));
 		const char* nconnections = GetOption(BString<100>("Server%i.Connections", n));
 		const char* nretention = GetOption(BString<100>("Server%i.Retention", n));
 
-		bool definition = nactive || nname || nlevel || ngroup || nhost || nport ||
+		bool definition = nactive || nname || nlevel || ngroup || nhost || nport || noptional ||
 			nusername || npassword || nconnections || njoingroup || ntls || ncipher || nretention;
 		bool completed = nhost && nport && nconnections;
 
@@ -968,12 +1038,14 @@ void Options::InitServers()
 				m_extender->AddNewsServer(n, active, nname,
 					nhost,
 					nport ? atoi(nport) : 119,
+					ipversion,
 					nusername, npassword,
 					joinGroup, tls, ncipher,
 					nconnections ? atoi(nconnections) : 1,
 					nretention ? atoi(nretention) : 0,
 					nlevel ? atoi(nlevel) : 0,
-					ngroup ? atoi(ngroup) : 0);
+					ngroup ? atoi(ngroup) : 0,
+					optional);
 			}
 		}
 		else
@@ -1000,10 +1072,10 @@ void Options::InitCategories()
 			unpack = (bool)ParseEnumValue(BString<100>("Category%i.Unpack", n), BoolCount, BoolNames, BoolValues);
 		}
 
-		const char* npostscript = GetOption(BString<100>("Category%i.PostScript", n));
+		const char* nextensions = GetOption(BString<100>("Category%i.Extensions", n));
 		const char* naliases = GetOption(BString<100>("Category%i.Aliases", n));
 
-		bool definition = nname || ndestdir || nunpack || npostscript || naliases;
+		bool definition = nname || ndestdir || nunpack || nextensions || naliases;
 		bool completed = nname && strlen(nname) > 0;
 
 		if (!definition)
@@ -1019,7 +1091,7 @@ void Options::InitCategories()
 				CheckDir(destDir, BString<100>("Category%i.DestDir", n), m_destDir, false, false);
 			}
 
-			m_categories.emplace_back(nname, destDir, unpack, npostscript);
+			m_categories.emplace_back(nname, destDir, unpack, nextensions);
 			Category& category = m_categories.back();
 
 			// split Aliases into tokens and create items for each token
@@ -1050,7 +1122,7 @@ void Options::InitFeeds()
 		const char* nurl = GetOption(BString<100>("Feed%i.URL", n));
 		const char* nfilter = GetOption(BString<100>("Feed%i.Filter", n));
 		const char* ncategory = GetOption(BString<100>("Feed%i.Category", n));
-		const char* nfeedscript = GetOption(BString<100>("Feed%i.FeedScript", n));
+		const char* nextensions = GetOption(BString<100>("Feed%i.Extensions", n));
 
 		const char* nbacklog = GetOption(BString<100>("Feed%i.Backlog", n));
 		bool backlog = true;
@@ -1070,7 +1142,7 @@ void Options::InitFeeds()
 		const char* npriority = GetOption(BString<100>("Feed%i.Priority", n));
 
 		bool definition = nname || nurl || nfilter || ncategory || nbacklog || npausenzb ||
-			ninterval || npriority || nfeedscript;
+			ninterval || npriority || nextensions;
 		bool completed = nurl;
 
 		if (!definition)
@@ -1083,7 +1155,7 @@ void Options::InitFeeds()
 			if (m_extender)
 			{
 				m_extender->AddFeed(n, nname, nurl, ninterval ? atoi(ninterval) : 0, nfilter,
-					backlog, pauseNzb, ncategory, npriority ? atoi(npriority) : 0, nfeedscript);
+					backlog, pauseNzb, ncategory, npriority ? atoi(npriority) : 0, nextensions);
 			}
 		}
 		else
@@ -1152,13 +1224,6 @@ void Options::InitScheduler()
 			continue;
 		}
 
-		int weekDaysVal = 0;
-		if (weekDays && !ParseWeekDays(weekDays, &weekDaysVal))
-		{
-			ConfigError("Invalid value for option \"Task%i.WeekDays\": \"%s\"", n, weekDays);
-			continue;
-		}
-
 		if (taskCommand == scDownloadRate)
 		{
 			if (param)
@@ -1189,36 +1254,60 @@ void Options::InitScheduler()
 			continue;
 		}
 
-		int hours, minutes;
-		Tokenizer tok(time, ";,");
-		while (const char* oneTime = tok.Next())
-		{
-			if (!ParseTime(oneTime, &hours, &minutes))
-			{
-				ConfigError("Invalid value for option \"Task%i.Time\": \"%s\"", n, oneTime);
-				break;
-			}
+		CreateSchedulerTask(n, time, weekDays, taskCommand, param);
+	}
+}
 
-			if (m_extender)
+void Options::CreateSchedulerTask(int id, const char* time, const char* weekDays,
+	ESchedulerCommand command, const char* param)
+{
+	if (!id)
+	{
+		m_configLine = 0;
+	}
+
+	int weekDaysVal = 0;
+	if (weekDays && !ParseWeekDays(weekDays, &weekDaysVal))
+	{
+		ConfigError("Invalid value for option \"Task%i.WeekDays\": \"%s\"", id, weekDays);
+		return;
+	}
+
+	int hours, minutes;
+	Tokenizer tok(time, ";,");
+	while (const char* oneTime = tok.Next())
+	{
+		if (!ParseTime(oneTime, &hours, &minutes))
+		{
+			ConfigError("Invalid value for option \"Task%i.Time\": \"%s\"", id, oneTime);
+			return;
+		}
+
+		if (m_extender)
+		{
+			if (hours == -2)
 			{
-				if (hours == -1)
+				for (int everyHour = 0; everyHour < 24; everyHour++)
 				{
-					for (int everyHour = 0; everyHour < 24; everyHour++)
-					{
-						m_extender->AddTask(n, everyHour, minutes, weekDaysVal, taskCommand, param);
-					}
-				}
-				else
-				{
-					m_extender->AddTask(n, hours, minutes, weekDaysVal, taskCommand, param);
+					m_extender->AddTask(id, everyHour, minutes, weekDaysVal, command, param);
 				}
 			}
+			else
+			{
+				m_extender->AddTask(id, hours, minutes, weekDaysVal, command, param);
+			}
 		}
 	}
 }
 
 bool Options::ParseTime(const char* time, int* hours, int* minutes)
 {
+	if (!strcmp(time, "*"))
+	{
+		*hours = -1;
+		return true;
+	}
+
 	int colons = 0;
 	const char* p = time;
 	while (*p)
@@ -1247,7 +1336,7 @@ bool Options::ParseTime(const char* time, int* hours, int* minutes)
 
 	if (time[0] == '*')
 	{
-		*hours = -1;
+		*hours = -2;
 	}
 	else
 	{
@@ -1411,7 +1500,7 @@ bool Options::SplitOptionString(const char* option, CString& optName, CString& o
 		return false;
 	}
 
-	optName.Set(option, eq - option);
+	optName.Set(option, (int)(eq - option));
 	optValue.Set(eq + 1);
 
 	ConvertOldOption(optName, optValue);
@@ -1446,7 +1535,8 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 			!strcasecmp(p, ".password") || !strcasecmp(p, ".joingroup") ||
 			!strcasecmp(p, ".encryption") || !strcasecmp(p, ".connections") ||
 			!strcasecmp(p, ".cipher") || !strcasecmp(p, ".group") ||
-			!strcasecmp(p, ".retention")))
+			!strcasecmp(p, ".retention") || !strcasecmp(p, ".optional") ||
+			!strcasecmp(p, ".notes") || !strcasecmp(p, ".ipversion")))
 		{
 			return true;
 		}
@@ -1468,7 +1558,7 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 	{
 		char* p = (char*)optname + 8;
 		while (*p >= '0' && *p <= '9') p++;
-		if (p && (!strcasecmp(p, ".name") || !strcasecmp(p, ".destdir") || !strcasecmp(p, ".postscript") ||
+		if (p && (!strcasecmp(p, ".name") || !strcasecmp(p, ".destdir") || !strcasecmp(p, ".extensions") ||
 			!strcasecmp(p, ".unpack") || !strcasecmp(p, ".aliases")))
 		{
 			return true;
@@ -1481,7 +1571,7 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 		while (*p >= '0' && *p <= '9') p++;
 		if (p && (!strcasecmp(p, ".name") || !strcasecmp(p, ".url") || !strcasecmp(p, ".interval") ||
 			 !strcasecmp(p, ".filter") || !strcasecmp(p, ".backlog") || !strcasecmp(p, ".pausenzb") ||
-			 !strcasecmp(p, ".category") || !strcasecmp(p, ".priority") || !strcasecmp(p, ".feedscript")))
+			 !strcasecmp(p, ".category") || !strcasecmp(p, ".priority") || !strcasecmp(p, ".extensions")))
 		{
 			return true;
 		}
@@ -1506,26 +1596,41 @@ bool Options::ValidateOptionName(const char* optname, const char* optvalue)
 		!strcasecmp(optname, OPTION_MERGENZB) ||
 		!strcasecmp(optname, OPTION_STRICTPARNAME) ||
 		!strcasecmp(optname, OPTION_RELOADURLQUEUE) ||
-		!strcasecmp(optname, OPTION_RELOADPOSTQUEUE))
+		!strcasecmp(optname, OPTION_RELOADPOSTQUEUE) ||
+		!strcasecmp(optname, OPTION_PARCLEANUPQUEUE) ||
+		!strcasecmp(optname, OPTION_DELETECLEANUPDISK) ||
+		!strcasecmp(optname, OPTION_HISTORYCLEANUPDISK) ||
+		!strcasecmp(optname, OPTION_SAVEQUEUE) ||
+		!strcasecmp(optname, OPTION_RELOADQUEUE) ||
+		!strcasecmp(optname, OPTION_TERMINATETIMEOUT) ||
+		!strcasecmp(optname, OPTION_ACCURATERATE) ||
+		!strcasecmp(optname, OPTION_CREATEBROKENLOG) ||
+		!strcasecmp(optname, OPTION_BROKENLOG))
 	{
 		ConfigWarn("Option \"%s\" is obsolete, ignored", optname);
 		return true;
 	}
+
 	if (!strcasecmp(optname, OPTION_POSTPROCESS) ||
 		!strcasecmp(optname, OPTION_NZBPROCESS) ||
 		!strcasecmp(optname, OPTION_NZBADDEDPROCESS))
 	{
 		if (optvalue && strlen(optvalue) > 0)
 		{
-			ConfigError("Option \"%s\" is obsolete, ignored, use \"%s\" and \"%s\" instead", optname, OPTION_SCRIPTDIR,
-				!strcasecmp(optname, OPTION_POSTPROCESS) ? OPTION_POSTSCRIPT :
-				!strcasecmp(optname, OPTION_NZBPROCESS) ? OPTION_SCANSCRIPT :
-				!strcasecmp(optname, OPTION_NZBADDEDPROCESS) ? OPTION_QUEUESCRIPT :
-				"ERROR");
+			ConfigError("Option \"%s\" is obsolete, ignored, use \"%s\" and \"%s\" instead",
+				optname, OPTION_SCRIPTDIR, OPTION_EXTENSIONS);
 		}
 		return true;
 	}
 
+	if (!strcasecmp(optname, OPTION_SCANSCRIPT) ||
+		!strcasecmp(optname, OPTION_QUEUESCRIPT) ||
+		!strcasecmp(optname, OPTION_FEEDSCRIPT))
+	{
+		// will be automatically converted into "Extensions"
+		return true;
+	}
+
 	if (!strcasecmp(optname, OPTION_CREATELOG) || !strcasecmp(optname, OPTION_RESETLOG))
 	{
 		ConfigWarn("Option \"%s\" is obsolete, ignored, use \"%s\" instead", optname, OPTION_WRITELOG);
@@ -1579,16 +1684,22 @@ void Options::ConvertOldOption(CString& option, CString& value)
 		value = "extended";
 	}
 
-	if (!strcasecmp(option, "DefScript"))
+	if (!strcasecmp(option, "DefScript") || !strcasecmp(option, "PostScript"))
 	{
-		option = "PostScript";
+		option = "Extensions";
 	}
 
 	int nameLen = strlen(option);
-	if (!strncasecmp(option, "Category", 8) && nameLen > 10 &&
-		!strcasecmp(option + nameLen - 10, ".DefScript"))
+	if (!strncasecmp(option, "Category", 8) &&
+		((nameLen > 10 && !strcasecmp(option + nameLen - 10, ".DefScript")) ||
+		 (nameLen > 11 && !strcasecmp(option + nameLen - 11, ".PostScript"))))
 	{
-		option.Replace(".DefScript", ".PostScript");
+		option.Replace(".DefScript", ".Extensions");
+		option.Replace(".PostScript", ".Extensions");
+	}
+	if (!strncasecmp(option, "Feed", 4) && nameLen > 11 && !strcasecmp(option + nameLen - 11, ".FeedScript"))
+	{
+		option.Replace(".FeedScript", ".Extensions");
 	}
 
 	if (!strcasecmp(option, "WriteBufferSize"))
@@ -1604,9 +1715,30 @@ void Options::ConvertOldOption(CString& option, CString& value)
 		option = "ArticleTimeout";
 	}
 
-	if (!strcasecmp(option, "CreateBrokenLog"))
+	if (!strcasecmp(option, "Retries"))
 	{
-		option = "BrokenLog";
+		option = "ArticleRetries";
+	}
+
+	if (!strcasecmp(option, "RetryInterval"))
+	{
+		option = "ArticleInterval";
+	}
+
+	if (!strcasecmp(option, "DumpCore"))
+	{
+		option = OPTION_CRASHDUMP;
+	}
+
+	if (!strcasecmp(option, OPTION_DECODE))
+	{
+		option = OPTION_RAWARTICLE;
+		value = !strcasecmp(value, "no") ? "yes" : "no";
+	}
+
+	if (!strcasecmp(option, "LogBufferSize"))
+	{
+		option = OPTION_LOGBUFFER;
 	}
 }
 
@@ -1623,6 +1755,11 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_PARRENAME);
 		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_PARRENAME);
 	}
+	if (m_directRename)
+	{
+		LocateOptionSrcPos(OPTION_DIRECTRENAME);
+		ConfigError("Invalid value for option \"%s\": program was compiled without parcheck-support", OPTION_DIRECTRENAME);
+	}
 #endif
 
 #ifdef DISABLE_CURSES
@@ -1639,13 +1776,53 @@ void Options::CheckOptions()
 		LocateOptionSrcPos(OPTION_SECURECONTROL);
 		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_SECURECONTROL);
 	}
+
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Invalid value for option \"%s\": program was compiled without TLS/SSL-support", OPTION_CERTCHECK);
+	}
 #endif
 
-	if (!m_decode)
+#ifdef HAVE_OPENSSL
+#ifndef HAVE_X509_CHECK_HOST
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is limited because the program "
+			"was compiled with older OpenSSL version not supporting hostname validation (at least OpenSSL 1.0.2d is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+#ifdef HAVE_LIBGNUTLS
+#if	GNUTLS_VERSION_NUMBER < 0x030104
+	if (m_certCheck)
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigWarn("TLS certificate verification (option \"%s\") is disabled because the program "
+			"was compiled with older GnuTLS version not supporting certificate validation (at least GnuTLS 3.1.4 is required)", OPTION_CERTCHECK);
+	}
+#endif
+#endif
+
+	if (m_certCheck && m_certStore.Empty())
+	{
+		LocateOptionSrcPos(OPTION_CERTCHECK);
+		ConfigError("Option \"%s\" requires proper configuration of option \"%s\"", OPTION_CERTCHECK, OPTION_CERTSTORE);
+		m_certCheck = false;
+	}
+
+	if (m_rawArticle)
 	{
 		m_directWrite = false;
 	}
 
+	if (m_skipWrite)
+	{
+		m_directRename = false;
+	}
+
 	// if option "ConfigTemplate" is not set, use "WebDir" as default location for template
 	// (for compatibility with versions 9 and 10).
 	if (m_configTemplate.Empty() && !m_noDiskAccess)
@@ -1675,7 +1852,7 @@ void Options::CheckOptions()
 	if (sizeof(void*) == 4 && m_parBuffer + m_articleCache > 1900)
 	{
 		ConfigError("Options \"ArticleCache\" and \"ParBuffer\" in total cannot use more than 1900MB of memory in 32-Bit mode. Changed to 1500 and 400");
-		m_articleCache = 1900;
+		m_articleCache = 1500;
 		m_parBuffer = 400;
 	}
 
@@ -1684,3 +1861,77 @@ void Options::CheckOptions()
 		ConfigError("Invalid value for option \"UnpackPassFile\": %s. File not found", *m_unpackPassFile);
 	}
 }
+
+void Options::ConvertOldOptions(OptEntries* optEntries)
+{
+	MergeOldScriptOption(optEntries, OPTION_SCANSCRIPT, true);
+	MergeOldScriptOption(optEntries, OPTION_QUEUESCRIPT, true);
+	MergeOldScriptOption(optEntries, OPTION_FEEDSCRIPT, false);
+}
+
+void Options::MergeOldScriptOption(OptEntries* optEntries, const char* optname, bool mergeCategories)
+{
+	OptEntry* optEntry = optEntries->FindOption(optname);
+	if (!optEntry || Util::EmptyStr(optEntry->GetValue()))
+	{
+		return;
+	}
+
+	OptEntry* extensionsOpt = optEntries->FindOption(OPTION_EXTENSIONS);
+	if (!extensionsOpt)
+	{
+		optEntries->emplace_back(OPTION_EXTENSIONS, "");
+		extensionsOpt = optEntries->FindOption(OPTION_EXTENSIONS);
+	}
+
+	const char* scriptList = optEntry->GetValue();
+
+	Tokenizer tok(scriptList, ",;");
+	while (const char* scriptName = tok.Next())
+	{
+		// merge into global "Extensions"
+		if (!HasScript(extensionsOpt->m_value, scriptName))
+		{
+			if (!extensionsOpt->m_value.Empty())
+			{
+				extensionsOpt->m_value.Append(",");
+			}
+			extensionsOpt->m_value.Append(scriptName);
+		}
+
+		// merge into categories' "Extensions" (if not empty)
+		if (mergeCategories)
+		{
+			for (OptEntry& opt : optEntries)
+			{
+				const char* catoptname = opt.GetName();
+				if (!strncasecmp(catoptname, "category", 8))
+				{
+					char* p = (char*)catoptname + 8;
+					while (*p >= '0' && *p <= '9') p++;
+					if (p && (!strcasecmp(p, ".extensions")))
+					{
+						if (!opt.m_value.Empty() && !HasScript(opt.m_value, scriptName))
+						{
+							opt.m_value.Append(",");
+							opt.m_value.Append(scriptName);
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+bool Options::HasScript(const char* scriptList, const char* scriptName)
+{
+	Tokenizer tok(scriptList, ",;");
+	while (const char* scriptName2 = tok.Next())
+	{
+		if (!strcasecmp(scriptName2, scriptName))
+		{
+			return true;
+		}
+	}
+	return false;
+};

daemon/main/Options.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -67,6 +67,7 @@ public:
 	{
 		hcPause,
 		hcDelete,
+		hcPark,
 		hcNone
 	};
 	enum ESchedulerCommand
@@ -84,6 +85,19 @@ public:
 		scDeactivateServer,
 		scFetchFeed
 	};
+	enum EPostStrategy
+	{
+		ppSequential,
+		ppBalanced,
+		ppAggressive,
+		ppRocket
+	};
+	enum EFileNaming
+	{
+		nfAuto,
+		nfArticle,
+		nfNzb
+	};
 
 	class OptEntry
 	{
@@ -125,19 +139,19 @@ public:
 	class Category
 	{
 	public:
-		Category(const char* name, const char* destDir, bool unpack, const char* postScript) :
-			m_name(name), m_destDir(destDir), m_unpack(unpack), m_postScript(postScript) {}
+		Category(const char* name, const char* destDir, bool unpack, const char* extensions) :
+			m_name(name), m_destDir(destDir), m_unpack(unpack), m_extensions(extensions) {}
 		const char* GetName() { return m_name; }
 		const char* GetDestDir() { return m_destDir; }
 		bool GetUnpack() { return m_unpack; }
-		const char* GetPostScript() { return m_postScript; }
+		const char* GetExtensions() { return m_extensions; }
 		NameList* GetAliases() { return &m_aliases; }
 
 	private:
 		CString m_name;
 		CString m_destDir;
 		bool m_unpack;
-		CString m_postScript;
+		CString m_extensions;
 		NameList m_aliases;
 	};
 
@@ -153,12 +167,12 @@ public:
 	{
 	public:
 		virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-			int port, const char* user, const char* pass, bool joinGroup,
+			int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 			bool tls, const char* cipher, int maxConnections, int retention,
-			int level, int group) = 0;
+			int level, int group, bool optional) = 0;
 		virtual void AddFeed(int id, const char* name, const char* url, int interval,
 			const char* filter, bool backlog, bool pauseNzb, const char* category,
-			int priority, const char* feedScript) {}
+			int priority, const char* extensions) {}
 		virtual void AddTask(int id, int hours, int minutes, int weekDaysBits, ESchedulerCommand command,
 			const char* param) {}
 		virtual void SetupFirstStart() {}
@@ -169,9 +183,12 @@ public:
 	Options(CmdOptList* commandLineOptions, Extender* extender);
 	~Options();
 
-	bool SplitOptionString(const char* option, CString& optName, CString& optValue);
+	static bool SplitOptionString(const char* option, CString& optName, CString& optValue);
+	static void ConvertOldOptions(OptEntries* optEntries);
 	bool GetFatalError() { return m_fatalError; }
 	GuardedOptEntries GuardOptEntries() { return GuardedOptEntries(&m_optEntries, &m_optEntriesMutex); }
+	void CreateSchedulerTask(int id, const char* time, const char* weekDays,
+		ESchedulerCommand command, const char* param);
 
 	// Options
 	const char* GetConfigFilename() { return m_configFilename; }
@@ -186,7 +203,6 @@ public:
 	const char* GetConfigTemplate() { return m_configTemplate; }
 	const char* GetScriptDir() { return m_scriptDir; }
 	const char* GetRequiredDir() { return m_requiredDir; }
-	bool GetBrokenLog() const { return m_brokenLog; }
 	bool GetNzbLog() const { return m_nzbLog; }
 	EMessageTarget GetInfoTarget() const { return m_infoTarget; }
 	EMessageTarget GetWarningTarget() const { return m_warningTarget; }
@@ -195,13 +211,15 @@ public:
 	EMessageTarget GetDetailTarget() const { return m_detailTarget; }
 	int GetArticleTimeout() { return m_articleTimeout; }
 	int GetUrlTimeout() { return m_urlTimeout; }
-	int GetTerminateTimeout() { return m_terminateTimeout; }
-	bool GetDecode() { return m_decode; };
+	int GetRemoteTimeout() { return m_remoteTimeout; }
+	bool GetRawArticle() { return m_rawArticle; };
+	bool GetSkipWrite() { return m_skipWrite; };
 	bool GetAppendCategoryDir() { return m_appendCategoryDir; }
 	bool GetContinuePartial() { return m_continuePartial; }
-	int GetRetries() { return m_retries; }
-	int GetRetryInterval() { return m_retryInterval; }
-	bool GetSaveQueue() { return m_saveQueue; }
+	int GetArticleRetries() { return m_articleRetries; }
+	int GetArticleInterval() { return m_articleInterval; }
+	int GetUrlRetries() { return m_urlRetries; }
+	int GetUrlInterval() { return m_urlInterval; }
 	bool GetFlushQueue() { return m_flushQueue; }
 	bool GetDupeCheck() { return m_dupeCheck; }
 	const char* GetControlIp() { return m_controlIp; }
@@ -212,17 +230,19 @@ public:
 	const char* GetAddUsername() { return m_addUsername; }
 	const char* GetAddPassword() { return m_addPassword; }
 	int GetControlPort() { return m_controlPort; }
+	bool GetFormAuth() { return m_formAuth; }
 	bool GetSecureControl() { return m_secureControl; }
 	int GetSecurePort() { return m_securePort; }
 	const char* GetSecureCert() { return m_secureCert; }
 	const char* GetSecureKey() { return m_secureKey; }
+	const char* GetCertStore() { return m_certStore; }
+	bool GetCertCheck() { return m_certCheck; }
 	const char* GetAuthorizedIp() { return m_authorizedIp; }
 	const char* GetLockFile() { return m_lockFile; }
 	const char* GetDaemonUsername() { return m_daemonUsername; }
 	EOutputMode GetOutputMode() { return m_outputMode; }
-	bool GetReloadQueue() { return m_reloadQueue; }
 	int GetUrlConnections() { return m_urlConnections; }
-	int GetLogBufferSize() { return m_logBufferSize; }
+	int GetLogBuffer() { return m_logBuffer; }
 	EWriteLog GetWriteLog() { return m_writeLog; }
 	const char* GetLogFile() { return m_logFile; }
 	int GetRotateLog() { return m_rotateLog; }
@@ -230,15 +250,14 @@ public:
 	bool GetParRepair() { return m_parRepair; }
 	EParScan GetParScan() { return m_parScan; }
 	bool GetParQuick() { return m_parQuick; }
+	EPostStrategy GetPostStrategy() { return m_postStrategy; }
 	bool GetParRename() { return m_parRename; }
 	int GetParBuffer() { return m_parBuffer; }
 	int GetParThreads() { return m_parThreads; }
+	bool GetRarRename() { return m_rarRename; }
 	EHealthCheck GetHealthCheck() { return m_healthCheck; }
 	const char* GetScriptOrder() { return m_scriptOrder; }
-	const char* GetPostScript() { return m_postScript; }
-	const char* GetScanScript() { return m_scanScript; }
-	const char* GetQueueScript() { return m_queueScript; }
-	const char* GetFeedScript() { return m_feedScript; }
+	const char* GetExtensions() { return m_extensions; }
 	int GetUMask() { return m_umask; }
 	int GetUpdateInterval() {return m_updateInterval; }
 	bool GetCursesNzbName() { return m_cursesNzbName; }
@@ -249,18 +268,17 @@ public:
 	int GetWriteBuffer() { return m_writeBuffer; }
 	int GetNzbDirInterval() { return m_nzbDirInterval; }
 	int GetNzbDirFileAge() { return m_nzbDirFileAge; }
-	bool GetParCleanupQueue() { return m_parCleanupQueue; }
 	int GetDiskSpace() { return m_diskSpace; }
 	bool GetTls() { return m_tls; }
-	bool GetDumpCore() { return m_dumpCore; }
+	bool GetCrashTrace() { return m_crashTrace; }
+	bool GetCrashDump() { return m_crashDump; }
 	bool GetParPauseQueue() { return m_parPauseQueue; }
 	bool GetScriptPauseQueue() { return m_scriptPauseQueue; }
 	bool GetNzbCleanupDisk() { return m_nzbCleanupDisk; }
-	bool GetDeleteCleanupDisk() { return m_deleteCleanupDisk; }
 	int GetParTimeLimit() { return m_parTimeLimit; }
 	int GetKeepHistory() { return m_keepHistory; }
-	bool GetAccurateRate() { return m_accurateRate; }
 	bool GetUnpack() { return m_unpack; }
+	bool GetDirectUnpack() { return m_directUnpack; }
 	bool GetUnpackCleanupDisk() { return m_unpackCleanupDisk; }
 	const char* GetUnrarCmd() { return m_unrarCmd; }
 	const char* GetSevenZipCmd() { return m_sevenZipCmd; }
@@ -268,12 +286,21 @@ public:
 	bool GetUnpackPauseQueue() { return m_unpackPauseQueue; }
 	const char* GetExtCleanupDisk() { return m_extCleanupDisk; }
 	const char* GetParIgnoreExt() { return m_parIgnoreExt; }
+	const char* GetUnpackIgnoreExt() { return m_unpackIgnoreExt; }
 	int GetFeedHistory() { return m_feedHistory; }
 	bool GetUrlForce() { return m_urlForce; }
 	int GetTimeCorrection() { return m_timeCorrection; }
 	int GetPropagationDelay() { return m_propagationDelay; }
 	int GetArticleCache() { return m_articleCache; }
 	int GetEventInterval() { return m_eventInterval; }
+	const char* GetShellOverride() { return m_shellOverride; }
+	int GetMonthlyQuota() { return m_monthlyQuota; }
+	int GetQuotaStartDay() { return m_quotaStartDay; }
+	int GetDailyQuota() { return m_dailyQuota; }
+	bool GetDirectRename() { return m_directRename; }
+	bool GetReorderFiles() { return m_reorderFiles; }
+	EFileNaming GetFileNaming() { return m_fileNaming; }
+	int GetDownloadRate() const { return m_downloadRate; }
 
 	Categories* GetCategories() { return &m_categories; }
 	Category* FindCategory(const char* name, bool searchAliases) { return m_categories.FindCategory(name, searchAliases); }
@@ -285,22 +312,6 @@ public:
 	bool GetDaemonMode() { return m_daemonMode; }
 	void SetRemoteClientMode(bool remoteClientMode) { m_remoteClientMode = remoteClientMode; }
 	bool GetRemoteClientMode() { return m_remoteClientMode; }
-	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; }
-	bool GetPauseDownload() const { return m_pauseDownload; }
-	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; }
-	bool GetPausePostProcess() const { return m_pausePostProcess; }
-	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; }
-	bool GetPauseScan() const { return m_pauseScan; }
-	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; }
-	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
-	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
-	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; }
-	void SetDownloadRate(int rate) { m_downloadRate = rate; }
-	int GetDownloadRate() const { return m_downloadRate; }
-	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; }
-	time_t GetResumeTime() const { return m_resumeTime; }
-	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; }
-	int GetLocalTimeOffset() { return m_localTimeOffset; }
 
 private:
 	OptEntries m_optEntries;
@@ -330,17 +341,18 @@ private:
 	EMessageTarget m_errorTarget = mtScreen;
 	EMessageTarget m_debugTarget = mtNone;
 	EMessageTarget m_detailTarget = mtScreen;
-	bool m_decode = true;
-	bool m_brokenLog = false;
+	bool m_skipWrite = false;
+	bool m_rawArticle = false;
 	bool m_nzbLog = false;
 	int m_articleTimeout = 0;
 	int m_urlTimeout = 0;
-	int m_terminateTimeout = 0;
+	int m_remoteTimeout = 0;
 	bool m_appendCategoryDir = false;
 	bool m_continuePartial = false;
-	int m_retries = 0;
-	int m_retryInterval = 0;
-	bool m_saveQueue = false;
+	int m_articleRetries = 0;
+	int m_articleInterval = 0;
+	int m_urlRetries = 0;
+	int m_urlInterval = 0;
 	bool m_flushQueue = false;
 	bool m_dupeCheck = false;
 	CString m_controlIp;
@@ -350,18 +362,20 @@ private:
 	CString m_restrictedPassword;
 	CString m_addUsername;
 	CString m_addPassword;
+	bool m_formAuth = false;
 	int m_controlPort = 0;
 	bool m_secureControl = false;
 	int m_securePort = 0;
 	CString m_secureCert;
 	CString m_secureKey;
+	CString m_certStore;
+	bool m_certCheck = false;
 	CString m_authorizedIp;
 	CString m_lockFile;
 	CString m_daemonUsername;
 	EOutputMode m_outputMode = omLoggable;
-	bool m_reloadQueue = false;
 	int m_urlConnections = 0;
-	int m_logBufferSize = 0;
+	int m_logBuffer = 0;
 	EWriteLog m_writeLog = wlAppend;
 	int m_rotateLog = 0;
 	CString m_logFile;
@@ -369,15 +383,15 @@ private:
 	bool m_parRepair = false;
 	EParScan m_parScan = psLimited;
 	bool m_parQuick = true;
+	EPostStrategy m_postStrategy = ppSequential;
 	bool m_parRename = false;
 	int m_parBuffer = 0;
 	int m_parThreads = 0;
+	bool m_rarRename = false;
+	bool m_directRename = false;
 	EHealthCheck m_healthCheck = hcNone;
-	CString m_postScript;
+	CString m_extensions;
 	CString m_scriptOrder;
-	CString m_scanScript;
-	CString m_queueScript;
-	CString m_feedScript;
 	int m_umask = 0;
 	int m_updateInterval = 0;
 	bool m_cursesNzbName = false;
@@ -388,18 +402,17 @@ private:
 	int m_writeBuffer = 0;
 	int m_nzbDirInterval = 0;
 	int m_nzbDirFileAge = 0;
-	bool m_parCleanupQueue = false;
 	int m_diskSpace = 0;
 	bool m_tls = false;
-	bool m_dumpCore = false;
+	bool m_crashTrace = false;
+	bool m_crashDump = false;
 	bool m_parPauseQueue = false;
 	bool m_scriptPauseQueue = false;
 	bool m_nzbCleanupDisk = false;
-	bool m_deleteCleanupDisk = false;
 	int m_parTimeLimit = 0;
 	int m_keepHistory = 0;
-	bool m_accurateRate = false;
 	bool m_unpack = false;
+	bool m_directUnpack = false;
 	bool m_unpackCleanupDisk = false;
 	CString m_unrarCmd;
 	CString m_sevenZipCmd;
@@ -407,25 +420,25 @@ private:
 	bool m_unpackPauseQueue;
 	CString m_extCleanupDisk;
 	CString m_parIgnoreExt;
+	CString m_unpackIgnoreExt;
 	int m_feedHistory = 0;
 	bool m_urlForce = false;
 	int m_timeCorrection = 0;
 	int m_propagationDelay = 0;
 	int m_articleCache = 0;
 	int m_eventInterval = 0;
+	CString m_shellOverride;
+	int m_monthlyQuota = 0;
+	int m_quotaStartDay = 0;
+	int m_dailyQuota = 0;
+	bool m_reorderFiles = false;
+	EFileNaming m_fileNaming = nfArticle;
+	int m_downloadRate = 0;
 
-	// Current state
+	// Application mode
 	bool m_serverMode = false;
 	bool m_daemonMode = false;
 	bool m_remoteClientMode = false;
-	bool m_pauseDownload = false;
-	bool m_pausePostProcess = false;
-	bool m_pauseScan = false;
-	bool m_tempPauseDownload = true;
-	bool m_tempPausePostprocess = true;
-	int m_downloadRate = 0;
-	time_t m_resumeTime = 0;
-	int m_localTimeOffset = 0;
 
 	void Init(const char* exeName, const char* configFilename, bool noConfig,
 		CmdOptList* commandLineOptions, bool noDiskAccess, Extender* extender);
@@ -453,7 +466,9 @@ private:
 	void ConfigError(const char* msg, ...);
 	void ConfigWarn(const char* msg, ...);
 	void LocateOptionSrcPos(const char *optionName);
-	void ConvertOldOption(CString& option, CString& value);
+	static void ConvertOldOption(CString& option, CString& value);
+	static void MergeOldScriptOption(OptEntries* optEntries, const char* optname, bool mergeCategories);
+	static bool HasScript(const char* scriptList, const char* scriptName);
 };
 
 extern Options* g_Options;

daemon/main/Scheduler.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Scheduler.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "NewsServer.h"
 #include "ServerPool.h"
@@ -40,7 +41,7 @@ void Scheduler::FirstCheck()
 		Guard guard(m_taskListMutex);
 
 		std::sort(m_taskList.begin(), m_taskList.end(),
-			[](std::unique_ptr<Task>& task1, std::unique_ptr<Task>& task2)
+			[](const std::unique_ptr<Task>& task1, const std::unique_ptr<Task>& task2)
 		{
 			return (task1->m_hours < task2->m_hours) ||
 				((task1->m_hours == task2->m_hours) && (task1->m_minutes < task2->m_minutes));
@@ -51,13 +52,33 @@ void Scheduler::FirstCheck()
 	CheckTasks();
 }
 
+void Scheduler::ScheduleNextWork()
+{
+	// Ideally we should calculate wait time until next scheduler task or until resume time.
+	// The first isn't trivial and the second requires watching/reaction on changed scheduled resume time.
+	// We do it simpler instead: check once per minute, when seconds are changing from 59 to 00.
+
+	time_t curTime = Util::CurrentTime();
+	tm sched;
+	gmtime_r(&curTime, &sched);
+	sched.tm_min++;
+	sched.tm_sec = 0;
+	time_t nextMinute = Util::Timegm(&sched);
+
+	m_serviceInterval = nextMinute - curTime;
+}
+
 void Scheduler::ServiceWork()
 {
+	debug("Scheduler service work");
+
 	if (!DownloadQueue::IsLoaded())
 	{
 		return;
 	}
 
+	debug("Scheduler service work: doing work");
+
 	if (!m_firstChecked)
 	{
 		FirstCheck();
@@ -68,6 +89,7 @@ void Scheduler::ServiceWork()
 	m_executeProcess = true;
 	CheckTasks();
 	CheckScheduledResume();
+	ScheduleNextWork();
 }
 
 void Scheduler::CheckTasks()
@@ -93,12 +115,15 @@ void Scheduler::CheckTasks()
 
 				for (Task* task : &m_taskList)
 				{
-					task->m_lastExecuted = 0;
+					if (task->m_hours != Task::STARTUP_TASK)
+					{
+						task->m_lastExecuted = 0;
+					}
 				}
 			}
 
-			time_t localCurrent = current + g_Options->GetLocalTimeOffset();
-			time_t localLastCheck = m_lastCheck + g_Options->GetLocalTimeOffset();
+			time_t localCurrent = current + g_WorkState->GetLocalTimeOffset();
+			time_t localLastCheck = m_lastCheck + g_WorkState->GetLocalTimeOffset();
 
 			tm tmCurrent;
 			gmtime_r(&localCurrent, &tmCurrent);
@@ -133,9 +158,8 @@ void Scheduler::CheckTasks()
 						}
 
 						bool weekDayOK = task->m_weekDaysBits == 0 || (task->m_weekDaysBits & (1 << (weekDay - 1)));
-						bool doTask = weekDayOK && localLastCheck < appoint && appoint <= localCurrent;
-
-						//debug("TEMP: 1) m_tLastCheck=%i, tLocalCurrent=%i, tLoop=%i, tAppoint=%i, bWeekDayOK=%i, bDoTask=%i", m_tLastCheck, tLocalCurrent, tLoop, tAppoint, (int)bWeekDayOK, (int)bDoTask);
+						bool doTask = (task->m_hours >= 0 && weekDayOK && localLastCheck < appoint && appoint <= localCurrent) ||
+							(task->m_hours == Task::STARTUP_TASK && task->m_lastExecuted == 0);
 
 						if (doTask)
 						{
@@ -157,42 +181,46 @@ void Scheduler::CheckTasks()
 
 void Scheduler::ExecuteTask(Task* task)
 {
+#ifdef DEBUG
 	const char* commandName[] = { "Pause", "Unpause", "Pause Post-processing", "Unpause Post-processing",
 		"Set download rate", "Execute process", "Execute script",
 		"Pause Scan", "Unpause Scan", "Enable Server", "Disable Server", "Fetch Feed" };
 	debug("Executing scheduled command: %s", commandName[task->m_command]);
+#endif
+
+	bool executeProcess = m_executeProcess || task->m_hours == Task::STARTUP_TASK;
 
 	switch (task->m_command)
 	{
 		case scDownloadRate:
 			if (!task->m_param.Empty())
 			{
-				g_Options->SetDownloadRate(atoi(task->m_param) * 1024);
+				g_WorkState->SetSpeedLimit(atoi(task->m_param) * 1024);
 				m_downloadRateChanged = true;
 			}
 			break;
 
 		case scPauseDownload:
 		case scUnpauseDownload:
-			g_Options->SetPauseDownload(task->m_command == scPauseDownload);
+			g_WorkState->SetPauseDownload(task->m_command == scPauseDownload);
 			m_pauseDownloadChanged = true;
 			break;
 
 		case scPausePostProcess:
 		case scUnpausePostProcess:
-			g_Options->SetPausePostProcess(task->m_command == scPausePostProcess);
+			g_WorkState->SetPausePostProcess(task->m_command == scPausePostProcess);
 			m_pausePostProcessChanged = true;
 			break;
 
 		case scPauseScan:
 		case scUnpauseScan:
-			g_Options->SetPauseScan(task->m_command == scPauseScan);
+			g_WorkState->SetPauseScan(task->m_command == scPauseScan);
 			m_pauseScanChanged = true;
 			break;
 
-		case scScript:
+		case scExtensions:
 		case scProcess:
-			if (m_executeProcess)
+			if (executeProcess)
 			{
 				SchedulerScriptController::StartScript(task->m_param, task->m_command == scProcess, task->m_id);
 			}
@@ -204,7 +232,7 @@ void Scheduler::ExecuteTask(Task* task)
 			break;
 
 		case scFetchFeed:
-			if (m_executeProcess)
+			if (executeProcess)
 			{
 				FetchFeed(task->m_param);
 				break;
@@ -225,19 +253,19 @@ void Scheduler::PrintLog()
 {
 	if (m_downloadRateChanged)
 	{
-		info("Scheduler: setting download rate to %i KB/s", g_Options->GetDownloadRate() / 1024);
+		info("Scheduler: setting download rate to %i KB/s", g_WorkState->GetSpeedLimit() / 1024);
 	}
 	if (m_pauseDownloadChanged)
 	{
-		info("Scheduler: %s download", g_Options->GetPauseDownload() ? "pausing" : "unpausing");
+		info("Scheduler: %s download", g_WorkState->GetPauseDownload() ? "pausing" : "unpausing");
 	}
 	if (m_pausePostProcessChanged)
 	{
-		info("Scheduler: %s post-processing", g_Options->GetPausePostProcess() ? "pausing" : "unpausing");
+		info("Scheduler: %s post-processing", g_WorkState->GetPausePostProcess() ? "pausing" : "unpausing");
 	}
 	if (m_pauseScanChanged)
 	{
-		info("Scheduler: %s scan", g_Options->GetPauseScan() ? "pausing" : "unpausing");
+		info("Scheduler: %s scan", g_WorkState->GetPauseScan() ? "pausing" : "unpausing");
 	}
 	if (m_serverChanged)
 	{
@@ -304,14 +332,14 @@ void Scheduler::FetchFeed(const char* feedList)
 
 void Scheduler::CheckScheduledResume()
 {
-	time_t resumeTime = g_Options->GetResumeTime();
+	time_t resumeTime = g_WorkState->GetResumeTime();
 	time_t currentTime = Util::CurrentTime();
 	if (resumeTime > 0 && currentTime >= resumeTime)
 	{
 		info("Autoresume");
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(false);
-		g_Options->SetPausePostProcess(false);
-		g_Options->SetPauseScan(false);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(false);
+		g_WorkState->SetPausePostProcess(false);
+		g_WorkState->SetPauseScan(false);
 	}
 }

daemon/main/Scheduler.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -35,7 +35,7 @@ public:
 		scPausePostProcess,
 		scUnpausePostProcess,
 		scDownloadRate,
-		scScript,
+		scExtensions,
 		scProcess,
 		scPauseScan,
 		scUnpauseScan,
@@ -52,7 +52,7 @@ public:
 			m_id(id), m_hours(hours), m_minutes(minutes),
 			m_weekDaysBits(weekDaysBits), m_command(command), m_param(param) {}
 		friend class Scheduler;
-
+		static const int STARTUP_TASK = -1;
 	private:
 		int m_id;
 		int m_hours;
@@ -66,7 +66,7 @@ public:
 	void AddTask(std::unique_ptr<Task> task);
 
 protected:
-	virtual int ServiceInterval() { return 1000; }
+	virtual int ServiceInterval() { return m_serviceInterval; }
 	virtual void ServiceWork();
 
 private:
@@ -84,6 +84,7 @@ private:
 	bool m_serverChanged;
 	ServerStatusList m_serverStatusList;
 	bool m_firstChecked = false;
+	int m_serviceInterval = 1;
 
 	void ExecuteTask(Task* task);
 	void CheckTasks();
@@ -93,6 +94,7 @@ private:
 	void FetchFeed(const char* feedList);
 	void CheckScheduledResume();
 	void FirstCheck();
+	void ScheduleNextWork();
 };
 
 #endif

daemon/main/StackTrace.cpp

@@ -147,17 +147,20 @@ LONG __stdcall ExceptionFilter(EXCEPTION_POINTERS* exPtrs)
 
 #ifdef DEBUG
 	PrintBacktrace(exPtrs->ContextRecord);
-#else
-	 info("Detailed exception information can be printed by debug version of NZBGet (available from download page)");
-#endif
-
-	ExitProcess(-1);
 	return EXCEPTION_CONTINUE_SEARCH;
+#else
+	info("Detailed crash information can be printed by debug version of NZBGet."
+		" For more info visit http://nzbget.net/crash-dump");
+	return EXCEPTION_EXECUTE_HANDLER;
+#endif
 }
 
 void InstallErrorHandler()
 {
-	SetUnhandledExceptionFilter(ExceptionFilter);
+	if (g_Options->GetCrashTrace())
+	{
+		SetUnhandledExceptionFilter(ExceptionFilter);
+	}
 }
 
 #else
@@ -171,7 +174,7 @@ std::vector<sighandler> SignalProcList;
 /**
 * activates the creation of core-files
 */
-void EnableDumpCore()
+void EnableCoreDump()
 {
 	rlimit rlim;
 	rlim.rlim_cur= RLIM_INFINITY;
@@ -248,9 +251,9 @@ void SignalProc(int signum)
 void InstallErrorHandler()
 {
 #ifdef HAVE_SYS_PRCTL_H
-	if (g_Options->GetDumpCore())
+	if (g_Options->GetCrashDump())
 	{
-		EnableDumpCore();
+		EnableCoreDump();
 	}
 #endif
 
@@ -258,7 +261,10 @@ void InstallErrorHandler()
 	signal(SIGTERM, SignalProc);
 	signal(SIGPIPE, SIG_IGN);
 #ifdef DEBUG
-	signal(SIGSEGV, SignalProc);
+	if (g_Options->GetCrashTrace())
+	{
+		signal(SIGSEGV, SignalProc);
+	}
 #endif
 #ifdef SIGCHLD_HANDLER
 	// it could be necessary on some systems to activate a handler for SIGCHLD
@@ -276,7 +282,7 @@ public:
 	void DoSegFault()
 	{
 		char* N = nullptr;
-		strcpy(N, "");
+		*N = '\0';
 	}
 };
 

daemon/main/WorkState.cpp

@@ -0,0 +1,27 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "WorkState.h"
+
+void WorkState::Changed()
+{
+	Notify(nullptr);
+}

daemon/main/WorkState.h

@@ -0,0 +1,76 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef WORKSTATE_H
+#define WORKSTATE_H
+
+#include "Observer.h"
+
+// WorkState is observable but notifications are not 100% reliable.
+// The changes via Set-methods and readings via Get-methods are not synchronized throughout the program.
+// As result race conditions may occur and some changes may go unnoticed.
+// When waiting for changes don't wait too long to avoid lock ups.
+
+class WorkState : public Subject
+{
+public:
+	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; Changed(); }
+	bool GetPauseDownload() const { return m_pauseDownload; }
+	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; Changed(); }
+	bool GetPausePostProcess() const { return m_pausePostProcess; }
+	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; Changed(); }
+	bool GetPauseScan() const { return m_pauseScan; }
+	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; Changed(); }
+	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
+	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; Changed(); }
+	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
+	void SetPauseFrontend(bool pauseFrontend) { m_pauseFrontend = pauseFrontend; Changed(); }
+	bool GetPauseFrontend() const { return m_pauseFrontend; }
+	void SetSpeedLimit(int speedLimit) { m_speedLimit = speedLimit; Changed(); }
+	int GetSpeedLimit() const { return m_speedLimit; }
+	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; Changed(); }
+	time_t GetResumeTime() const { return m_resumeTime; }
+	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; Changed(); }
+	int GetLocalTimeOffset() { return m_localTimeOffset; }
+	void SetQuotaReached(bool quotaReached) { m_quotaReached = quotaReached; Changed(); }
+	bool GetQuotaReached() { return m_quotaReached; }
+	void SetDownloading(bool downloading) { m_downloading = downloading; Changed(); }
+	bool GetDownloading() { return m_downloading; }
+
+private:
+	bool m_pauseDownload = false;
+	bool m_pausePostProcess = false;
+	bool m_pauseScan = false;
+	bool m_tempPauseDownload = true;
+	bool m_tempPausePostprocess = true;
+	bool m_pauseFrontend = false;
+	int m_downloadRate = 0;
+	time_t m_resumeTime = 0;
+	int m_localTimeOffset = 0;
+	bool m_quotaReached = false;
+	int m_speedLimit = 0;
+	bool m_downloading = false;
+
+	void Changed();
+};
+
+extern WorkState* g_WorkState;
+
+#endif

daemon/main/nzbget.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "Log.h"
 #include "NzbFile.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "CommandLineParser.h"
 #include "ScriptConfig.h"
 #include "Thread.h"
@@ -39,7 +40,6 @@
 #include "PrePostProcessor.h"
 #include "HistoryCoordinator.h"
 #include "DupeCoordinator.h"
-#include "ParChecker.h"
 #include "Scheduler.h"
 #include "Scanner.h"
 #include "FeedCoordinator.h"
@@ -52,6 +52,8 @@
 #include "Util.h"
 #include "FileSystem.h"
 #include "StackTrace.h"
+#include "CommandScript.h"
+#include "YEncode.h"
 #ifdef WIN32
 #include "WinService.h"
 #include "WinConsole.h"
@@ -60,6 +62,9 @@
 #ifdef ENABLE_TESTS
 #include "TestMain.h"
 #endif
+#ifndef DISABLE_NSERV
+#include "NServMain.h"
+#endif
 
 // Prototypes
 void RunMain();
@@ -67,6 +72,7 @@ void RunMain();
 // Globals
 Log* g_Log;
 Options* g_Options;
+WorkState* g_WorkState;
 ServerPool* g_ServerPool;
 QueueCoordinator* g_QueueCoordinator;
 UrlCoordinator* g_UrlCoordinator;
@@ -82,6 +88,7 @@ ArticleCache* g_ArticleCache;
 QueueScriptCoordinator* g_QueueScriptCoordinator;
 ServiceCoordinator* g_ServiceCoordinator;
 ScriptConfig* g_ScriptConfig;
+CommandScriptLog* g_CommandScriptLog; 
 #ifdef WIN32
 WinConsole* g_WinConsole;
 #endif
@@ -108,6 +115,7 @@ int main(int argc, char *argv[], char *argp[])
 #endif
 
 	Util::Init();
+	YEncode::init();
 
 	g_ArgumentCount = argc;
 	g_Arguments = (char*(*)[])argv;
@@ -127,17 +135,28 @@ int main(int argc, char *argv[], char *argp[])
 	TestCleanup();
 #endif
 
+	if (argc > 1 && (!strcmp(argv[1], "--nserv")))
+	{
+#ifndef DISABLE_NSERV
+		return NServMain(argc, argv);
+#else
+		printf("ERROR: Could not start NServ, the program was compiled without NServ\n");
+		return 1;
+#endif
+	}
+
 #ifdef WIN32
 	InstallUninstallServiceCheck(argc, argv);
 #endif
 
-	srand(Util::CurrentTime());
+	srand((unsigned int)Util::CurrentTime());
 
 #ifdef WIN32
 	for (int i=0; i < argc; i++)
 	{
 		if (!strcmp(argv[i], "-D"))
 		{
+			AllocConsole(); // needed for sending CTRL+BREAK signal to child processes
 			StartService(RunMain);
 			return 0;
 		}
@@ -160,9 +179,9 @@ public:
 
 	// Options::Extender
 	virtual void AddNewsServer(int id, bool active, const char* name, const char* host,
-		int port, const char* user, const char* pass, bool joinGroup,
+		int port, int ipVersion, const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
-		int level, int group);
+		int level, int group, bool optional);
 	virtual void AddFeed(int id, const char* name, const char* url, int interval,
 		const char* filter, bool backlog, bool pauseNzb, const char* category,
 		int priority, const char* feedScript);
@@ -176,6 +195,7 @@ private:
 	// globals
 	std::unique_ptr<Log> m_log;
 	std::unique_ptr<Options> m_options;
+	std::unique_ptr<WorkState> m_workState;
 	std::unique_ptr<ServerPool> m_serverPool;
 	std::unique_ptr<QueueCoordinator> m_queueCoordinator;
 	std::unique_ptr<UrlCoordinator> m_urlCoordinator;
@@ -191,6 +211,7 @@ private:
 	std::unique_ptr<QueueScriptCoordinator> m_queueScriptCoordinator;
 	std::unique_ptr<ServiceCoordinator> m_serviceCoordinator;
 	std::unique_ptr<ScriptConfig> m_scriptConfig;
+	std::unique_ptr<CommandScriptLog> m_commandScriptLog;
 #ifdef WIN32
 	std::unique_ptr<WinConsole> m_winConsole;
 #endif
@@ -205,13 +226,17 @@ private:
 
 	bool m_reloading = false;
 	bool m_daemonized = false;
+	bool m_stopped = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	void Init();
+	void Final();
 	void BootConfig();
 	void CreateGlobals();
 	void Cleanup();
 	void PrintOptions();
-	bool ProcessDirect();
+	void ProcessDirect();
 	void ProcessClientRequest();
 	void ProcessWebGet();
 	void ProcessSigVerify();
@@ -266,6 +291,9 @@ void NZBGet::Init()
 
 	m_scanner->InitOptions();
 	m_queueScriptCoordinator->InitOptions();
+#ifndef DISABLE_TLS
+	TlsSocket::InitOptions(g_Options->GetCertCheck() ? g_Options->GetCertStore() : nullptr);
+#endif
 
 	if (m_commandLineParser->GetDaemonMode())
 	{
@@ -299,6 +327,17 @@ void NZBGet::Init()
 	InstallErrorHandler();
 }
 
+void NZBGet::Final()
+{
+	if (!m_reloading)
+	{
+#ifndef DISABLE_TLS
+		TlsSocket::Final();
+#endif
+		Connection::Final();
+	}
+}
+
 void NZBGet::CreateGlobals()
 {
 #ifdef WIN32
@@ -306,6 +345,9 @@ void NZBGet::CreateGlobals()
 	g_WinConsole = m_winConsole.get();
 #endif
 
+	m_workState = std::make_unique<WorkState>();
+	g_WorkState = m_workState.get();
+
 	m_serviceCoordinator = std::make_unique<ServiceCoordinator>();
 	g_ServiceCoordinator = m_serviceCoordinator.get();
 
@@ -351,6 +393,9 @@ void NZBGet::CreateGlobals()
 	m_scriptConfig = std::make_unique<ScriptConfig>();
 	g_ScriptConfig = m_scriptConfig.get();
 
+	m_commandScriptLog = std::make_unique<CommandScriptLog>();
+	g_CommandScriptLog = m_commandScriptLog.get();
+
 	m_scheduler = std::make_unique<Scheduler>();
 
 	m_diskService = std::make_unique<DiskService>();
@@ -368,7 +413,7 @@ void NZBGet::BootConfig()
 	if (m_commandLineParser->GetPrintUsage() || m_commandLineParser->GetErrors() || g_ArgumentCount <= 1)
 	{
 		m_commandLineParser->PrintUsage(((const char**)(*g_Arguments))[0]);
-		exit(0);
+		exit(m_commandLineParser->GetPrintUsage() ? 0 : 1);
 	}
 
 	debug("Reading options");
@@ -376,7 +421,8 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetNoConfig(), (Options::CmdOptList*)m_commandLineParser->GetOptionList(), this);
 	m_options->SetRemoteClientMode(m_commandLineParser->GetRemoteClientMode());
 	m_options->SetServerMode(m_commandLineParser->GetServerMode());
-	m_options->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetSpeedLimit(g_Options->GetDownloadRate());
 
 	m_log->InitOptions();
 
@@ -388,13 +434,13 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetClientOperation() == CommandLineParser::opClientNoOperation)
 	{
 		info("Pausing all activities due to errors in configuration");
-		m_options->SetPauseDownload(true);
-		m_options->SetPausePostProcess(true);
-		m_options->SetPauseScan(true);
+		m_workState->SetPauseDownload(true);
+		m_workState->SetPausePostProcess(true);
+		m_workState->SetPauseScan(true);
 	}
 
 	m_serverPool->SetTimeout(m_options->GetArticleTimeout());
-	m_serverPool->SetRetryInterval(m_options->GetRetryInterval());
+	m_serverPool->SetRetryInterval(m_options->GetArticleInterval());
 
 	m_scriptConfig->InitOptions();
 }
@@ -423,46 +469,41 @@ void NZBGet::Cleanup()
 	g_QueueScriptCoordinator = nullptr;
 	g_Maintenance = nullptr;
 	g_StatMeter = nullptr;
+	g_CommandScriptLog = nullptr;
 #ifdef WIN32
 	g_WinConsole = nullptr;
 #endif
 }
 
-bool NZBGet::ProcessDirect()
+void NZBGet::ProcessDirect()
 {
 #ifdef DEBUG
 	if (m_commandLineParser->GetTestBacktrace())
 	{
-		TestSegFault();
+		TestSegFault(); // never returns
 	}
 #endif
 
 	if (m_commandLineParser->GetWebGet())
 	{
-		ProcessWebGet();
-		return true;
+		ProcessWebGet(); // never returns
 	}
 
 	if (m_commandLineParser->GetSigVerify())
 	{
-		ProcessSigVerify();
-		return true;
+		ProcessSigVerify(); // never returns
 	}
 
 	// client request
 	if (m_commandLineParser->GetClientOperation() != CommandLineParser::opClientNoOperation)
 	{
-		ProcessClientRequest();
-		return true;
+		ProcessClientRequest(); // never returns
 	}
 
 	if (m_commandLineParser->GetPrintOptions())
 	{
-		PrintOptions();
-		return true;
+		PrintOptions(); // never returns
 	}
-
-	return false;
 }
 
 void NZBGet::StartRemoteServer()
@@ -476,7 +517,11 @@ void NZBGet::StartRemoteServer()
 	m_remoteServer = std::make_unique<RemoteServer>(false);
 	m_remoteServer->Start();
 
-	if (m_options->GetSecureControl())
+	if (m_options->GetSecureControl()
+#ifndef WIN32
+		&& !(m_options->GetControlIp() && m_options->GetControlIp()[0] == '/')
+#endif
+)
 	{
 		m_remoteSecureServer = std::make_unique<RemoteServer>(true);
 		m_remoteSecureServer->Start();
@@ -489,37 +534,55 @@ void NZBGet::StopRemoteServer()
 	{
 		debug("stopping RemoteServer");
 		m_remoteServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteServer->IsRunning())
-		{
-			debug("Killing RemoteServer");
-			m_remoteServer->Kill();
-		}
-		debug("RemoteServer stopped");
 	}
 
 	if (m_remoteSecureServer)
 	{
 		debug("stopping RemoteSecureServer");
 		m_remoteSecureServer->Stop();
-		int maxWaitMSec = 1000;
-		while (m_remoteSecureServer->IsRunning() && maxWaitMSec > 0)
-		{
-			usleep(100 * 1000);
-			maxWaitMSec -= 100;
-		}
-		if (m_remoteSecureServer->IsRunning())
-		{
-			debug("Killing RemoteSecureServer");
-			m_remoteSecureServer->Kill();
-		}
-		debug("RemoteSecureServer stopped");
 	}
+
+	int maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		Util::Sleep(100);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		m_remoteServer->ForceStop();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		m_remoteSecureServer->ForceStop();
+	}
+
+	maxWaitMSec = 5000;
+	while (((m_remoteServer && m_remoteServer->IsRunning()) ||
+		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
+		maxWaitMSec > 0)
+	{
+		Util::Sleep(100);
+		maxWaitMSec -= 100;
+	}
+
+	if (m_remoteServer && m_remoteServer->IsRunning())
+	{
+		debug("Killing RemoteServer");
+		m_remoteServer->Kill();
+	}
+
+	if (m_remoteSecureServer && m_remoteSecureServer->IsRunning())
+	{
+		debug("Killing RemoteSecureServer");
+		m_remoteSecureServer->Kill();
+	}
+
+	debug("RemoteServer stopped");
 }
 
 void NZBGet::StartFrontend()
@@ -561,7 +624,7 @@ void NZBGet::StopFrontend()
 		}
 		while (m_frontend->IsRunning())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 		}
 		debug("Frontend stopped");
 	}
@@ -641,7 +704,14 @@ void NZBGet::DoMainLoop()
 				m_serviceCoordinator->Stop();
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
+
+		if (m_options->GetServerMode() && !m_stopped)
+		{
+			// wait for stop signal
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&]{ return m_stopped; });
+		}
 	}
 
 	debug("Main program loop terminated");
@@ -653,10 +723,7 @@ void NZBGet::Run(bool reload)
 
 	Init();
 
-	if (ProcessDirect())
-	{
-		return;
-	}
+	ProcessDirect();
 
 	StartRemoteServer();
 	StartFrontend();
@@ -675,111 +742,116 @@ void NZBGet::Run(bool reload)
 
 	StopRemoteServer();
 	StopFrontend();
+
+	Final();
 }
 
 void NZBGet::ProcessClientRequest()
 {
 	RemoteClient Client;
+	bool ok = false;
 
 	switch (m_commandLineParser->GetClientOperation())
 	{
 		case CommandLineParser::opClientRequestListFiles:
-			Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(true, false, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListGroups:
-			Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
+			ok = Client.RequestServerList(false, true, m_commandLineParser->GetMatchMode() == CommandLineParser::mmRegEx ? m_commandLineParser->GetEditQueueText() : nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestListStatus:
-			Client.RequestServerList(false, false, nullptr);
+			ok = Client.RequestServerList(false, false, nullptr);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadPause:
-			Client.RequestServerPauseUnpause(true, rpDownload);
+			ok = Client.RequestServerPauseUnpause(true, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestDownloadUnpause:
-			Client.RequestServerPauseUnpause(false, rpDownload);
+			ok = Client.RequestServerPauseUnpause(false, rpDownload);
 			break;
 
 		case CommandLineParser::opClientRequestSetRate:
-			Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
+			ok = Client.RequestServerSetDownloadRate(m_commandLineParser->GetSetRate());
 			break;
 
 		case CommandLineParser::opClientRequestDumpDebug:
-			Client.RequestServerDumpDebug();
+			ok = Client.RequestServerDumpDebug();
 			break;
 
 		case CommandLineParser::opClientRequestEditQueue:
-			Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
+			ok = Client.RequestServerEditQueue((DownloadQueue::EEditAction)m_commandLineParser->GetEditQueueAction(),
 				m_commandLineParser->GetEditQueueOffset(), m_commandLineParser->GetEditQueueText(),
 				m_commandLineParser->GetEditQueueIdList(), m_commandLineParser->GetEditQueueNameList(),
 				(ERemoteMatchMode)m_commandLineParser->GetMatchMode());
 			break;
 
 		case CommandLineParser::opClientRequestLog:
-			Client.RequestServerLog(m_commandLineParser->GetLogLines());
+			ok = Client.RequestServerLog(m_commandLineParser->GetLogLines());
 			break;
 
 		case CommandLineParser::opClientRequestShutdown:
-			Client.RequestServerShutdown();
+			ok = Client.RequestServerShutdown();
 			break;
 
 		case CommandLineParser::opClientRequestReload:
-			Client.RequestServerReload();
+			ok = Client.RequestServerReload();
 			break;
 
 		case CommandLineParser::opClientRequestDownload:
-			Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
+			ok = Client.RequestServerDownload(m_commandLineParser->GetAddNzbFilename(), m_commandLineParser->GetArgFilename(),
 				m_commandLineParser->GetAddCategory(), m_commandLineParser->GetAddTop(), m_commandLineParser->GetAddPaused(), m_commandLineParser->GetAddPriority(),
 				m_commandLineParser->GetAddDupeKey(), m_commandLineParser->GetAddDupeMode(), m_commandLineParser->GetAddDupeScore());
 			break;
 
 		case CommandLineParser::opClientRequestVersion:
-			Client.RequestServerVersion();
+			ok = Client.RequestServerVersion();
 			break;
 
 		case CommandLineParser::opClientRequestPostQueue:
-			Client.RequestPostQueue();
+			ok = Client.RequestPostQueue();
 			break;
 
 		case CommandLineParser::opClientRequestWriteLog:
-			Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
+			ok = Client.RequestWriteLog(m_commandLineParser->GetWriteLogKind(), m_commandLineParser->GetLastArg());
 			break;
 
 		case CommandLineParser::opClientRequestScanAsync:
-			Client.RequestScan(false);
+			ok = Client.RequestScan(false);
 			break;
 
 		case CommandLineParser::opClientRequestScanSync:
-			Client.RequestScan(true);
+			ok = Client.RequestScan(true);
 			break;
 
 		case CommandLineParser::opClientRequestPostPause:
-			Client.RequestServerPauseUnpause(true, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(true, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestPostUnpause:
-			Client.RequestServerPauseUnpause(false, rpPostProcess);
+			ok = Client.RequestServerPauseUnpause(false, rpPostProcess);
 			break;
 
 		case CommandLineParser::opClientRequestScanPause:
-			Client.RequestServerPauseUnpause(true, rpScan);
+			ok = Client.RequestServerPauseUnpause(true, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestScanUnpause:
-			Client.RequestServerPauseUnpause(false, rpScan);
+			ok = Client.RequestServerPauseUnpause(false, rpScan);
 			break;
 
 		case CommandLineParser::opClientRequestHistory:
 		case CommandLineParser::opClientRequestHistoryAll:
-			Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
+			ok = Client.RequestHistory(m_commandLineParser->GetClientOperation() == CommandLineParser::opClientRequestHistoryAll);
 			break;
 
 		case CommandLineParser::opClientNoOperation:
-			break;
+			return;
 	}
+
+	exit(ok ? 0 : 1);
 }
 
 void NZBGet::ProcessWebGet()
@@ -842,14 +914,20 @@ void NZBGet::Stop(bool reload)
 #endif
 		}
 	}
+
+	// trigger stop/reload signal
+	Guard guard(m_waitMutex);
+	m_stopped = true;
+	m_waitCond.NotifyAll();
 }
 
 void NZBGet::PrintOptions()
 {
-	for (Options::OptEntry& optEntry : *g_Options->GuardOptEntries())
+	for (Options::OptEntry& optEntry : g_Options->GuardOptEntries())
 	{
 		printf("%s = \"%s\"\n", optEntry.GetName(), optEntry.GetValue());
 	}
+	exit(0);
 }
 
 #ifndef WIN32
@@ -865,16 +943,12 @@ void NZBGet::Daemonize()
 	// obtain a new process group
 	setsid();
 
-	// close all descriptors
-	for (int i = getdtablesize(); i >= 0; --i)
-	{
-		close(i);
-	}
-
 	// handle standart I/O
 	int d = open("/dev/null", O_RDWR);
-	dup(d);
-	dup(d);
+	dup2(d, 0);
+	dup2(d, 1);
+	dup2(d, 2);
+	close(d);
 
 	// set up lock-file
 	int lfp = -1;
@@ -886,7 +960,12 @@ void NZBGet::Daemonize()
 			error("Starting daemon failed: could not create lock-file %s", m_options->GetLockFile());
 			exit(1);
 		}
+
+#ifdef HAVE_LOCKF
 		if (lockf(lfp, F_TLOCK, 0) < 0)
+#else
+		if (flock(lfp, LOCK_EX) < 0)
+#endif
 		{
 			error("Starting daemon failed: could not acquire lock on lock-file %s", m_options->GetLockFile());
 			exit(1);
@@ -928,11 +1007,11 @@ void NZBGet::Daemonize()
 #endif
 
 void NZBGet::AddNewsServer(int id, bool active, const char* name, const char* host,
-	int port, const char* user, const char* pass, bool joinGroup, bool tls,
-	const char* cipher, int maxConnections, int retention, int level, int group)
+	int port, int ipVersion, const char* user, const char* pass, bool joinGroup, bool tls,
+	const char* cipher, int maxConnections, int retention, int level, int group, bool optional)
 {
-	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, user, pass, joinGroup,
-		tls, cipher, maxConnections, retention, level, group));
+	m_serverPool->AddServer(std::make_unique<NewsServer>(id, active, name, host, port, ipVersion, user, pass, joinGroup,
+		tls, cipher, maxConnections, retention, level, group, optional));
 }
 
 void NZBGet::AddFeed(int id, const char* name, const char* url, int interval, const char* filter,

daemon/main/nzbget.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -40,6 +40,9 @@
 //#define HAVE_LIBGNUTLS
 #endif
 
+/* Define to 1 if OpenSSL supports function "X509_check_host". */
+#define HAVE_X509_CHECK_HOST 1
+
 /* Define to the name of macro which returns the name of function being
 compiled */
 #define FUNCTION_MACRO_NAME __FUNCTION__
@@ -56,9 +59,6 @@ compiled */
 /* Define to 1 if variadic macros are supported */
 #define HAVE_VARIADIC_MACROS
 
-/* Define to 1 if libpar2 supports cancelling (needs a special patch) */
-#define HAVE_PAR2_CANCEL
-
 /* Define to 1 if function GetAddrInfo is supported */
 #define HAVE_GETADDRINFO
 
@@ -66,7 +66,16 @@ compiled */
 #define SOCKLEN_T socklen_t
 
 /* Define to 1 if you have the <regex.h> header file. */
+#ifndef DISABLE_REGEX
 #define HAVE_REGEX_H 1
+// Static linking to regex library
+#define REGEX_STATIC
+#endif
+
+#ifndef DISABLE_GZIP
+// Static linking to zlib library
+#define ZLIB_WINAPI
+#endif
 
 /* Suppress warnings */
 #define _CRT_SECURE_NO_DEPRECATE
@@ -74,13 +83,21 @@ compiled */
 /* Suppress warnings */
 #define _CRT_NONSTDC_NO_WARNINGS
 
+#ifndef _WIN64
 #define _USE_32BIT_TIME_T
+#endif
 
 #if _WIN32_WINNT < 0x0501
 #undef _WIN32_WINNT
 #define _WIN32_WINNT 0x0501
 #endif
 
+#ifdef _WIN64
+#define __amd64__
+#else
+#define __i686__
+#endif
+
 #ifdef _DEBUG
 // detection of memory leaks
 #define _CRTDBG_MAP_ALLOC
@@ -98,7 +115,7 @@ compiled */
 
 // WINDOWS INCLUDES
 
-// Using "WIN32_LEAN_AND_MEAN" to disable including on many unneeded headers
+// Using "WIN32_LEAN_AND_MEAN" to disable including of many unneeded headers
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 
@@ -151,21 +168,29 @@ using namespace MSXML;
 #include <sys/resource.h>
 #include <sys/statvfs.h>
 #include <sys/wait.h>
+#include <sys/un.h>
+#include <sys/file.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <stdint.h>
 #include <pwd.h>
 #include <dirent.h>
 
+#ifndef DISABLE_LIBXML2
 #include <libxml/parser.h>
 #include <libxml/xmlreader.h>
 #include <libxml/xmlerror.h>
 #include <libxml/entities.h>
+#endif
 
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
 #endif
 
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#endif
+
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
 #endif
@@ -184,6 +209,7 @@ using namespace MSXML;
 #include <stdarg.h>
 #include <time.h>
 #include <ctype.h>
+#include <inttypes.h>
 
 #include <string>
 #include <vector>
@@ -191,11 +217,22 @@ using namespace MSXML;
 #include <list>
 #include <set>
 #include <map>
+#include <unordered_map>
 #include <iterator>
 #include <algorithm>
-#include <iostream>
 #include <fstream>
 #include <memory>
+#include <functional>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <chrono>
+
+// NOTE: do not include <iostream> in "nzbget.h". <iostream> contains objects requiring
+// intialization, causing every unit in nzbget to have initialization routine. This in particular
+// is causing fatal problems in SIMD units which must not have static initialization because
+// they contain code with runtime CPU dispatching.
+//#include <iostream>
 
 #ifdef HAVE_LIBGNUTLS
 #ifdef WIN32
@@ -204,12 +241,16 @@ typedef SSIZE_T ssize_t;
 typedef int pid_t;
 #endif
 #include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
 #if GNUTLS_VERSION_NUMBER <= 0x020b00
 #define NEED_GCRYPT_LOCKING
 #endif
 #ifdef NEED_GCRYPT_LOCKING
 #include <gcrypt.h>
 #endif /* NEED_GCRYPT_LOCKING */
+#include <nettle/sha.h>
+#include <nettle/pbkdf2.h>
+#include <nettle/aes.h>
 #endif /* HAVE_LIBGNUTLS */
 
 #ifdef HAVE_OPENSSL
@@ -218,6 +259,8 @@ typedef int pid_t;
 #include <openssl/rsa.h>
 #include <openssl/sha.h>
 #include <openssl/pem.h>
+#include <openssl/x509v3.h>
+#include <openssl/comp.h>
 #endif /* HAVE_OPENSSL */
 
 #ifdef HAVE_REGEX_H
@@ -235,9 +278,6 @@ typedef int pid_t;
 #ifdef HAVE_MEMORY_H
 # include <memory.h>
 #endif
-#ifdef HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
 #endif /* NOT DISABLE_PARCHECK */
 
 
@@ -264,7 +304,6 @@ typedef int pid_t;
 #define S_ISDIR(mode) __S_ISTYPE((mode), _S_IFDIR)
 #define S_ISREG(mode) __S_ISTYPE((mode), _S_IFREG)
 #define S_DIRMODE nullptr
-#define usleep(usec) Sleep((usec) / 1000)
 #define socklen_t int
 #define SHUT_WR 0x01
 #define SHUT_RDWR 0x02
@@ -286,11 +325,22 @@ typedef int pid_t;
 #define FOPEN_WB "wbN"
 #define FOPEN_AB "abN"
 
+#define __SSE2__
+#define __SSSE3__
+#define __PCLMUL__
+
 #ifdef DEBUG
 // redefine "exit" to avoid printing memory leaks report when terminated because of wrong command line switches
 #define exit(code) ExitProcess(code)
 #endif
 
+#ifdef HAVE_OPENSSL
+FILE _iob[] = {*stdin, *stdout, *stderr};
+extern "C" FILE * __cdecl __iob_func(void) { return _iob; }
+// For static linking of OpenSSL libraries:
+#pragma comment (lib, "legacy_stdio_definitions.lib")
+#endif /* HAVE_OPENSSL */
+
 #else
 
 // POSIX
@@ -318,24 +368,62 @@ typedef int pid_t;
 
 #ifdef HAVE_STDINT_H
 typedef uint8_t uint8;
+typedef int16_t int16;
+typedef uint16_t uint16;
 typedef uint32_t int32;
 typedef uint32_t uint32;
 typedef int64_t int64;
 typedef uint64_t uint64;
 #else
 typedef unsigned char uint8;
+typedef signed short int16;
+typedef unsigned short uint16;
 typedef signed int int32;
 typedef unsigned int uint32;
 typedef signed long long int64;
 typedef unsigned long long uint64;
 #endif
 
+#ifndef PRId64
+#define PRId64 "lld"
+#endif
+#ifndef PRIi64
+#define PRIi64 "lli"
+#endif
+#ifndef PRIu64
+#define PRIu64 "llu"
+#endif
+
 typedef unsigned char uchar;
 
+// Assume little endian if byte order is not defined
+#ifndef __BYTE_ORDER
+#define __LITTLE_ENDIAN 1234
+#define __BIG_ENDIAN    4321
+#define __BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
 #ifdef __GNUC__
 #define PRINTF_SYNTAX(strindex) __attribute__ ((format (printf, strindex, strindex+1)))
+#define SCANF_SYNTAX(strindex) __attribute__ ((format (scanf, strindex, strindex+1)))
 #else
 #define PRINTF_SYNTAX(strindex)
+#define SCANF_SYNTAX(strindex)
+#endif
+
+// providing "std::make_unique" for GCC 4.8.x (only 4.8.x)
+#if __GNUC__ && __cplusplus < 201402L && __cpp_generic_lambdas < 201304
+namespace std {
+template<class T> struct _Unique_if { typedef unique_ptr<T> _Single_object; };
+template<class T> struct _Unique_if<T[]> { typedef unique_ptr<T[]> _Unknown_bound; };
+template<class T, class... Args> typename _Unique_if<T>::_Single_object make_unique(Args&&... args) {
+	return unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+template<class T> typename _Unique_if<T>::_Unknown_bound make_unique(size_t n) {
+	typedef typename remove_extent<T>::type U;
+	return unique_ptr<T>(new U[n]());
+}
+}
 #endif
 
 #endif /* NZBGET_H */

daemon/nntp/ArticleDownloader.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "StatMeter.h"
 #include "Util.h"
@@ -33,7 +34,6 @@ ArticleDownloader::ArticleDownloader()
 {
 	debug("Creating ArticleDownloader");
 
-	m_articleWriter.SetOwner(this);
 	SetLastUpdateTimeNow();
 }
 
@@ -63,8 +63,8 @@ void ArticleDownloader::SetInfoName(const char* infoName)
 		- if download fails with error "Not-Found" (article or group not found) or with CRC error,
 		  add the server to failed server list;
 		- if download fails with general failure error (article incomplete, other unknown error
-		  codes), try the same server again as many times as defined by option <Retries>; if all attempts
-		  fail, add the server to failed server list;
+		  codes), try the same server again as many times as defined by option <ArticleRetries>;
+		  if all attempts fail, add the server to failed server list;
 		- if all servers from current level were tried, increase level;
 		- if all servers from all levels were tried, break the loop with failure status.
 	<end-loop>
@@ -80,7 +80,7 @@ void ArticleDownloader::Run()
 	m_articleWriter.Prepare();
 
 	EStatus status = adFailed;
-	int retries = g_Options->GetRetries() > 0 ? g_Options->GetRetries() : 1;
+	int retries = g_Options->GetArticleRetries() > 0 ? g_Options->GetArticleRetries() : 1;
 	int remainedRetries = retries;
 	ServerPool::RawServerList failedServers;
 	failedServers.reserve(g_ServerPool->GetServers()->size());
@@ -98,13 +98,13 @@ void ArticleDownloader::Run()
 		while (!m_connection && !(IsStopped() || serverConfigGeneration != g_ServerPool->GetGeneration()))
 		{
 			m_connection = g_ServerPool->GetConnection(level, wantServer, &failedServers);
-			usleep(5 * 1000);
+			Util::Sleep(5);
 		}
 		SetLastUpdateTimeNow();
 		SetStatus(adRunning);
 
-		if (IsStopped() || (g_Options->GetPauseDownload() && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
@@ -112,6 +112,7 @@ void ArticleDownloader::Run()
 		}
 
 		lastServer = m_connection->GetNewsServer();
+		level = lastServer->GetNormLevel();
 
 		m_connection->SetSuppressErrors(false);
 
@@ -172,9 +173,11 @@ void ArticleDownloader::Run()
 			remainedRetries--;
 		}
 
+		bool optionalBlocked = false;
 		if (!connected && m_connection && !IsStopped())
 		{
 			g_ServerPool->BlockServer(lastServer);
+			optionalBlocked = lastServer->GetOptional();
 		}
 
 		wantServer = nullptr;
@@ -192,17 +195,20 @@ void ArticleDownloader::Run()
 			break;
 		}
 
-		if (IsStopped() || (g_Options->GetPauseDownload() && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
 			break;
 		}
 
-		if (!wantServer && (connected || retentionFailure))
+		if (!wantServer && (connected || retentionFailure || optionalBlocked))
 		{
-			failedServers.push_back(lastServer);
+			if (!optionalBlocked)
+			{
+				failedServers.push_back(lastServer);
+			}
 
 			// if all servers from current level were tried, increase level
 			// if all servers from all levels were tried, break the loop with failure status
@@ -212,7 +218,8 @@ void ArticleDownloader::Run()
 			{
 				if (candidateServer->GetNormLevel() == level)
 				{
-					bool serverFailed = !candidateServer->GetActive() || candidateServer->GetMaxConnections() == 0;
+					bool serverFailed = !candidateServer->GetActive() || candidateServer->GetMaxConnections() == 0 ||
+						(candidateServer->GetOptional() && g_ServerPool->IsServerBlocked(candidateServer));
 					if (!serverFailed)
 					{
 						for (NewsServer* ignoreServer : failedServers)
@@ -289,6 +296,11 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	m_writingStarted = false;
 	m_articleInfo->SetCrc(0);
 
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Reset();
+	}
+
 	if (m_connection->GetNewsServer()->GetJoinGroup())
 	{
 		// change group
@@ -309,14 +321,8 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	}
 
 	// retrieve article
-	for (int retry = 3; retry > 0; retry--)
-	{
-		response = m_connection->Request(BString<1024>("ARTICLE %s\r\n", m_articleInfo->GetMessageId()));
-		if ((response && !strncmp(response, "2", 1)) || m_connection->GetAuthError())
-		{
-			break;
-		}
-	}
+	response = m_connection->Request(BString<1024>("%s %s\r\n",
+		g_Options->GetRawArticle() ? "ARTICLE" : "BODY", m_articleInfo->GetMessageId()));
 
 	status = CheckResponse(response, "could not fetch article");
 	if (status != adFinished)
@@ -324,47 +330,35 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 		return status;
 	}
 
-	if (g_Options->GetDecode())
-	{
-		m_yDecoder.Clear();
-		m_yDecoder.SetCrcCheck(g_Options->GetCrcCheck());
-		m_uDecoder.Clear();
-	}
+	m_decoder.Clear();
+	m_decoder.SetCrcCheck(g_Options->GetCrcCheck());
+	m_decoder.SetRawMode(g_Options->GetRawArticle());
 
-	bool body = false;
-	bool end = false;
-	CharBuffer lineBuf(1024*10);
 	status = adRunning;
+	CharBuffer lineBuf(1024*4);
 
-	while (!IsStopped())
+	while (!IsStopped() && !m_decoder.GetEof())
 	{
-		time_t oldTime = m_lastUpdateTime;
-		SetLastUpdateTimeNow();
-		if (oldTime != m_lastUpdateTime)
-		{
-			AddServerData();
-		}
-
-		// Throttle the bandwidth
-		while (!IsStopped() && (g_Options->GetDownloadRate() > 0.0f) &&
-			(g_StatMeter->CalcCurrentDownloadSpeed() > g_Options->GetDownloadRate() ||
-			g_StatMeter->CalcMomentaryDownloadSpeed() > g_Options->GetDownloadRate()))
+		// throttle the bandwidth
+		while (!IsStopped() && (g_WorkState->GetSpeedLimit() > 0.0f) &&
+			(g_StatMeter->CalcCurrentDownloadSpeed() > g_WorkState->GetSpeedLimit() ||
+			g_StatMeter->CalcMomentaryDownloadSpeed() > g_WorkState->GetSpeedLimit()))
 		{
 			SetLastUpdateTimeNow();
-			usleep(10 * 1000);
+			Util::Sleep(10);
 		}
 
-		int len = 0;
-		char* line = m_connection->ReadLine(lineBuf, lineBuf.Size(), &len);
-
-		g_StatMeter->AddSpeedReading(len);
-		if (g_Options->GetAccurateRate())
+		char* buffer;
+		int len;
+		m_connection->ReadBuffer(&buffer, &len);
+		if (len == 0)
 		{
-			AddServerData();
+			len = m_connection->TryRecv(lineBuf, lineBuf.Size());
+			buffer = lineBuf;
 		}
 
-		// Have we encountered a timeout?
-		if (!line)
+		// have we encountered a timeout?
+		if (len <= 0)
 		{
 			if (!IsStopped())
 			{
@@ -374,67 +368,25 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 			break;
 		}
 
-		//detect end of article
-		if (!strcmp(line, ".\r\n") || !strcmp(line, ".\n"))
+		g_StatMeter->AddSpeedReading(len);
+		time_t oldTime = m_lastUpdateTime;
+		SetLastUpdateTimeNow();
+		if (oldTime != m_lastUpdateTime)
 		{
-			end = true;
-			break;
+			AddServerData();
 		}
 
-		//detect lines starting with "." (marked as "..")
-		if (!strncmp(line, "..", 2))
-		{
-			line++;
-			len--;
-		}
-
-		if (!body)
-		{
-			// detect body of article
-			if (*line == '\r' || *line == '\n')
-			{
-				body = true;
-			}
-			// check id of returned article
-			else if (!strncmp(line, "Message-ID: ", 12))
-			{
-				char* p = line + 12;
-				if (strncmp(p, m_articleInfo->GetMessageId(), strlen(m_articleInfo->GetMessageId())))
-				{
-					if (char* e = strrchr(p, '\r')) *e = '\0'; // remove trailing CR-character
-					detail("Article %s @ %s failed: Wrong message-id, expected %s, returned %s", *m_infoName,
-						*m_connectionName, m_articleInfo->GetMessageId(), p);
-					status = adFailed;
-					break;
-				}
-			}
-		}
-
-		if (m_format == Decoder::efUnknown && g_Options->GetDecode())
-		{
-			m_format = Decoder::DetectFormat(line, len, body);
-			if (m_format != Decoder::efUnknown)
-			{
-				// sometimes news servers misbehave and send article body without new line separator between headers and body
-				// if we found decoder signature we know the body is already arrived
-				body = true;
-			}
-		}
+		// decode article data
+		len = m_decoder.DecodeBuffer(buffer, len);
 
 		// write to output file
-		if (((body && m_format != Decoder::efUnknown) || !g_Options->GetDecode()) && !Write(line, len))
+		if (len > 0 && !Write(buffer, len))
 		{
 			status = adFatalError;
 			break;
 		}
 	}
 
-	if (!end && status == adRunning && !IsStopped())
-	{
-		detail("Article %s @ %s failed: article incomplete", *m_infoName, *m_connectionName);
-		status = adFailed;
-	}
-
 	if (IsStopped())
 	{
 		status = adFailed;
@@ -493,89 +445,69 @@ ArticleDownloader::EStatus ArticleDownloader::CheckResponse(const char* response
 	}
 }
 
-bool ArticleDownloader::Write(char* line, int len)
+bool ArticleDownloader::Write(char* buffer, int len)
 {
 	const char* articleFilename = nullptr;
 	int64 articleFileSize = 0;
 	int64 articleOffset = 0;
 	int articleSize = 0;
 
-	if (g_Options->GetDecode())
+	if (!m_writingStarted)
 	{
-		if (m_format == Decoder::efYenc)
+		if (!g_Options->GetRawArticle())
 		{
-			len = m_yDecoder.DecodeBuffer(line, len);
-			articleFilename = m_yDecoder.GetArticleFilename();
-			articleFileSize = m_yDecoder.GetSize();
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			len = m_uDecoder.DecodeBuffer(line, len);
-			articleFilename = m_uDecoder.GetArticleFilename();
-		}
-		else
-		{
-			detail("Decoding %s failed: unsupported encoding", *m_infoName);
-			return false;
-		}
-
-		if (len > 0 && m_format == Decoder::efYenc)
-		{
-			if (m_yDecoder.GetBegin() == 0 || m_yDecoder.GetEnd() == 0)
+			articleFilename = m_decoder.GetArticleFilename();
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
-				return false;
+				if (m_decoder.GetBeginPos() == 0 || m_decoder.GetEndPos() == 0)
+				{
+					return false;
+				}
+				articleFileSize = m_decoder.GetSize();
+				articleOffset = m_decoder.GetBeginPos() - 1;
+				articleSize = (int)(m_decoder.GetEndPos() - m_decoder.GetBeginPos() + 1);
+				if (articleSize <= 0 || articleSize > 1024*1024*1024)
+				{
+					warn("Malformed article %s: size %i out of range", *m_infoName, articleSize);
+					return false;
+				}
 			}
-			articleOffset = m_yDecoder.GetBegin() - 1;
-			articleSize = (int)(m_yDecoder.GetEnd() - m_yDecoder.GetBegin() + 1);
 		}
-	}
 
-	if (!m_writingStarted && len > 0)
-	{
-		if (!m_articleWriter.Start(m_format, articleFilename, articleFileSize, articleOffset, articleSize))
+		if (!m_articleWriter.Start(m_decoder.GetFormat(), articleFilename, articleFileSize, articleOffset, articleSize))
 		{
 			return false;
 		}
 		m_writingStarted = true;
 	}
 
-	bool ok = len == 0 || m_articleWriter.Write(line, len);
+	bool ok = m_articleWriter.Write(buffer, len);
+
+	if (m_contentAnalyzer)
+	{
+		m_contentAnalyzer->Append(buffer, len);
+	}
 
 	return ok;
 }
 
 ArticleDownloader::EStatus ArticleDownloader::DecodeCheck()
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
-		Decoder* decoder = nullptr;
-		if (m_format == Decoder::efYenc)
-		{
-			decoder = &m_yDecoder;
-		}
-		else if (m_format == Decoder::efUx)
-		{
-			decoder = &m_uDecoder;
-		}
-		else
-		{
-			detail("Decoding %s failed: no binary data or unsupported encoding format", *m_infoName);
-			return adFailed;
-		}
-
-		Decoder::EStatus status = decoder->Check();
+		Decoder::EStatus status = m_decoder.Check();
 
 		if (status == Decoder::dsFinished)
 		{
-			if (decoder->GetArticleFilename())
+			if (m_decoder.GetArticleFilename())
 			{
-				m_articleFilename = decoder->GetArticleFilename();
+				m_articleFilename = m_decoder.GetArticleFilename();
 			}
 
-			if (m_format == Decoder::efYenc)
+			if (m_decoder.GetFormat() == Decoder::efYenc)
 			{
 				m_articleInfo->SetCrc(g_Options->GetCrcCheck() ?
-					m_yDecoder.GetCalculatedCrc() : m_yDecoder.GetExpectedCrc());
+					m_decoder.GetCalculatedCrc() : m_decoder.GetExpectedCrc());
 			}
 
 			return adFinished;
@@ -636,22 +568,6 @@ void ArticleDownloader::Stop()
 	debug("ArticleDownloader stopped successfully");
 }
 
-bool ArticleDownloader::Terminate()
-{
-	NntpConnection* connection = m_connection;
-	bool terminated = Kill();
-	if (terminated && connection)
-	{
-		debug("Terminating connection");
-		connection->SetSuppressErrors(true);
-		connection->Cancel();
-		connection->Disconnect();
-		g_StatMeter->AddServerData(connection->FetchTotalBytesRead(), connection->GetNewsServer()->GetId());
-		g_ServerPool->FreeConnection(connection, true);
-	}
-	return terminated;
-}
-
 void ArticleDownloader::FreeConnection(bool keepConnected)
 {
 	if (m_connection)

daemon/nntp/ArticleDownloader.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -29,6 +29,15 @@
 #include "NntpConnection.h"
 #include "Decoder.h"
 #include "ArticleWriter.h"
+#include "Util.h"
+
+class ArticleContentAnalyzer
+{
+public:
+	virtual ~ArticleContentAnalyzer() {};
+	virtual void Reset() = 0;
+	virtual void Append(const void* buffer, int len) = 0;
+};
 
 class ArticleDownloader : public Thread, public Subject
 {
@@ -47,16 +56,6 @@ public:
 		adFatalError
 	};
 
-	class ArticleWriterImpl : public ArticleWriter
-	{
-	public:
-		void SetOwner(ArticleDownloader* owner) { m_owner = owner; }
-	protected:
-		virtual void SetLastUpdateTimeNow() { m_owner->SetLastUpdateTimeNow(); }
-	private:
-		ArticleDownloader* m_owner;
-	};
-
 	ArticleDownloader();
 	virtual ~ArticleDownloader();
 	void SetFileInfo(FileInfo* fileInfo) { m_fileInfo = fileInfo; }
@@ -67,7 +66,6 @@ public:
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 	virtual void Run();
 	virtual void Stop();
-	bool Terminate();
 	time_t GetLastUpdateTime() { return m_lastUpdateTime; }
 	void SetLastUpdateTimeNow();
 	const char* GetArticleFilename() { return m_articleFilename; }
@@ -77,6 +75,8 @@ public:
 	void SetConnection(NntpConnection* connection) { m_connection = connection; }
 	void CompleteFileParts() { m_articleWriter.CompleteFileParts(); }
 	int GetDownloadedSize() { return m_downloadedSize; }
+	void SetContentAnalyzer(std::unique_ptr<ArticleContentAnalyzer> contentAnalyzer) { m_contentAnalyzer = std::move(contentAnalyzer); }
+	ArticleContentAnalyzer* GetContentAnalyzer() { return m_contentAnalyzer.get(); }
 
 	void LogDebugInfo();
 
@@ -90,20 +90,19 @@ private:
 	CString m_connectionName;
 	CString m_articleFilename;
 	time_t m_lastUpdateTime;
-	Decoder::EFormat m_format = Decoder::efUnknown;
-	YDecoder m_yDecoder;
-	UDecoder m_uDecoder;
-	ArticleWriterImpl m_articleWriter;
+	Decoder m_decoder;
+	ArticleWriter m_articleWriter;
 	ServerStatList m_serverStats;
 	bool m_writingStarted;
 	int m_downloadedSize = 0;
+	std::unique_ptr<ArticleContentAnalyzer> m_contentAnalyzer;
 
 	EStatus Download();
 	EStatus DecodeCheck();
 	void FreeConnection(bool keepConnected);
 	EStatus CheckResponse(const char* response, const char* comment);
 	void SetStatus(EStatus status) { m_status = status; }
-	bool Write(char* line, int len);
+	bool Write(char* buffer, int len);
 	void AddServerData();
 };
 

daemon/nntp/ArticleWriter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -84,7 +84,7 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 			}
 
 			if (!outputInitialized && filename &&
-				FileSystem::FileExists(m_fileInfo->GetNzbInfo()->GetDestDir(), filename))
+				FileSystem::FileExists(BString<1024>("%s%c%s", m_fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, filename)))
 			{
 				m_duplicate = true;
 				return false;
@@ -106,14 +106,14 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 	}
 
 	// allocate cache buffer
-	if (g_Options->GetArticleCache() > 0 && g_Options->GetDecode() &&
+	if (g_Options->GetArticleCache() > 0 && !g_Options->GetRawArticle() &&
 		(!g_Options->GetDirectWrite() || m_format == Decoder::efYenc))
 	{
 		m_articleData = g_ArticleCache->Alloc(m_articleSize);
 
 		while (!m_articleData.GetData() && g_ArticleCache->GetFlushing())
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			m_articleData = g_ArticleCache->Alloc(m_articleSize);
 		}
 
@@ -125,18 +125,18 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 
 	if (!m_articleData.GetData())
 	{
-		bool directWrite = g_Options->GetDirectWrite() && m_format == Decoder::efYenc;
-		const char* filename = directWrite ? m_outputFilename : m_tempFilename;
-		if (!m_outFile.Open(filename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
+		bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
+		const char* outFilename = directWrite ? m_outputFilename : m_tempFilename;
+		if (!m_outFile.Open(outFilename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
 		{
 			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-				"Could not %s file %s: %s", directWrite ? "open" : "create", filename,
+				"Could not %s file %s: %s", directWrite ? "open" : "create", outFilename,
 				*FileSystem::GetLastErrorMessage());
 			return false;
 		}
 		SetWriteBuffer(m_outFile, m_articleInfo->GetSize());
 
-		if (g_Options->GetDirectWrite() && m_format == Decoder::efYenc)
+		if (directWrite)
 		{
 			m_outFile.Seek(m_articleOffset);
 		}
@@ -147,22 +147,31 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 
 bool ArticleWriter::Write(char* buffer, int len)
 {
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		m_articlePtr += len;
 	}
 
-	if (g_Options->GetDecode() && m_articleData.GetData())
+	if (m_articlePtr > m_articleSize)
+	{
+		// An attempt to write beyond article border is detected.
+		// That's an error condition (damaged article).
+		// We return 'false' since this isn't a fatal disk error and
+		// article size mismatch will be detected in decoder check anyway.
+		return true;
+	}
+
+	if (!g_Options->GetRawArticle() && m_articleData.GetData())
 	{
-		if (m_articlePtr > m_articleSize)
-		{
-			detail("Decoding %s failed: article size mismatch", *m_infoName);
-			return false;
-		}
 		memcpy(m_articleData.GetData() + m_articlePtr - len, buffer, len);
 		return true;
 	}
 
+	if (g_Options->GetSkipWrite())
+	{
+		return true;
+	}
+
 	return m_outFile.Write(buffer, len) > 0;
 }
 
@@ -177,9 +186,9 @@ void ArticleWriter::Finish(bool success)
 		return;
 	}
 
-	bool directWrite = g_Options->GetDirectWrite() && m_format == Decoder::efYenc;
+	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
 
-	if (g_Options->GetDecode())
+	if (!g_Options->GetRawArticle())
 	{
 		if (!directWrite && !m_articleData.GetData())
 		{
@@ -189,10 +198,9 @@ void ArticleWriter::Finish(bool success)
 					"Could not rename file %s to %s: %s", *m_tempFilename, m_resultFilename,
 					*FileSystem::GetLastErrorMessage());
 			}
+			FileSystem::DeleteFile(m_tempFilename);
 		}
 
-		FileSystem::DeleteFile(m_tempFilename);
-
 		if (m_articleData.GetData())
 		{
 			if (m_articleSize != m_articlePtr)
@@ -224,19 +232,20 @@ void ArticleWriter::Finish(bool success)
 /* creates output file and subdirectores */
 bool ArticleWriter::CreateOutputFile(int64 size)
 {
-	if (g_Options->GetDirectWrite() && FileSystem::FileExists(m_outputFilename) &&
-		FileSystem::FileSize(m_outputFilename) == size)
+	if (FileSystem::FileExists(m_outputFilename))
 	{
-		// keep existing old file from previous program session
-		return true;
+		if (FileSystem::FileSize(m_outputFilename) == size)
+		{
+			// keep existing old file from previous program session
+			return true;
+		}
+		// delete existing old file from previous program session
+		FileSystem::DeleteFile(m_outputFilename);
 	}
 
-	// delete eventually existing old file from previous program session
-	FileSystem::DeleteFile(m_outputFilename);
-
 	// ensure the directory exist
 	BString<1024> destDir;
-	destDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+	destDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 	CString errmsg;
 
 	if (!FileSystem::ForceDirectories(destDir, errmsg))
@@ -246,7 +255,7 @@ bool ArticleWriter::CreateOutputFile(int64 size)
 		return false;
 	}
 
-	if (!FileSystem::CreateSparseFile(m_outputFilename, size, errmsg))
+	if (!FileSystem::AllocateFile(m_outputFilename, size, g_Options->GetArticleCache() == 0, errmsg))
 	{
 		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 			"Could not create file %s: %s", *m_outputFilename, *errmsg);
@@ -264,7 +273,7 @@ void ArticleWriter::BuildOutputFilename()
 	m_articleInfo->SetResultFilename(filename);
 	m_tempFilename.Format("%s.tmp", *filename);
 
-	if (g_Options->GetDirectWrite())
+	if (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite())
 	{
 		Guard guard = m_fileInfo->GuardOutputFile();
 
@@ -275,7 +284,7 @@ void ArticleWriter::BuildOutputFilename()
 		else
 		{
 			filename.Format("%s%c%i.out.tmp", m_fileInfo->GetNzbInfo()->GetDestDir(),
-				(int)PATH_SEPARATOR, m_fileInfo->GetId());
+				PATH_SEPARATOR, m_fileInfo->GetId());
 			m_fileInfo->SetOutputFilename(filename);
 		}
 
@@ -288,22 +297,24 @@ void ArticleWriter::CompleteFileParts()
 	debug("Completing file parts");
 	debug("ArticleFilename: %s", m_fileInfo->GetFilename());
 
-	bool directWrite = g_Options->GetDirectWrite() && m_fileInfo->GetOutputInitialized();
+	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_fileInfo->GetOutputInitialized();
 
 	BString<1024> nzbName;
 	BString<1024> nzbDestDir;
+	BString<1024> filename;
 
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
 		nzbName = m_fileInfo->GetNzbInfo()->GetName();
 		nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir();
+		filename = m_fileInfo->GetFilename();
 	}
 
-	BString<1024> infoFilename("%s%c%s", *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
+	BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename);
 
 	bool cached = m_fileInfo->GetCachedArticles() > 0;
 
-	if (!g_Options->GetDecode())
+	if (g_Options->GetRawArticle())
 	{
 		detail("Moving articles for %s", *infoFilename);
 	}
@@ -329,12 +340,20 @@ void ArticleWriter::CompleteFileParts()
 		return;
 	}
 
-	CString ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+	CString ofn;
+	if (m_fileInfo->GetForceDirectWrite())
+	{
+		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename);
+	}
+	else
+	{
+		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename);
+	}
 
 	DiskFile outfile;
 	BString<1024> tmpdestfile("%s.tmp", *ofn);
 
-	if (g_Options->GetDecode() && !directWrite)
+	if (!g_Options->GetRawArticle() && !directWrite)
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!outfile.Open(tmpdestfile, DiskFile::omWrite))
@@ -354,7 +373,7 @@ void ArticleWriter::CompleteFileParts()
 		}
 		tmpdestfile = *m_outputFilename;
 	}
-	else if (!g_Options->GetDecode())
+	else if (g_Options->GetRawArticle())
 	{
 		FileSystem::DeleteFile(tmpdestfile);
 		if (!FileSystem::CreateDirectory(ofn))
@@ -382,7 +401,7 @@ void ArticleWriter::CompleteFileParts()
 		CharBuffer buffer;
 		bool firstArticle = true;
 
-		if (g_Options->GetDecode() && !directWrite)
+		if (!g_Options->GetRawArticle() && !directWrite)
 		{
 			buffer.Reserve(1024 * 64);
 		}
@@ -394,22 +413,27 @@ void ArticleWriter::CompleteFileParts()
 				continue;
 			}
 
-			if (g_Options->GetDecode() && !directWrite && pa->GetSegmentOffset() > -1 &&
+			if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 &&
 				pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1)
 			{
 				memset(buffer, 0, buffer.Size());
-				while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
-					outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				if (!g_Options->GetSkipWrite())
+				{
+					while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
+						outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
+				}
 			}
 
 			if (pa->GetSegmentContent())
 			{
-				outfile.Seek(pa->GetSegmentOffset());
-				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				if (!g_Options->GetSkipWrite())
+				{
+					outfile.Seek(pa->GetSegmentOffset());
+					outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+				}
 				pa->DiscardSegment();
-				SetLastUpdateTimeNow();
 			}
-			else if (g_Options->GetDecode() && !directWrite)
+			else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite())
 			{
 				DiskFile infile;
 				if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead))
@@ -419,7 +443,6 @@ void ArticleWriter::CompleteFileParts()
 					{
 						cnt = (int)infile.Read(buffer, buffer.Size());
 						outfile.Write(buffer, cnt);
-						SetLastUpdateTimeNow();
 					}
 					infile.Close();
 				}
@@ -428,14 +451,14 @@ void ArticleWriter::CompleteFileParts()
 					m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1);
 					m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1);
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-						"Could not find file %s for %s%c%s [%i/%i]",
-						pa->GetResultFilename(), *nzbName, (int)PATH_SEPARATOR, m_fileInfo->GetFilename(),
-						pa->GetPartNumber(), (int)m_fileInfo->GetArticles()->size());
+						"Could not find file %s for %s [%i/%i]",
+						pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(),
+						(int)m_fileInfo->GetArticles()->size());
 				}
 			}
-			else if (!g_Options->GetDecode())
+			else if (g_Options->GetRawArticle())
 			{
-				BString<1024> dstFileName("%s%c%03i", *ofn, (int)PATH_SEPARATOR, pa->GetPartNumber());
+				BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber());
 				if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName))
 				{
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
@@ -446,7 +469,7 @@ void ArticleWriter::CompleteFileParts()
 
 			if (m_format == Decoder::efYenc)
 			{
-				crc = firstArticle ? pa->GetCrc() : Util::Crc32Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
+				crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
 				firstArticle = false;
 			}
 		}
@@ -467,7 +490,8 @@ void ArticleWriter::CompleteFileParts()
 
 	if (directWrite)
 	{
-		if (!FileSystem::MoveFile(m_outputFilename, ofn))
+		if (!FileSystem::SameFilename(m_outputFilename, ofn) &&
+			!FileSystem::MoveFile(m_outputFilename, ofn))
 		{
 			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 				"Could not move file %s to %s: %s", *m_outputFilename, *ofn,
@@ -481,7 +505,7 @@ void ArticleWriter::CompleteFileParts()
 		{
 			debug("Checking old dir for: %s", *m_outputFilename);
 			BString<1024> oldDestDir;
-			oldDestDir.Set(m_outputFilename, FileSystem::BaseFileName(m_outputFilename) - m_outputFilename);
+			oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
 			if (FileSystem::DirEmpty(oldDestDir))
 			{
 				debug("Deleting old dir: %s", *oldDestDir);
@@ -494,51 +518,48 @@ void ArticleWriter::CompleteFileParts()
 	{
 		for (ArticleInfo* pa : m_fileInfo->GetArticles())
 		{
-			FileSystem::DeleteFile(pa->GetResultFilename());
+			if (pa->GetResultFilename())
+			{
+				FileSystem::DeleteFile(pa->GetResultFilename());
+			}
 		}
 	}
 
-	if (m_fileInfo->GetMissedArticles() == 0 && m_fileInfo->GetFailedArticles() == 0)
+	if (m_fileInfo->GetTotalArticles() == m_fileInfo->GetSuccessArticles())
 	{
 		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Successfully downloaded %s", *infoFilename);
 	}
-	else
+	else if (m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles() > 0)
 	{
 		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
 			"%i of %i article downloads failed for \"%s\"",
 			m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(),
 			m_fileInfo->GetTotalArticles(), *infoFilename);
-
-		if (g_Options->GetBrokenLog())
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", *nzbDestDir, (int)PATH_SEPARATOR);
-			DiskFile file;
-			if (file.Open(brokenLogName, DiskFile::omAppend))
-			{
-				file.Print("%s (%i/%i)%s", m_fileInfo->GetFilename(), m_fileInfo->GetSuccessArticles(),
-					m_fileInfo->GetTotalArticles(), LINE_ENDING);
-				file.Close();
-			}
-		}
-
-		crc = 0;
-
-		if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
-		{
-			g_DiskState->DiscardFile(m_fileInfo, false, true, false);
-			g_DiskState->SaveFileState(m_fileInfo, true);
-		}
 	}
-
-	CompletedFile::EStatus fileStatus = m_fileInfo->GetMissedArticles() == 0 &&
-		m_fileInfo->GetFailedArticles() == 0 ? CompletedFile::cfSuccess :
-		m_fileInfo->GetSuccessArticles() > 0 ? CompletedFile::cfPartial :
-		CompletedFile::cfFailure;
+	else
+	{
+		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename);
+	}
 
 	{
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
-		m_fileInfo->GetNzbInfo()->GetCompletedFiles()->emplace_back(
-			m_fileInfo->GetId(), FileSystem::BaseFileName(ofn), fileStatus, crc);
+
+		m_fileInfo->SetCrc(crc);
+		m_fileInfo->SetOutputFilename(ofn);
+
+		if (strcmp(m_fileInfo->GetFilename(), filename))
+		{
+			// file was renamed during completion, need to move the file
+			ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
+			if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn))
+			{
+				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
+					"Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(),
+					*ofn, *FileSystem::GetLastErrorMessage());
+			}
+			m_fileInfo->SetOutputFilename(ofn);
+		}
+
 		if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir))
 		{
 			// destination directory was changed during completion, need to move the file
@@ -561,10 +582,18 @@ void ArticleWriter::FlushCache()
 		ArticleCache::FlushGuard flushGuard = g_ArticleCache->GuardFlush();
 
 		std::vector<ArticleInfo*> cachedArticles;
-		cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
+
+			if (m_fileInfo->GetFlushLocked())
+			{
+				return;
+			}
+
+			m_fileInfo->SetFlushLocked(true);
+
+			cachedArticles.reserve(m_fileInfo->GetArticles()->size());
 			for (ArticleInfo* pa : m_fileInfo->GetArticles())
 			{
 				if (pa->GetSegmentContent())
@@ -576,7 +605,7 @@ void ArticleWriter::FlushCache()
 
 		for (ArticleInfo* pa : cachedArticles)
 		{
-			if (m_fileInfo->GetDeleted())
+			if (m_fileInfo->GetDeleted() && !m_fileInfo->GetNzbInfo()->GetParking())
 			{
 				// the file was deleted during flushing: stop flushing immediately
 				break;
@@ -589,6 +618,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not open file %s: %s", m_fileInfo->GetOutputFilename(),
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -604,6 +636,9 @@ void ArticleWriter::FlushCache()
 					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
 						"Could not create file %s: %s", *destFile,
 						*FileSystem::GetLastErrorMessage());
+					// prevent multiple error messages
+					pa->DiscardSegment();
+					flushedArticles++;
 					break;
 				}
 				needBufFile = true;
@@ -620,7 +655,10 @@ void ArticleWriter::FlushCache()
 				outfile.Seek(pa->GetSegmentOffset());
 			}
 
-			outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			if (!g_Options->GetSkipWrite())
+			{
+				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
+			}
 
 			flushedSize += pa->GetSegmentSize();
 			flushedArticles++;
@@ -645,6 +683,7 @@ void ArticleWriter::FlushCache()
 		{
 			Guard contentGuard = g_ArticleCache->GuardContent();
 			m_fileInfo->SetCachedArticles(m_fileInfo->GetCachedArticles() - flushedArticles);
+			m_fileInfo->SetFlushLocked(false);
 		}
 	}
 
@@ -670,14 +709,14 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 	// move already downloaded files to new destination
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		BString<1024> oldFileName("%s%c%s", oldDestDir, (int)PATH_SEPARATOR, completedFile.GetFileName());
-		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName());
+		BString<1024> oldFileName("%s%c%s", oldDestDir, PATH_SEPARATOR, completedFile.GetFilename());
+		BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename());
 
 		// check if file was not moved already
 		if (strcmp(oldFileName, newFileName))
 		{
 			// prevent overwriting of existing files
-			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFileName());
+			newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFilename());
 
 			detail("Moving file %s to %s", *oldFileName, *newFileName);
 			if (!FileSystem::MoveFile(oldFileName, newFileName))
@@ -688,57 +727,6 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 		}
 	}
 
-	// move brokenlog.txt
-	if (g_Options->GetBrokenLog())
-	{
-		BString<1024> oldBrokenLogName("%s%c_brokenlog.txt", oldDestDir, (int)PATH_SEPARATOR);
-		if (FileSystem::FileExists(oldBrokenLogName))
-		{
-			BString<1024> brokenLogName("%s%c_brokenlog.txt", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR);
-
-			detail("Moving file %s to %s", *oldBrokenLogName, *brokenLogName);
-			if (FileSystem::FileExists(brokenLogName))
-			{
-				// copy content to existing new file, then delete old file
-				DiskFile outfile;
-				if (outfile.Open(brokenLogName, DiskFile::omAppend))
-				{
-					DiskFile infile;
-					if (infile.Open(oldBrokenLogName, DiskFile::omRead))
-					{
-						CharBuffer buffer(1024 * 50);
-						int cnt = buffer.Size();
-						while (cnt == buffer.Size())
-						{
-							cnt = (int)infile.Read(buffer, buffer.Size());
-							outfile.Write(buffer, cnt);
-						}
-						infile.Close();
-						FileSystem::DeleteFile(oldBrokenLogName);
-					}
-					else
-					{
-						nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *oldBrokenLogName);
-					}
-					outfile.Close();
-				}
-				else
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
-				}
-			}
-			else
-			{
-				// move to new destination
-				if (!FileSystem::MoveFile(oldBrokenLogName, brokenLogName))
-				{
-					nzbInfo->PrintMessage(Message::mkError, "Could not move file %s to %s: %s",
-						*oldBrokenLogName, *brokenLogName, *FileSystem::GetLastErrorMessage());
-				}
-			}
-		}
-	}
-
 	// delete old directory (if empty)
 	if (FileSystem::DirEmpty(oldDestDir))
 	{
@@ -772,13 +760,6 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir)
 }
 
 
-ArticleCache::ArticleCache()
-{
-	m_allocated = 0;
-	m_flushing = false;
-	m_fileInfo = nullptr;
-}
-
 CachedSegmentData ArticleCache::Alloc(int size)
 {
 	Guard guard(m_allocMutex);
@@ -790,15 +771,20 @@ CachedSegmentData ArticleCache::Alloc(int size)
 		p = malloc(size);
 		if (p)
 		{
-			if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+			if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 			{
 				g_DiskState->WriteCacheFlag();
 			}
+			if (!m_allocated)
+			{
+				// Resume Run(), the notification arrives later, after releasing m_allocMutex
+				m_allocCond.NotifyAll();
+			}
 			m_allocated += size;
 		}
 	}
 
-	return CachedSegmentData((char*)p, size);
+	return CachedSegmentData((char*)p, p ? size : 0);
 }
 
 bool ArticleCache::Realloc(CachedSegmentData* segment, int newSize)
@@ -809,6 +795,8 @@ bool ArticleCache::Realloc(CachedSegmentData* segment, int newSize)
 	if (p)
 	{
 		m_allocated += newSize - segment->m_size;
+		segment->m_size = newSize;
+		segment->m_data = (char*)p;
 	}
 
 	return p;
@@ -822,7 +810,7 @@ void ArticleCache::Free(CachedSegmentData* segment)
 
 		Guard guard(m_allocMutex);
 		m_allocated -= segment->m_size;
-		if (!m_allocated && g_Options->GetSaveQueue() && g_Options->GetServerMode() && g_Options->GetContinuePartial())
+		if (!m_allocated && g_Options->GetServerMode() && g_Options->GetContinuePartial())
 		{
 			g_DiskState->DeleteCacheFlag();
 		}
@@ -838,42 +826,60 @@ void ArticleCache::Run()
 	bool justFlushed = false;
 	while (!IsStopped() || m_allocated > 0)
 	{
-		if ((justFlushed || resetCounter >= 1000  || IsStopped() ||
+		if ((justFlushed || resetCounter >= 1000 || IsStopped() ||
 			 (g_Options->GetDirectWrite() && m_allocated >= fillThreshold)) &&
 			m_allocated > 0)
 		{
 			justFlushed = CheckFlush(m_allocated >= fillThreshold);
 			resetCounter = 0;
 		}
+		else if (!m_allocated)
+		{
+			Guard guard(m_allocMutex);
+			m_allocCond.Wait(m_allocMutex, [&]{ return IsStopped() || m_allocated > 0; });
+			resetCounter = 0;
+		}
 		else
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			resetCounter += 5;
 		}
 	}
 }
 
+void ArticleCache::Stop()
+{
+	Thread::Stop();
+
+	// Resume Run() to exit it
+	Guard guard(m_allocMutex);
+	m_allocCond.NotifyAll();
+}
+
 bool ArticleCache::CheckFlush(bool flushEverything)
 {
 	debug("Checking cache, Allocated: %i, FlushEverything: %i", (int)m_allocated, (int)flushEverything);
 
 	BString<1024> infoName;
 
-	for (NzbInfo* nzbInfo : DownloadQueue::Guard()->GetQueue())
 	{
-		if (m_fileInfo)
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 		{
-			break;
-		}
-
-		for (FileInfo* fileInfo : nzbInfo->GetFileList())
-		{
-			if (fileInfo->GetCachedArticles() > 0 && (fileInfo->GetActiveDownloads() == 0 || flushEverything))
+			if (m_fileInfo)
 			{
-				m_fileInfo = fileInfo;
-				infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, m_fileInfo->GetFilename());
 				break;
 			}
+
+			for (FileInfo* fileInfo : nzbInfo->GetFileList())
+			{
+				if (fileInfo->GetCachedArticles() > 0 && (fileInfo->GetActiveDownloads() == 0 || flushEverything))
+				{
+					m_fileInfo = fileInfo;
+					infoName.Format("%s%c%s", m_fileInfo->GetNzbInfo()->GetName(), PATH_SEPARATOR, m_fileInfo->GetFilename());
+					break;
+				}
+			}
 		}
 	}
 

daemon/nntp/ArticleWriter.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -60,9 +60,6 @@ public:
 	static bool MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir);
 	void FlushCache();
 
-protected:
-	virtual void SetLastUpdateTimeNow() {}
-
 private:
 	FileInfo* m_fileInfo;
 	ArticleInfo* m_articleInfo;
@@ -97,8 +94,8 @@ public:
 		friend class ArticleCache;
 	};
 
-	ArticleCache();
 	virtual void Run();
+	virtual void Stop();
 	CachedSegmentData Alloc(int size);
 	bool Realloc(CachedSegmentData* segment, int newSize);
 	void Free(CachedSegmentData* segment);
@@ -109,12 +106,13 @@ public:
 	bool FileBusy(FileInfo* fileInfo) { return fileInfo == m_fileInfo; }
 
 private:
-	size_t m_allocated;
-	bool m_flushing;
+	size_t m_allocated = 0;
+	bool m_flushing = false;
 	Mutex m_allocMutex;
 	Mutex m_flushMutex;
 	Mutex m_contentMutex;
-	FileInfo* m_fileInfo;
+	FileInfo* m_fileInfo = nullptr;
+	ConditionVar m_allocCond;
 
 	bool CheckFlush(bool flushEverything);
 };

daemon/nntp/Decoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,22 +22,129 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Util.h"
+#include "YEncode.h"
 
-const char* Decoder::FormatNames[] = { "Unknown", "yEnc", "UU" };
+Decoder::Decoder()
+{
+	debug("%s", YEncode::decode_simd ? "SIMD yEnc decoder can be used" : "SIMD yEnc decoder isn't available for this CPU");
+	debug("%s", YEncode::crc_simd ? "SIMD Crc routine can be used" : "SIMD Crc routine isn't available for this CPU");
+
+	Clear();
+}
 
 void Decoder::Clear()
 {
 	m_articleFilename.Clear();
+	m_body = false;
+	m_begin = false;
+	m_part = false;
+	m_end = false;
+	m_crc = false;
+	m_eof = false;
+	m_expectedCRC = 0;
+	m_crc32.Reset();
+	m_beginPos = 0;
+	m_endPos = 0;
+	m_size = 0;
+	m_endSize = 0;
+	m_outSize = 0;
+	m_state = 0;
+	m_crcCheck = false;
+	m_lineBuf.Reserve(1024*8);
+	m_lineBuf.SetLength(0);
 }
 
-Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
+/* At the beginning of article the processing goes line by line to find '=ybegin'-marker.
+ * Once the yEnc-data is started switches to blockwise processing.
+ * At the end of yEnc-data switches back to line by line mode to
+ * process '=yend'-marker and EOF-marker.
+ * UU-encoded articles are processed completely in line by line mode.
+ */
+int Decoder::DecodeBuffer(char* buffer, int len)
+{
+	if (m_rawMode)
+	{
+		ProcessRaw(buffer, len);
+		return len;
+	}
+
+	int outlen = 0;
+
+	if (m_body && m_format == efYenc)
+	{
+		outlen = DecodeYenc(buffer, buffer, len);
+		if (m_body)
+		{
+			return outlen;
+		}
+	}
+	else
+	{
+		m_lineBuf.Append(buffer, len);
+	}
+
+	char* line = (char*)m_lineBuf;
+	while (char* end = strchr(line, '\n'))
+	{
+		int llen = (int)(end - line + 1);
+
+		if (line[0] == '.' && line[1] == '\r')
+		{
+			m_eof = true;
+			m_lineBuf.SetLength(0);
+			return outlen;
+		}
+
+		if (m_format == efUnknown)
+		{
+			m_format = DetectFormat(line, llen);
+		}
+
+		if (m_format == efYenc)
+		{
+			ProcessYenc(line, llen);
+			if (m_body)
+			{
+				outlen = DecodeYenc(end + 1, buffer, m_lineBuf.Length() - (int)(end + 1 - m_lineBuf));
+				if (m_body)
+				{
+					m_lineBuf.SetLength(0);
+					return outlen;
+				}
+				line = (char*)m_lineBuf;
+				continue;
+			}
+		}
+		else if (m_format == efUx)
+		{
+			outlen += DecodeUx(line, llen);
+		}
+
+		line = end + 1;
+	}
+
+	if (*line)
+	{
+		len = m_lineBuf.Length() - (int)(line - m_lineBuf);
+		memmove((char*)m_lineBuf, line, len);
+		m_lineBuf.SetLength(len);
+	}
+	else
+	{
+		m_lineBuf.SetLength(0);
+	}
+
+	return outlen;
+}
+
+Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len)
 {
 	if (!strncmp(buffer, "=ybegin ", 8))
 	{
 		return efYenc;
 	}
 
-	if (inBody && (len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
+	if ((len == 62 || len == 63) && (buffer[62] == '\n' || buffer[62] == '\r') && *buffer == 'M')
 	{
 		return efUx;
 	}
@@ -64,139 +171,122 @@ Decoder::EFormat Decoder::DetectFormat(const char* buffer, int len, bool inBody)
 	return efUnknown;
 }
 
-/**
-  * YDecoder: fast implementation of yEnc-Decoder
-  */
-
-YDecoder::YDecoder()
+void Decoder::ProcessYenc(char* buffer, int len)
 {
-	Clear();
-}
-
-void YDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_begin = false;
-	m_part = false;
-	m_end = false;
-	m_crc = false;
-	m_expectedCRC = 0;
-	m_calculatedCRC = 0xFFFFFFFF;
-	m_beginPos = 0;
-	m_endPos = 0;
-	m_size = 0;
-	m_endSize = 0;
-	m_crcCheck = false;
-}
-
-int YDecoder::DecodeBuffer(char* buffer, int len)
-{
-	if (m_body && !m_end)
+	if (!strncmp(buffer, "=ybegin ", 8))
 	{
-		if (!strncmp(buffer, "=yend ", 6))
+		m_begin = true;
+		char* pb = strstr(buffer, " name=");
+		if (pb)
 		{
-			m_end = true;
-			char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
-			if (pb)
-			{
-				m_crc = true;
-				pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
-				m_expectedCRC = strtoul(pb, nullptr, 16);
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_endSize = (int64)atoll(pb);
-			}
-			return 0;
+			pb += 6; //=strlen(" name=")
+			char* pe;
+			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++);
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 		}
-
-		char* iptr = buffer;
-		char* optr = buffer;
-		while (true)
+		pb = strstr(buffer, " size=");
+		if (pb)
 		{
-			switch (*iptr)
-			{
-				case '=':	//escape-sequence
-					iptr++;
-					*optr = *iptr - 64 - 42;
-					optr++;
-					break;
-				case '\n':	// ignored char
-				case '\r':	// ignored char
-					break;
-				case '\0':
-					goto BreakLoop;
-				default:	// normal char
-					*optr = *iptr - 42;
-					optr++;
-					break;
-			}
-			iptr++;
+			pb += 6; //=strlen(" size=")
+			m_size = (int64)atoll(pb);
 		}
-BreakLoop:
-
-		if (m_crcCheck)
+		m_part = strstr(buffer, " part=");
+		if (!m_part)
 		{
-			m_calculatedCRC = Util::Crc32m(m_calculatedCRC, (uchar *)buffer, (uint32)(optr - buffer));
-		}
-		return optr - buffer;
-	}
-	else
-	{
-		if (!m_part && !strncmp(buffer, "=ybegin ", 8))
-		{
-			m_begin = true;
-			char* pb = strstr(buffer, " name=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" name=")
-				char* pe;
-				for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-				m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
-			}
-			pb = strstr(buffer, " size=");
-			if (pb)
-			{
-				pb += 6; //=strlen(" size=")
-				m_size = (int64)atoll(pb);
-			}
-			m_part = strstr(buffer, " part=");
-			if (!m_part)
-			{
-				m_body = true;
-				m_beginPos = 1;
-				m_endPos = m_size;
-			}
-		}
-		else if (m_part && !strncmp(buffer, "=ypart ", 7))
-		{
-			m_part = true;
 			m_body = true;
-			char* pb = strstr(buffer, " begin=");
-			if (pb)
-			{
-				pb += 7; //=strlen(" begin=")
-				m_beginPos = (int64)atoll(pb);
-			}
-			pb = strstr(buffer, " end=");
-			if (pb)
-			{
-				pb += 5; //=strlen(" end=")
-				m_endPos = (int64)atoll(pb);
-			}
+			m_beginPos = 1;
+			m_endPos = m_size;
+		}
+	}
+	else if (!strncmp(buffer, "=ypart ", 7))
+	{
+		m_part = true;
+		m_body = true;
+		char* pb = strstr(buffer, " begin=");
+		if (pb)
+		{
+			pb += 7; //=strlen(" begin=")
+			m_beginPos = (int64)atoll(pb);
+		}
+		pb = strstr(buffer, " end=");
+		if (pb)
+		{
+			pb += 5; //=strlen(" end=")
+			m_endPos = (int64)atoll(pb);
+		}
+	}
+	else if (!strncmp(buffer, "=yend ", 6))
+	{
+		m_end = true;
+		char* pb = strstr(buffer, m_part ? " pcrc32=" : " crc32=");
+		if (pb)
+		{
+			m_crc = true;
+			pb += 7 + (int)m_part; //=strlen(" crc32=") or strlen(" pcrc32=")
+			m_expectedCRC = strtoul(pb, nullptr, 16);
+		}
+		pb = strstr(buffer, " size=");
+		if (pb)
+		{
+			pb += 6; //=strlen(" size=")
+			m_endSize = (int64)atoll(pb);
 		}
 	}
-
-	return 0;
 }
 
-Decoder::EStatus YDecoder::Check()
+int Decoder::DecodeYenc(char* buffer, char* outbuf, int len)
 {
-	m_calculatedCRC ^= 0xFFFFFFFF;
+	const unsigned char* src = (unsigned char*)buffer;
+	unsigned char* dst = (unsigned char*)outbuf;
+
+	int endseq = YEncode::decode(&src, &dst, len, (YEncode::YencDecoderState*)&m_state);
+	int outlen = (int)((char*)dst - outbuf);
+
+	// endseq:
+	//   0: no end sequence found
+	//   1: \r\n=y sequence found, src points to byte after 'y'
+	//   2: \r\n.\r\n sequence found, src points to byte after last '\n'
+	if (endseq != 0)
+	{
+		// switch back to line mode to process '=yend'- or eof- marker
+		m_lineBuf.SetLength(0);
+		m_lineBuf.Append(endseq == 1 ? "=y" : ".\r\n");
+		int rem = len - (int)((const char*)src - buffer);
+		if (rem > 0)
+		{
+			m_lineBuf.Append((const char*)src, rem);
+		}
+		m_body = false;
+	}
+
+	if (m_crcCheck)
+	{
+		m_crc32.Append((uchar*)outbuf, (uint32)outlen);
+	}
+
+	m_outSize += outlen;
+
+	return outlen;
+}
+
+Decoder::EStatus Decoder::Check()
+{
+	switch (m_format)
+	{
+		case efYenc:
+			return CheckYenc();
+			 
+		case efUx:
+			return CheckUx();
+
+		default:
+			return dsUnknownError;
+	}
+}
+
+Decoder::EStatus Decoder::CheckYenc()
+{
+	m_calculatedCRC = m_crc32.Finish();
 
 	debug("Expected crc32=%x", m_expectedCRC);
 	debug("Calculated crc32=%x", m_calculatedCRC);
@@ -209,7 +299,7 @@ Decoder::EStatus YDecoder::Check()
 	{
 		return dsArticleIncomplete;
 	}
-	else if (!m_part && m_size != m_endSize)
+	else if ((!m_part && m_size != m_endSize) || (m_endSize != m_outSize))
 	{
 		return dsInvalidSize;
 	}
@@ -222,24 +312,7 @@ Decoder::EStatus YDecoder::Check()
 }
 
 
-/**
-  * UDecoder: supports UU encoding formats
-  */
-
-UDecoder::UDecoder()
-{
-	Clear();
-}
-
-void UDecoder::Clear()
-{
-	Decoder::Clear();
-
-	m_body = false;
-	m_end = false;
-}
-
-/* DecodeBuffer-function uses portions of code from tool UUDECODE by Clem Dye
+/* DecodeUx-function uses portions of code from tool UUDECODE by Clem Dye
  * UUDECODE.c (http://www.bastet.com/uue.zip)
  * Copyright (C) 1998 Clem Dye
  *
@@ -248,7 +321,7 @@ void UDecoder::Clear()
 
 #define UU_DECODE_CHAR(c) (c == '`' ? 0 : (((c) - ' ') & 077))
 
-int UDecoder::DecodeBuffer(char* buffer, int len)
+int Decoder::DecodeUx(char* buffer, int len)
 {
 	if (!m_body)
 	{
@@ -264,7 +337,7 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			// extracting filename
 			char* pe;
 			for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ;
-			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, pe - pb));
+			m_articleFilename = WebUtil::Latin1ToUtf8(CString(pb, (int)(pe - pb)));
 
 			m_body = true;
 			return 0;
@@ -301,10 +374,7 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			}
 			else
 			{
-				if (effLen >= 1)
-				{
-					*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
-				}
+				*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
 				if (effLen >= 2)
 				{
 					*optr++ = UU_DECODE_CHAR (iptr[1]) << 4 | UU_DECODE_CHAR (iptr[2]) >> 2;
@@ -312,13 +382,13 @@ int UDecoder::DecodeBuffer(char* buffer, int len)
 			}
 		}
 
-		return optr - buffer;
+		return (int)(optr - buffer);
 	}
 
 	return 0;
 }
 
-Decoder::EStatus UDecoder::Check()
+Decoder::EStatus Decoder::CheckUx()
 {
 	if (!m_body)
 	{
@@ -327,3 +397,50 @@ Decoder::EStatus UDecoder::Check()
 
 	return dsFinished;
 }
+
+void Decoder::ProcessRaw(char* buffer, int len)
+{
+	switch (m_state)
+	{
+		case 1:
+			m_eof = len >= 4 && buffer[0] == '\n' &&
+				buffer[1] == '.' && buffer[2] == '\r' && buffer[3] == '\n';
+			break;
+
+		case 2:
+			m_eof = len >= 3 && buffer[0] == '.' && buffer[1] == '\r' && buffer[2] == '\n';
+			break;
+
+		case 3:
+			m_eof = len >= 2 && buffer[0] == '\r' && buffer[1] == '\n';
+			break;
+
+		case 4:
+			m_eof = len >= 1 && buffer[0] == '\n';
+			break;
+	}
+
+	m_eof |= len >= 5 && strstr(buffer, "\r\n.\r\n");
+
+	if (len >= 4 && buffer[len-4] == '\r' && buffer[len-3] == '\n' &&
+		buffer[len-2] == '.' && buffer[len-1] == '\r')
+	{
+		m_state = 4;
+	}
+	else if (len >= 3 && buffer[len-3] == '\r' && buffer[len-2] == '\n' && buffer[len-1] == '.')
+	{
+		m_state = 3;
+	}
+	else if (len >= 2 && buffer[len-2] == '\r' && buffer[len-1] == '\n')
+	{
+		m_state = 2;
+	}
+	else if (len >= 1 && buffer[len-1] == '\r')
+	{
+		m_state = 1;
+	}
+	else
+	{
+		m_state = 0;
+	}
+}

daemon/nntp/Decoder.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #define DECODER_H
 
 #include "NString.h"
+#include "Util.h"
 
 class Decoder
 {
@@ -43,37 +44,26 @@ public:
 		efUx,
 	};
 
-	static const char* FormatNames[];
-
-	virtual ~Decoder() {}
-	virtual EStatus Check() = 0;
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len) = 0;
-	const char* GetArticleFilename() { return m_articleFilename; }
-	static EFormat DetectFormat(const char* buffer, int len, bool inBody);
-
-protected:
-	CString m_articleFilename;
-};
-
-class YDecoder: public Decoder
-{
-public:
-	YDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
+	Decoder();
+	EStatus Check();
+	void Clear();
+	int DecodeBuffer(char* buffer, int len);
 	void SetCrcCheck(bool crcCheck) { m_crcCheck = crcCheck; }
-	int64 GetBegin() { return m_beginPos; }
-	int64 GetEnd() { return m_endPos; }
+	void SetRawMode(bool rawMode) { m_rawMode = rawMode; }
+	EFormat GetFormat() { return m_format; }
+	int64 GetBeginPos() { return m_beginPos; }
+	int64 GetEndPos() { return m_endPos; }
 	int64 GetSize() { return m_size; }
 	uint32 GetExpectedCrc() { return m_expectedCRC; }
 	uint32 GetCalculatedCrc() { return m_calculatedCRC; }
+	bool GetEof() { return m_eof; }
+	const char* GetArticleFilename() { return m_articleFilename; }
 
-private:
+private: 
+	EFormat m_format = efUnknown;
 	bool m_begin;
 	bool m_part;
-	bool m_body;
+	bool m_body; 
 	bool m_end;
 	bool m_crc;
 	uint32 m_expectedCRC;
@@ -82,20 +72,22 @@ private:
 	int64 m_endPos;
 	int64 m_size;
 	int64 m_endSize;
+	int64 m_outSize;
+	bool m_eof;
 	bool m_crcCheck;
-};
+	char m_state;
+	bool m_rawMode = false;
+	CString m_articleFilename;
+	StringBuilder m_lineBuf;
+	Crc32 m_crc32;
 
-class UDecoder: public Decoder
-{
-public:
-	UDecoder();
-	virtual EStatus Check();
-	virtual void Clear();
-	virtual int DecodeBuffer(char* buffer, int len);
-
-private:
-	bool m_body;
-	bool m_end;
+	EFormat DetectFormat(const char* buffer, int len);
+	void ProcessYenc(char* buffer, int len);
+	int DecodeYenc(char* buffer, char* outbuf, int len);
+	EStatus CheckYenc();
+	int DecodeUx(char* buffer, int len);
+	EStatus CheckUx();
+	void ProcessRaw(char* buffer, int len);
 };
 
 #endif

daemon/nntp/NewsServer.cpp

@@ -22,13 +22,13 @@
 #include "nzbget.h"
 #include "NewsServer.h"
 
-NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port,
-	const char* user, const char* pass, bool joinGroup, bool tls,
-	const char* cipher, int maxConnections, int retention, int level, int group) :
-		m_id(id), m_active(active), m_port(port), m_level(level), m_normLevel(level),
-		m_group(group), m_maxConnections(maxConnections), m_joinGroup(joinGroup),
-		m_tls(tls), m_name(name), m_host(host ? host : ""), m_user(user ? user : ""),
-		m_password(pass ? pass : ""), m_cipher(cipher ? cipher : ""), m_retention(retention)
+NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
+	const char* user, const char* pass, bool joinGroup, bool tls, const char* cipher,
+	int maxConnections, int retention, int level, int group, bool optional) :
+		m_id(id), m_active(active), m_name(name), m_host(host ? host : ""), m_port(port), m_ipVersion(ipVersion),
+		m_user(user ? user : ""), m_password(pass ? pass : ""), m_joinGroup(joinGroup), m_tls(tls),
+		m_cipher(cipher ? cipher : ""), m_maxConnections(maxConnections), m_retention(retention),
+		m_level(level), m_normLevel(level), m_group(group), m_optional(optional)
 {
 	if (m_name.Empty())
 	{

daemon/nntp/NewsServer.h

@@ -32,10 +32,10 @@
 class NewsServer
 {
 public:
-	NewsServer(int id, bool active, const char* name, const char* host, int port,
+	NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 		const char* user, const char* pass, bool joinGroup,
 		bool tls, const char* cipher, int maxConnections, int retention,
-		int level, int group);
+		int level, int group, bool optional);
 	int GetId() { return m_id; }
 	int GetStateId() { return m_stateId; }
 	void SetStateId(int stateId) { m_stateId = stateId; }
@@ -45,6 +45,7 @@ public:
 	int GetGroup() { return m_group; }
 	const char* GetHost() { return m_host; }
 	int GetPort() { return m_port; }
+	int GetIpVersion() { return m_ipVersion; }
 	const char* GetUser() { return m_user; }
 	const char* GetPassword() { return m_password; }
 	int GetMaxConnections() { return m_maxConnections; }
@@ -55,6 +56,7 @@ public:
 	bool GetTls() { return m_tls; }
 	const char* GetCipher() { return m_cipher; }
 	int GetRetention() { return m_retention; }
+	bool GetOptional() { return m_optional; }
 	time_t GetBlockTime() { return m_blockTime; }
 	void SetBlockTime(time_t blockTime) { m_blockTime = blockTime; }
 
@@ -63,18 +65,20 @@ private:
 	int m_stateId = 0;
 	bool m_active;
 	CString m_name;
-	int m_group;
 	CString m_host;
 	int m_port;
+	int m_ipVersion;
 	CString m_user;
 	CString m_password;
-	int m_maxConnections;
-	int m_level;
-	int m_normLevel;
 	bool m_joinGroup;
 	bool m_tls;
 	CString m_cipher;
+	int m_maxConnections;
 	int m_retention;
+	int m_level;
+	int m_normLevel;
+	int m_group;
+	bool m_optional = false;
 	time_t m_blockTime = 0;
 };
 

daemon/nntp/NntpConnection.cpp

@@ -27,10 +27,13 @@
 
 static const int CONNECTION_LINEBUFFER_SIZE = 1024*10;
 
-NntpConnection::NntpConnection(NewsServer* newsServer) : Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
+NntpConnection::NntpConnection(NewsServer* newsServer) :
+	Connection(newsServer->GetHost(), newsServer->GetPort(), newsServer->GetTls()), m_newsServer(newsServer)
 {
 	m_lineBuf.Reserve(CONNECTION_LINEBUFFER_SIZE);
 	SetCipher(newsServer->GetCipher());
+	SetIPVersion(newsServer->GetIpVersion() == 4 ? Connection::ipV4 :
+		newsServer->GetIpVersion() == 6 ? Connection::ipV6 : Connection::ipAuto);
 }
 
 const char* NntpConnection::Request(const char* req)
@@ -224,10 +227,7 @@ bool NntpConnection::Disconnect()
 {
 	if (m_status == csConnected)
 	{
-		if (!m_broken)
-		{
-			Request("quit\r\n");
-		}
+		Request("quit\r\n");
 		m_activeGroup = nullptr;
 	}
 	return Connection::Disconnect();

daemon/nntp/ServerPool.cpp

@@ -137,70 +137,93 @@ void ServerPool::InitConnections()
 	m_generation++;
 }
 
+/* Returns connection from any server on a given level or nullptr if there is no free connection at the moment.
+ * If all servers are blocked and all are optional a connection from the next level is returned instead.
+ */
 NntpConnection* ServerPool::GetConnection(int level, NewsServer* wantServer, RawServerList* ignoreServers)
 {
 	Guard guard(m_connectionsMutex);
 
-	PooledConnection* connection = nullptr;
-
-	time_t curTime = Util::CurrentTime();
-
-	if (level < (int)m_levels.size() && m_levels[level] > 0)
+	for (; level < (int)m_levels.size() && m_levels[level] > 0; level++)
 	{
-		std::vector<PooledConnection*> candidates;
-		candidates.reserve(m_connections.size());
-
-		for (PooledConnection* candidateConnection : &m_connections)
-		{
-			NewsServer* candidateServer = candidateConnection->GetNewsServer();
-			if (!candidateConnection->GetInUse() && candidateServer->GetActive() &&
-				candidateServer->GetNormLevel() == level &&
-				(!wantServer || candidateServer == wantServer ||
-				 (wantServer->GetGroup() > 0 && wantServer->GetGroup() == candidateServer->GetGroup())) &&
-				(candidateConnection->GetStatus() == Connection::csConnected ||
-				 !candidateServer->GetBlockTime() ||
-				 candidateServer->GetBlockTime() + m_retryInterval <= curTime ||
-				 candidateServer->GetBlockTime() > curTime))
-			{
-				// free connection found, check if it's not from the server which should be ignored
-				bool useConnection = true;
-				if (ignoreServers && !wantServer)
-				{
-					for (NewsServer* ignoreServer : ignoreServers)
-					{
-						if (ignoreServer == candidateServer ||
-							(ignoreServer->GetGroup() > 0 && ignoreServer->GetGroup() == candidateServer->GetGroup() &&
-							 ignoreServer->GetNormLevel() == candidateServer->GetNormLevel()))
-						{
-							useConnection = false;
-							break;
-						}
-					}
-				}
-
-				candidateServer->SetBlockTime(0);
-
-				if (useConnection)
-				{
-					candidates.push_back(candidateConnection);
-				}
-			}
-		}
-
-		if (!candidates.empty())
-		{
-			// Peeking a random free connection. This is better than taking the first
-			// available connection because provides better distribution across news servers,
-			// especially when one of servers becomes unavailable or doesn't have requested articles.
-			int randomIndex = rand() % candidates.size();
-			connection = candidates[randomIndex];
-			connection->SetInUse(true);
-		}
-
+		NntpConnection* connection = LockedGetConnection(level, wantServer, ignoreServers);
 		if (connection)
 		{
-			m_levels[level]--;
+			return connection;
 		}
+
+		for (NewsServer* newsServer : m_sortedServers)
+		{
+			if (newsServer->GetNormLevel() == level && newsServer->GetActive() &&
+				!(newsServer->GetOptional() && IsServerBlocked(newsServer)))
+			{
+				return nullptr;
+			}
+		}
+	}
+
+	return nullptr;
+}
+
+NntpConnection* ServerPool::LockedGetConnection(int level, NewsServer* wantServer, RawServerList* ignoreServers)
+{
+	if (level >= (int)m_levels.size() || m_levels[level] == 0)
+	{
+		return nullptr;
+	}
+
+	PooledConnection* connection = nullptr;
+	std::vector<PooledConnection*> candidates;
+	candidates.reserve(m_connections.size());
+
+	for (PooledConnection* candidateConnection : &m_connections)
+	{
+		NewsServer* candidateServer = candidateConnection->GetNewsServer();
+		if (!candidateConnection->GetInUse() && candidateServer->GetActive() &&
+			candidateServer->GetNormLevel() == level &&
+			(!wantServer || candidateServer == wantServer ||
+			 (wantServer->GetGroup() > 0 && wantServer->GetGroup() == candidateServer->GetGroup())) &&
+			(candidateConnection->GetStatus() == Connection::csConnected ||
+			 !IsServerBlocked(candidateServer)))
+		{
+			// free connection found, check if it's not from the server which should be ignored
+			bool useConnection = true;
+			if (ignoreServers && !wantServer)
+			{
+				for (NewsServer* ignoreServer : ignoreServers)
+				{
+					if (ignoreServer == candidateServer ||
+						(ignoreServer->GetGroup() > 0 && ignoreServer->GetGroup() == candidateServer->GetGroup() &&
+						 ignoreServer->GetNormLevel() == candidateServer->GetNormLevel()))
+					{
+						useConnection = false;
+						break;
+					}
+				}
+			}
+
+			candidateServer->SetBlockTime(0);
+
+			if (useConnection)
+			{
+				candidates.push_back(candidateConnection);
+			}
+		}
+	}
+
+	if (!candidates.empty())
+	{
+		// Peeking a random free connection. This is better than taking the first
+		// available connection because provides better distribution across news servers,
+		// especially when one of servers becomes unavailable or doesn't have requested articles.
+		int randomIndex = rand() % candidates.size();
+		connection = candidates[randomIndex];
+		connection->SetInUse(true);
+	}
+
+	if (connection)
+	{
+		m_levels[level]--;
 	}
 
 	return connection;
@@ -243,6 +266,19 @@ void ServerPool::BlockServer(NewsServer* newsServer)
 	}
 }
 
+bool ServerPool::IsServerBlocked(NewsServer* newsServer)
+{
+	if (!newsServer->GetBlockTime())
+	{
+		return false;
+	}
+
+	time_t curTime = Util::CurrentTime();
+	bool blocked = newsServer->GetBlockTime() <= curTime &&
+		curTime < newsServer->GetBlockTime() + m_retryInterval;
+	return blocked;
+}
+
 void ServerPool::CloseUnusedConnections()
 {
 	Guard guard(m_connectionsMutex);

daemon/nntp/ServerPool.h

@@ -45,6 +45,7 @@ public:
 	void Changed();
 	int GetGeneration() { return m_generation; }
 	void BlockServer(NewsServer* newsServer);
+	bool IsServerBlocked(NewsServer* newsServer);
 
 protected:
 	virtual void LogDebugInfo();
@@ -77,6 +78,7 @@ private:
 	int m_generation = 0;
 
 	void NormalizeLevels();
+	NntpConnection* LockedGetConnection(int level, NewsServer* wantServer, RawServerList* ignoreServers);
 };
 
 extern ServerPool* g_ServerPool;

daemon/nntp/StatMeter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "StatMeter.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "DiskState.h"
 #include "Util.h"
@@ -57,8 +58,8 @@ void ServerVolume::CalcSlots(time_t locCurTime)
 void ServerVolume::AddData(int bytes)
 {
 	time_t curTime = Util::CurrentTime();
-	time_t locCurTime = curTime + g_Options->GetLocalTimeOffset();
-	time_t locDataTime = m_dataTime + g_Options->GetLocalTimeOffset();
+	time_t locCurTime = curTime + g_WorkState->GetLocalTimeOffset();
+	time_t locDataTime = m_dataTime + g_WorkState->GetLocalTimeOffset();
 
 	int lastMinSlot = m_minSlot;
 	int lastHourSlot = m_hourSlot;
@@ -138,28 +139,28 @@ void ServerVolume::LogDebugInfo()
 
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerSeconds[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerSeconds[i]);
 	}
 	info("Secs: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerMinutes[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerMinutes[i]);
 	}
 	info("Mins: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 24; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerHours[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerHours[i]);
 	}
 	info("Hours: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < (int)m_bytesPerDays.size(); i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", m_firstDay + i, m_bytesPerDays[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", m_firstDay + i, m_bytesPerDays[i]);
 	}
 	info("Days: %s", *msg);
 }
@@ -188,10 +189,10 @@ void StatMeter::AdjustTimeOffset()
 	tmSplittedTime.tm_isdst = -1;
 	time_t locTime = mktime(&tmSplittedTime);
 	time_t localTimeDelta = utcTime - locTime;
-	g_Options->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
+	g_WorkState->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
 	m_lastTimeOffset = utcTime;
 
-	debug("UTC delta: %i (%i+%i)", g_Options->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
+	debug("UTC delta: %i (%i+%i)", g_WorkState->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
 }
 
 /*
@@ -224,6 +225,8 @@ void StatMeter::IntervalCheck()
 
 	m_lastCheck = m_curTime;
 
+	CheckQuota();
+
 	if (m_statChanged)
 	{
 		Save();
@@ -306,8 +309,6 @@ void StatMeter::AddSpeedReading(int bytes)
 	time_t curTime = Util::CurrentTime();
 	int nowSlot = (int)curTime / SPEEDMETER_SLOTSIZE;
 
-	Guard guard(g_Options->GetAccurateRate() ? &m_speedMutex : nullptr);
-
 	if (curTime != m_curSecTime)
 	{
 		m_curSecTime =	curTime;
@@ -380,9 +381,9 @@ void StatMeter::LogDebugInfo()
 	int timeDiff = (int)Util::CurrentTime() - m_speedStartTime * SPEEDMETER_SLOTSIZE;
 	info("      Speed: %i", speed);
 	info("      SpeedStartTime: %i", m_speedStartTime);
-	info("      SpeedTotalBytes: %lli", m_speedTotalBytes);
+	info("      SpeedTotalBytes: %" PRIi64, m_speedTotalBytes);
 	info("      SpeedBytesIndex: %i", m_speedBytesIndex);
-	info("      AllBytes: %lli", m_allBytes);
+	info("      AllBytes: %" PRIi64, m_allBytes);
 	info("      Time: %i", (int)Util::CurrentTime());
 	info("      TimeDiff: %i", timeDiff);
 	for (int i=0; i < SPEEDMETER_SLOTS; i++)
@@ -446,8 +447,98 @@ bool StatMeter::Load(bool* perfectServerMatch)
 
 	for (ServerVolume& serverVolume : m_serverVolumes)
 	{
-		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_Options->GetLocalTimeOffset());
+		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_WorkState->GetLocalTimeOffset());
 	}
 
 	return ok;
 }
+
+void StatMeter::CheckQuota()
+{
+	if ((g_Options->GetDailyQuota() == 0 && g_Options->GetMonthlyQuota() == 0))
+	{
+		return;
+	}
+
+	int64 monthBytes, dayBytes;
+	CalcQuotaUsage(monthBytes, dayBytes);
+
+	bool monthlyQuotaReached = g_Options->GetMonthlyQuota() > 0 && monthBytes >= (int64)g_Options->GetMonthlyQuota() * 1024 * 1024;
+	bool dailyQuotaReached = g_Options->GetDailyQuota() > 0 && dayBytes >= (int64)g_Options->GetDailyQuota() * 1024 * 1024;
+
+	if (monthlyQuotaReached && !g_WorkState->GetQuotaReached())
+	{
+		warn("Monthly quota reached at %s", *Util::FormatSize(monthBytes));
+	}
+	else if (dailyQuotaReached && !g_WorkState->GetQuotaReached())
+	{
+		warn("Daily quota reached at %s", *Util::FormatSize(dayBytes));
+	}
+	else if (!monthlyQuotaReached && !dailyQuotaReached && g_WorkState->GetQuotaReached())
+	{
+		info("Quota lifted");
+	}
+
+	g_WorkState->SetQuotaReached(monthlyQuotaReached || dailyQuotaReached);
+}
+
+void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
+{
+	Guard guard(m_volumeMutex);
+
+	ServerVolume totalVolume = m_serverVolumes[0];
+
+	time_t locTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
+	int daySlot = (int)(locTime / 86400) - totalVolume.GetFirstDay();
+
+	dayBytes = 0;
+	if (daySlot < (int)totalVolume.BytesPerDays()->size())
+	{
+		dayBytes = totalVolume.BytesPerDays()->at(daySlot);
+	}
+
+	int elapsedSlots = CalcMonthSlots(totalVolume);
+	monthBytes = 0;
+	int endSlot = std::max(daySlot - elapsedSlots, -1);
+	for (int slot = daySlot; slot >= 0 && slot > endSlot; slot--)
+	{
+		if (slot < (int)totalVolume.BytesPerDays()->size())
+		{
+			monthBytes += totalVolume.BytesPerDays()->at(slot);
+			debug("adding slot %i: %i", slot, (int)(totalVolume.BytesPerDays()->at(slot) / 1024 / 1024));
+		}
+	}
+
+	debug("month volume: %i MB", (int)(monthBytes / 1024 / 1024));
+}
+
+int StatMeter::CalcMonthSlots(ServerVolume& volume)
+{
+	int elapsedDays;
+
+	time_t locCurTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
+	tm dayparts;
+	gmtime_r(&locCurTime, &dayparts);
+
+	if (g_Options->GetQuotaStartDay() > dayparts.tm_mday)
+	{
+		dayparts.tm_mon--;
+		dayparts.tm_mday = g_Options->GetQuotaStartDay();
+		time_t prevMonth = Util::Timegm(&dayparts);
+		tm prevparts;
+		gmtime_r(&prevMonth, &prevparts);
+		if (prevparts.tm_mday != g_Options->GetQuotaStartDay())
+		{
+			dayparts.tm_mday = 1;
+			dayparts.tm_mon++;
+			prevMonth = Util::Timegm(&dayparts);
+		}
+		elapsedDays = (int)(locCurTime - prevMonth) / 60 / 60 / 24 + 1;
+	}
+	else
+	{
+		elapsedDays = dayparts.tm_mday - g_Options->GetQuotaStartDay() + 1;
+	}
+
+	return elapsedDays;
+}

daemon/nntp/StatMeter.h

@@ -55,9 +55,9 @@ public:
 	void LogDebugInfo();
 
 private:
-	VolumeArray m_bytesPerSeconds{60};
-	VolumeArray m_bytesPerMinutes{60};
-	VolumeArray m_bytesPerHours{24};
+	VolumeArray m_bytesPerSeconds = VolumeArray(60);
+	VolumeArray m_bytesPerMinutes = VolumeArray(60);
+	VolumeArray m_bytesPerHours = VolumeArray(24);
 	VolumeArray m_bytesPerDays;
 	int m_firstDay = 0;
 	int64 m_totalBytes = 0;
@@ -75,6 +75,24 @@ typedef GuardedPtr<ServerVolumes> GuardedServerVolumes;
 
 class StatMeter : public Debuggable
 {
+public:
+	StatMeter();
+	void Init();
+	int CalcCurrentDownloadSpeed();
+	int CalcMomentaryDownloadSpeed();
+	void AddSpeedReading(int bytes);
+	void AddServerData(int bytes, int serverId);
+	void CalcTotalStat(int* upTimeSec, int* dnTimeSec, int64* allBytes, bool* standBy);
+	void CalcQuotaUsage(int64& monthBytes, int64& dayBytes);
+	void IntervalCheck();
+	void EnterLeaveStandBy(bool enter);
+	GuardedServerVolumes GuardServerVolumes();
+	void Save();
+	bool Load(bool* perfectServerMatch);
+
+protected:
+	virtual void LogDebugInfo();
+
 private:
 	// speed meter
 	static const int SPEEDMETER_SLOTS = 30;
@@ -87,7 +105,6 @@ private:
 	int m_speedBytesIndex;
 	int m_curSecBytes;
 	time_t m_curSecTime;
-	Mutex m_speedMutex;
 
 	// time
 	int64 m_allBytes = 0;
@@ -106,24 +123,8 @@ private:
 
 	void ResetSpeedStat();
 	void AdjustTimeOffset();
-
-protected:
-	virtual void LogDebugInfo();
-
-public:
-	StatMeter();
-	void Init();
-	int CalcCurrentDownloadSpeed();
-	int CalcMomentaryDownloadSpeed();
-	void AddSpeedReading(int bytes);
-	void AddServerData(int bytes, int serverId);
-	void CalcTotalStat(int* upTimeSec, int* dnTimeSec, int64* allBytes, bool* standBy);
-	bool GetStandBy() { return m_standBy; }
-	void IntervalCheck();
-	void EnterLeaveStandBy(bool enter);
-	GuardedServerVolumes GuardServerVolumes();
-	void Save();
-	bool Load(bool* perfectServerMatch);
+	void CheckQuota();
+	int CalcMonthSlots(ServerVolume& volume);
 };
 
 extern StatMeter* g_StatMeter;

daemon/nserv/NServFrontend.cpp

@@ -0,0 +1,146 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "NServFrontend.h"
+#include "Util.h"
+
+NServFrontend::NServFrontend()
+{
+#ifdef WIN32
+	m_console = GetStdHandle(STD_OUTPUT_HANDLE);
+#endif
+}
+
+void NServFrontend::Run()
+{
+	while (!IsStopped())
+	{
+		Update();
+		Util::Sleep(100);
+	}
+	// Printing the last messages
+	Update();
+}
+
+void NServFrontend::Update()
+{
+	BeforePrint();
+
+	{
+		GuardedMessageList messages = g_Log->GuardMessages();
+		if (!messages->empty())
+		{
+			Message& firstMessage = messages->front();
+			int start = m_neededLogFirstId - firstMessage.GetId() + 1;
+			if (start < 0)
+			{
+				PrintSkip();
+				start = 0;
+			}
+			for (uint32 i = (uint32)start; i < messages->size(); i++)
+			{
+				PrintMessage(messages->at(i));
+				m_neededLogFirstId = messages->at(i).GetId();
+			}
+		}
+	}
+
+	fflush(stdout);
+}
+
+void NServFrontend::BeforePrint()
+{
+	if (m_needGoBack)
+	{
+		// go back one line
+#ifdef WIN32
+		CONSOLE_SCREEN_BUFFER_INFO BufInfo;
+		GetConsoleScreenBufferInfo(m_console, &BufInfo);
+		BufInfo.dwCursorPosition.Y--;
+		SetConsoleCursorPosition(m_console, BufInfo.dwCursorPosition);
+#else
+		printf("\r\033[1A");
+#endif
+		m_needGoBack = false;
+	}
+}
+
+void NServFrontend::PrintMessage(Message& message)
+{
+#ifdef WIN32
+	switch (message.GetKind())
+	{
+		case Message::mkDebug:
+			SetConsoleTextAttribute(m_console, 8);
+			printf("[DEBUG]  ");
+			break;
+		case Message::mkError:
+			SetConsoleTextAttribute(m_console, 4);
+			printf("[ERROR] ");
+			break;
+		case Message::mkWarning:
+			SetConsoleTextAttribute(m_console, 5);
+			printf("[WARNING]");
+			break;
+		case Message::mkInfo:
+			SetConsoleTextAttribute(m_console, 2);
+			printf("[INFO]  ");
+			break;
+		case Message::mkDetail:
+			SetConsoleTextAttribute(m_console, 2);
+			printf("[DETAIL]");
+			break;
+	}
+	SetConsoleTextAttribute(m_console, 7);
+	CString msg = message.GetText();
+	CharToOem(msg, msg);
+	printf(" %s\n", *msg);
+#else
+	const char* msg = message.GetText();
+	switch (message.GetKind())
+	{
+		case Message::mkDebug:
+			printf("[DEBUG] %s\033[K\n", msg);
+			break;
+		case Message::mkError:
+			printf("\033[31m[ERROR]\033[39m %s\033[K\n", msg);
+			break;
+		case Message::mkWarning:
+			printf("\033[35m[WARNING]\033[39m %s\033[K\n", msg);
+			break;
+		case Message::mkInfo:
+			printf("\033[32m[INFO]\033[39m %s\033[K\n", msg);
+			break;
+		case Message::mkDetail:
+			printf("\033[32m[DETAIL]\033[39m %s\033[K\n", msg);
+			break;
+	}
+#endif
+}
+
+void NServFrontend::PrintSkip()
+{
+#ifdef WIN32
+	printf(".....\n");
+#else
+	printf(".....\033[K\n");
+#endif
+}

daemon/nserv/NServFrontend.h

@@ -0,0 +1,48 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef NSERVFRONTEND_H
+#define NSERVFRONTEND_H
+
+#include "Thread.h"
+#include "Log.h"
+
+class NServFrontend : public Thread
+{
+public:
+	NServFrontend();
+
+private:
+	uint32 m_neededLogEntries = 0;
+	uint32 m_neededLogFirstId = 0;
+	bool m_needGoBack = false;
+
+#ifdef WIN32
+	HANDLE m_console;
+#endif
+
+	void Run();
+	void Update();
+	void BeforePrint();
+	void PrintMessage(Message& message);
+	void PrintSkip();
+};
+
+#endif

daemon/nserv/NServMain.cpp

@@ -0,0 +1,280 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+
+#include "Thread.h"
+#include "Connection.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "NServFrontend.h"
+#include "NntpServer.h"
+#include "NzbGenerator.h"
+#include "Options.h"
+
+struct NServOpts
+{
+	CString dataDir;
+	CString cacheDir;
+	CString bindAddress;
+	int firstPort;
+	int instances;
+	CString logFile;
+	CString secureCert;
+	CString secureKey;
+	BString<1024> logOpt;
+	bool generateNzb;
+	int segmentSize;
+	bool quit;
+	int latency;
+	int speed;
+	bool memCache;
+	bool paramError;
+
+	NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts);
+};
+
+void NServPrintUsage(const char* com);
+
+int NServMain(int argc, char* argv[])
+{
+	Log log;
+
+	info("NServ %s (Test NNTP server)", Util::VersionRevision());
+
+	Options::CmdOptList cmdOpts;
+	NServOpts opts(argc, argv, cmdOpts);
+
+	if (opts.dataDir.Empty() || opts.paramError)
+	{
+		NServPrintUsage(argv[0]);
+		return 1;
+	}
+
+	if (!FileSystem::DirectoryExists(opts.dataDir))
+	{
+		// dataDir does not exist. Let's find out a bit more, and report:
+		if (FileSystem::FileExists(opts.dataDir))
+		{
+			error("Specified data-dir %s is not a directory, but a file", *opts.dataDir );
+		} else {
+			error("Specified data-dir %s does not exist", *opts.dataDir );
+		}
+	}
+
+	Options options(&cmdOpts, nullptr);
+
+	log.InitOptions();
+	Thread::Init();
+	Connection::Init();
+#ifndef DISABLE_TLS
+	TlsSocket::Init();
+#endif
+
+#ifndef WIN32
+	signal(SIGPIPE, SIG_IGN);
+#endif
+
+	NServFrontend frontend;
+	frontend.Start();
+
+	if (opts.generateNzb)
+	{
+		NzbGenerator gen(opts.dataDir, opts.segmentSize);
+		gen.Execute();
+		if (opts.quit)
+		{
+			return 0;
+		}
+	}
+
+	CString errmsg;
+	if (opts.cacheDir && !FileSystem::ForceDirectories(opts.cacheDir, errmsg))
+	{
+		error("Could not create directory %s: %s", *opts.cacheDir, *errmsg);
+	}
+
+	std::vector<std::unique_ptr<NntpServer>> instances;
+	NntpCache cache;
+
+	for (int i = 0; i < opts.instances; i++)
+	{
+		instances.emplace_back(std::make_unique<NntpServer>(i + 1, opts.bindAddress,
+			opts.firstPort + i, opts.secureCert, opts.secureKey, opts.dataDir, opts.cacheDir,
+			opts.latency, opts.speed, opts.memCache ? &cache : nullptr));
+		instances.back()->Start();
+	}
+
+	info("Press Ctrl+C to quit");
+	while (getchar()) Util::Sleep(200);
+
+	for (std::unique_ptr<NntpServer>& serv: instances)
+	{
+		serv->Stop();
+	}
+	frontend.Stop();
+
+	bool hasRunning = false;
+	do
+	{
+		hasRunning = frontend.IsRunning();
+		for (std::unique_ptr<NntpServer>& serv : instances)
+		{
+			hasRunning |= serv->IsRunning();
+		}
+		Util::Sleep(50);
+	} while (hasRunning);
+
+	return 0;
+}
+
+void NServPrintUsage(const char* com)
+{
+	printf("Usage:\n"
+		" %s --nserv -d <data-dir> [optional switches] \n"
+		"    -d <data-dir>   - directory whose files will be served\n"
+		"  Optional switches:\n"
+		"    -c <cache-dir>  - directory to store encoded articles\n"
+		"    -m              - in-memory cache (unlimited, use with care)\n"
+		"    -l <log-file>   - write into log-file (disabled by default)\n"
+		"    -i <instances>  - number of server instances (default is 1)\n"
+		"    -b <address>    - ip address to bind to (default is 0.0.0.0)\n"
+		"    -p <port>       - port number for the first instance (default is 6791)\n"
+		"    -s <cert> <key> - paths to SSL certificate and key files\n"
+		"    -v <verbose>    - verbosity level 0..3 (default is 2)\n"
+		"    -w <msec>       - response latency (in milliseconds)\n"
+		"    -r <KB/s>       - speed throttling (in kilobytes per second)\n"
+		"    -z <seg-size>   - generate nzbs for all files in data-dir (size in bytes)\n"
+		"    -q              - quit after generating nzbs (in combination with -z)\n"
+		, FileSystem::BaseFileName(com));
+}
+
+NServOpts::NServOpts(int argc, char* argv[], Options::CmdOptList& cmdOpts)
+{
+	instances = 1;
+	bindAddress = "0.0.0.0";
+	firstPort = 6791;
+	generateNzb = false;
+	segmentSize = 500000;
+	quit = false;
+	latency = 0;
+	memCache = false;
+	speed = 0;
+	paramError = false;
+	int verbosity = 2;
+
+	char short_options[] = "b:c:d:l:p:i:ms:v:w:r:z:q";
+
+	optind = 2;
+	while (true)
+	{
+		int c = getopt(argc, argv, short_options);
+		if (c == -1) break;
+		switch (c)
+		{
+			case 'd':
+				dataDir = optind > argc ? nullptr : argv[optind - 1];
+				break;
+
+			case 'c':
+				cacheDir = optind > argc ? nullptr : argv[optind - 1];
+				break;
+
+			case 'm':
+				memCache = true;
+				break;
+
+			case 'l':
+				logFile = optind > argc ? nullptr : argv[optind - 1];
+				break;
+
+			case 'b':
+				bindAddress= optind > argc ? "0.0.0.0" : argv[optind - 1];
+				break;
+
+			case 'p':
+				firstPort = atoi(optind > argc ? "6791" : argv[optind - 1]);
+				break;
+
+			case 's':
+				secureCert = optind > argc ? nullptr : argv[optind - 1];
+				optind++;
+				secureKey = optind > argc ? nullptr : argv[optind - 1];
+				break;
+
+			case 'i':
+				instances = atoi(optind > argc ? "1" : argv[optind - 1]);
+				break;
+
+			case 'v':
+				verbosity = atoi(optind > argc ? "1" : argv[optind - 1]);
+				break;
+
+			case 'w':
+				latency = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
+			case 'r':
+				speed = atoi(optind > argc ? "0" : argv[optind - 1]);
+				break;
+
+			case 'z':
+				generateNzb = true;
+				segmentSize = atoi(optind > argc ? "500000" : argv[optind - 1]);
+				break;
+
+			case 'q':
+				quit = true;
+				break;
+		}
+	}
+
+	if (optind < argc)
+	{
+		paramError = true;
+	}
+
+	if (logFile.Empty())
+	{
+		cmdOpts.push_back("WriteLog=none");
+	}
+	else
+	{
+		cmdOpts.push_back("WriteLog=append");
+		logOpt.Format("LogFile=%s", *logFile);
+		cmdOpts.push_back(logOpt);
+	}
+
+	if (verbosity < 1)
+	{
+		cmdOpts.push_back("InfoTarget=none");
+		cmdOpts.push_back("WarningTarget=none");
+		cmdOpts.push_back("ErrorTarget=none");
+	}
+	if (verbosity < 2)
+	{
+		cmdOpts.push_back("DetailTarget=none");
+	}
+	if (verbosity > 2)
+	{
+		cmdOpts.push_back("DebugTarget=both");
+	}
+}

daemon/nserv/NServMain.h

@@ -0,0 +1,26 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef NSERVMAIN_H
+#define NSERVMAIN_H
+
+int NServMain(int argc, char * argv[]);
+
+#endif

daemon/nserv/NntpServer.cpp

@@ -0,0 +1,443 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "NntpServer.h"
+#include "Log.h"
+#include "Util.h"
+#include "YEncoder.h"
+
+class NntpProcessor : public Thread
+{
+public:
+	NntpProcessor(int id, int serverId, const char* dataDir, const char* cacheDir,
+		const char* secureCert, const char* secureKey, int latency, int speed, NntpCache* cache) :
+		m_id(id), m_serverId(serverId), m_dataDir(dataDir), m_cacheDir(cacheDir),
+		m_secureCert(secureCert), m_secureKey(secureKey), m_latency(latency),
+		m_speed(speed), m_cache(cache) {}
+	~NntpProcessor() { m_connection->Disconnect(); }
+	virtual void Run();
+	void SetConnection(std::unique_ptr<Connection>&& connection) { m_connection = std::move(connection); }
+
+private:
+	int m_id;
+	int m_serverId;
+	std::unique_ptr<Connection> m_connection;
+	const char* m_dataDir;
+	const char* m_cacheDir;
+	const char* m_secureCert;
+	const char* m_secureKey;
+	int m_latency;
+	int m_speed;
+	const char* m_messageid;
+	CString m_filename;
+	int m_part;
+	int64 m_offset;
+	int m_size;
+	bool m_sendHeaders;
+	int64 m_start;
+	NntpCache* m_cache;
+
+	void ServArticle();
+	void SendSegment();
+	bool ServerInList(const char* servList);
+	void SendData(const char* buffer, int size);
+};
+
+
+void NntpServer::Run()
+{
+	debug("Entering NntpServer-loop");
+
+	info("Listening on port %i", m_port);
+
+#ifdef WIN32
+	if (m_speed > 0)
+	{
+		timeBeginPeriod(1);
+	}
+#endif
+
+	int num = 1;
+
+	while (!IsStopped())
+	{
+		bool bind = true;
+
+		if (!m_connection)
+		{
+			m_connection = std::make_unique<Connection>(m_host, m_port, m_secureCert);
+			m_connection->SetTimeout(10);
+			m_connection->SetSuppressErrors(false);
+			bind = m_connection->Bind();
+		}
+
+		// Accept connections and store the new Connection
+		std::unique_ptr<Connection> acceptedConnection;
+		if (bind)
+		{
+			acceptedConnection = m_connection->Accept();
+		}
+		if (!bind || !acceptedConnection)
+		{
+			// Server could not bind or accept connection, waiting 1/2 sec and try again
+			if (IsStopped())
+			{
+				break;
+			}
+			m_connection.reset();
+			Util::Sleep(500);
+			continue;
+		}
+		
+		NntpProcessor* commandThread = new NntpProcessor(num++, m_id, m_dataDir,
+			m_cacheDir, m_secureCert, m_secureKey, m_latency, m_speed, m_cache);
+		commandThread->SetAutoDestroy(true);
+		commandThread->SetConnection(std::move(acceptedConnection));
+		commandThread->Start();
+	}
+
+	if (m_connection)
+	{
+		m_connection->Disconnect();
+	}
+
+	debug("Exiting NntpServer-loop");
+}
+
+void NntpServer::Stop()
+{
+	Thread::Stop();
+	if (m_connection)
+	{
+		m_connection->SetSuppressErrors(true);
+		m_connection->Cancel();
+#ifdef WIN32
+		m_connection->Disconnect();
+#endif
+	}
+}
+
+
+void NntpProcessor::Run()
+{
+	m_connection->SetSuppressErrors(false);
+
+#ifndef DISABLE_TLS
+	if (m_secureCert && !m_connection->StartTls(false, m_secureCert, m_secureKey))
+	{
+		error("Could not establish secure connection to nntp-client: Start TLS failed");
+		return;
+	}
+#endif
+
+	info("[%i] Incoming connection from: %s", m_id, m_connection->GetHost() );
+	m_connection->WriteLine("200 Welcome (NServ)\r\n");
+
+	CharBuffer buf(1024);
+	int bytesRead = 0;
+	while (CString line = m_connection->ReadLine(buf, 1024, &bytesRead))
+	{
+		line.TrimRight();
+		detail("[%i] Received: %s", m_id, *line);
+
+		if (!strncasecmp(line, "ARTICLE ", 8))
+		{
+			m_messageid = line + 8;
+			m_sendHeaders = true;
+			ServArticle();
+		}
+		else if (!strncasecmp(line, "BODY ", 5))
+		{
+			m_messageid = line + 5;
+			m_sendHeaders = false;
+			ServArticle();
+		}
+		else if (!strncasecmp(line, "GROUP ", 6))
+		{
+			m_connection->WriteLine(CString::FormatStr("211 0 0 0 %s\r\n", line + 6));
+		}
+		else if (!strncasecmp(line, "AUTHINFO ", 9))
+		{
+			m_connection->WriteLine("281 Authentication accepted\r\n");
+		}
+		else if (!strcasecmp(line, "QUIT"))
+		{
+			detail("[%i] Closing connection", m_id);
+			m_connection->WriteLine("205 Connection closing\r\n");
+			break;
+		}
+		else
+		{
+			warn("[%i] Unknown command: %s", m_id, *line);
+			m_connection->WriteLine("500 Unknown command\r\n");
+		}
+	}
+
+	m_connection->SetGracefull(true);
+	m_connection->Disconnect();
+}
+
+/* 
+ Message-id format:
+   <file-path-relative-to-dataDir?xxx=yyy:zzz!1,2,3>
+where:
+   xxx   - part number (integer)
+   xxx   - offset from which to read the files (integer)
+   yyy   - size of file block to return (integer)
+   1,2,3 - list of server ids, which have the article (optional),
+           if the list is given and current server is not in the list
+           the "article not found"-error is returned.
+ Examples:
+	<parchecker/testfile.dat?1=0:50000>	       - return first 50000 bytes starting from beginning
+	<parchecker/testfile.dat?2=50000:50000>      - return 50000 bytes starting from offset 50000
+	<parchecker/testfile.dat?2=50000:50000!2>    - article is missing on server 1
+*/
+void NntpProcessor::ServArticle()
+{
+	detail("[%i] Serving: %s", m_id, m_messageid);
+
+	if (m_latency)
+	{
+		Util::Sleep(m_latency);
+	}
+
+	bool ok = false;
+
+	const char* from = strchr(m_messageid, '?');
+	const char* off = strchr(m_messageid, '=');
+	const char* to = strchr(m_messageid, ':');
+	const char* end = strchr(m_messageid, '>');
+	const char* serv = strchr(m_messageid, '!');
+
+	if (from && off && to && end)
+	{
+		m_filename.Set(m_messageid + 1, (int)(from - m_messageid - 1));
+		m_part = atoi(from + 1);
+		m_offset = atoll(off + 1);
+		m_size = atoi(to + 1);
+
+		ok = !serv || ServerInList(serv + 1);
+
+		if (ok)
+		{
+			SendSegment();
+			return;
+		}
+
+		if (!ok)
+		{
+			m_connection->WriteLine("430 No Such Article Found\r\n");
+		}
+	}
+	else
+	{
+		m_connection->WriteLine("430 No Such Article Found (invalid message id format)\r\n");
+	}
+}
+
+bool NntpProcessor::ServerInList(const char* servList)
+{
+	Tokenizer tok(servList, ",");
+	while (const char* servid = tok.Next())
+	{
+		if (atoi(servid) == m_serverId)
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+void NntpProcessor::SendSegment()
+{
+	detail("[%i] Sending segment %s (%i=%" PRIi64 ":%i)", m_id, *m_filename, m_part, m_offset, m_size);
+
+	if (m_speed > 0)
+	{
+		m_start = Util::CurrentTicks();
+	}
+
+	BString<1024> fullFilename("%s/%s", m_dataDir, *m_filename);
+	BString<1024> cacheFileDir("%s/%s", m_cacheDir, *m_filename);
+	BString<1024> cacheFileName("%i=%" PRIi64 "-%i", m_part, m_offset, m_size);
+	BString<1024> cacheFullFilename("%s/%s", *cacheFileDir, *cacheFileName);
+	BString<1024> cacheKey("%s/%s", *m_filename, *cacheFileName);
+
+	const char* cachedData = nullptr;
+	int cachedSize;
+	if (m_cache)
+	{
+		m_cache->Find(cacheKey, cachedData, cachedSize);
+	}
+
+	DiskFile cacheFile;
+	bool readCache = !cachedData && m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
+	bool writeCache = !cachedData && m_cacheDir && !readCache;
+	StringBuilder cacheMem;
+	if (m_cache && !cachedData)
+	{
+		cacheMem.Reserve((int)(m_size * 1.1));
+	}
+
+	CString errmsg;
+	if (writeCache && !FileSystem::ForceDirectories(cacheFileDir, errmsg))
+	{
+		error("Could not create directory %s: %s", *cacheFileDir, *errmsg);
+	}
+
+	if (writeCache && !cacheFile.Open(cacheFullFilename, DiskFile::omWrite))
+	{
+		error("Could not create file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
+	}
+
+	if (!cachedData && !readCache && !FileSystem::FileExists(fullFilename))
+	{
+		m_connection->WriteLine(CString::FormatStr("430 Article not found\r\n"));
+		return;
+	}
+
+	YEncoder encoder(fullFilename, m_part, m_offset, m_size, 
+		[proc = this, writeCache, &cacheFile, &cacheMem](const char* buf, int size)
+		{
+			if (proc->m_cache)
+			{
+				cacheMem.Append(buf);
+			}
+			if (writeCache)
+			{
+				cacheFile.Write(buf, size);
+			}
+			proc->SendData(buf, size);
+		});
+
+	if (!cachedData && !readCache && !encoder.OpenFile(errmsg))
+	{
+		m_connection->WriteLine(CString::FormatStr("403 %s\r\n", *errmsg));
+		return;
+	}
+
+	m_connection->WriteLine(CString::FormatStr("%i, 0 %s\r\n", m_sendHeaders ? 222 : 220, m_messageid));
+	if (m_sendHeaders)
+	{
+		m_connection->WriteLine(CString::FormatStr("Message-ID: %s\r\n", m_messageid));
+		m_connection->WriteLine(CString::FormatStr("Subject: \"%s\"\r\n", FileSystem::BaseFileName(m_filename)));
+		m_connection->WriteLine("\r\n");
+	}
+
+	if (cachedData)
+	{
+		SendData(cachedData, cachedSize);
+	}
+	else if (readCache)
+	{
+		cacheFile.Seek(0, DiskFile::soEnd);
+		int size = (int)cacheFile.Position();
+		CharBuffer buf(size);
+		cacheFile.Seek(0);
+		if (cacheFile.Read((char*)buf, size) != size)
+		{
+			error("Could not read file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
+		}
+		if (m_cache)
+		{
+			cacheMem.Append(buf, size);
+		}
+		SendData(buf, size);
+	}
+	else
+	{
+		encoder.WriteSegment();
+	}
+
+	if (!cachedData && cacheMem.Length() > 0)
+	{
+		m_cache->Append(cacheKey, cacheMem, cacheMem.Length());
+	}
+
+	m_connection->WriteLine(".\r\n");
+}
+
+void NntpProcessor::SendData(const char* buffer, int size)
+{
+	if (m_speed == 0)
+	{
+		m_connection->Send(buffer, size);
+		return;
+	}
+
+	int64 expectedTime = (int64)1000 * size / (m_speed * 1024) - (Util::CurrentTicks() - m_start) / 1000;
+
+	int chunkNum = 21;
+	int chunkSize = size;
+	int pause = 0;
+
+	while (pause < 1 && chunkNum > 1)
+	{
+		chunkNum--;
+		chunkSize = size / chunkNum;
+		pause = (int)(expectedTime / chunkNum);
+	}
+
+	int sent = 0;
+	for (int i = 0; i < chunkNum; i++)
+	{
+		int len = sent + chunkSize < size ? chunkSize : size - sent;
+
+		while (sent + len < size && *(buffer + sent + len) != '\r')
+		{
+			len++;
+		}
+		
+		m_connection->Send(buffer + sent, len);
+		int64 now = Util::CurrentTicks();
+		if (now + pause * 1000 < m_start + expectedTime * 1000)
+		{
+			Util::Sleep(pause);
+		}
+		sent += len;
+	}
+}
+
+
+void NntpCache::Append(const char* key, const char* data, int len)
+{
+	Guard guard(m_lock);
+	if (!len)
+	{
+		len = strlen(data);
+	}
+	m_items.emplace(key, std::make_unique<CacheItem>(key, data, len));
+}
+
+bool NntpCache::Find(const char* key, const char*& data, int& size)
+{
+	Guard guard(m_lock);
+
+	CacheMap::iterator pos = m_items.find(key);
+	if (pos != m_items.end())
+	{
+		data = (*pos).second->m_data;
+		size = (*pos).second->m_size;
+		return true;
+	}
+
+	return false;
+}

daemon/nserv/NntpServer.h

@@ -0,0 +1,78 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef NNTPSERVER_H
+#define NNTPSERVER_H
+
+#include "Thread.h"
+#include "Connection.h"
+#include "Util.h"
+
+class NntpCache
+{
+public:
+	void Append(const char* key, const char* data, int len = 0);
+	bool Find(const char* key, const char*& data, int& size);
+
+private:
+	class CacheItem
+	{
+	public:
+		CacheItem(const char* key, const char* data, int size) :
+			m_key(key), m_data(data), m_size(size) {}
+
+		CString m_key;
+		CString m_data;
+		int m_size = 0;
+	};
+
+	typedef std::unordered_map<std::string, std::unique_ptr<CacheItem>> CacheMap;
+
+	CacheMap m_items;
+	Mutex m_lock;
+};
+
+class NntpServer : public Thread
+{
+public:
+	NntpServer(int id, const char* host, int port, const char* secureCert,
+		const char* secureKey, const char* dataDir, const char* cacheDir,
+		int latency, int speed, NntpCache* cache) :
+		m_id(id), m_host(host), m_port(port), m_secureCert(secureCert),
+		m_secureKey(secureKey), m_dataDir(dataDir), m_cacheDir(cacheDir),
+		m_latency(latency), m_speed(speed), m_cache(cache) {}
+	virtual void Run();
+	virtual void Stop();
+
+private:
+	int m_id;
+	CString m_host;
+	int m_port;
+	CString m_secureCert;
+	CString m_secureKey;
+	std::unique_ptr<Connection> m_connection;
+	CString m_dataDir;
+	CString m_cacheDir;
+	int m_latency;
+	int m_speed;
+	NntpCache* m_cache;
+};
+
+#endif

daemon/nserv/NzbGenerator.cpp

@@ -0,0 +1,131 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "NzbGenerator.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "Log.h"
+
+void NzbGenerator::Execute()
+{
+	info("Generating nzbs for %s", *m_dataDir);
+
+	DirBrowser dir(m_dataDir);
+	while (const char* filename = dir.Next())
+	{
+		BString<1024> fullFilename("%s%c%s", *m_dataDir, PATH_SEPARATOR, filename);
+
+		int len = strlen(filename);
+		if (len > 4 && !strcasecmp(filename + len - 4, ".nzb"))
+		{
+			// skip nzb-files
+			continue;
+		}
+
+		GenerateNzb(fullFilename);
+	}
+
+	info("Nzb generation finished");
+}
+
+void NzbGenerator::GenerateNzb(const char* path)
+{
+	BString<1024> nzbFilename("%s%c%s.nzb", *m_dataDir, PATH_SEPARATOR, FileSystem::BaseFileName(path));
+
+	if (FileSystem::FileExists(nzbFilename))
+	{
+		return;
+	}
+
+	info("Generating nzb for %s", FileSystem::BaseFileName(path));
+
+	DiskFile outfile;
+	if (!outfile.Open(nzbFilename, DiskFile::omWrite))
+	{
+		error("Could not create file %s", *nzbFilename);
+		return;
+	}
+
+	outfile.Print("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+	outfile.Print("<!DOCTYPE nzb PUBLIC \"-//newzBin//DTD NZB 1.0//EN\" \"http://www.newzbin.com/DTD/nzb/nzb-1.0.dtd\">\n");
+	outfile.Print("<nzb xmlns=\"http://www.newzbin.com/DTD/2003/nzb\">\n");
+
+	bool isDir = FileSystem::DirectoryExists(path);
+	if (isDir)
+	{
+		AppendDir(outfile, path);
+	}
+	else
+	{
+		AppendFile(outfile, path, nullptr);
+	}
+
+	outfile.Print("</nzb>\n");
+
+	outfile.Close();
+}
+
+void NzbGenerator::AppendDir(DiskFile& outfile, const char* path)
+{
+	DirBrowser dir(path);
+	while (const char* filename = dir.Next())
+	{
+		BString<1024> fullFilename("%s%c%s", path, PATH_SEPARATOR, filename);
+
+		bool isDir = FileSystem::DirectoryExists(fullFilename);
+		if (!isDir)
+		{
+			AppendFile(outfile, fullFilename, FileSystem::BaseFileName(path));
+		}
+	}
+}
+
+void NzbGenerator::AppendFile(DiskFile& outfile, const char* filename, const char* relativePath)
+{
+	detail("Processing %s", FileSystem::BaseFileName(filename));
+
+	int64 fileSize = FileSystem::FileSize(filename);
+	time_t timestamp = Util::CurrentTime();
+
+	int segmentCount = (int)((fileSize + m_segmentSize - 1) / m_segmentSize);
+
+	outfile.Print("<file poster=\"nserv\" date=\"%i\" subject=\"&quot;%s&quot; yEnc (1/%i)\">\n",
+		(int)timestamp, FileSystem::BaseFileName(filename), segmentCount);
+	outfile.Print("<groups>\n");
+	outfile.Print("<group>alt.binaries.test</group>\n");
+	outfile.Print("</groups>\n");
+	outfile.Print("<segments>\n");
+
+	int64 segOffset = 0;
+	for (int segno = 1; segno <= segmentCount; segno++)
+	{
+		int segSize = (int)(segOffset + m_segmentSize < fileSize ? m_segmentSize : fileSize - segOffset);
+		outfile.Print("<segment bytes=\"%i\" number=\"%i\">%s%s%s?%i=%" PRIi64 ":%i</segment>\n",
+			m_segmentSize, segno,
+			relativePath ? relativePath : "",
+			relativePath ? "/" : "",
+			FileSystem::BaseFileName(filename), segno, segOffset, segSize);
+		segOffset += segSize;
+	}
+
+	outfile.Print("</segments>\n");
+	outfile.Print("</file>\n");
+}

daemon/nserv/NzbGenerator.h

@@ -0,0 +1,43 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef NZBGENERATOR_H
+#define NZBGENERATOR_H
+
+#include "NString.h"
+#include "FileSystem.h"
+
+class NzbGenerator
+{
+public:
+	NzbGenerator(const char* dataDir, int segmentSize) :
+		m_dataDir(dataDir), m_segmentSize(segmentSize) {};
+	void Execute();
+
+private:
+	CString m_dataDir;
+	int m_segmentSize;
+
+	void GenerateNzb(const char* path);
+	void AppendFile(DiskFile& outfile, const char* filename, const char* relativePath);
+	void AppendDir(DiskFile& outfile, const char* path);
+};
+
+#endif

daemon/nserv/YEncoder.cpp

@@ -0,0 +1,129 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "YEncoder.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "Log.h"
+
+bool YEncoder::OpenFile(CString& errmsg)
+{
+	if (m_size < 0)
+	{
+		errmsg = "Invalid segment size";
+		return false;
+	}
+
+	if (!m_diskfile.Open(m_filename, DiskFile::omRead) || !m_diskfile.Seek(0, DiskFile::soEnd))
+	{
+		errmsg = "File not found";
+		return false;
+	}
+
+	m_fileSize = m_diskfile.Position();
+	if (m_size == 0)
+	{
+		m_size = (int)(m_fileSize - m_offset + 1);
+	}
+
+	if (m_fileSize < m_offset + m_size)
+	{
+		errmsg = "Invalid segment size";
+		return false;
+	}
+
+	if (!m_diskfile.Seek(m_offset))
+	{
+		errmsg = "Invalid segment offset";
+		return false;
+	}
+
+	return true;
+}
+
+void YEncoder::WriteSegment()
+{
+	StringBuilder outbuf;
+	outbuf.Reserve(std::max(2048, std::min((int)(m_size * 1.1), 16 * 1024 * 1024)));
+
+	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%" PRIi64 " name=%s\r\n", m_part, m_fileSize, FileSystem::BaseFileName(m_filename)));
+	outbuf.Append(CString::FormatStr("=ypart begin=%" PRIi64 " end=%" PRIi64 "\r\n", m_offset + 1, m_offset + m_size));
+
+	Crc32 crc;
+	CharBuffer inbuf(std::min(m_size, 16 * 1024 * 1024));
+	int lnsz = 0;
+	char* out = (char*)outbuf + outbuf.Length();
+
+	while (m_diskfile.Position() < m_offset + m_size)
+	{
+		int64 needBytes = std::min((int64)inbuf.Size(), m_offset + m_size - m_diskfile.Position());
+		int64 readBytes = m_diskfile.Read(inbuf, needBytes);
+		bool lastblock = m_diskfile.Position() == m_offset + m_size;
+		if (readBytes == 0)
+		{
+			return; // error;
+		}
+
+		crc.Append((uchar*)(const char*)inbuf, (int)readBytes);
+
+		char* in = inbuf;
+		while (readBytes > 0)
+		{
+			char ch = *in++;
+			readBytes--;
+			ch = (char)(((uchar)(ch) + 42) % 256);
+			if (ch == '\0' || ch == '\n' || ch == '\r' || ch == '=' || ch == ' ' || ch == '\t')
+			{
+				*out++ = '=';
+				lnsz++;
+				ch = (char)(((uchar)ch + 64) % 256);
+			}
+			if (ch == '.' && lnsz == 0)
+			{
+				*out++ = '.';
+				lnsz++;
+			}
+			*out++ = ch;
+			lnsz++;
+
+			if (lnsz >= 128 || (readBytes == 0 && lastblock))
+			{
+				*out++ = '\r';
+				*out++ = '\n';
+				lnsz += 2;
+				outbuf.SetLength(outbuf.Length() + lnsz);
+
+				if (outbuf.Length() > outbuf.Capacity() - 200)
+				{
+					m_writeFunc(outbuf, outbuf.Length());
+					outbuf.SetLength(0);
+					out = (char*)outbuf;
+				}
+
+				lnsz = 0;
+			}
+		}
+	}
+	m_diskfile.Close();
+
+	outbuf.Append(CString::FormatStr("=yend size=%i part=0 pcrc32=%08x\r\n", m_size, (unsigned int)crc.Finish()));
+	m_writeFunc(outbuf, outbuf.Length());
+}

daemon/nserv/YEncoder.h

@@ -0,0 +1,47 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef YENCODER_H
+#define YENCODER_H
+
+#include "NString.h"
+#include "FileSystem.h"
+
+class YEncoder
+{
+public:
+	typedef std::function<void(const char* buf, int size)> WriteFunc;
+
+	YEncoder(const char* filename, int part, int64 offset, int size, WriteFunc writeFunc) :
+		m_filename(filename), m_part(part), m_offset(offset), m_size(size), m_writeFunc(writeFunc) {};
+	bool OpenFile(CString& errmsg);
+	void WriteSegment();
+
+private:
+	DiskFile m_diskfile;
+	CString m_filename;
+	int m_part;
+	int64 m_offset;
+	int m_size;
+	int64 m_fileSize;
+	WriteFunc m_writeFunc;
+};
+
+#endif

daemon/postprocess/Cleanup.cpp

@@ -76,7 +76,6 @@ void MoveController::Run()
 		m_postInfo->GetNzbInfo()->SetMoveStatus(NzbInfo::msFailure);
 	}
 
-	m_postInfo->SetStage(PostInfo::ptQueued);
 	m_postInfo->SetWorking(false);
 }
 
@@ -96,7 +95,7 @@ bool MoveController::MoveFiles()
 		while (const char* filename = dir.Next())
 		{
 			BString<1024> srcFile("%s%c%s",* m_interDir, PATH_SEPARATOR, filename);
-			CString dstFile = FileSystem::MakeUniqueFilename(m_destDir, filename);
+			CString dstFile = FileSystem::MakeUniqueFilename(m_destDir, FileSystem::MakeValidFilename(filename));
 			bool hiddenFile = filename[0] == '.';
 
 			if (!hiddenFile)
@@ -182,7 +181,6 @@ void CleanupController::Run()
 		m_postInfo->GetNzbInfo()->SetCleanupStatus(NzbInfo::csFailure);
 	}
 
-	m_postInfo->SetStage(PostInfo::ptQueued);
 	m_postInfo->SetWorking(false);
 }
 

daemon/postprocess/DirectUnpack.cpp

@@ -0,0 +1,558 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "DirectUnpack.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+#include "Options.h"
+
+void DirectUnpack::StartJob(NzbInfo* nzbInfo)
+{
+	DirectUnpack* directUnpack = new DirectUnpack();
+	directUnpack->m_nzbId = nzbInfo->GetId();
+	directUnpack->SetAutoDestroy(true);
+
+	nzbInfo->SetUnpackThread(directUnpack);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsRunning);
+
+	directUnpack->Start();
+}
+
+void DirectUnpack::Run()
+{
+	debug("Entering DirectUnpack-loop for %i", m_nzbId);
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo)
+		{
+			debug("Could not find NzbInfo for %i", m_nzbId);
+			return;
+		}
+
+		m_name = nzbInfo->GetName();
+		m_destDir = nzbInfo->GetDestDir();
+		m_finalDir = nzbInfo->BuildFinalDirName();
+
+		NzbParameter* parameter = nzbInfo->GetParameters()->Find("*Unpack:Password");
+		if (parameter)
+		{
+			m_password = parameter->GetValue();
+		}
+	}
+
+	m_infoName.Format("direct unpack for %s", *m_name);
+	m_infoNameUp.Format("Direct unpack for %s", *m_name); // first letter in upper case
+
+	FindArchiveFiles();
+
+	SetInfoName(m_infoName);
+	SetWorkingDir(m_destDir);
+
+	while (!IsStopped())
+	{
+		CString archive;
+		{
+			Guard guard(m_volumeMutex);
+			if (!m_archives.empty())
+			{
+				archive = std::move(m_archives.front());
+				m_archives.pop_front();
+			}
+		}
+
+		if (archive)
+		{
+			if (!m_processed)
+			{
+				PrintMessage(Message::mkInfo, "Directly unpacking %s", *m_name);
+				Cleanup();
+				CreateUnpackDir();
+				m_processed = true;
+			}
+
+			ExecuteUnrar(archive);
+
+			if (!m_unpackOk)
+			{
+				break;
+			}
+		}
+		else
+		{
+			if (m_nzbCompleted)
+			{
+				break;
+			}
+			Util::Sleep(100);
+		}
+	}
+
+	if (!m_unpackOk)
+	{
+		Cleanup();
+	}
+
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (!nzbInfo || nzbInfo->GetUnpackThread() != this)
+		{
+			debug("Could not find NzbInfo for %s", *m_infoName);
+			return;
+		}
+
+		nzbInfo->SetUnpackThread(nullptr);
+		nzbInfo->SetDirectUnpackStatus(m_processed ? m_unpackOk && !IsStopped() ? NzbInfo::nsSuccess : NzbInfo::nsFailure : NzbInfo::nsNone);
+
+		if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			nzbInfo->AddMessage(Message::mkInfo, BString<1024>("%s successful", *m_infoNameUp));
+		}
+		else if (nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsFailure)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("%s failed", *m_infoNameUp));
+		}
+
+		if (nzbInfo->GetPostInfo() && nzbInfo->GetDirectUnpackStatus() == NzbInfo::nsSuccess)
+		{
+			std::move(m_extractedArchives.begin(), m_extractedArchives.end(),
+				std::back_inserter(*nzbInfo->GetPostInfo()->GetExtractedArchives()));
+		}
+
+		AddExtraTime(nzbInfo);
+
+		if (nzbInfo->GetPostInfo())
+		{
+			nzbInfo->GetPostInfo()->SetWorking(false);
+		}
+	}
+
+	debug("Exiting DirectUnpack-loop for %i", m_nzbId);
+}
+
+void DirectUnpack::ExecuteUnrar(const char* archiveName)
+{
+	// Format:
+	//   unrar x -y -p- -o+ -vp <archive.part0001.rar> <dest-dir>/_unpack/
+
+	ParamList params;
+	if (!PrepareCmdParams(g_Options->GetUnrarCmd(), &params, "unrar"))
+	{
+		return;
+	}
+
+	if (!params.Exists("x") && !params.Exists("e"))
+	{
+		params.emplace_back("x");
+	}
+
+	params.emplace_back("-y");
+
+	if (!m_password.Empty())
+	{
+		params.push_back(CString::FormatStr("-p%s", *m_password));
+	}
+	else
+	{
+		params.emplace_back("-p-");
+	}
+
+	if (!params.Exists("-o+") && !params.Exists("-o-"))
+	{
+		params.emplace_back("-o+");
+	}
+
+	params.emplace_back("-vp");
+
+	params.emplace_back(archiveName);
+	m_unpackExtendedDir = FileSystem::MakeExtendedPath(m_unpackDir, true);
+	params.push_back(*BString<1024>("%s%c", *m_unpackExtendedDir, PATH_SEPARATOR));
+	SetArgs(std::move(params));
+	SetLogPrefix("Unrar");
+	ResetEnv();
+
+	m_allOkMessageReceived = false;
+
+	SetNeedWrite(true);
+	m_unpacking = true;
+	int exitCode = Execute();
+	m_unpacking = false;
+	SetLogPrefix(nullptr);
+
+	m_unpackOk = exitCode == 0 && m_allOkMessageReceived && !GetTerminated();
+
+	if (!m_unpackOk && exitCode > 0)
+	{
+		PrintMessage(Message::mkError, "Unrar error code: %i", exitCode);
+	}
+}
+
+bool DirectUnpack::PrepareCmdParams(const char* command, ParamList* params, const char* infoName)
+{
+	if (FileSystem::FileExists(command))
+	{
+		params->emplace_back(command);
+		return true;
+	}
+
+	std::vector<CString> cmdArgs = Util::SplitCommandLine(command);
+	if (cmdArgs.empty())
+	{
+		PrintMessage(Message::mkError, "Could not start %s, failed to parse command line: %s", infoName, command);
+		m_unpackOk = false;
+		return false;
+	}
+
+	std::move(cmdArgs.begin(), cmdArgs.end(), std::back_inserter(*params));
+
+	return true;
+}
+
+void DirectUnpack::CreateUnpackDir()
+{
+	bool useInterDir = !Util::EmptyStr(g_Options->GetInterDir()) &&
+		!strncmp(m_destDir, g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
+		m_destDir[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
+
+	m_finalDirCreated = useInterDir && !FileSystem::DirectoryExists(m_finalDir);
+
+	const char* destDir = useInterDir && !m_finalDir.Empty() ? *m_finalDir : *m_destDir;
+
+	m_unpackDir.Format("%s%c%s", destDir, PATH_SEPARATOR, "_unpack");
+
+	detail("Unpacking into %s", *m_unpackDir);
+
+	CString errmsg;
+	if (!FileSystem::ForceDirectories(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not create directory %s: %s", *m_unpackDir, *errmsg);
+	}
+}
+
+void DirectUnpack::FindArchiveFiles()
+{
+	Guard guard(m_volumeMutex);
+
+	DirBrowser dir(m_destDir);
+	while (const char* filename = dir.Next())
+	{
+		if (IsMainArchive(filename))
+		{
+			BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+			if (!FileSystem::DirectoryExists(fullFilename))
+			{
+				m_archives.emplace_back(filename);
+			}
+		}
+	}
+}
+
+bool DirectUnpack::IsMainArchive(const char* filename)
+{
+	RegEx regExRarPart(".*\\.part([0-9]+)\\.rar$");
+	bool mainPart = Util::EndsWith(filename, ".rar", false) &&
+		(!regExRarPart.Match(filename) || atoi(filename + regExRarPart.GetMatchStart(1)) == 1);
+	return mainPart;
+}
+
+/**
+ * Unrar prints "Insert disk"-message without new line terminator.
+ * In order to become the message we analyze the output after every char.
+ */
+bool DirectUnpack::ReadLine(char* buf, int bufSize, FILE* stream)
+{
+	int i = 0;
+	bool backspace = false;
+
+	for (; i < bufSize - 1; i++)
+	{
+		int ch = fgetc(stream);
+
+		if (ch == '\b')
+		{
+			backspace = true;
+		}
+		if (!backspace)
+		{
+			buf[i] = ch;
+			buf[i+1] = '\0';
+		}
+		if (ch == EOF)
+		{
+			break;
+		}
+		if (ch == '\n')
+		{
+			i++;
+			break;
+		}
+		if (i > 35 && ch == ' ' &&
+			!strncmp(buf, "Insert disk with", 16) && strstr(buf, " [C]ontinue, [Q]uit "))
+		{
+			i++;
+			break;
+		}
+	}
+
+	// skip unnecessary progress messages
+	if (!strncmp(buf, "...", 3))
+	{
+		buf[0] = '\0';
+	}
+
+	buf[i] = '\0';
+
+	return i > 0;
+}
+
+void DirectUnpack::AddMessage(Message::EKind kind, const char* text)
+{
+	debug("%s", text);
+
+	BString<1024> msgText = text;
+	int len = strlen(text);
+
+	// Modify unrar messages for better readability:
+	// remove the destination path part from message "Extracting file.xxx"
+	if (!strncmp(text, "Unrar: Extracting  ", 19) &&
+		!strncmp(text + 19, m_unpackExtendedDir, strlen(m_unpackExtendedDir)))
+	{
+		msgText.Format("Unrar: Extracting %s", text + 19 + strlen(m_unpackExtendedDir) + 1);
+	}
+
+	if (!strncmp(text, "Unrar: Insert disk with", 23) && strstr(text, " [C]ontinue, [Q]uit"))
+	{
+		BString<1024> filename;
+		filename.Set(text + 24, (int)(strstr(text, " [C]ontinue, [Q]uit") - text - 24));
+		WaitNextVolume(filename);
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+	if (nzbInfo)
+	{
+		nzbInfo->AddMessage(kind, msgText);
+	}
+	else
+	{
+		ScriptController::AddMessage(kind, msgText);
+	}
+
+	if (!strncmp(msgText, "Unrar: Extracting ", 18) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, msgText + 7);
+	}
+
+	if (!strncmp(text, "Unrar: Extracting from ", 23) && nzbInfo)
+	{
+		SetProgressLabel(nzbInfo, text + 7);
+		m_extractedArchives.emplace_back(text + 23);
+	}
+
+	if (!IsStopped() && (
+		!strncmp(text, "Unrar: Checksum error in the encrypted file", 42) ||
+		!strncmp(text, "Unrar: CRC failed in the encrypted file", 39) ||
+		!strncmp(text, "Unrar: The specified password is incorrect.", 43) ||
+		strstr(text, " : packed data CRC failed in volume") ||
+		strstr(text, " : packed data checksum error in volume") ||
+		(len > 13 && !strncmp(text + len - 13, " - CRC failed", 13)) ||
+		(len > 18 && !strncmp(text + len - 18, " - checksum failed", 18)) ||
+		!strncmp(text, "Unrar: WARNING: You need to start extraction from a previous volume", 67)))
+	{
+		Stop(downloadQueue, nzbInfo);
+	}
+
+	if (!strncmp(text, "Unrar: All OK", 13))
+	{
+		m_allOkMessageReceived = true;
+	}
+}
+
+void DirectUnpack::Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("Stopping direct unpack for %s", *m_infoName);
+	if (m_processed)
+	{
+		if (nzbInfo)
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Cancelling %s", *m_infoName));
+		}
+		else
+		{
+			warn("Cancelling %s", *m_infoName);
+		}
+	}
+	AddExtraTime(nzbInfo);
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetWorking(false);
+	}
+	Thread::Stop();
+	if (m_unpacking)
+	{
+		Terminate();
+	}
+}
+
+void DirectUnpack::WaitNextVolume(const char* filename)
+{
+	debug("WaitNextVolume for %s", filename);
+
+	// Stop direct unpack if destination directory was changed during unpack
+	{
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+		NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+		if (nzbInfo && (strcmp(m_destDir, nzbInfo->GetDestDir()) ||
+			strcmp(m_finalDir, nzbInfo->BuildFinalDirName())))
+		{
+			nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Destination directory changed for %s", nzbInfo->GetName()));
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+
+	BString<1024> fullFilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+	if (FileSystem::FileExists(fullFilename))
+	{
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+	else
+	{
+		Guard guard(m_volumeMutex);
+		m_waitingFile = filename;
+		if (m_nzbCompleted)
+		{
+			// nzb completed but unrar waits for another volume
+			PrintMessage(Message::mkWarning, "Could not find volume %s", filename);
+			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+			NzbInfo* nzbInfo = downloadQueue->GetQueue()->Find(m_nzbId);
+			Stop(downloadQueue, nzbInfo);
+		}
+	}
+}
+
+void DirectUnpack::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo)
+{
+	debug("FileDownloaded for %s/%s", fileInfo->GetNzbInfo()->GetName(), fileInfo->GetFilename());
+
+	if (fileInfo->GetNzbInfo()->GetFailedArticles() > 0)
+	{
+		Stop(downloadQueue, fileInfo->GetNzbInfo());
+		return;
+	}
+
+	Guard guard(m_volumeMutex);
+	if (m_waitingFile && !strcasecmp(fileInfo->GetFilename(), m_waitingFile))
+	{
+		m_waitingFile = nullptr;
+		Write("\n"); // emulating click on Enter-key for "continue"
+	}
+
+	if (IsMainArchive(fileInfo->GetFilename()))
+	{
+		m_archives.emplace_back(fileInfo->GetFilename());
+	}
+}
+
+void DirectUnpack::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDownloaded for %s", nzbInfo->GetName());
+
+	Guard guard(m_volumeMutex);
+	m_nzbCompleted = true;
+	if (m_waitingFile)
+	{
+		// nzb completed but unrar waits for another volume
+		nzbInfo->AddMessage(Message::mkWarning, BString<1024>("Unrar: Could not find volume %s", *m_waitingFile));
+		Stop(downloadQueue, nzbInfo);
+		return;
+	}
+
+	m_extraStartTime = Util::CurrentTime();
+
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(m_progressLabel);
+	}
+}
+
+void DirectUnpack::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
+{
+	debug("NzbDeleted for %s", nzbInfo->GetName());
+
+	nzbInfo->SetUnpackThread(nullptr);
+	nzbInfo->SetDirectUnpackStatus(NzbInfo::nsFailure);
+	Stop(downloadQueue, nzbInfo);
+}
+
+// Remove _unpack-dir
+void DirectUnpack::Cleanup()
+{
+	debug("Cleanup for %s", *m_infoName);
+
+	CString errmsg;
+	if (FileSystem::DirectoryExists(m_unpackDir) &&
+		!FileSystem::DeleteDirectoryWithContent(m_unpackDir, errmsg))
+	{
+		PrintMessage(Message::mkError, "Could not delete temporary directory %s: %s", *m_unpackDir, *errmsg);
+	}
+
+	if (m_finalDirCreated)
+	{
+		FileSystem::DeleteDirectory(m_finalDir);
+	}
+}
+
+void DirectUnpack::SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel)
+{
+	m_progressLabel = progressLabel;
+	if (nzbInfo->GetPostInfo())
+	{
+		nzbInfo->GetPostInfo()->SetProgressLabel(progressLabel);
+	}
+}
+
+void DirectUnpack::AddExtraTime(NzbInfo* nzbInfo)
+{
+	if (m_extraStartTime)
+	{
+		int extraTime = (int)(Util::CurrentTime() - m_extraStartTime);
+		nzbInfo->SetUnpackSec(nzbInfo->GetUnpackSec() + extraTime);
+		nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() + extraTime);
+		m_extraStartTime = 0;
+	}
+}
+
+bool DirectUnpack::IsArchiveFilename(const char* filename)
+{
+	if (Util::EndsWith(filename, ".rar", false))
+	{
+		return true;
+	}
+
+	RegEx regExRarMultiSeq(".*\\.[r-z][0-9][0-9]$");
+	return regExRarMultiSeq.Match(filename);
+}

daemon/postprocess/DirectUnpack.h

@@ -0,0 +1,86 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef DIRECTUNPACK_H
+#define DIRECTUNPACK_H
+
+#include "Log.h"
+#include "Thread.h"
+#include "DownloadInfo.h"
+#include "Script.h"
+
+class DirectUnpack : public Thread, public ScriptController
+{
+public:
+	virtual void Run();
+	static void StartJob(NzbInfo* nzbInfo);
+	void Stop(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo);
+	void NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	void NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	static bool IsArchiveFilename(const char* filename);
+
+protected:
+	virtual bool ReadLine(char* buf, int bufSize, FILE* stream);
+	virtual void AddMessage(Message::EKind kind, const char* text);
+
+private:
+	class ParamList : public std::vector<CString>
+	{
+	public:
+		bool Exists(const char* param) { return std::find(begin(), end(), param) != end(); }
+	};
+
+	typedef std::deque<CString> ArchiveList;
+
+	int m_nzbId;
+	CString m_name;
+	CString m_infoName;
+	CString m_infoNameUp;
+	CString m_destDir;
+	CString m_finalDir;
+	CString m_unpackDir;
+	CString m_unpackExtendedDir;
+	CString m_password;
+	CString m_waitingFile;
+	CString m_progressLabel;
+	bool m_allOkMessageReceived = false;
+	bool m_unpackOk = false;
+	bool m_finalDirCreated = false;
+	bool m_nzbCompleted = false;
+	Mutex m_volumeMutex;
+	ArchiveList m_archives;
+	bool m_processed = false;
+	bool m_unpacking = false;
+	time_t m_extraStartTime = 0;
+	ArchiveList m_extractedArchives;
+
+	void CreateUnpackDir();
+	void FindArchiveFiles();
+	void ExecuteUnrar(const char* archiveName);
+	bool PrepareCmdParams(const char* command, ParamList* params, const char* infoName);
+	void WaitNextVolume(const char* filename);
+	void Cleanup();
+	bool IsMainArchive(const char* filename);
+	void SetProgressLabel(NzbInfo* nzbInfo, const char* progressLabel);
+	void AddExtraTime(NzbInfo* nzbInfo);
+};
+
+#endif

daemon/postprocess/DupeMatcher.cpp

@@ -68,7 +68,6 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 		return false;
 	}
 	const char* unrarPath = cmdArgs[0];
-	unrar.SetScript(unrarPath);
 	unrar.SetArgs({unrarPath, "lt", "*.rar"});
 	unrar.SetWorkingDir(directory);
 
@@ -81,7 +80,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 		curTime + timeoutSec > Util::CurrentTime() &&
 		curTime >= Util::CurrentTime())					// in a case clock was changed
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	if (unrar.IsRunning())
@@ -92,7 +91,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 	// wait until terminated or killed
 	while (unrar.IsRunning())
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	*maxSize = unrar.m_maxSize;
@@ -162,7 +161,7 @@ bool DupeMatcher::Prepare()
 	char filename[1024];
 	FindLargestFile(m_destDir, filename, sizeof(filename), &m_maxSize, &m_compressed);
 	bool sizeOK = SizeDiffOK(m_maxSize, m_expectedSize, 20);
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, sizeOK ? "" : ", size mismatch");
 	return sizeOK;
 }
@@ -174,7 +173,7 @@ bool DupeMatcher::MatchDupeContent(const char* dupeDir)
 	char filename[1024];
 	FindLargestFile(dupeDir, filename, sizeof(filename), &dupeMaxSize, &dupeCompressed);
 	bool ok = dupeMaxSize == m_maxSize && dupeCompressed == m_compressed;
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, ok ? "" : ", size mismatch");
 	return ok;
 }

daemon/postprocess/ParChecker.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -42,20 +42,14 @@ const char* Par2CmdLineErrStr[] = { "OK",
 	"internal error occurred",
 	"out of memory" };
 
-// Sleep interval for synchronisation (microseconds)
-#ifdef WIN32
-// Windows doesn't allow sleep intervals less than one millisecond
-#define SYNC_SLEEP_INTERVAL 1000
-#else
-#define SYNC_SLEEP_INTERVAL 100
-#endif
-
 class RepairThread;
 
 class Repairer : public Par2::Par2Repairer, public ParChecker::AbstractRepairer
 {
 public:
-	Repairer(ParChecker* owner) { m_owner = owner; }
+	Repairer(ParChecker* owner):
+		Par2::Par2Repairer(owner->m_parCout, owner->m_parCerr),
+		m_owner(owner), commandLine(owner->m_parCout, owner->m_parCerr) {}
 	Par2::Result PreProcess(const char *parFilename);
 	Par2::Result Process(bool dorepair);
 	virtual Repairer* GetRepairer() { return this; }
@@ -72,8 +66,8 @@ protected:
 private:
 	typedef vector<Thread*> Threads;
 
-	Par2::CommandLine commandLine;
 	ParChecker* m_owner;
+	Par2::CommandLine commandLine;
 	Threads m_threads;
 	bool m_parallel;
 	Mutex progresslock;
@@ -81,6 +75,7 @@ private:
 	virtual void BeginRepair();
 	virtual void EndRepair();
 	void RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength);
+	static void SyncSleep();
 
 	friend class ParChecker;
 	friend class RepairThread;
@@ -104,6 +99,11 @@ private:
 	volatile bool m_working = false;
 };
 
+class RepairCreatorPacket : public Par2::CreatorPacket
+{
+	friend class ParChecker;
+};
+
 Par2::Result Repairer::PreProcess(const char *parFilename)
 {
 	BString<100> memParam("-m%i", g_Options->GetParBuffer());
@@ -246,7 +246,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 
 		if (!jobAdded)
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			SyncSleep();
 		}
 	}
 
@@ -261,7 +261,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 			if (repairThread->IsWorking())
 			{
 				working = true;
-				usleep(SYNC_SLEEP_INTERVAL);
+				SyncSleep();
 				break;
 			}
 		}
@@ -298,6 +298,17 @@ void Repairer::RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t b
 	}
 }
 
+// Sleep for synchronisation
+void Repairer::SyncSleep()
+{
+#ifdef WIN32
+	// Windows doesn't allow sleep intervals less than one millisecond
+	Sleep(1);
+#else
+	usleep(100);
+#endif
+}
+
 void RepairThread::Run()
 {
 	while (!IsStopped())
@@ -309,7 +320,7 @@ void RepairThread::Run()
 		}
 		else
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			Repairer::SyncSleep();
 		}
 	}
 }
@@ -364,15 +375,9 @@ int ParChecker::StreamBuf::overflow(int ch)
 }
 
 
-ParChecker::~ParChecker()
-{
-	debug("Destroying ParChecker");
-
-	Cleanup();
-}
-
 void ParChecker::Cleanup()
 {
+	Guard guard(m_repairerMutex);
 	m_repairer.reset();
 	m_queuedParFiles.clear();
 	m_processedFiles.clear();
@@ -381,14 +386,11 @@ void ParChecker::Cleanup()
 	m_errMsg = nullptr;
 }
 
-void ParChecker::Run()
+void ParChecker::Execute()
 {
-	Par2::cout.rdbuf(&m_parOutStream);
-	Par2::cerr.rdbuf(&m_parErrStream);
-
 	m_status = RunParCheckAll();
 
-	if (m_status == psRepairNotNeeded && m_parQuick && m_forceRepair && !m_cancelled)
+	if (m_status == psRepairNotNeeded && m_parQuick && m_forceRepair && !IsStopped())
 	{
 		PrintMessage(Message::mkInfo, "Performing full par-check for %s", *m_nzbName);
 		m_parQuick = false;
@@ -396,9 +398,6 @@ void ParChecker::Run()
 	}
 
 	Completed();
-
-	Par2::cout.rdbuf(&Par2::nullStreamBuf);
-	Par2::cerr.rdbuf(&Par2::nullStreamBuf);
 }
 
 ParChecker::EStatus ParChecker::RunParCheckAll()
@@ -411,23 +410,22 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 	}
 
 	EStatus allStatus = psRepairNotNeeded;
-	m_cancelled = false;
 	m_parFull = true;
 
 	for (CString& parFilename : fileList)
 	{
 		debug("Found par: %s", *parFilename);
 
-		if (!IsStopped() && !m_cancelled)
+		if (!IsStopped())
 		{
-			BString<1024> fullParFilename( "%s%c%s", *m_destDir, (int)PATH_SEPARATOR, *parFilename);
+			BString<1024> fullParFilename( "%s%c%s", *m_destDir, PATH_SEPARATOR, *parFilename);
 
 			int baseLen = 0;
-			ParParser::ParseParFilename(parFilename, &baseLen, nullptr);
+			ParParser::ParseParFilename(parFilename, true, &baseLen, nullptr);
 			BString<1024> infoName;
 			infoName.Set(parFilename, baseLen);
 
-			BString<1024> parInfoName("%s%c%s", *m_nzbName, (int)PATH_SEPARATOR, *infoName);
+			BString<1024> parInfoName("%s%c%s", *m_nzbName, PATH_SEPARATOR, *infoName);
 			SetInfoName(parInfoName);
 
 			EStatus status = RunParCheck(fullParFilename);
@@ -437,11 +435,6 @@ ParChecker::EStatus ParChecker::RunParCheckAll()
 			{
 				allStatus = status;
 			}
-
-			if (g_Options->GetBrokenLog())
-			{
-				WriteBrokenLog(status);
-			}
 		}
 	}
 
@@ -476,6 +469,9 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		return psFailed;
 	}
 
+	CString creator = GetPacketCreator();
+	info("Recovery files created by: %s", creator.Empty() ? "<unknown program>" : *creator);
+
 	m_stage = ptVerifyingSources;
 	res = GetRepairer()->Process(false);
 
@@ -571,7 +567,7 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		}
 	}
 
-	if (m_cancelled)
+	if (IsStopped())
 	{
 		if (m_stage >= ptRepairing)
 		{
@@ -592,7 +588,8 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename)
 		{
 			m_errMsg = Par2CmdLineErrStr[res];
 		}
-		PrintMessage(Message::mkError, "Repair failed for %s: %s", *m_infoName, *m_errMsg);
+		PrintMessage(Message::mkError, "Repair failed for %s: %s. Recovery files created by: %s",
+			*m_infoName, *m_errMsg, creator.Empty() ? "<unknown program>" : *creator);
 	}
 
 	Cleanup();
@@ -606,7 +603,10 @@ int ParChecker::PreProcessPar()
 	{
 		Cleanup();
 
-		m_repairer = std::make_unique<Repairer>(this);
+		{
+			Guard guard(m_repairerMutex);
+			m_repairer = std::make_unique<Repairer>(this);
+		}
 
 		res = GetRepairer()->PreProcess(m_parFilename);
 		debug("ParChecker: PreProcess-result=%i", res);
@@ -688,13 +688,13 @@ bool ParChecker::LoadMainParBak()
 		{
 			// wait until new files are added by "AddParFile" or a change is signaled by "QueueChanged"
 			bool queuedParFilesChanged = false;
-			while (!queuedParFilesChanged && !IsStopped() && !m_cancelled)
+			while (!queuedParFilesChanged && !IsStopped())
 			{
 				{
 					Guard guard(m_queuedParFilesMutex);
 					queuedParFilesChanged = m_queuedParFilesChanged;
 				}
-				usleep(100 * 1000);
+				Util::Sleep(100);
 			}
 		}
 	}
@@ -754,18 +754,18 @@ int ParChecker::ProcessMorePars()
 			{
 				// wait until new files are added by "AddParFile" or a change is signaled by "QueueChanged"
 				bool queuedParFilesChanged = false;
-				while (!queuedParFilesChanged && !IsStopped() && !m_cancelled)
+				while (!queuedParFilesChanged && !IsStopped())
 				{
 					{
 						Guard guard(m_queuedParFilesMutex);
 						queuedParFilesChanged = m_queuedParFilesChanged;
 					}
-					usleep(100 * 1000);
+					Util::Sleep(100);
 				}
 			}
 		}
 
-		if (IsStopped() || m_cancelled)
+		if (IsStopped())
 		{
 			break;
 		}
@@ -826,29 +826,25 @@ bool ParChecker::AddSplittedFragments()
 	DirBrowser dir(m_destDir);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") && !IsParredFile(filename) && !IsProcessedFile(filename))
+		if (!IsParredFile(filename) && !IsProcessedFile(filename))
 		{
 			for (Par2::Par2RepairerSourceFile *sourcefile : GetRepairer()->sourcefiles)
 			{
 				std::string target = sourcefile->TargetFileName();
-				const char* filename2 = target.c_str();
-				const char* basename2 = FileSystem::BaseFileName(filename2);
-				int baseLen = strlen(basename2);
+				const char* current = FileSystem::BaseFileName(target.c_str());
 
-				if (!strncasecmp(filename, basename2, baseLen))
+				// if file was renamed by par-renamer we also check the original filename
+				const char* original = FindFileOrigname(current);
+
+				if (MaybeSplittedFragement(filename, current) ||
+					(!Util::EmptyStr(original) && strcasecmp(original, current) &&
+					MaybeSplittedFragement(filename, original)))
 				{
-					const char* p = filename + baseLen;
-					if (*p == '.')
-					{
-						for (p++; *p && strchr("0123456789", *p); p++) ;
-						if (!*p)
-						{
-							debug("Found splitted fragment %s", filename);
-							BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
-							Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
-							extrafiles.push_back(extrafile);
-						}
-					}
+					detail("Found splitted fragment %s", filename);
+					BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+					Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
+					extrafiles.push_back(extrafile);
+					break;
 				}
 			}
 		}
@@ -869,6 +865,40 @@ bool ParChecker::AddSplittedFragments()
 	return fragmentsAdded;
 }
 
+bool ParChecker::MaybeSplittedFragement(const char* filename1, const char* filename2)
+{
+	// check if name is same but the first name has additional numerical extension
+	int len = strlen(filename2);
+	if (!strncasecmp(filename1, filename2, len))
+	{
+		const char* p = filename1 + len;
+		if (*p == '.')
+		{
+			for (p++; *p && strchr("0123456789", *p); p++) ;
+			if (!*p)
+			{
+				return true;
+			}
+		}
+	}
+
+	// check if same name (without extension) and extensions are numerical and exactly 3 characters long
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+	if (ext1 && ext2 && (strlen(ext1) == 4) && (strlen(ext2) == 4) &&
+		!strncasecmp(filename1, filename2, ext1 - filename1))
+	{
+		for (ext1++; *ext1 && strchr("0123456789", *ext1); ext1++) ;
+		for (ext2++; *ext2 && strchr("0123456789", *ext2); ext2++) ;
+		if (!*ext1 && !*ext2)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
 bool ParChecker::AddMissingFiles()
 {
 	return AddExtraFiles(true, false, m_destDir);
@@ -954,8 +984,7 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d
 	DirBrowser dir(directory);
 	while (const char* filename = dir.Next())
 	{
-		if (strcmp(filename, "_brokenlog.txt") &&
-			(externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename))))
+		if (externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename)))
 		{
 			BString<1024> fullfilename("%s%c%s", directory, PATH_SEPARATOR, filename);
 			extrafiles.emplace_back(*fullfilename, FileSystem::FileSize(fullfilename));
@@ -973,7 +1002,7 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d
 
 		// adding files one by one until all missing files are found
 
-		while (!IsStopped() && !m_cancelled && extrafiles.size() > 0)
+		while (!IsStopped() && extrafiles.size() > 0)
 		{
 			std::list<Par2::CommandLine::ExtraFile> extrafiles1;
 			extrafiles1.splice(extrafiles1.end(), extrafiles, extrafiles.begin());
@@ -1177,7 +1206,7 @@ void ParChecker::CheckEmptyFiles()
 		if (sourcefile && sourcefile->GetDescriptionPacket())
 		{
 			// GetDescriptionPacket()->FileName() returns a temp string object, which we need to hold for a while
-			std::string filenameObj = sourcefile->GetDescriptionPacket()->FileName();
+			std::string filenameObj = Par2::DiskFile::TranslateFilename(sourcefile->GetDescriptionPacket()->FileName());
 			const char* filename = filenameObj.c_str();
 			if (!Util::EmptyStr(filename) && !IsProcessedFile(filename))
 			{
@@ -1198,50 +1227,14 @@ void ParChecker::CheckEmptyFiles()
 
 void ParChecker::Cancel()
 {
-	GetRepairer()->cancelled = true;
-	m_cancelled = true;
-	QueueChanged();
-}
-
-void ParChecker::WriteBrokenLog(EStatus status)
-{
-	BString<1024> brokenLogName("%s%c_brokenlog.txt", *m_destDir, (int)PATH_SEPARATOR);
-
-	if (status != psRepairNotNeeded || FileSystem::FileExists(brokenLogName))
 	{
-		DiskFile file;
-		if (file.Open(brokenLogName, DiskFile::omAppend))
+		Guard guard(m_repairerMutex);
+		if (m_repairer)
 		{
-			if (status == psFailed)
-			{
-				if (m_cancelled)
-				{
-					file.Print("Repair cancelled for %s\n", *m_infoName);
-				}
-				else
-				{
-					file.Print("Repair failed for %s: %s\n", *m_infoName, *m_errMsg);
-				}
-			}
-			else if (status == psRepairPossible)
-			{
-				file.Print("Repair possible for %s\n", *m_infoName);
-			}
-			else if (status == psRepaired)
-			{
-				file.Print("Successfully repaired %s\n", *m_infoName);
-			}
-			else if (status == psRepairNotNeeded)
-			{
-				file.Print("Repair not needed for %s\n", *m_infoName);
-			}
-			file.Close();
-		}
-		else
-		{
-			PrintMessage(Message::mkError, "Could not open file %s", *brokenLogName);
+			m_repairer->GetRepairer()->cancelled = true;
 		}
 	}
+	QueueChanged();
 }
 
 void ParChecker::SaveSourceList()
@@ -1412,7 +1405,7 @@ bool ParChecker::VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32
 	{
 		const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 		Par2::u32 blockCrc = entry->crc;
-		parCrc = i == 0 ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+		parCrc = i == 0 ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 	}
 	debug("Block-CRC: %x, filename: %s", parCrc, FileSystem::BaseFileName(sourceFile->GetTargetFile()->FileName().c_str()));
 
@@ -1436,7 +1429,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 		int64 blockEnd = blockStart + blocksize < fileSize - 1 ? blockStart + blocksize : fileSize - 1;
 		bool blockOK = false;
 		bool blockEndFound = false;
-		Par2::u64 curOffset = 0;
+		int64 curOffset = 0;
 		for (Segment& segment : segments)
 		{
 			if (!blockOK && segment.GetSuccess() && segment.GetOffset() <= blockStart &&
@@ -1490,7 +1483,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 			}
 			const Par2::FILEVERIFICATIONENTRY* entry = packet->VerificationEntry(i);
 			Par2::u32 blockCrc = entry->crc;
-			parCrc = blockStart == i ? blockCrc : Util::Crc32Combine(parCrc, blockCrc, (uint32)blocksize);
+			parCrc = blockStart == i ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize);
 		}
 		else
 		{
@@ -1550,7 +1543,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 
 		if (segment.GetOffset() >= start && segment.GetOffset() + segment.GetSize() <= end)
 		{
-			downloadCrc = !started ? segment.GetCrc() : Util::Crc32Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
+			downloadCrc = !started ? segment.GetCrc() : Crc32::Combine(downloadCrc, segment.GetCrc(), (uint32)segment.GetSize());
 			started = true;
 		}
 
@@ -1568,7 +1561,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S
 				return false;
 			}
 
-			downloadCrc = Util::Crc32Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
+			downloadCrc = Crc32::Combine(downloadCrc, (uint32)partialCrc, (uint32)(end - segment.GetOffset() + 1));
 
 			break;
 		}
@@ -1589,21 +1582,37 @@ bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, ui
 	}
 
 	CharBuffer buffer(1024 * 64);
-	uint32 downloadCrc = 0xFFFFFFFF;
+	Crc32 downloadCrc;
 
 	int cnt = buffer.Size();
 	while (cnt == buffer.Size() && start < end)
 	{
 		int needBytes = end - start + 1 > buffer.Size() ? buffer.Size() : (int)(end - start + 1);
 		cnt = (int)file.Read(buffer, needBytes);
-		downloadCrc = Util::Crc32m(downloadCrc, (uchar*)(char*)buffer, cnt);
+		downloadCrc.Append((uchar*)(char*)buffer, cnt);
 		start += cnt;
 	}
 
-	downloadCrc ^= 0xFFFFFFFF;
-
-	*downloadCrcOut = downloadCrc;
+	*downloadCrcOut = downloadCrc.Finish();
 	return true;
 }
 
+CString ParChecker::GetPacketCreator()
+{
+	Par2::CREATORPACKET* creatorpacket;
+	if (GetRepairer()->creatorpacket &&
+		(creatorpacket = (Par2::CREATORPACKET*)(((RepairCreatorPacket*)GetRepairer()->creatorpacket)->packetdata)))
+	{
+		int len = (int)(creatorpacket->header.length - sizeof(Par2::PACKET_HEADER));
+		BString<1024> creator;
+		if (len > 0)
+		{
+			creator.Set((const char*)creatorpacket->client, len);
+		}
+		return *creator;
+	}
+
+	return nullptr;
+}
+
 #endif

daemon/postprocess/ParChecker.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,13 +25,12 @@
 
 #include "NString.h"
 #include "Container.h"
-#include "Thread.h"
 #include "FileSystem.h"
 #include "Log.h"
 
 class Repairer;
 
-class ParChecker : public Thread
+class ParChecker
 {
 public:
 	enum EStatus
@@ -57,8 +56,7 @@ public:
 		virtual Repairer* GetRepairer() = 0;
 	};
 
-	virtual ~ParChecker();
-	virtual void Run();
+	void Execute();
 	void SetDestDir(const char* destDir) { m_destDir = destDir; }
 	const char* GetParFilename() { return m_parFilename; }
 	const char* GetInfoName() { return m_infoName; }
@@ -74,7 +72,6 @@ public:
 	void AddParFile(const char* parFilename);
 	void QueueChanged();
 	void Cancel();
-	bool GetCancelled() { return m_cancelled; }
 
 protected:
 	class Segment
@@ -129,11 +126,13 @@ protected:
 	*/
 	virtual bool RequestMorePars(int blockNeeded, int* blockFound) = 0;
 	virtual void UpdateProgress() {}
+	virtual bool IsStopped() { return false; };
 	virtual void Completed() {}
 	virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3) {}
 	virtual void RegisterParredFile(const char* filename) {}
 	virtual bool IsParredFile(const char* filename) { return false; }
 	virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) { return fsUnknown; }
+	virtual const char* FindFileOrigname(const char* filename) { return nullptr; }
 	virtual void RequestDupeSources(DupeSourceList* dupeSourceList) {}
 	virtual void StatDupeSources(DupeSourceList* dupeSourceList) {}
 	EStage GetStage() { return m_stage; }
@@ -186,6 +185,9 @@ private:
 	DupeSourceList m_dupeSources;
 	StreamBuf m_parOutStream{this, Message::mkDetail};
 	StreamBuf m_parErrStream{this, Message::mkError};
+	std::ostream m_parCout{&m_parOutStream};
+	std::ostream m_parCerr{&m_parErrStream};
+	Mutex m_repairerMutex;
 
 	// "m_repairer" should be of type "Par2::Par2Repairer", however to prevent the
 	// including of libpar2-headers into this header-file we use an empty abstract class.
@@ -204,7 +206,6 @@ private:
 	bool AddDupeFiles();
 	bool AddExtraFiles(bool onlyMissing, bool externalDir, const char* directory);
 	bool IsProcessedFile(const char* filename);
-	void WriteBrokenLog(EStatus status);
 	void SaveSourceList();
 	void DeleteLeftovers();
 	void signal_filename(std::string str);
@@ -220,6 +221,8 @@ private:
 		uint32* downloadCrc);
 	bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrc);
 	void CheckEmptyFiles();
+	CString GetPacketCreator();
+	bool MaybeSplittedFragement(const char* filename1, const char* filename2);
 
 	friend class Repairer;
 };

daemon/postprocess/ParParser.cpp

@@ -33,7 +33,7 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 	while (const char* filename = dir.Next())
 	{
 		int baseLen = 0;
-		if (ParseParFilename(filename, &baseLen, nullptr))
+		if (ParseParFilename(filename, true, &baseLen, nullptr))
 		{
 			if (!fileList)
 			{
@@ -44,7 +44,7 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 			bool exists = false;
 			for (CString& filename2 : fileList)
 			{
-				exists = SameParCollection(filename, filename2);
+				exists = SameParCollection(filename, filename2, true);
 				if (exists)
 				{
 					break;
@@ -59,16 +59,16 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList)
 	return fileList && !fileList->empty();
 }
 
-bool ParParser::SameParCollection(const char* filename1, const char* filename2)
+bool ParParser::SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames)
 {
 	int baseLen1 = 0, baseLen2 = 0;
-	return ParseParFilename(filename1, &baseLen1, nullptr) &&
-		ParseParFilename(filename2, &baseLen2, nullptr) &&
+	return ParseParFilename(filename1, confirmedFilenames, &baseLen1, nullptr) &&
+		ParseParFilename(filename2, confirmedFilenames, &baseLen2, nullptr) &&
 		baseLen1 == baseLen2 &&
 		!strncasecmp(filename1, filename2, baseLen1);
 }
 
-bool ParParser::ParseParFilename(const char* parFilename, int* baseNameLen, int* blocks)
+bool ParParser::ParseParFilename(const char* parFilename, bool confirmedFilename, int* baseNameLen, int* blocks)
 {
 	BString<1024> filename = parFilename;
 	for (char* p = filename; *p; p++) *p = tolower(*p); // convert string to lowercase
@@ -79,10 +79,13 @@ bool ParParser::ParseParFilename(const char* parFilename, int* baseNameLen, int*
 		return false;
 	}
 
-	// find last occurence of ".par2" and trim filename after it
-	char* end = filename;
-	while (char* p = strstr(end, ".par2")) end = p + 5;
-	*end = '\0';
+	if (!confirmedFilename)
+	{
+		// find last occurence of ".par2" and trim filename after it
+		char* end = filename;
+		while (char* p = strstr(end, ".par2")) end = p + 5;
+		*end = '\0';
+	}
 
 	len = strlen(filename);
 	if (len < 6)

daemon/postprocess/ParParser.h

@@ -30,8 +30,8 @@ public:
 	typedef std::vector<CString> ParFileList;
 
 	static bool FindMainPars(const char* path, ParFileList* fileList);
-	static bool ParseParFilename(const char* parFilename, int* baseNameLen, int* blocks);
-	static bool SameParCollection(const char* filename1, const char* filename2);
+	static bool ParseParFilename(const char* parFilename, bool confirmedFilename, int* baseNameLen, int* blocks);
+	static bool SameParCollection(const char* filename1, const char* filename2, bool confirmedFilenames);
 };
 
 #endif

daemon/postprocess/ParRenamer.cpp

@@ -36,31 +36,17 @@
 class ParRenamerRepairer : public Par2::Par2Repairer
 {
 public:
+	ParRenamerRepairer() : Par2::Par2Repairer(m_nout, m_nout) {};
 	friend class ParRenamer;
+private:
+	class NullStreamBuf : public std::streambuf {};
+	NullStreamBuf m_nullbuf;
+	std::ostream m_nout{&m_nullbuf};
 };
 
 
-void ParRenamer::Cleanup()
+void ParRenamer::Execute()
 {
-	m_dirList.clear();
-	m_fileHashList.clear();
-}
-
-void ParRenamer::Cancel()
-{
-	m_cancelled = true;
-}
-
-void ParRenamer::Run()
-{
-	Cleanup();
-	m_cancelled = false;
-	m_fileCount = 0;
-	m_curFile = 0;
-	m_renamedCount = 0;
-	m_hasMissedFiles = false;
-	m_status = psFailed;
-
 	m_progressLabel.Format("Checking renamed files for %s", *m_infoName);
 	m_stageProgress = 0;
 	UpdateProgress();
@@ -71,14 +57,17 @@ void ParRenamer::Run()
 	{
 		debug("Checking %s", *destDir);
 		m_fileHashList.clear();
-		LoadParFiles(destDir);
+		m_parInfoList.clear();
+		m_badParList.clear();
+		m_loadedParList.clear();
 
-		if (m_fileHashList.empty())
+		CheckFiles(destDir, true);
+		RenameParFiles(destDir);
+
+		LoadMainParFiles(destDir);
+		if (m_hasDamagedParFiles)
 		{
-			int savedCurFile = m_curFile;
-			CheckFiles(destDir, true);
-			m_curFile = savedCurFile; // restore progress indicator
-			LoadParFiles(destDir);
+			LoadExtraParFiles(destDir);
 		}
 
 		CheckFiles(destDir, false);
@@ -87,24 +76,12 @@ void ParRenamer::Run()
 		{
 			CheckMissing();
 		}
-	}
 
-	if (m_cancelled)
-	{
-		PrintMessage(Message::mkWarning, "Renaming cancelled for %s", *m_infoName);
+		if (m_renamedCount > 0 && !m_badParList.empty())
+		{
+			RenameBadParFiles();
+		}
 	}
-	else if (m_renamedCount > 0)
-	{
-		PrintMessage(Message::mkInfo, "Successfully renamed %i file(s) for %s", m_renamedCount, *m_infoName);
-		m_status = psSuccess;
-	}
-	else
-	{
-		PrintMessage(Message::mkInfo, "No renamed files found for %s", *m_infoName);
-	}
-
-	Cleanup();
-	Completed();
 }
 
 void ParRenamer::BuildDirList(const char* destDir)
@@ -115,7 +92,7 @@ void ParRenamer::BuildDirList(const char* destDir)
 
 	while (const char* filename = dirBrowser.Next())
 	{
-		if (!m_cancelled)
+		if (!IsStopped())
 		{
 			BString<1024> fullFilename("%s%c%s", destDir, PATH_SEPARATOR, filename);
 			if (FileSystem::DirectoryExists(fullFilename))
@@ -130,7 +107,7 @@ void ParRenamer::BuildDirList(const char* destDir)
 	}
 }
 
-void ParRenamer::LoadParFiles(const char* destDir)
+void ParRenamer::LoadMainParFiles(const char* destDir)
 {
 	ParParser::ParFileList parFileList;
 	ParParser::FindMainPars(destDir, &parFileList);
@@ -142,19 +119,52 @@ void ParRenamer::LoadParFiles(const char* destDir)
 	}
 }
 
+void ParRenamer::LoadExtraParFiles(const char* destDir)
+{
+	DirBrowser dir(destDir);
+	while (const char* filename = dir.Next())
+	{
+		BString<1024> fullParFilename("%s%c%s", destDir, PATH_SEPARATOR, filename);
+		if (ParParser::ParseParFilename(fullParFilename, true, nullptr, nullptr))
+		{
+			bool knownBadParFile = std::find_if(m_badParList.begin(), m_badParList.end(),
+				[&fullParFilename](CString& filename)
+				{
+					return !strcmp(filename, fullParFilename);
+				}) != m_badParList.end();
+
+			bool loadedParFile = std::find_if(m_loadedParList.begin(), m_loadedParList.end(),
+				[&fullParFilename](CString& filename)
+				{
+					return !strcmp(filename, fullParFilename);
+				}) != m_loadedParList.end();
+
+			if (!knownBadParFile && !loadedParFile)
+			{
+				LoadParFile(fullParFilename);
+			}
+		}
+	}
+}
+
 void ParRenamer::LoadParFile(const char* parFilename)
 {
 	ParRenamerRepairer repairer;
 
-	if (!repairer.LoadPacketsFromFile(parFilename))
+	if (!repairer.LoadPacketsFromFile(parFilename) || FileSystem::FileSize(parFilename) == 0)
 	{
 		PrintMessage(Message::mkWarning, "Could not load par2-file %s", parFilename);
+		m_hasDamagedParFiles = true;
+		m_badParList.emplace_back(parFilename);
 		return;
 	}
 
+	m_loadedParList.emplace_back(parFilename);
+	PrintMessage(Message::mkInfo, "Loaded par2-file %s for par-rename", FileSystem::BaseFileName(parFilename));
+
 	for (std::pair<const Par2::MD5Hash, Par2::Par2RepairerSourceFile*>& entry : repairer.sourcefilemap)
 	{
-		if (m_cancelled)
+		if (IsStopped())
 		{
 			break;
 		}
@@ -162,32 +172,46 @@ void ParRenamer::LoadParFile(const char* parFilename)
 		Par2::Par2RepairerSourceFile* sourceFile = entry.second;
 		if (!sourceFile || !sourceFile->GetDescriptionPacket())
 		{
-			PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", parFilename);
+			PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", FileSystem::BaseFileName(parFilename));
+			m_badParList.emplace_back(parFilename);
+			m_hasDamagedParFiles = true;
 			continue;
 		}
-		m_fileHashList.emplace_back(sourceFile->GetDescriptionPacket()->FileName().c_str(),
-			sourceFile->GetDescriptionPacket()->Hash16k().print().c_str());
-		RegisterParredFile(sourceFile->GetDescriptionPacket()->FileName().c_str());
+		std::string filename = Par2::DiskFile::TranslateFilename(sourceFile->GetDescriptionPacket()->FileName());
+		std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print();
+
+		bool exists = std::find_if(m_fileHashList.begin(), m_fileHashList.end(),
+			[&hash](FileHash& fileHash)
+			{
+				return !strcmp(fileHash.GetHash(), hash.c_str());
+			})
+			!= m_fileHashList.end();
+
+		if (!exists)
+		{
+			m_fileHashList.emplace_back(filename.c_str(), hash.c_str());
+			RegisterParredFile(filename.c_str());
+		}
 	}
 }
 
-void ParRenamer::CheckFiles(const char* destDir, bool renamePars)
+void ParRenamer::CheckFiles(const char* destDir, bool checkPars)
 {
 	DirBrowser dir(destDir);
 	while (const char* filename = dir.Next())
 	{
-		if (!m_cancelled)
+		if (!IsStopped())
 		{
 			BString<1024> fullFilename("%s%c%s", destDir, PATH_SEPARATOR, filename);
 
 			if (!FileSystem::DirectoryExists(fullFilename))
 			{
 				m_progressLabel.Format("Checking file %s", filename);
-				m_stageProgress = m_fileCount > 0 ? m_curFile * 1000 / m_fileCount : 1000;
+				m_stageProgress = m_fileCount > 0 ? m_curFile * 1000 / m_fileCount / 2 : 1000;
 				UpdateProgress();
 				m_curFile++;
 
-				if (renamePars)
+				if (checkPars)
 				{
 					CheckParFile(destDir, fullFilename);
 				}
@@ -289,22 +313,10 @@ void ParRenamer::CheckRegularFile(const char* destDir, const char* filename)
 	}
 }
 
-/*
-* For files not having par2-extensions: checks if the file is a par2-file and renames
-* it according to its set-id.
-*/
 void ParRenamer::CheckParFile(const char* destDir, const char* filename)
 {
 	debug("Checking par2-header for %s", filename);
 
-	const char* basename = FileSystem::BaseFileName(filename);
-	const char* extension = strrchr(basename, '.');
-	if (extension && !strcasecmp(extension, ".par2"))
-	{
-		// do not process files already having par2-extension
-		return;
-	}
-
 	DiskFile file;
 	if (!file.Open(filename, DiskFile::omRead))
 	{
@@ -337,13 +349,70 @@ void ParRenamer::CheckParFile(const char* destDir, const char* filename)
 	BString<100> setId = header.setid.print().c_str();
 	for (char* p = setId; *p; p++) *p = tolower(*p); // convert string to lowercase
 
-	debug("Renaming: %s; setid: %s", FileSystem::BaseFileName(filename), *setId);
+	debug("Storing: %s; setid: %s", FileSystem::BaseFileName(filename), *setId);
+
+	m_parInfoList.emplace_back(filename, setId);
+}
+
+void ParRenamer::RenameParFiles(const char* destDir)
+{
+	if (NeedRenameParFiles())
+	{
+		for (ParInfo& parInfo : m_parInfoList)
+		{
+			RenameParFile(destDir, parInfo.GetFilename(), parInfo.GetSetId());
+		}
+	}
+}
+
+bool ParRenamer::NeedRenameParFiles()
+{
+	for (ParInfoList::iterator it1 = m_parInfoList.begin(); it1 != m_parInfoList.end(); it1++)
+	{
+		ParInfo& parInfo1 = *it1;
+
+		const char* baseName1 = FileSystem::BaseFileName(parInfo1.GetFilename());
+
+		const char* extension = strrchr(baseName1, '.');
+		if (!extension || strcasecmp(extension, ".par2"))
+		{
+			// file doesn't have "par2" extension
+			return true;
+		}
+
+		int baseLen1;
+		ParParser::ParseParFilename(baseName1, true, &baseLen1, nullptr);
+
+		for (ParInfoList::iterator it2 = it1 + 1; it2 != m_parInfoList.end(); it2++)
+		{
+			ParInfo& parInfo2 = *it2;
+
+			if (!strcmp(parInfo1.GetSetId(), parInfo2.GetSetId()))
+			{
+				const char* baseName2 = FileSystem::BaseFileName(parInfo2.GetFilename());
+				int baseLen2;
+				ParParser::ParseParFilename(baseName2, true, &baseLen2, nullptr);
+				if (baseLen1 != baseLen2 || strncasecmp(baseName1, baseName2, baseLen1))
+				{
+					// same setid but different base file names
+					return true;
+				}
+			}
+		}
+	}
+
+	return false;
+}
+
+void ParRenamer::RenameParFile(const char* destDir, const char* filename, const char* setId)
+{
+	debug("Renaming: %s; setid: %s", FileSystem::BaseFileName(filename), setId);
 
 	BString<1024> destFileName;
 	int num = 1;
 	while (num == 1 || FileSystem::FileExists(destFileName))
 	{
-		destFileName.Format("%s%c%s.vol%03i+01.PAR2", destDir, PATH_SEPARATOR, *setId, num);
+		destFileName.Format("%s%c%s.vol%03i+01.PAR2", destDir, PATH_SEPARATOR, setId, num);
 		num++;
 	}
 
@@ -366,4 +435,13 @@ void ParRenamer::RenameFile(const char* srcFilename, const char* destFileName)
 	RegisterRenamedFile(FileSystem::BaseFileName(srcFilename), FileSystem::BaseFileName(destFileName));
 }
 
+void ParRenamer::RenameBadParFiles()
+{
+	for (CString& parFilename : m_badParList)
+	{
+		BString<1024> destFileName("%s.bad", *parFilename);
+		RenameFile(parFilename, destFileName);
+	}
+}
+
 #endif

daemon/postprocess/ParRenamer.h

@@ -24,32 +24,23 @@
 #ifndef DISABLE_PARCHECK
 
 #include "NString.h"
-#include "Thread.h"
 #include "Log.h"
 
-class ParRenamer : public Thread
+class ParRenamer
 {
 public:
-	enum EStatus
-	{
-		psFailed,
-		psSuccess
-	};
-
-	virtual void Run();
+	void Execute();
 	void SetDestDir(const char* destDir) { m_destDir = destDir; }
 	const char* GetInfoName() { return m_infoName; }
 	void SetInfoName(const char* infoName) { m_infoName = infoName; }
-	void SetStatus(EStatus status);
-	EStatus GetStatus() { return m_status; }
-	void Cancel();
-	bool GetCancelled() { return m_cancelled; }
+	int GetRenamedCount() { return m_renamedCount; }
 	bool HasMissedFiles() { return m_hasMissedFiles; }
+	bool HasDamagedParFiles() { return m_hasDamagedParFiles; }
 	void SetDetectMissing(bool detectMissing) { m_detectMissing = detectMissing; }
 
 protected:
 	virtual void UpdateProgress() {}
-	virtual void Completed() {}
+	virtual bool IsStopped() { return false; };
 	virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3) {}
 	virtual void RegisterParredFile(const char* filename) {}
 	virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName) {}
@@ -72,34 +63,52 @@ private:
 		bool m_fileExists = false;
 	};
 
+	class ParInfo
+	{
+	public:
+		ParInfo(const char* filename, const char* setId) :
+			m_filename(filename), m_setId(setId) {}
+		const char* GetFilename() { return m_filename; }
+		const char* GetSetId() { return m_setId; }
+	private:
+		CString m_filename;
+		CString m_setId;
+	};
+
 	typedef std::deque<FileHash> FileHashList;
-	typedef std::deque<CString> DirList;
+	typedef std::deque<ParInfo> ParInfoList;
+	typedef std::deque<CString> NameList;
 
 	CString m_infoName;
 	CString m_destDir;
-	EStatus m_status;
 	CString m_progressLabel;
-	int m_stageProgress;
-	bool m_cancelled;
-	DirList m_dirList;
+	int m_stageProgress = 0;
+	NameList m_dirList;
 	FileHashList m_fileHashList;
-	int m_fileCount;
-	int m_curFile;
-	int m_renamedCount;
-	bool m_hasMissedFiles;
+	ParInfoList m_parInfoList;
+	NameList m_badParList;
+	NameList m_loadedParList;
+	int m_fileCount = 0;
+	int m_curFile = 0;
+	int m_renamedCount = 0;
+	bool m_hasMissedFiles = false;
 	bool m_detectMissing = false;
+	bool m_hasDamagedParFiles = false;
 
 	void BuildDirList(const char* destDir);
-	void CheckDir(const char* destDir);
-	void LoadParFiles(const char* destDir);
+	void LoadMainParFiles(const char* destDir);
+	void LoadExtraParFiles(const char* destDir);
 	void LoadParFile(const char* parFilename);
-	void CheckFiles(const char* destDir, bool renamePars);
+	void CheckFiles(const char* destDir, bool checkPars);
 	void CheckRegularFile(const char* destDir, const char* filename);
 	void CheckParFile(const char* destDir, const char* filename);
 	bool IsSplittedFragment(const char* filename, const char* correctName);
 	void CheckMissing();
+	void RenameParFiles(const char* destDir);
+	void RenameParFile(const char* destDir, const char* filename, const char* setId);
+	bool NeedRenameParFiles();
 	void RenameFile(const char* srcFilename, const char* destFileName);
-	void Cleanup();
+	void RenameBadParFiles();
 };
 
 #endif

daemon/postprocess/PrePostProcessor.h

@@ -24,50 +24,47 @@
 #include "Thread.h"
 #include "Observer.h"
 #include "DownloadInfo.h"
-#include "ParCoordinator.h"
 
-class PrePostProcessor : public Thread
+class PrePostProcessor : public Thread, public Observer
 {
 public:
 	PrePostProcessor();
 	virtual void Run();
 	virtual void Stop();
-	bool HasMoreJobs() { return m_jobCount > 0; }
-	int GetJobCount() { return m_jobCount; }
+	bool HasMoreJobs() { return m_queuedJobs > 0; }
+	int GetJobCount() { return m_queuedJobs; }
 	bool EditList(DownloadQueue* downloadQueue, IdList* idList, DownloadQueue::EEditAction action,
-		int offset, const char* text);
+		const char* args);
 	void NzbAdded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
 	void NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
 
+protected:
+	virtual void Update(Subject* caller, void* aspect) { DownloadQueueUpdate(aspect); }
+
 private:
-	class DownloadQueueObserver: public Observer
-	{
-	public:
-		PrePostProcessor* m_owner;
-		virtual void Update(Subject* Caller, void* Aspect) { m_owner->DownloadQueueUpdate(Caller, Aspect); }
-	};
+	int m_queuedJobs = 0;
+	RawNzbList m_activeJobs;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
-	ParCoordinator m_parCoordinator;
-	DownloadQueueObserver m_downloadQueueObserver;
-	int m_jobCount = 0;
-	NzbInfo* m_curJob = nullptr;
-	const char* m_pauseReason = nullptr;
-
-	bool IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars, bool allowOnlyOneDeleted);
-	bool IsNzbFileDownloading(NzbInfo* nzbInfo);
 	void CheckPostQueue();
-	void JobCompleted(DownloadQueue* downloadQueue, PostInfo* postInfo);
-	void StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo);
+	void CheckRequestPar(DownloadQueue* downloadQueue);
+	void CleanupJobs(DownloadQueue* downloadQueue);
+	bool CanRunMoreJobs(bool* allowPar);
+	NzbInfo* PickNextJob(DownloadQueue* downloadQueue, bool allowPar);
+	void StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo, bool allowPar);
+	void EnterStage(DownloadQueue* downloadQueue, PostInfo* postInfo, PostInfo::EStage stage);
 	void SanitisePostQueue();
-	void UpdatePauseState(bool needPause, const char* reason);
+	void UpdatePauseState();
 	void NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
 	void NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
 	void NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool saveQueue);
+	void JobCompleted(DownloadQueue* downloadQueue, PostInfo* postInfo);
 	bool PostQueueDelete(DownloadQueue* downloadQueue, IdList* idList);
-	void DeletePostThread(PostInfo* postInfo);
-	NzbInfo* GetNextJob(DownloadQueue* downloadQueue);
-	void DownloadQueueUpdate(Subject* Caller, void* Aspect);
+	void DownloadQueueUpdate(void* aspect);
 	void DeleteCleanup(NzbInfo* nzbInfo);
+	void WaitJobs();
+	void FileDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileInfo* fileInfo);
 };
 
 extern PrePostProcessor* g_PrePostProcessor;

daemon/postprocess/RarReader.cpp

@@ -0,0 +1,742 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+
+#include "RarReader.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+
+// RAR3 constants
+
+static const uint16 RAR3_MAIN_VOLUME = 0x0001;
+static const uint16 RAR3_MAIN_NEWNUMBERING = 0x0010;
+static const uint16 RAR3_MAIN_PASSWORD = 0x0080;
+
+static const uint8 RAR3_BLOCK_MAIN = 0x73; // s
+static const uint8 RAR3_BLOCK_FILE = 0x74; // t
+static const uint8 RAR3_BLOCK_ENDARC = 0x7b; // {
+
+static const uint16 RAR3_BLOCK_ADDSIZE = 0x8000;
+
+static const uint16 RAR3_FILE_ADDSIZE = 0x0100;
+static const uint16 RAR3_FILE_SPLITBEFORE = 0x0001;
+static const uint16 RAR3_FILE_SPLITAFTER = 0x0002;
+
+static const uint16 RAR3_ENDARC_NEXTVOL = 0x0001;
+static const uint16 RAR3_ENDARC_DATACRC = 0x0002;
+static const uint16 RAR3_ENDARC_VOLNUMBER = 0x0008;
+
+// RAR5 constants
+
+static const uint8 RAR5_BLOCK_MAIN = 1;
+static const uint8 RAR5_BLOCK_FILE = 2;
+static const uint8 RAR5_BLOCK_ENCRYPTION = 4;
+static const uint8 RAR5_BLOCK_ENDARC = 5;
+
+static const uint8 RAR5_BLOCK_EXTRADATA = 0x01;
+static const uint8 RAR5_BLOCK_DATAAREA = 0x02;
+static const uint8 RAR5_BLOCK_SPLITBEFORE = 0x08;
+static const uint8 RAR5_BLOCK_SPLITAFTER = 0x10;
+
+static const uint8 RAR5_MAIN_ISVOL = 0x01;
+static const uint8 RAR5_MAIN_VOLNR = 0x02;
+
+static const uint8 RAR5_FILE_TIME = 0x02;
+static const uint8 RAR5_FILE_CRC = 0x04;
+static const uint8 RAR5_FILE_EXTRATIME = 0x03;
+static const uint8 RAR5_FILE_EXTRATIMEUNIXFORMAT = 0x01;
+
+static const uint8 RAR5_ENDARC_NEXTVOL = 0x01;
+
+
+bool RarVolume::Read()
+{
+	debug("Checking file %s", *m_filename);
+
+	DiskFile file;
+	if (!file.Open(m_filename, DiskFile::omRead))
+	{
+		return false;
+	}
+
+	m_version = DetectRarVersion(file);
+	file.Seek(0);
+
+	bool ok = false;
+
+	switch (m_version)
+	{
+		case 3:
+			ok = ReadRar3Volume(file);
+			break;
+
+		case 5:
+			ok = ReadRar5Volume(file);
+			break;
+	}
+
+	file.Close();
+	DecryptFree();
+
+	LogDebugInfo();
+
+	return ok;
+}
+
+int RarVolume::DetectRarVersion(DiskFile& file)
+{
+	static char RAR3_SIGNATURE[] = { 0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00 };
+	static char RAR5_SIGNATURE[] = { 0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00 };
+
+	char fileSignature[8];
+
+	int cnt = 0;
+	cnt = (int)file.Read(fileSignature, sizeof(fileSignature));
+
+	bool rar5 = cnt == sizeof(fileSignature) && !strcmp(RAR5_SIGNATURE, fileSignature);
+	bool rar3 = !rar5 && cnt == sizeof(fileSignature) && !strcmp(RAR3_SIGNATURE, fileSignature);
+
+	return rar3 ? 3 : rar5 ? 5 : 0;
+}
+
+bool RarVolume::Read(DiskFile& file, RarBlock* block, void* buffer, int64 size)
+{
+	if (m_encrypted)
+	{
+		if (!DecryptRead(file, buffer, size)) return false;
+	}
+	else
+	{
+		if (file.Read(buffer, size) != size) return false;
+	}
+
+	if (block)
+	{
+		block->trailsize -= size;
+	}
+
+	return true;
+}
+
+bool RarVolume::Read16(DiskFile& file, RarBlock* block, uint16* result)
+{
+	uint8 buf[2];
+	if (!Read(file, block, buf, sizeof(buf))) return false;
+	*result = ((uint16)buf[1] << 8) + buf[0];
+	return true;
+}
+
+bool RarVolume::Read32(DiskFile& file, RarBlock* block, uint32* result)
+{
+	uint8 buf[4];
+	if (!Read(file, block, buf, sizeof(buf))) return false;
+	*result = ((uint32)buf[3] << 24) + ((uint32)buf[2] << 16) + ((uint32)buf[1] << 8) + buf[0];
+	return true;
+}
+
+bool RarVolume::ReadV(DiskFile& file, RarBlock* block, uint64* result)
+{
+	*result = 0;
+	uint8 val;
+	uint8 bits = 0;
+	do
+	{
+		if (Read(file, block, &val, sizeof(val)) != sizeof(val)) return false;
+		*result += (uint64)(val & 0x7f) << bits;
+		bits += 7;
+	} while (val & 0x80);
+
+	return true;
+}
+
+bool RarVolume::Skip(DiskFile& file, RarBlock* block, int64 size)
+{
+	uint8 buf[256];
+	while (size > 0)
+	{
+		int64 len = size <= (int64)sizeof(buf) ? size : (int64)sizeof(buf);
+		if (!Read(file, block, buf, len)) return false;
+		size -= len;
+	}
+	return true;
+}
+
+bool RarVolume::ReadRar3Volume(DiskFile& file)
+{
+	debug("Reading rar3-file %s", *m_filename);
+
+	while (!file.Eof())
+	{
+		RarBlock block = ReadRar3Block(file);
+		if (!block.type)
+		{
+			return false;
+		}
+
+		if (block.type == RAR3_BLOCK_MAIN)
+		{
+			if (block.flags & RAR3_MAIN_PASSWORD)
+			{
+				m_encrypted = true;
+				if (m_password.Empty()) return false;
+			}
+			m_newNaming = block.flags & RAR3_MAIN_NEWNUMBERING;
+			m_multiVolume = block.flags & RAR3_MAIN_VOLUME;
+		}
+
+		else if (block.type == RAR3_BLOCK_FILE)
+		{
+			RarFile innerFile;
+			if (!ReadRar3File(file, block, innerFile)) return false;
+			m_files.push_back(std::move(innerFile));
+		}
+
+		else if (block.type == RAR3_BLOCK_ENDARC)
+		{
+			if (block.flags & RAR3_ENDARC_DATACRC)
+			{
+				if (!Skip(file, &block, 4)) return false;
+			}
+			if (block.flags & RAR3_ENDARC_VOLNUMBER)
+			{
+				if (!Read32(file, &block, &m_volumeNo)) return false;
+				m_hasNextVolume = (block.flags & RAR3_ENDARC_NEXTVOL) != 0;
+			}
+			break;
+		}
+
+		else if (block.type < 0x72 || block.type > 0x7b)
+		{
+			// inlvaid block type
+			return false;
+		}
+
+		uint64 skip = block.trailsize;
+		if (m_encrypted)
+		{
+			skip -= 16 - m_decryptPos;
+			m_decryptPos = 16;
+			DecryptFree();
+		}
+
+		if (!file.Seek(skip, DiskFile::soCur))
+		{
+			return false;
+		}
+	}
+
+	return true;
+}
+
+RarVolume::RarBlock RarVolume::ReadRar3Block(DiskFile& file)
+{
+	RarBlock block {0};
+	uint8 salt[8];
+
+	if (m_encrypted &&
+		!(file.Read(salt, sizeof(salt)) == sizeof(salt) &&
+		  DecryptRar3Prepare(salt) && DecryptInit(128)))
+	{
+		return {0};
+	}
+
+	uint8 buf[7];
+
+	if (!Read(file, nullptr, &buf, sizeof(buf))) return {0};
+	block.crc = ((uint16)buf[1] << 8) + buf[0];
+	block.type = buf[2];
+	block.flags = ((uint16)buf[4] << 8) + buf[3];
+	uint16 size = ((uint16)buf[6] << 8) + buf[5];
+
+	uint32 blocksize = size;
+	if (m_encrypted)
+	{
+		// Align to 16 bytes
+		blocksize = (blocksize + ((~blocksize + 1) & (16 - 1)));
+	}
+
+	block.trailsize = blocksize - sizeof(buf);
+
+	uint8 addbuf[4];
+	if ((block.flags & RAR3_BLOCK_ADDSIZE) && !Read(file, nullptr, &addbuf, sizeof(addbuf)))
+	{
+		return {0};
+	}
+	block.addsize = ((uint32)addbuf[3] << 24) + ((uint32)addbuf[2] << 16) + ((uint32)addbuf[1] << 8) + addbuf[0];
+
+	if (block.flags & RAR3_BLOCK_ADDSIZE)
+	{
+		blocksize += (uint32)block.addsize;
+		block.trailsize = blocksize - sizeof(buf) - 4;
+	}
+
+#ifdef DEBUG
+	static int num = 0;
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
+
+	return block;
+}
+
+bool RarVolume::ReadRar3File(DiskFile& file, RarBlock& block, RarFile& innerFile)
+{
+	innerFile.m_splitBefore = block.flags & RAR3_FILE_SPLITBEFORE;
+	innerFile.m_splitAfter = block.flags & RAR3_FILE_SPLITAFTER;
+
+	uint16 namelen;
+
+	uint32 size;
+	if (!Read32(file, &block, &size)) return false;
+	innerFile.m_size = size;
+
+	if (!Skip(file, &block, 1)) return false;
+	if (!Skip(file, &block, 4)) return false;
+	if (!Read32(file, &block, &innerFile.m_time)) return false;
+	if (!Skip(file, &block, 2)) return false;
+	if (!Read16(file, &block, &namelen)) return false;
+	if (!Read32(file, &block, &innerFile.m_attr)) return false;
+
+	if (block.flags & RAR3_FILE_ADDSIZE)
+	{
+		uint32 highsize;
+		if (!Read32(file, &block, &highsize)) return false;
+		block.trailsize += (uint64)highsize << 32;
+
+		if (!Read32(file, &block, &highsize)) return false;
+		innerFile.m_size += (uint64)highsize << 32;
+	}
+
+	if (namelen > 8192) return false; // an error
+	CharBuffer name;
+	name.Reserve(namelen + 1);
+	if (!Read(file, &block, (char*)name, namelen)) return false;
+	name[namelen] = '\0';
+	innerFile.m_filename = name;
+	debug("%i, %i, %s", (int)block.trailsize, (int)namelen, (const char*)name);
+
+	return true;
+}
+
+bool RarVolume::ReadRar5Volume(DiskFile& file)
+{
+	debug("Reading rar5-file %s", *m_filename);
+
+	file.Seek(8);
+
+	while (!file.Eof())
+	{
+		RarBlock block = ReadRar5Block(file);
+		if (!block.type)
+		{
+			return false;
+		}
+
+		if (block.type == RAR5_BLOCK_MAIN)
+		{
+			uint64 arcflags;
+			if (!ReadV(file, &block, &arcflags)) return false;
+			if (arcflags & RAR5_MAIN_VOLNR)
+			{
+				uint64 volnr;
+				if (!ReadV(file, &block, &volnr)) return false;
+				m_volumeNo = (uint32)volnr;
+			}
+			m_newNaming = true;
+			m_multiVolume = (arcflags & RAR5_MAIN_ISVOL) != 0;
+		}
+
+		else if (block.type == RAR5_BLOCK_ENCRYPTION)
+		{
+			uint64 val;
+			if (!ReadV(file, &block, &val)) return false;
+			if (val != 0) return false; // supporting only AES
+			if (!ReadV(file, &block, &val)) return false;
+			uint8 kdfCount;
+			uint8 salt[16];
+			if (!Read(file, &block, &kdfCount, sizeof(kdfCount))) return false;
+			if (!Read(file, &block, &salt, sizeof(salt))) return false;
+			m_encrypted = true;
+			if (m_password.Empty()) return false;
+			if (!DecryptRar5Prepare(kdfCount, salt)) return false;
+		}
+		
+		else if (block.type == RAR5_BLOCK_FILE)
+		{
+			RarFile innerFile;
+			if (!ReadRar5File(file, block, innerFile)) return false;
+			m_files.push_back(std::move(innerFile));
+		}
+
+		else if (block.type == RAR5_BLOCK_ENDARC)
+		{
+			uint64 endflags;
+			if (!ReadV(file, &block, &endflags)) return false;
+			m_hasNextVolume = (endflags & RAR5_ENDARC_NEXTVOL) != 0;
+			break;
+		}
+
+		else if (block.type < 1 || block.type > 5)
+		{
+			// inlvaid block type
+			return false;
+		}
+
+		uint64 skip = block.trailsize;
+		if (m_encrypted)
+		{
+			skip -= 16 - m_decryptPos;
+			if (m_decryptPos < 16)
+			{
+				skip += skip % 16 > 0 ? 16 - skip % 16 : 0;
+				m_decryptPos = 16;
+			}
+			DecryptFree();
+		}
+
+		if (!file.Seek(skip, DiskFile::soCur))
+		{
+			return false;
+		}
+	}
+
+	return true;
+}
+
+RarVolume::RarBlock RarVolume::ReadRar5Block(DiskFile& file)
+{
+	RarBlock block {0};
+	uint64 buf = 0;
+
+	if (m_encrypted &&
+		!(file.Read(m_decryptIV, sizeof(m_decryptIV)) == sizeof(m_decryptIV) &&
+		  DecryptInit(256)))
+	{
+		return {0};
+	}
+
+	if (!Read32(file, nullptr, &block.crc)) return {0};
+
+	if (!ReadV(file, nullptr, &buf)) return {0};
+	uint32 size = (uint32)buf;
+	block.trailsize = size;
+
+	if (!ReadV(file, &block, &buf)) return {0};
+	block.type = (uint8)buf;
+
+	if (!ReadV(file, &block, &buf)) return {0};
+	block.flags = (uint16)buf;
+
+	block.addsize = 0;
+	if ((block.flags & RAR5_BLOCK_EXTRADATA) && !ReadV(file, &block, &block.addsize)) return {0};
+
+	uint64 datasize = 0;
+	if ((block.flags & RAR5_BLOCK_DATAAREA) && !ReadV(file, &block, &datasize)) return {0};
+	block.trailsize += datasize;
+
+#ifdef DEBUG
+	static int num = 0;
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
+
+	return block;
+}
+
+bool RarVolume::ReadRar5File(DiskFile& file, RarBlock& block, RarFile& innerFile)
+{
+	innerFile.m_splitBefore = block.flags & RAR5_BLOCK_SPLITBEFORE;
+	innerFile.m_splitAfter = block.flags & RAR5_BLOCK_SPLITAFTER;
+
+	uint64 val;
+
+	uint64 fileflags;
+	if (!ReadV(file, &block, &fileflags)) return false;
+
+	if (!ReadV(file, &block, &val)) return false; // skip
+	innerFile.m_size = (int64)val;
+
+	if (!ReadV(file, &block, &val)) return false;
+	innerFile.m_attr = (uint32)val;
+
+	if (fileflags & RAR5_FILE_TIME && !Read32(file, &block, &innerFile.m_time)) return false;
+	if (fileflags & RAR5_FILE_CRC && !Skip(file, &block, 4)) return false;
+
+	if (!ReadV(file, &block, &val)) return false; // skip
+	if (!ReadV(file, &block, &val)) return false; // skip
+
+	uint64 namelen;
+	if (!ReadV(file, &block, &namelen)) return false;
+	if (namelen > 8192) return false; // an error
+	CharBuffer name;
+	name.Reserve((uint32)namelen + 1);
+	if (!Read(file, &block, (char*)name, namelen)) return false;
+	name[namelen] = '\0';
+	innerFile.m_filename = name;
+
+	// reading extra headers to find file time
+	if (block.flags & RAR5_BLOCK_EXTRADATA)
+	{
+		uint64 remsize = block.addsize;
+		while (remsize > 0)
+		{
+			uint64 trailsize = block.trailsize;
+
+			uint64 len;
+			if (!ReadV(file, &block, &len)) return false;
+			remsize -= trailsize - block.trailsize + len;
+			trailsize = block.trailsize;
+
+			uint64 type;
+			if (!ReadV(file, &block, &type)) return false;
+
+			if (type == RAR5_FILE_EXTRATIME)
+			{
+				uint64 flags;
+				if (!ReadV(file, &block, &flags)) return false;
+				if (flags & RAR5_FILE_EXTRATIMEUNIXFORMAT)
+				{
+					if (!Read32(file, &block, &innerFile.m_time)) return false;
+				}
+				else
+				{
+					uint32 timelow, timehigh;
+					if (!Read32(file, &block, &timelow)) return false;
+					if (!Read32(file, &block, &timehigh)) return false;
+					uint64 wintime = ((uint64)timehigh << 32) + timelow;
+					innerFile.m_time = (uint32)(wintime / 10000000 - 11644473600LL);
+				}
+			}
+
+			len -= trailsize - block.trailsize;
+
+			if (!Skip(file, &block, len)) return false;
+		}
+	}
+
+	debug("%" PRIu64 ", %" PRIu64 ", %s", block.trailsize, namelen, (const char*)name);
+
+	return true;
+}
+
+void RarVolume::LogDebugInfo()
+{
+#ifdef DEBUG
+	debug("Volume: version:%i, multi:%i, vol-no:%i, new-naming:%i, has-next:%i, encrypted:%i, file-count:%i, [%s]",
+		(int)m_version, (int)m_multiVolume, m_volumeNo, (int)m_newNaming, (int)m_hasNextVolume,
+		(int)m_encrypted, (int)m_files.size(), FileSystem::BaseFileName(m_filename));
+
+	for (RarFile& file : m_files)
+	{
+		debug("  time:%i, size:%" PRIi64 ", attr:%i, split-before:%i, split-after:%i, [%s]",
+			file.m_time, file.m_size, file.m_attr,
+			file.m_splitBefore, file.m_splitAfter, *file.m_filename);
+	}
+#endif
+}
+
+bool RarVolume::DecryptRar3Prepare(const uint8 salt[8])
+{
+	WString wstr(*m_password);
+	int len = wstr.Length();
+	if (len == 0) return false;
+
+	CharBuffer seed(len * 2 + 8);
+	for (int i = 0; i < len; i++)
+	{
+		wchar_t ch = wstr[i];
+		seed[i * 2] = ch & 0xFF;
+		seed[i * 2 + 1] = (ch & 0xFF00) >> 8;
+	}
+	memcpy(seed + len * 2, salt, 8);
+
+	debug("seed: %s", *Util::FormatBuffer((const char*)seed, seed.Size()));
+
+#ifdef HAVE_OPENSSL
+	EVP_MD_CTX* context = EVP_MD_CTX_create();
+
+	if (!EVP_DigestInit(context, EVP_sha1()))
+	{
+		EVP_MD_CTX_destroy(context);
+		return false;
+	}
+#elif defined(HAVE_NETTLE)
+	sha1_ctx context;
+	sha1_init(&context);
+#else
+	return false;
+#endif
+
+	uint8 digest[20];
+	const int rounds = 0x40000;
+
+	for (int i = 0; i < rounds; i++)
+	{
+#ifdef HAVE_OPENSSL
+		EVP_DigestUpdate(context, *seed, seed.Size());
+#elif defined(HAVE_NETTLE)
+		sha1_update(&context, seed.Size(), (const uint8_t*)*seed);
+#endif
+
+		uint8 buf[3];
+		buf[0] = (uint8)i;
+		buf[1] = (uint8)(i >> 8);
+		buf[2] = (uint8)(i >> 16);
+
+#ifdef HAVE_OPENSSL
+		EVP_DigestUpdate(context, buf, sizeof(buf));
+#elif defined(HAVE_NETTLE)
+		sha1_update(&context, sizeof(buf), buf);
+#endif
+
+		if (i % (rounds / 16) == 0)
+		{
+#ifdef HAVE_OPENSSL
+			EVP_MD_CTX* ivContext = EVP_MD_CTX_create();
+			EVP_MD_CTX_copy(ivContext, context);
+			EVP_DigestFinal(ivContext, digest, nullptr);
+			EVP_MD_CTX_destroy(ivContext);
+#elif defined(HAVE_NETTLE)
+			sha1_ctx ivContext = context;
+			sha1_digest(&ivContext, sizeof(digest), digest);
+#endif
+			m_decryptIV[i / (rounds / 16)] = digest[sizeof(digest) - 1];
+		}
+	}
+
+#ifdef HAVE_OPENSSL
+	EVP_DigestFinal(context, digest, nullptr);
+	EVP_MD_CTX_destroy(context);
+#elif defined(HAVE_NETTLE)
+	sha1_digest(&context, sizeof(digest), digest);
+#endif
+
+	debug("digest: %s", *Util::FormatBuffer((const char*)digest, sizeof(digest)));
+
+	for (int i = 0; i < 4; i++)
+	{
+		for (int j = 0; j < 4; j++)
+		{
+			m_decryptKey[i * 4 + j] = digest[i * 4 + 3 - j];
+		}
+	}
+
+	debug("key: %s", *Util::FormatBuffer((const char*)m_decryptKey, sizeof(m_decryptKey)));
+	debug("iv: %s", *Util::FormatBuffer((const char*)m_decryptIV, sizeof(m_decryptIV)));
+
+	return true;
+}
+
+bool RarVolume::DecryptRar5Prepare(uint8 kdfCount, const uint8 salt[16])
+{
+	if (kdfCount > 24) return false;
+
+	int iterations = 1 << kdfCount;
+
+#ifdef HAVE_OPENSSL
+	if (!PKCS5_PBKDF2_HMAC(m_password, m_password.Length(), salt, 16,
+		iterations, EVP_sha256(), sizeof(m_decryptKey), m_decryptKey)) return false;
+	return true;
+#elif defined(HAVE_NETTLE)
+	pbkdf2_hmac_sha256(m_password.Length(), (const uint8_t*)*m_password,
+		iterations, 16, salt, sizeof(m_decryptKey), m_decryptKey);
+	return true;
+#else
+	return false;
+#endif
+}
+
+bool RarVolume::DecryptInit(int keyLength)
+{
+#ifdef HAVE_OPENSSL
+	if (!(m_context = EVP_CIPHER_CTX_new())) return false;
+	if (!EVP_DecryptInit((EVP_CIPHER_CTX*)m_context,
+		keyLength == 128 ? EVP_aes_128_cbc() : EVP_aes_256_cbc(),
+		m_decryptKey, m_decryptIV))
+		return false;
+	return true;
+#elif defined(HAVE_NETTLE)
+	m_context = new aes_ctx;
+	aes_set_decrypt_key((aes_ctx*)m_context, keyLength == 128 ? 16 : 32, m_decryptKey);
+	return true;
+#else
+	return false;
+#endif
+}
+
+bool RarVolume::DecryptBuf(const uint8 in[16], uint8 out[16])
+{
+#ifdef HAVE_OPENSSL
+	uint8 outbuf[32];
+	int outlen = 0;
+	if (!EVP_DecryptUpdate((EVP_CIPHER_CTX*)m_context, outbuf, &outlen, in, 16)) return false;
+	memcpy(out, outbuf + outlen, 16);
+	debug("decrypted: %s", *Util::FormatBuffer((const char*)out, 16));
+	return true;
+#elif defined(HAVE_NETTLE)
+	aes_decrypt((aes_ctx*)m_context, 16, out, in);
+	for (int i = 0; i < 16; i++)
+	{
+		out[i] ^= m_decryptIV[i];
+	}
+	memcpy(m_decryptIV, in, 16);
+	debug("decrypted: %s", *Util::FormatBuffer((const char*)out, 16));
+	return true;
+#else
+	return false;
+#endif
+}
+
+void RarVolume::DecryptFree()
+{
+	if (m_context)
+	{
+#ifdef HAVE_OPENSSL
+		EVP_CIPHER_CTX_free((EVP_CIPHER_CTX*)m_context);
+#elif defined(HAVE_NETTLE)
+		delete (aes_ctx*)m_context;
+#endif
+		m_context = nullptr;
+	}
+}
+
+bool RarVolume::DecryptRead(DiskFile& file, void* buffer, int64 size)
+{
+	while (size > 0)
+	{
+		if (m_decryptPos >= 16)
+		{
+			uint8 buf[16];
+			if (file.Read(&buf, sizeof(buf)) != sizeof(buf)) return false;
+			m_decryptPos = 0;
+			if (!DecryptBuf(buf, m_decryptBuf)) return false;
+		}
+
+		uint8 remainingBuf = 16 - m_decryptPos;
+		uint8 len = size <= remainingBuf ? (uint8)size : remainingBuf;
+		memcpy(buffer, m_decryptBuf + m_decryptPos, len);
+		m_decryptPos += len;
+		size -= len;
+		buffer = (char*)buffer + len;
+	}
+
+	return true;
+}
+

daemon/postprocess/RarReader.h

@@ -0,0 +1,114 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef RARREADER_H
+#define RARREADER_H
+
+#include "NString.h"
+#include "Log.h"
+#include "FileSystem.h"
+
+class RarFile
+{
+public:
+	const char* GetFilename() { return m_filename; }
+	uint32 GetTime() { return m_time; }
+	uint32 GetAttr() { return m_attr; }
+	int64 GetSize() { return m_size; }
+	bool GetSplitBefore() { return m_splitBefore; }
+	bool GetSplitAfter() { return m_splitAfter; }
+private:
+	CString m_filename;
+	uint32 m_time = 0;
+	uint32 m_attr = 0;
+	int64 m_size = 0;
+	bool m_splitBefore = false;
+	bool m_splitAfter = false;
+	friend class RarVolume;
+};
+
+class RarVolume
+{
+public:
+	typedef std::deque<RarFile> FileList;
+
+	RarVolume(const char* filename) : m_filename(filename) {}
+	bool Read();
+
+	const char* GetFilename() { return m_filename; }
+	int GetVersion() { return m_version; }
+	uint32 GetVolumeNo() { return m_volumeNo; }
+	bool GetNewNaming() { return m_newNaming; }
+	bool GetHasNextVolume() { return m_hasNextVolume; }
+	bool GetMultiVolume() { return m_multiVolume; }
+	bool GetEncrypted() { return m_encrypted; }
+	void SetPassword(const char* password) { m_password = password; }
+	FileList* GetFiles() { return &m_files; }
+
+private:
+	struct RarBlock
+	{
+		uint32 crc;
+		uint8 type;
+		uint16 flags;
+		uint64 addsize;
+		uint64 trailsize;
+	};
+
+	CString m_filename;
+	int m_version = 0;
+	uint32 m_volumeNo = 0;
+	bool m_newNaming = false;
+	bool m_hasNextVolume = false;
+	bool m_multiVolume = false;
+	FileList m_files;
+	bool m_encrypted = false;
+	CString m_password;
+	uint8 m_decryptKey[32];
+	uint8 m_decryptIV[16];
+	uint8 m_decryptBuf[16];
+	uint8 m_decryptPos = 16;
+
+	// using "void*" to prevent the including of GnuTLS/OpenSSL header files into TlsSocket.h
+	void* m_context = nullptr;
+	void* m_session = nullptr;
+
+	int DetectRarVersion(DiskFile& file);
+	void LogDebugInfo();
+	bool Skip(DiskFile& file, RarBlock* block, int64 size);
+	bool Read(DiskFile& file, RarBlock* block, void* buffer, int64 size);
+	bool Read16(DiskFile& file, RarBlock* block, uint16* result);
+	bool Read32(DiskFile& file, RarBlock* block, uint32* result);
+	bool ReadV(DiskFile& file, RarBlock* block, uint64* result);
+	bool ReadRar3Volume(DiskFile& file);
+	bool ReadRar5Volume(DiskFile& file);
+	RarBlock ReadRar3Block(DiskFile& file);
+	RarBlock ReadRar5Block(DiskFile& file);
+	bool ReadRar3File(DiskFile& file, RarBlock& block, RarFile& innerFile);
+	bool ReadRar5File(DiskFile& file, RarBlock& block, RarFile& innerFile);
+	bool DecryptRar3Prepare(const uint8 salt[8]);
+	bool DecryptRar5Prepare(uint8 kdfCount, const uint8 salt[16]);
+	bool DecryptInit(int keyLength);
+	bool DecryptBuf(const uint8 in[16], uint8 out[16]);
+	void DecryptFree();
+	bool DecryptRead(DiskFile& file, void* buffer, int64 size);
+};
+
+#endif

daemon/postprocess/RarRenamer.cpp

@@ -0,0 +1,393 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+
+#include "RarRenamer.h"
+#include "Log.h"
+#include "Util.h"
+#include "FileSystem.h"
+
+void RarRenamer::Execute()
+{
+	m_progressLabel.Format("Checking renamed rar-files for %s", *m_infoName);
+	m_stageProgress = 0;
+	UpdateProgress();
+
+	BuildDirList(m_destDir);
+
+	for (CString& destDir : m_dirList)
+	{
+		debug("Checking %s", *destDir);
+		CheckFiles(destDir);
+	}
+}
+
+void RarRenamer::BuildDirList(const char* destDir)
+{
+	m_dirList.push_back(destDir);
+
+	DirBrowser dirBrowser(destDir);
+
+	while (const char* filename = dirBrowser.Next())
+	{
+		if (!IsStopped())
+		{
+			BString<1024> fullFilename("%s%c%s", destDir, PATH_SEPARATOR, filename);
+			if (FileSystem::DirectoryExists(fullFilename))
+			{
+				BuildDirList(fullFilename);
+			}
+			else
+			{
+				m_fileCount++;
+			}
+		}
+	}
+}
+
+void RarRenamer::CheckFiles(const char* destDir)
+{
+	DirBrowser dir(destDir);
+	while (const char* filename = dir.Next())
+	{
+		if (!IsStopped())
+		{
+			BString<1024> fullFilename("%s%c%s", destDir, PATH_SEPARATOR, filename);
+
+			if (!FileSystem::DirectoryExists(fullFilename))
+			{
+				m_progressLabel.Format("Checking file %s", filename);
+				m_stageProgress = m_fileCount > 0 ? m_curFile * 1000 / m_fileCount : 1000;
+				UpdateProgress();
+				m_curFile++;
+
+				CheckOneFile(fullFilename);
+			}
+		}
+	}
+
+	if (!m_volumes.empty())
+	{
+		RenameFiles(destDir);
+	}
+}
+
+void RarRenamer::CheckOneFile(const char* filename)
+{
+	if (m_ignoreExt && Util::MatchFileExt(FileSystem::BaseFileName(filename), m_ignoreExt, ",;"))
+	{
+		return;
+	}
+	
+	RarVolume volume(filename);
+	volume.SetPassword(m_password);
+	if (volume.Read())
+	{
+		m_volumes.push_back(std::move(volume));
+	}
+}
+
+void RarRenamer::RenameFile(const char* srcFilename, const char* destFileName)
+{
+	PrintMessage(Message::mkInfo, "Renaming %s to %s", FileSystem::BaseFileName(srcFilename), FileSystem::BaseFileName(destFileName));
+	if (!FileSystem::MoveFile(srcFilename, destFileName))
+	{
+		PrintMessage(Message::mkError, "Could not rename %s to %s: %s", srcFilename, destFileName,
+			*FileSystem::GetLastErrorMessage());
+		return;
+	}
+
+	m_renamedCount++;
+
+	// notify about new file name
+	RegisterRenamedFile(FileSystem::BaseFileName(srcFilename), FileSystem::BaseFileName(destFileName));
+}
+
+void RarRenamer::RenameFiles(const char* destDir)
+{
+	MakeSets();
+
+	for (RarVolumeSet& set : m_sets)
+	{
+		if (!IsSetProperlyNamed(set))
+		{
+			RarFile* mainFile = FindMainFile(set);
+			BString<1024> mainBasename = FileSystem::BaseFileName(mainFile->GetFilename());
+			char* ext = strrchr(mainBasename, '.');
+			// strip extension if its length is 3 chars
+			if (ext && strlen(ext) == 4)
+			{
+				*ext = '\0';
+			}
+
+			BString<1024> newBasename = *mainBasename;
+			int num = 0;
+			bool willOverwrite = true;
+			while (willOverwrite)
+			{
+				if (num++)
+				{
+					newBasename.Format("%s-%i", *mainBasename, num);
+				}
+
+				for (RarVolume* volume : set)
+				{
+					CString destfilename = GenNewVolumeFilename(destDir, newBasename, volume);
+					willOverwrite = strcmp(volume->GetFilename(), destfilename) && FileSystem::FileExists(destfilename);
+					if (willOverwrite)
+					{
+						break;
+					}
+				}
+			}
+
+			for (RarVolume* volume : set)
+			{
+				CString destfilename = GenNewVolumeFilename(destDir, newBasename, volume);
+				if (strcmp(volume->GetFilename(), destfilename))
+				{
+					RenameFile(volume->GetFilename(), destfilename);
+				}
+			}
+		}
+	}
+}
+
+CString RarRenamer::GenNewVolumeFilename(const char* destDir, const char* newBasename, RarVolume* volume)
+{
+	CString extension = volume->GetNewNaming() ? GenNewExtension(volume->GetVolumeNo()) : GenOldExtension(volume->GetVolumeNo());
+	return CString::FormatStr("%s%c%s.%s", destDir, PATH_SEPARATOR, newBasename, *extension);
+}
+
+CString RarRenamer::GenNewExtension(int volumeNo)
+{
+	return CString::FormatStr("part%04i.rar", volumeNo + 1);
+}
+
+CString RarRenamer::GenOldExtension(int volumeNo)
+{
+	if (volumeNo == 0)
+	{
+		return "rar";
+	}
+	else
+	{
+		unsigned char ch = 'r' + (volumeNo - 1) / 100;
+		return CString::FormatStr("%c%02d", ch, (volumeNo - 1) % 100);
+	}
+}
+
+void RarRenamer::MakeSets()
+{
+	m_sets.clear();
+
+	// find first volumes and create initial incomplete sets
+	for (RarVolume& volume : m_volumes)
+	{
+		if (!volume.GetFiles()->empty() && volume.GetVolumeNo() == 0 &&
+			!volume.GetFiles()->front().GetSplitBefore())
+		{
+			m_sets.push_back({&volume});
+		}
+	}
+
+	// complete sets, discard sets which cannot be completed
+	m_sets.erase(std::remove_if(m_sets.begin(), m_sets.end(),
+		[volumes = &m_volumes](RarVolumeSet& set)
+		{
+			debug("*** Building set %s", FileSystem::BaseFileName(set[0]->GetFilename()));
+			bool found = true;
+			while (found)
+			{
+				found = false;
+				std::vector<RarVolume*> candidates;
+
+				RarVolume* lastVolume = set.back();
+				for (RarVolume& volume : *volumes)
+				{
+					if (!volume.GetFiles()->empty() && volume.GetMultiVolume() &&
+						volume.GetVolumeNo() == lastVolume->GetVolumeNo() + 1 &&
+						volume.GetVersion() == lastVolume->GetVersion() &&
+						lastVolume->GetHasNextVolume() &&
+						((volume.GetFiles()->front().GetSplitBefore() &&
+						 lastVolume->GetFiles()->back().GetSplitAfter() &&
+						 !strcmp(volume.GetFiles()->front().GetFilename(), lastVolume->GetFiles()->back().GetFilename())) ||
+						 (!volume.GetFiles()->front().GetSplitBefore() && !lastVolume->GetFiles()->back().GetSplitAfter())))
+					{
+						debug("   found candidate %s", FileSystem::BaseFileName(volume.GetFilename()));
+						candidates.push_back(&volume);
+					}
+				}
+
+				RarVolume* nextVolume = nullptr;
+
+				if (candidates.size() > 1)
+				{
+					for (RarVolume* volume : candidates)
+					{
+						if (SameArchiveName(FileSystem::BaseFileName(set[0]->GetFilename()),
+							FileSystem::BaseFileName(volume->GetFilename()), set[0]->GetNewNaming()))
+						{
+							nextVolume = volume;
+							break;
+						}
+					}
+				}
+
+				if (!nextVolume && !candidates.empty())
+				{
+					nextVolume = candidates.front();
+				}
+
+				if (nextVolume)
+				{
+					debug("   adding %s", FileSystem::BaseFileName(nextVolume->GetFilename()));
+					set.push_back(nextVolume);
+					found = true;
+				}
+			}
+
+			RarVolume* lastVolume = set.back();
+			bool completed = !lastVolume->GetHasNextVolume() &&
+				(lastVolume->GetFiles()->empty() || !lastVolume->GetFiles()->back().GetSplitAfter());
+
+			return !completed;
+		}),
+		m_sets.end());
+
+#ifdef DEBUG
+	// debug log
+	for (RarVolumeSet& set : m_sets)
+	{
+		debug("*** Set ***");
+		for (RarVolume* volume : set)
+		{
+			debug("   %s", FileSystem::BaseFileName(volume->GetFilename()));
+		}
+	}
+#endif
+}
+
+bool RarRenamer::SameArchiveName(const char* filename1, const char* filename2, bool newNaming)
+{
+	if (strlen(filename1) != strlen(filename2))
+	{
+		return false;
+	}
+
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+
+	if (!(ext1 && ext2 && strlen(ext1) == strlen(ext2)))
+	{
+		return false;
+	}
+
+	if (newNaming)
+	{
+		if (ext1 == filename1 || ext2 == filename2)
+		{
+			return false;
+		}
+		BString<1024> name1, name2;
+		name1.Set(filename1, (int)(ext1 - filename1));
+		name2.Set(filename2, (int)(ext2 - filename2));
+		ext1 = strrchr(name1, '.');
+		ext2 = strrchr(name2, '.');
+		return ext1 && ext2 && strlen(ext1) == strlen(ext2) &&
+			!strncmp(ext1, ".part", 5) && !strncmp(ext2, ".part", 5) &&
+			!strncmp(name1, name2, ext1 - name1);
+	}
+	else
+	{
+		return !strncmp(filename1, filename2, ext1 - filename1);
+	}
+}
+
+bool RarRenamer::IsSetProperlyNamed(RarVolumeSet& set)
+{
+	RegEx regExPart(".*.part([0-9]+)\\.rar$");
+
+	const char* setBasename = FileSystem::BaseFileName(set[0]->GetFilename());
+	int setPartLen = 0;
+	for (RarVolume* volume : set)
+	{
+		const char* filename = FileSystem::BaseFileName(volume->GetFilename());
+
+		if (strlen(setBasename) != strlen(filename))
+		{
+			return false;
+		}
+
+		if (volume->GetNewNaming())
+		{
+			if (!regExPart.Match(filename))
+			{
+				return false;
+			}
+			BString<1024> partNo(filename + regExPart.GetMatchStart(1), regExPart.GetMatchLen(1));
+			if (setPartLen == 0)
+			{
+				setPartLen = partNo.Length();
+			}
+			bool ok = (uint32)atoi(partNo) == volume->GetVolumeNo() + 1 &&
+				partNo.Length() == setPartLen &&
+				!strncmp(setBasename, filename, regExPart.GetMatchStart(1));
+			if (!ok)
+			{
+				return false;
+			}
+		}
+		else
+		{
+			const char* ext = strrchr(filename, '.');
+			if (!ext || strcmp(ext + 1, GenOldExtension(volume->GetVolumeNo())) ||
+				strncmp(setBasename, filename, ext - filename))
+			{
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+RarFile* RarRenamer::FindMainFile(RarVolumeSet& set)
+{
+	std::deque<RarFile*> allFiles;
+
+	for (RarVolume* volume : set)
+	{
+		for (RarFile& file : *volume->GetFiles())
+		{
+			allFiles.push_back(&file);
+		}
+
+	}
+
+	std::deque<RarFile*>::iterator it = std::max_element(allFiles.begin(), allFiles.end(),
+		[](RarFile* file1, RarFile* file2)
+		{
+			return file1->GetSize() < file2->GetSize();
+		});
+
+	return *it;
+}

daemon/postprocess/RarRenamer.h

@@ -0,0 +1,82 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef RARRENAMER_H
+#define RARRENAMER_H
+
+#include "NString.h"
+#include "Log.h"
+#include "FileSystem.h"
+#include "RarReader.h"
+
+class RarRenamer
+{
+public:
+	void Execute();
+	void SetDestDir(const char* destDir) { m_destDir = destDir; }
+	const char* GetInfoName() { return m_infoName; }
+	void SetInfoName(const char* infoName) { m_infoName = infoName; }
+	void SetPassword(const char* password) { m_password = password; }
+	void SetIgnoreExt(const char* ignoreExt) { m_ignoreExt = ignoreExt; }
+	int GetRenamedCount() { return m_renamedCount; }
+
+protected:
+	virtual void UpdateProgress() {}
+	virtual bool IsStopped() { return false; };
+	virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3) {}
+	virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName) {}
+	const char* GetProgressLabel() { return m_progressLabel; }
+	int GetStageProgress() { return m_stageProgress; }
+
+private:
+	typedef std::deque<CString> DirList;
+	typedef std::deque<RarVolume> RarVolumeList;
+	typedef std::deque<RarVolume*> RarVolumeSet;
+	typedef std::deque<RarVolumeSet> RarSets;
+
+	CString m_infoName;
+	CString m_destDir;
+	CString m_progressLabel;
+	int m_stageProgress = 0;
+	bool m_cancelled = false;
+	DirList m_dirList;
+	int m_fileCount = 0;
+	int m_curFile = 0;
+	int m_renamedCount = 0;
+	RarVolumeList m_volumes;
+	RarSets m_sets;
+	CString m_password;
+	CString m_ignoreExt;
+
+	void BuildDirList(const char* destDir);
+	void CheckFiles(const char* destDir);
+	void CheckOneFile(const char* filename);
+	void RenameFile(const char* srcFilename, const char* destFileName);
+	void RenameFiles(const char* destDir);
+	CString GenNewVolumeFilename(const char* destDir, const char* newBasename, RarVolume* volume);
+	CString GenNewExtension(int volumeNo);
+	CString GenOldExtension(int volumeNo);
+	void MakeSets();
+	bool IsSetProperlyNamed(RarVolumeSet& set);
+	RarFile* FindMainFile(RarVolumeSet& set);
+	static bool SameArchiveName(const char* filename1, const char* filename2, bool newNaming);
+};
+
+#endif

daemon/postprocess/Rename.cpp

@@ -0,0 +1,225 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "Options.h"
+#include "DiskState.h"
+#include "Log.h"
+#include "FileSystem.h"
+#include "Rename.h"
+
+#ifndef DISABLE_PARCHECK
+void RenameController::PostParRenamer::PrintMessage(Message::EKind kind, const char* format, ...)
+{
+	char text[1024];
+	va_list args;
+	va_start(args, format);
+	vsnprintf(text, 1024, format, args);
+	va_end(args);
+	text[1024-1] = '\0';
+
+	m_owner->m_postInfo->GetNzbInfo()->AddMessage(kind, text);
+}
+#endif
+
+
+void RenameController::PostRarRenamer::PrintMessage(Message::EKind kind, const char* format, ...)
+{
+	char text[1024];
+	va_list args;
+	va_start(args, format);
+	vsnprintf(text, 1024, format, args);
+	va_end(args);
+	text[1024 - 1] = '\0';
+
+	m_owner->m_postInfo->GetNzbInfo()->AddMessage(kind, text);
+}
+
+
+RenameController::RenameController()
+{
+	debug("Creating RenameController");
+
+#ifndef DISABLE_PARCHECK
+	m_parRenamer.m_owner = this;
+#endif
+
+	m_rarRenamer.m_owner = this;
+}
+
+void RenameController::StartJob(PostInfo* postInfo, EJobKind kind)
+{
+	RenameController* renameController = new RenameController();
+	renameController->m_postInfo = postInfo;
+	renameController->m_kind = kind;
+	renameController->SetAutoDestroy(false);
+
+	postInfo->SetPostThread(renameController);
+
+	renameController->Start();
+}
+
+void RenameController::Run()
+{
+	BString<1024> nzbName;
+	CString destDir;
+	CString finalDir;
+	{
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		nzbName = m_postInfo->GetNzbInfo()->GetName();
+		destDir = m_postInfo->GetNzbInfo()->GetDestDir();
+		finalDir = m_postInfo->GetNzbInfo()->GetFinalDir();
+	}
+
+	BString<1024> infoName("rename for %s", *nzbName);
+	SetInfoName(infoName);
+
+	PrintMessage(Message::mkInfo, "Checking renamed %sfiles for %s",
+		m_kind == jkRar ? "archive " : "", *nzbName);
+
+	ExecRename(destDir, finalDir, nzbName);
+
+	if (IsStopped())
+	{
+		PrintMessage(Message::mkWarning, "Renaming cancelled for %s", *nzbName);
+	}
+	else if (m_renamedCount > 0)
+	{
+		PrintMessage(Message::mkInfo, "Successfully renamed %i %sfile(s) for %s",
+			m_renamedCount, m_kind == jkRar ? "archive " : "", *nzbName);
+	}
+	else
+	{
+		PrintMessage(Message::mkInfo, "No renamed %sfiles found for %s",
+			m_kind == jkRar ? "archive " : "", *nzbName);
+	}
+
+	RenameCompleted();
+}
+
+void RenameController::AddMessage(Message::EKind kind, const char* text)
+{
+	m_postInfo->GetNzbInfo()->AddMessage(kind, text);
+}
+
+void RenameController::ExecRename(const char* destDir, const char* finalDir, const char* nzbName)
+{
+	if (m_kind == jkPar)
+	{
+#ifndef DISABLE_PARCHECK
+		m_parRenamer.SetDestDir(m_postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSuccess &&
+			!Util::EmptyStr(finalDir) ? finalDir : destDir);
+		m_parRenamer.SetInfoName(nzbName);
+		m_parRenamer.SetDetectMissing(m_postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone);
+		m_parRenamer.Execute();
+#endif
+	}
+	else if (m_kind == jkRar)
+	{
+		m_rarRenamer.SetDestDir(destDir);
+		m_rarRenamer.SetInfoName(nzbName);
+		m_rarRenamer.SetIgnoreExt(g_Options->GetUnpackIgnoreExt());
+
+		NzbParameter* parameter = m_postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:Password");
+		if (parameter)
+		{
+			m_rarRenamer.SetPassword(parameter->GetValue());
+		}
+
+		m_rarRenamer.Execute();
+	}
+}
+
+void RenameController::RenameCompleted()
+{
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	if (m_kind == jkPar)
+	{
+		m_postInfo->GetNzbInfo()->SetParRenameStatus(m_renamedCount > 0 ? NzbInfo::rsSuccess : NzbInfo::rsNothing);
+#ifndef DISABLE_PARCHECK
+		// request another par2-file if the renaming has failed due to damaged par2-files
+		if (m_renamedCount == 0 && m_parRenamer.HasDamagedParFiles() &&
+			m_postInfo->GetNzbInfo()->GetRemainingParCount() > 0)
+		{
+			m_parRenamer.PrintMessage(Message::mkInfo, "Requesting extra par2-files for %s to perform par-rename", m_parRenamer.GetInfoName());
+			downloadQueue->EditEntry(m_postInfo->GetNzbInfo()->GetId(), DownloadQueue::eaGroupResume, nullptr);
+			downloadQueue->EditEntry(m_postInfo->GetNzbInfo()->GetId(), DownloadQueue::eaGroupPauseExtraPars, nullptr);
+			if (m_postInfo->GetNzbInfo()->GetRemainingSize() > 0)
+			{
+				// reset rename status to execute renamer again, after the new par2-file is downloaded
+				m_postInfo->GetNzbInfo()->SetParRenameStatus(NzbInfo::rsNone);
+			}
+		}
+#endif
+	}
+	else if (m_kind == jkRar)
+	{
+		m_postInfo->GetNzbInfo()->SetRarRenameStatus(m_renamedCount > 0 ? NzbInfo::rsSuccess : NzbInfo::rsNothing);
+	}
+
+#ifndef DISABLE_PARCHECK
+	if (m_parRenamer.HasMissedFiles() && m_postInfo->GetNzbInfo()->GetParStatus() <= NzbInfo::psSkipped)
+	{
+		m_parRenamer.PrintMessage(Message::mkInfo, "Requesting par-check/repair for %s to restore missing files ", m_parRenamer.GetInfoName());
+		m_postInfo->SetRequestParCheck(true);
+	}
+#endif
+
+	m_postInfo->SetWorking(false);
+}
+
+#ifndef DISABLE_PARCHECK
+void RenameController::UpdateParRenameProgress()
+{
+	GuardedDownloadQueue guard = DownloadQueue::Guard();
+
+	m_postInfo->SetProgressLabel(m_parRenamer.GetProgressLabel());
+	m_postInfo->SetStageProgress(m_parRenamer.GetStageProgress());
+}
+#endif
+
+void RenameController::UpdateRarRenameProgress()
+{
+	GuardedDownloadQueue guard = DownloadQueue::Guard();
+
+	m_postInfo->SetProgressLabel(m_rarRenamer.GetProgressLabel());
+	m_postInfo->SetStageProgress(m_rarRenamer.GetStageProgress());
+}
+
+/**
+*  Update file name in the CompletedFiles-list of NZBInfo
+*/
+void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFilename)
+{
+	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
+	{
+		if (!strcasecmp(completedFile.GetFilename(), oldFilename))
+		{
+			if (Util::EmptyStr(completedFile.GetOrigname()))
+			{
+				completedFile.SetOrigname(completedFile.GetFilename());
+			}
+			completedFile.SetFilename(newFilename);
+			break;
+		}
+	}
+	m_renamedCount++;
+}

daemon/postprocess/Rename.h

@@ -0,0 +1,98 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef RENAME_H
+#define RENAME_H
+
+#include "Thread.h"
+#include "DownloadInfo.h"
+#include "Script.h"
+#include "RarRenamer.h"
+
+#ifndef DISABLE_PARCHECK
+#include "ParRenamer.h"
+#endif
+
+class RenameController : public Thread, public ScriptController
+{
+public:
+	enum EJobKind
+	{
+		jkPar,
+		jkRar
+	};
+
+	RenameController();
+	virtual void Run();
+	static void StartJob(PostInfo* postInfo, EJobKind kind);
+
+protected:
+	virtual void AddMessage(Message::EKind kind, const char* text);
+
+private:
+	PostInfo* m_postInfo;
+	CString m_destDir;
+	int m_renamedCount = 0;
+	EJobKind m_kind;
+
+#ifndef DISABLE_PARCHECK
+	class PostParRenamer : public ParRenamer
+	{
+	protected:
+		virtual void UpdateProgress() { m_owner->UpdateParRenameProgress(); }
+		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
+		virtual void RegisterParredFile(const char* filename) 
+			{ m_owner->m_postInfo->GetParredFiles()->push_back(filename); }
+		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName) 
+			{ m_owner->RegisterRenamedFile(oldFilename, newFileName); }
+		virtual bool IsStopped() { return m_owner->IsStopped(); };
+	private:
+		RenameController* m_owner;
+		friend class RenameController;
+	};
+
+	PostParRenamer m_parRenamer;
+
+	void UpdateParRenameProgress();
+#endif
+
+	class PostRarRenamer : public RarRenamer
+	{
+	protected:
+		virtual void UpdateProgress() { m_owner->UpdateRarRenameProgress(); }
+		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
+		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFilename)
+			{ m_owner->RegisterRenamedFile(oldFilename, newFilename); }
+		virtual bool IsStopped() { return m_owner->IsStopped(); };
+	private:
+		RenameController* m_owner;
+		friend class RenameController;
+	};
+
+	PostRarRenamer m_rarRenamer;
+
+	void UpdateRarRenameProgress();
+
+	void ExecRename(const char* destDir, const char* finalDir, const char* nzbName);
+	void RenameCompleted();
+	void RegisterRenamedFile(const char* oldFilename, const char* newFilename);
+};
+
+#endif

daemon/postprocess/Repair.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -19,26 +19,27 @@
 
 
 #include "nzbget.h"
-#include "ParCoordinator.h"
+#include "Repair.h"
 #include "DupeCoordinator.h"
 #include "ParParser.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "DiskState.h"
 #include "Log.h"
 #include "FileSystem.h"
 
 #ifndef DISABLE_PARCHECK
-bool ParCoordinator::PostParChecker::RequestMorePars(int blockNeeded, int* blockFound)
+bool RepairController::PostParChecker::RequestMorePars(int blockNeeded, int* blockFound)
 {
 	return m_owner->RequestMorePars(m_postInfo->GetNzbInfo(), GetParFilename(), blockNeeded, blockFound);
 }
 
-void ParCoordinator::PostParChecker::UpdateProgress()
+void RepairController::PostParChecker::UpdateProgress()
 {
 	m_owner->UpdateParCheckProgress();
 }
 
-void ParCoordinator::PostParChecker::PrintMessage(Message::EKind kind, const char* format, ...)
+void RepairController::PostParChecker::PrintMessage(Message::EKind kind, const char* format, ...)
 {
 	char text[1024];
 	va_list args;
@@ -50,12 +51,12 @@ void ParCoordinator::PostParChecker::PrintMessage(Message::EKind kind, const cha
 	m_postInfo->GetNzbInfo()->AddMessage(kind, text);
 }
 
-void ParCoordinator::PostParChecker::RegisterParredFile(const char* filename)
+void RepairController::PostParChecker::RegisterParredFile(const char* filename)
 {
 	m_postInfo->GetParredFiles()->push_back(filename);
 }
 
-bool ParCoordinator::PostParChecker::IsParredFile(const char* filename)
+bool RepairController::PostParChecker::IsParredFile(const char* filename)
 {
 	for (CString& parredFile : m_postInfo->GetParredFiles())
 	{
@@ -67,14 +68,14 @@ bool ParCoordinator::PostParChecker::IsParredFile(const char* filename)
 	return false;
 }
 
-ParChecker::EFileStatus ParCoordinator::PostParChecker::FindFileCrc(const char* filename,
+ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char* filename,
 	uint32* crc, SegmentList* segments)
 {
 	CompletedFile* completedFile = nullptr;
 
 	for (CompletedFile& completedFile2 : m_postInfo->GetNzbInfo()->GetCompletedFiles())
 	{
-		if (!strcasecmp(completedFile2.GetFileName(), filename))
+		if (!strcasecmp(completedFile2.GetFilename(), filename))
 		{
 			completedFile = &completedFile2;
 			break;
@@ -85,7 +86,7 @@ ParChecker::EFileStatus ParCoordinator::PostParChecker::FindFileCrc(const char*
 		return ParChecker::fsUnknown;
 	}
 
-	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFileName()), completedFile->GetCrc(), (int)completedFile->GetStatus());
+	debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFilename()), completedFile->GetCrc(), (int)completedFile->GetStatus());
 
 	*crc = completedFile->GetCrc();
 
@@ -114,7 +115,20 @@ ParChecker::EFileStatus ParCoordinator::PostParChecker::FindFileCrc(const char*
 		ParChecker::fsUnknown;
 }
 
-void ParCoordinator::PostParChecker::RequestDupeSources(DupeSourceList* dupeSourceList)
+const char* RepairController::PostParChecker::FindFileOrigname(const char* filename)
+{
+	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
+	{
+		if (!strcasecmp(completedFile.GetFilename(), filename))
+		{
+			return completedFile.GetOrigname();
+		}
+	}
+
+	return nullptr;
+}
+
+void RepairController::PostParChecker::RequestDupeSources(DupeSourceList* dupeSourceList)
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
@@ -145,7 +159,7 @@ void ParCoordinator::PostParChecker::RequestDupeSources(DupeSourceList* dupeSour
 	}
 }
 
-void ParCoordinator::PostParChecker::StatDupeSources(DupeSourceList* dupeSourceList)
+void RepairController::PostParChecker::StatDupeSources(DupeSourceList* dupeSourceList)
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
@@ -169,44 +183,8 @@ void ParCoordinator::PostParChecker::StatDupeSources(DupeSourceList* dupeSourceL
 	m_postInfo->GetNzbInfo()->SetExtraParBlocks(m_postInfo->GetNzbInfo()->GetExtraParBlocks() + totalExtraParBlocks);
 }
 
-void ParCoordinator::PostParRenamer::UpdateProgress()
-{
-	m_owner->UpdateParRenameProgress();
-}
 
-void ParCoordinator::PostParRenamer::PrintMessage(Message::EKind kind, const char* format, ...)
-{
-	char text[1024];
-	va_list args;
-	va_start(args, format);
-	vsnprintf(text, 1024, format, args);
-	va_end(args);
-	text[1024-1] = '\0';
-
-	m_postInfo->GetNzbInfo()->AddMessage(kind, text);
-}
-
-void ParCoordinator::PostParRenamer::RegisterParredFile(const char* filename)
-{
-	m_postInfo->GetParredFiles()->push_back(filename);
-}
-
-/**
- *  Update file name in the CompletedFiles-list of NZBInfo
- */
-void ParCoordinator::PostParRenamer::RegisterRenamedFile(const char* oldFilename, const char* newFileName)
-{
-	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
-	{
-		if (!strcasecmp(completedFile.GetFileName(), oldFilename))
-		{
-			completedFile.SetFileName(newFileName);
-			break;
-		}
-	}
-}
-
-void ParCoordinator::PostDupeMatcher::PrintMessage(Message::EKind kind, const char* format, ...)
+void RepairController::PostDupeMatcher::PrintMessage(Message::EKind kind, const char* format, ...)
 {
 	char text[1024];
 	va_list args;
@@ -220,130 +198,69 @@ void ParCoordinator::PostDupeMatcher::PrintMessage(Message::EKind kind, const ch
 
 #endif
 
-ParCoordinator::ParCoordinator()
+RepairController::RepairController()
 {
-	debug("Creating ParCoordinator");
+	debug("Creating RepairController");
 
 #ifndef DISABLE_PARCHECK
 	m_parChecker.m_owner = this;
-	m_parRenamer.m_owner = this;
 #endif
 }
 
-ParCoordinator::~ParCoordinator()
+void RepairController::Stop()
 {
-	debug("Destroying ParCoordinator");
+	debug("Stopping RepairController");
+	Thread::Stop();
+#ifndef DISABLE_PARCHECK
+	m_parChecker.Cancel();
+#endif
 }
 
 #ifndef DISABLE_PARCHECK
-void ParCoordinator::Stop()
+
+void RepairController::StartJob(PostInfo* postInfo)
 {
-	debug("Stopping ParCoordinator");
+	RepairController* repairController = new RepairController();
+	repairController->m_postInfo = postInfo;
+	repairController->SetAutoDestroy(false);
 
-	m_stopped = true;
+	postInfo->SetPostThread(repairController);
 
-	if (m_parChecker.IsRunning())
+	repairController->Start();
+}
+
+void RepairController::Run()
+{
+	BString<1024> nzbName;
+	CString destDir;
 	{
-		m_parChecker.Stop();
-		int mSecWait = 5000;
-		while (m_parChecker.IsRunning() && mSecWait > 0)
-		{
-			usleep(50 * 1000);
-			mSecWait -= 50;
-		}
-		if (m_parChecker.IsRunning())
-		{
-			warn("Terminating par-check for %s", m_parChecker.GetInfoName());
-			m_parChecker.Kill();
-		}
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		nzbName = m_postInfo->GetNzbInfo()->GetName();
+		destDir = m_postInfo->GetNzbInfo()->GetDestDir();
 	}
-}
-#endif
 
-void ParCoordinator::PausePars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
-{
-	debug("ParCoordinator: Pausing pars");
-
-	downloadQueue->EditEntry(nzbInfo->GetId(),
-		DownloadQueue::eaGroupPauseExtraPars, 0, nullptr);
-}
-
-#ifndef DISABLE_PARCHECK
-
-/**
- * DownloadQueue must be locked prior to call of this function.
- */
-void ParCoordinator::StartParCheckJob(PostInfo* postInfo)
-{
-	m_currentJob = jkParCheck;
-	m_parChecker.SetPostInfo(postInfo);
-	m_parChecker.SetDestDir(postInfo->GetNzbInfo()->GetDestDir());
-	m_parChecker.SetNzbName(postInfo->GetNzbInfo()->GetName());
+	m_parChecker.SetPostInfo(m_postInfo);
+	m_parChecker.SetDestDir(destDir);
+	m_parChecker.SetNzbName(nzbName);
 	m_parChecker.SetParTime(Util::CurrentTime());
-	m_parChecker.SetDownloadSec(postInfo->GetNzbInfo()->GetDownloadSec());
-	m_parChecker.SetParQuick(g_Options->GetParQuick() && !postInfo->GetForceParFull());
-	m_parChecker.SetForceRepair(postInfo->GetForceRepair());
-	m_parChecker.PrintMessage(Message::mkInfo, "Checking pars for %s", postInfo->GetNzbInfo()->GetName());
-	postInfo->SetWorking(true);
-	m_parChecker.Start();
+	m_parChecker.SetDownloadSec(m_postInfo->GetNzbInfo()->GetDownloadSec());
+	m_parChecker.SetParQuick(g_Options->GetParQuick() && !m_postInfo->GetForceParFull());
+	m_parChecker.SetForceRepair(m_postInfo->GetForceRepair());
+
+	m_parChecker.PrintMessage(Message::mkInfo, "Checking pars for %s", *nzbName);
+
+	m_parChecker.Execute();
 }
 
 /**
  * DownloadQueue must be locked prior to call of this function.
  */
-void ParCoordinator::StartParRenameJob(PostInfo* postInfo)
+bool RepairController::AddPar(FileInfo* fileInfo, bool deleted)
 {
-	const char* destDir = postInfo->GetNzbInfo()->GetDestDir();
-
-	if (postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSuccess &&
-		!Util::EmptyStr(postInfo->GetNzbInfo()->GetFinalDir()))
-	{
-		destDir = postInfo->GetNzbInfo()->GetFinalDir();
-	}
-
-	m_currentJob = jkParRename;
-	m_parRenamer.SetPostInfo(postInfo);
-	m_parRenamer.SetDestDir(destDir);
-	m_parRenamer.SetInfoName(postInfo->GetNzbInfo()->GetName());
-	m_parRenamer.SetDetectMissing(postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone);
-	m_parRenamer.PrintMessage(Message::mkInfo, "Checking renamed files for %s", postInfo->GetNzbInfo()->GetName());
-	postInfo->SetWorking(true);
-	m_parRenamer.Start();
-}
-
-bool ParCoordinator::Cancel()
-{
-	if (m_currentJob == jkParCheck)
-	{
-		if (!m_parChecker.GetCancelled())
-		{
-			debug("Cancelling par-repair for %s", m_parChecker.GetInfoName());
-			m_parChecker.Cancel();
-			return true;
-		}
-	}
-	else if (m_currentJob == jkParRename)
-	{
-		if (!m_parRenamer.GetCancelled())
-		{
-			debug("Cancelling par-rename for %s", m_parRenamer.GetInfoName());
-			m_parRenamer.Cancel();
-			return true;
-		}
-	}
-	return false;
-}
-
-/**
- * DownloadQueue must be locked prior to call of this function.
- */
-bool ParCoordinator::AddPar(FileInfo* fileInfo, bool deleted)
-{
-	bool sameCollection = m_parChecker.IsRunning() &&
-		fileInfo->GetNzbInfo() == m_parChecker.GetPostInfo()->GetNzbInfo();
+	bool sameCollection = fileInfo->GetNzbInfo() == m_parChecker.GetPostInfo()->GetNzbInfo();
 	if (sameCollection && !deleted)
 	{
-		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), (int)PATH_SEPARATOR, fileInfo->GetFilename());
+		BString<1024> fullFilename("%s%c%s", fileInfo->GetNzbInfo()->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename());
 		m_parChecker.AddParFile(fullFilename);
 	}
 	else
@@ -353,7 +270,7 @@ bool ParCoordinator::AddPar(FileInfo* fileInfo, bool deleted)
 	return sameCollection;
 }
 
-void ParCoordinator::ParCheckCompleted()
+void RepairController::ParCheckCompleted()
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
@@ -385,9 +302,6 @@ void ParCoordinator::ParCheckCompleted()
 	postInfo->GetNzbInfo()->SetParFull(m_parChecker.GetParFull());
 
 	postInfo->SetWorking(false);
-	postInfo->SetStage(PostInfo::ptQueued);
-
-	downloadQueue->Save();
 }
 
 /**
@@ -397,80 +311,68 @@ void ParCoordinator::ParCheckCompleted()
 * special case: returns true if there are any unpaused par2-files in the queue regardless
 * of the amount of blocks; this is to keep par-checker wait for download completion.
 */
-bool ParCoordinator::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename, int blockNeeded, int* blockFoundOut)
+bool RepairController::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename, int blockNeeded, int* blockFoundOut)
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
-	Blocks blocks;
-	blocks.clear();
+	Blocks availableBlocks;
+	Blocks selectedBlocks;
 	int blockFound = 0;
 	int curBlockFound = 0;
 
-	FindPars(downloadQueue, nzbInfo, parFilename, blocks, true, true, &curBlockFound);
+	FindPars(downloadQueue, nzbInfo, parFilename, availableBlocks, true, true, &curBlockFound);
 	blockFound += curBlockFound;
 	if (blockFound < blockNeeded)
 	{
-		FindPars(downloadQueue, nzbInfo, parFilename, blocks, true, false, &curBlockFound);
+		FindPars(downloadQueue, nzbInfo, parFilename, availableBlocks, true, false, &curBlockFound);
 		blockFound += curBlockFound;
 	}
 	if (blockFound < blockNeeded)
 	{
-		FindPars(downloadQueue, nzbInfo, parFilename, blocks, false, false, &curBlockFound);
+		FindPars(downloadQueue, nzbInfo, parFilename, availableBlocks, false, false, &curBlockFound);
 		blockFound += curBlockFound;
 	}
 
+	std::sort(availableBlocks.begin(), availableBlocks.end(),
+		[](const BlockInfo& block1, const BlockInfo& block2)
+		{
+			return block1.m_blockCount < block2.m_blockCount;
+		});
+
 	if (blockFound >= blockNeeded)
 	{
-		// 1. first unpause all files with par-blocks less or equal iBlockNeeded
-		// starting from the file with max block count.
-		// if par-collection was built exponentially and all par-files present,
-		// this step selects par-files with exact number of blocks we need.
-		while (blockNeeded > 0)
+		// collect as much blocks as needed
+		for (Blocks::iterator it = availableBlocks.begin(); blockNeeded > 0 && it != availableBlocks.end(); it++)
 		{
-			BlockInfo* bestBlockInfo = nullptr;
-			Blocks::iterator bestBlockIter;
-			for (Blocks::iterator it = blocks.begin(); it != blocks.end(); it++)
+			BlockInfo& blockInfo = *it;
+			selectedBlocks.push_front(blockInfo);
+			blockNeeded -= blockInfo.m_blockCount;
+		}
+
+		// discarding superfluous blocks
+		for (Blocks::iterator it = selectedBlocks.begin(); it != selectedBlocks.end(); )
+		{
+			BlockInfo& blockInfo = *it;
+			if (blockNeeded + blockInfo.m_blockCount <= 0)
 			{
-				BlockInfo& blockInfo = *it;
-				if (blockInfo.m_blockCount <= blockNeeded &&
-				   (!bestBlockInfo || bestBlockInfo->m_blockCount < blockInfo.m_blockCount))
-				{
-					bestBlockInfo = &blockInfo;
-					bestBlockIter = it;
-				}
-			}
-			if (bestBlockInfo)
-			{
-				if (bestBlockInfo->m_fileInfo->GetPaused())
-				{
-					m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), (int)PATH_SEPARATOR, bestBlockInfo->m_fileInfo->GetFilename());
-					bestBlockInfo->m_fileInfo->SetPaused(false);
-					bestBlockInfo->m_fileInfo->SetExtraPriority(true);
-				}
-				blockNeeded -= bestBlockInfo->m_blockCount;
-				blocks.erase(bestBlockIter);
+				blockNeeded += blockInfo.m_blockCount;
+				it = selectedBlocks.erase(it);
 			}
 			else
 			{
-				break;
+				it++;
 			}
 		}
 
-		// 2. then unpause other files
-		// this step only needed if the par-collection was built not exponentially
-		// or not all par-files present (or some of them were corrupted)
-		// this step is not optimal, but we hope, that the first step will work good
-		// in most cases and we will not need the second step often
-		while (blockNeeded > 0)
+		// unpause files with blocks
+		for (BlockInfo& blockInfo : selectedBlocks)
 		{
-			BlockInfo& blockInfo = blocks.front();
 			if (blockInfo.m_fileInfo->GetPaused())
 			{
-				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), (int)PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
+				m_parChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", nzbInfo->GetName(), PATH_SEPARATOR, blockInfo.m_fileInfo->GetFilename());
 				blockInfo.m_fileInfo->SetPaused(false);
 				blockInfo.m_fileInfo->SetExtraPriority(true);
 			}
-			blockNeeded -= blockInfo.m_blockCount;
 		}
 	}
 
@@ -494,7 +396,7 @@ bool ParCoordinator::RequestMorePars(NzbInfo* nzbInfo, const char* parFilename,
 	return ok;
 }
 
-void ParCoordinator::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, const char* parFilename,
+void RepairController::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, const char* parFilename,
 	Blocks& blocks, bool strictParName, bool exactParName, int* blockFound)
 {
 	*blockFound = 0;
@@ -502,7 +404,7 @@ void ParCoordinator::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, co
 	// extract base name from m_szParFilename (trim .par2-extension and possible .vol-part)
 	char* baseParFilename = FileSystem::BaseFileName(parFilename);
 	int mainBaseLen = 0;
-	if (!ParParser::ParseParFilename(baseParFilename, &mainBaseLen, nullptr))
+	if (!ParParser::ParseParFilename(baseParFilename, true, &mainBaseLen, nullptr))
 	{
 		// should not happen
 		nzbInfo->PrintMessage(Message::mkError, "Internal error: could not parse filename %s", baseParFilename);
@@ -515,14 +417,15 @@ void ParCoordinator::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, co
 	for (FileInfo* fileInfo : nzbInfo->GetFileList())
 	{
 		int blockCount = 0;
-		if (ParParser::ParseParFilename(fileInfo->GetFilename(), nullptr, &blockCount) &&
+		if (ParParser::ParseParFilename(fileInfo->GetFilename(), fileInfo->GetFilenameConfirmed(), nullptr, &blockCount) &&
 			blockCount > 0)
 		{
 			bool useFile = true;
 
 			if (exactParName)
 			{
-				useFile = ParParser::SameParCollection(fileInfo->GetFilename(), FileSystem::BaseFileName(parFilename));
+				useFile = ParParser::SameParCollection(fileInfo->GetFilename(),
+					FileSystem::BaseFileName(parFilename), fileInfo->GetFilenameConfirmed());
 			}
 			else if (strictParName)
 			{
@@ -566,7 +469,7 @@ void ParCoordinator::FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, co
 	}
 }
 
-void ParCoordinator::UpdateParCheckProgress()
+void RepairController::UpdateParCheckProgress()
 {
 	PostInfo* postInfo;
 
@@ -600,7 +503,7 @@ void ParCoordinator::UpdateParCheckProgress()
 		}
 
 		bool parCancel = false;
-		if (!m_parChecker.GetCancelled())
+		if (!IsStopped())
 		{
 			if ((g_Options->GetParTimeLimit() > 0) &&
 				m_parChecker.GetStage() == PostParChecker::ptRepairing &&
@@ -621,16 +524,16 @@ void ParCoordinator::UpdateParCheckProgress()
 
 		if (parCancel)
 		{
-			m_parChecker.Cancel();
+			Stop();
 		}
 	}
 
 	CheckPauseState(postInfo);
 }
 
-void ParCoordinator::CheckPauseState(PostInfo* postInfo)
+void RepairController::CheckPauseState(PostInfo* postInfo)
 {
-	if (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = postInfo->GetStageTime();
 		time_t startTime = postInfo->GetStartTime();
@@ -639,9 +542,9 @@ void ParCoordinator::CheckPauseState(PostInfo* postInfo)
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !m_stopped)
+		while (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 
 			// update time stamps
 
@@ -667,44 +570,4 @@ void ParCoordinator::CheckPauseState(PostInfo* postInfo)
 	}
 }
 
-void ParCoordinator::ParRenameCompleted()
-{
-	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
-
-	PostInfo* postInfo = m_parRenamer.GetPostInfo();
-	postInfo->GetNzbInfo()->SetRenameStatus(m_parRenamer.GetStatus() == ParRenamer::psSuccess ? NzbInfo::rsSuccess : NzbInfo::rsFailure);
-
-	if (m_parRenamer.HasMissedFiles() && postInfo->GetNzbInfo()->GetParStatus() <= NzbInfo::psSkipped)
-	{
-		m_parRenamer.PrintMessage(Message::mkInfo, "Requesting par-check/repair for %s to restore missing files ", m_parRenamer.GetInfoName());
-		postInfo->SetRequestParCheck(true);
-	}
-
-	postInfo->SetWorking(false);
-	postInfo->SetStage(PostInfo::ptQueued);
-
-	downloadQueue->Save();
-}
-
-void ParCoordinator::UpdateParRenameProgress()
-{
-	PostInfo* postInfo;
-	{
-		GuardedDownloadQueue guard = DownloadQueue::Guard();
-
-		postInfo = m_parRenamer.GetPostInfo();
-		postInfo->SetProgressLabel(m_parRenamer.GetProgressLabel());
-		postInfo->SetStageProgress(m_parRenamer.GetStageProgress());
-		time_t current = Util::CurrentTime();
-
-		if (postInfo->GetStage() != PostInfo::ptRenaming)
-		{
-			postInfo->SetStage(PostInfo::ptRenaming);
-			postInfo->SetStageTime(current);
-		}
-	}
-
-	CheckPauseState(postInfo);
-}
-
 #endif

daemon/postprocess/Repair.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,36 +18,32 @@
  */
 
 
-#ifndef PARCOORDINATOR_H
-#define PARCOORDINATOR_H
+#ifndef REPAIR_H
+#define REPAIR_H
 
 #include "DownloadInfo.h"
+#include "Thread.h"
+#include "Script.h"
 
 #ifndef DISABLE_PARCHECK
 #include "ParChecker.h"
-#include "ParRenamer.h"
 #include "DupeMatcher.h"
 #endif
 
-class ParCoordinator
+class RepairController : public Thread, public ScriptController
 {
 public:
-	ParCoordinator();
-	virtual ~ParCoordinator();
-	void PausePars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo);
+	RepairController();
+	virtual void Stop();
 
 #ifndef DISABLE_PARCHECK
+	virtual void Run();
+	static void StartJob(PostInfo* postInfo);
 	bool AddPar(FileInfo* fileInfo, bool deleted);
-	void StartParCheckJob(PostInfo* postInfo);
-	void StartParRenameJob(PostInfo* postInfo);
-	void Stop();
-	bool Cancel();
 
 protected:
 	void UpdateParCheckProgress();
-	void UpdateParRenameProgress();
 	void ParCheckCompleted();
-	void ParRenameCompleted();
 	void CheckPauseState(PostInfo* postInfo);
 	bool RequestMorePars(NzbInfo* nzbInfo, const char* parFilename, int blockNeeded, int* blockFound);
 
@@ -66,39 +62,23 @@ private:
 	protected:
 		virtual bool RequestMorePars(int blockNeeded, int* blockFound);
 		virtual void UpdateProgress();
+		virtual bool IsStopped() { return m_owner->IsStopped(); };
 		virtual void Completed() { m_owner->ParCheckCompleted(); }
 		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
 		virtual void RegisterParredFile(const char* filename);
 		virtual bool IsParredFile(const char* filename);
 		virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments);
+		virtual const char* FindFileOrigname(const char* filename);
 		virtual void RequestDupeSources(DupeSourceList* dupeSourceList);
 		virtual void StatDupeSources(DupeSourceList* dupeSourceList);
 	private:
-		ParCoordinator* m_owner;
+		RepairController* m_owner;
 		PostInfo* m_postInfo;
 		time_t m_parTime;
 		time_t m_repairTime;
 		int m_downloadSec;
 
-		friend class ParCoordinator;
-	};
-
-	class PostParRenamer: public ParRenamer
-	{
-	public:
-		PostInfo* GetPostInfo() { return m_postInfo; }
-		void SetPostInfo(PostInfo* postInfo) { m_postInfo = postInfo; }
-	protected:
-		virtual void UpdateProgress();
-		virtual void Completed() { m_owner->ParRenameCompleted(); }
-		virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
-		virtual void RegisterParredFile(const char* filename);
-		virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName);
-	private:
-		ParCoordinator* m_owner;
-		PostInfo* m_postInfo;
-
-		friend class ParCoordinator;
+		friend class RepairController;
 	};
 
 	class PostDupeMatcher: public DupeMatcher
@@ -124,16 +104,8 @@ private:
 
 	typedef std::deque<BlockInfo> Blocks;
 
-	enum EJobKind
-	{
-		jkParCheck,
-		jkParRename
-	};
-
+	PostInfo* m_postInfo;
 	PostParChecker m_parChecker;
-	bool m_stopped = false;
-	PostParRenamer m_parRenamer;
-	EJobKind m_currentJob;
 
 	void FindPars(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, const char* parFilename,
 		Blocks& blocks, bool strictParName, bool exactParName, int* blockFound);