#784: removed expired root certificate

from ca root certificate store: certificate “DST Root CA X3” used by
Lets Encrypt

.lgtm.yml

@@ -0,0 +1,17 @@
+# Configuration file for integration with http://lgtm.com
+
+path_classifiers:
+  library:
+    # exclude these directories from default alerts report:
+    - lib
+    - webui/lib
+
+extraction:
+  cpp:
+    configure:
+      command:
+        # compile with tests to activate scanning of C++ sources for tests
+        - ./configure --enable-tests
+
+queries:
+  - exclude: js/incomplete-sanitization  # this one gives false positives only and nothing useful

COPYING

@@ -1,12 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -15,7 +15,7 @@ software--to make sure the software is free for all its users.  This
 General Public License applies to most of the Free Software
 Foundation's software and to any other program whose authors commit to
 using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
+the GNU Lesser General Public License instead.)  You can apply it to
 your programs, too.
 
   When we speak of free software, we are referring to freedom, not
@@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
+
+                    GNU GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License applies to any program or other work which contains
@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
     License.  (Exception: if the Program itself is interactive but
     does not normally print such an announcement, your work based on
     the Program is not required to print an announcement.)
-
+
 These requirements apply to the modified work as a whole.  If
 identifiable sections of that work are not derived from the Program,
 and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
 access to copy the source code from the same place counts as
 distribution of the source code, even though third parties are not
 compelled to copy the source along with the object code.
-
+
   4. You may not copy, modify, sublicense, or distribute the Program
 except as expressly provided under this License.  Any attempt
 otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
 
 This section is intended to make thoroughly clear what is believed to
 be a consequence of the rest of this License.
-
+
   8. If the distribution and/or use of the Program is restricted in
 certain countries either by patents or by copyrighted interfaces, the
 original copyright holder who places the Program under this License
@@ -255,7 +255,7 @@ make exceptions for this.  Our decision will be guided by the two goals
 of preserving the free status of all derivatives of our free software and
 of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
 
   If you develop a new program, and you want it to be of the greatest
 possible use to the public, the best way to achieve this is to make it
@@ -303,10 +303,9 @@ the "copyright" line and a pointer to where the full notice is found.
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -336,5 +335,5 @@ necessary.  Here is a sample; alter the names:
 This General Public License does not permit incorporating your program into
 proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

ChangeLog

@@ -1,3 +1,97 @@
+nzbget-21.2-testing:
+  - please see repository change log at
+    https://github.com/nzbget/nzbget/commits/develop
+
+nzbget-21.1:
+  - fixed crash on systems with 64-bit time;
+  - corrected icon in Windows "uninstall program" list;
+  - allow special characters in URL for username and password;
+  - improved reporting for binding errors on Windows;
+  - fixed unicode space characters in javascript files, which could cause issues
+    with nginx proxy;
+  - fixed negative values for "FileSizeLo" in json-rpc;
+  - corrected url detection in rpc-method "append";
+  - added support for new error messages in unrar 5.80;
+  - now always using snapshots when reading directory contents:
+    - in previous versions snapshots were used on macOS only;
+    - now they are used on all OSes;
+    - this solves issue with leftovers during directory cleanup, which could
+      happen on certain OSes when working with network drives;
+  - fixed file allocating on file systems where sparse files are not supported:
+    - the issue could happen when InterDir was located on a network drive;
+  - fixed crash caused by malformed nzb files;
+  - fixed GROUP command in nserv;
+  - updated url of the global certificate storage file in the build scripts;
+  - fixed: file selector in WebKit based browsers doesn't allow to choose the
+    same file again;
+  - removed outdated links from web interface;
+  - fixed PC sleep mode not working (Windows only);
+  - set "SameSite" attribute for cookies;
+  - corrected typo in about dialog of web interface;
+  - updated license text: changed address of Free Software Foundation and minor
+    formatting changes.
+
+nzbget-21.0:
+  - reworked duplicate handling to support URLs, especially when using RSS
+    feeds:
+      - queue items added via URLs (to be fetched by nzbget) are no longer
+        immediately fetched;
+      - instead url-items are handled by duplicate check similar to nzb-items
+        and may be placed into history as duplicate backups;
+      - if an url-item needs to be downloaded as backup for a failed other item
+        the nzb-file is fetched via provided URL;
+      - this greatly reduces the number of nzbs fetched from indexers when using
+        RSS feeds and duplicate handling;
+  - improved support for Android devices:
+    - now providing a separate installer package for Android;
+    - the package contains binaries built using Android NDK;
+    - this improves compatibility with Android, in particular with Android 8,
+      where general Linux installer version of NZBGet didn't work anymore due
+      to security changes in Android;
+    - android installer app is updated to use the new android installer package
+      instead of general Linux package;
+  - thoroughly optimised the program to reduce power consumption in idle state:
+    - number of CPU wake ups in idle state has been reduced from hundreds times
+      per second to about only one per second;
+  - optimisations for large queues with thousands of items:
+    - speed up saving of queue state and reduced number of queue state savings;
+    - improved queue state format to reduce amount of state data saved during
+      downloading;
+    - in tests download speed for very large queue (16000 items) has been
+      increased from 45 MB/s to 300 MB/s (comparing to 400 MB/s with small
+      queue); 
+  - added native support for aarch64 architecture (ARM 64 Bit CPU) in Linux and
+    Android installers;
+  - force par-check for nzbs without archives;
+  - added functional tests for unpack CRC error;
+  - click on nzbget logo in web-interface now switches to downloads tab instead
+    of showing "About dialog" which has been moved into settings;
+  - improved handling of files splitted via par2;
+  - added python 3 compatibility to EMail.py script;
+  - added python 3 compatibility to Logger.py script;
+  - proper UTF-8 encoding of email content in EMail.py script;
+  - improved error reporting for queue disk state corruption;
+  - updated unrar to 5.7 and 7-zip to 19.0;
+  - Windows installer now includes unrar in 32 bit and 64 bit variants;
+  - allowing wildcards in option AuthorizedIP;
+  - removed suggestion of RC4 cipher;
+  - better description of option UMask;
+  - integrated LGTM code analyser tool into project;
+  - fixed: failed downloads not having any par2- or archive- files were moved to
+    DestDir instead of remaining in InterDir;
+  - fixed crash when using FIPS version of OpenSSL;
+  - fixed compatibility issue with OpenSSL compiled without compression support;
+  - fixed deprecated OpenSSL calls;
+  - fixed potential crash in built-in web-server;
+  - fixed: statistics for session download time and speed may be way off on high
+    load;
+  - fixed many compilation warnings in GCC;
+  - fixed: macOS menubar widget could not connect if password contained special
+    characters;
+  - fixed: remote clients not displaying current download speed;
+  - fixed: remote server could crash when feed with invalid api request;
+  - fixed trimming of relative paths in config.
+
 nzbget-20.0:
   - massive performance optimisations in downloader:
         - improved yEnc decoder;

Makefile.am

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -68,6 +68,8 @@ nzbget_SOURCES = \
 	daemon/main/nzbget.h \
 	daemon/main/Options.cpp \
 	daemon/main/Options.h \
+	daemon/main/WorkState.cpp \
+	daemon/main/WorkState.h \
 	daemon/main/Scheduler.cpp \
 	daemon/main/Scheduler.h \
 	daemon/main/StackTrace.cpp \
@@ -345,6 +347,7 @@ linux_FILES = \
 	linux/build-info.txt \
 	linux/build-nzbget \
 	linux/build-unpack \
+	linux/build-toolchain-android \
 	linux/build-toolchain-freebsd
 
 doc_FILES = \
@@ -413,6 +416,14 @@ testdata_FILES = \
 	tests/testdata/parchecker/testfile.vol00+1.PAR2 \
 	tests/testdata/parchecker/testfile.vol01+2.PAR2 \
 	tests/testdata/parchecker/testfile.vol03+3.PAR2 \
+	tests/testdata/parchecker2/crc.txt \
+	tests/testdata/parchecker2/testfile.7z.001 \
+	tests/testdata/parchecker2/testfile.7z.002 \
+	tests/testdata/parchecker2/testfile.7z.003 \
+	tests/testdata/parchecker2/testfile.7z.par2 \
+	tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \
 	tests/testdata/rarrenamer/testfile3.part01.rar \
 	tests/testdata/rarrenamer/testfile3.part02.rar \
 	tests/testdata/rarrenamer/testfile3.part03.rar \

Makefile.in

@@ -17,7 +17,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -232,7 +232,8 @@ am__nzbget_SOURCES_DIST = daemon/connect/Connection.cpp \
 	daemon/main/DiskService.h daemon/main/Maintenance.cpp \
 	daemon/main/Maintenance.h daemon/main/nzbget.cpp \
 	daemon/main/nzbget.h daemon/main/Options.cpp \
-	daemon/main/Options.h daemon/main/Scheduler.cpp \
+	daemon/main/Options.h daemon/main/WorkState.cpp \
+	daemon/main/WorkState.h daemon/main/Scheduler.cpp \
 	daemon/main/Scheduler.h daemon/main/StackTrace.cpp \
 	daemon/main/StackTrace.h daemon/nntp/ArticleDownloader.cpp \
 	daemon/nntp/ArticleDownloader.h daemon/nntp/ArticleWriter.cpp \
@@ -387,7 +388,8 @@ am_nzbget_OBJECTS = daemon/connect/Connection.$(OBJEXT) \
 	daemon/main/CommandLineParser.$(OBJEXT) \
 	daemon/main/DiskService.$(OBJEXT) \
 	daemon/main/Maintenance.$(OBJEXT) daemon/main/nzbget.$(OBJEXT) \
-	daemon/main/Options.$(OBJEXT) daemon/main/Scheduler.$(OBJEXT) \
+	daemon/main/Options.$(OBJEXT) daemon/main/WorkState.$(OBJEXT) \
+	daemon/main/Scheduler.$(OBJEXT) \
 	daemon/main/StackTrace.$(OBJEXT) \
 	daemon/nntp/ArticleDownloader.$(OBJEXT) \
 	daemon/nntp/ArticleWriter.$(OBJEXT) \
@@ -720,7 +722,8 @@ nzbget_SOURCES = daemon/connect/Connection.cpp \
 	daemon/main/DiskService.h daemon/main/Maintenance.cpp \
 	daemon/main/Maintenance.h daemon/main/nzbget.cpp \
 	daemon/main/nzbget.h daemon/main/Options.cpp \
-	daemon/main/Options.h daemon/main/Scheduler.cpp \
+	daemon/main/Options.h daemon/main/WorkState.cpp \
+	daemon/main/WorkState.h daemon/main/Scheduler.cpp \
 	daemon/main/Scheduler.h daemon/main/StackTrace.cpp \
 	daemon/main/StackTrace.h daemon/nntp/ArticleDownloader.cpp \
 	daemon/nntp/ArticleDownloader.h daemon/nntp/ArticleWriter.cpp \
@@ -859,6 +862,7 @@ linux_FILES = \
 	linux/build-info.txt \
 	linux/build-nzbget \
 	linux/build-unpack \
+	linux/build-toolchain-android \
 	linux/build-toolchain-freebsd
 
 doc_FILES = \
@@ -927,6 +931,14 @@ testdata_FILES = \
 	tests/testdata/parchecker/testfile.vol00+1.PAR2 \
 	tests/testdata/parchecker/testfile.vol01+2.PAR2 \
 	tests/testdata/parchecker/testfile.vol03+3.PAR2 \
+	tests/testdata/parchecker2/crc.txt \
+	tests/testdata/parchecker2/testfile.7z.001 \
+	tests/testdata/parchecker2/testfile.7z.002 \
+	tests/testdata/parchecker2/testfile.7z.003 \
+	tests/testdata/parchecker2/testfile.7z.par2 \
+	tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \
+	tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \
 	tests/testdata/rarrenamer/testfile3.part01.rar \
 	tests/testdata/rarrenamer/testfile3.part02.rar \
 	tests/testdata/rarrenamer/testfile3.part03.rar \
@@ -1149,6 +1161,8 @@ daemon/main/nzbget.$(OBJEXT): daemon/main/$(am__dirstamp) \
 	daemon/main/$(DEPDIR)/$(am__dirstamp)
 daemon/main/Options.$(OBJEXT): daemon/main/$(am__dirstamp) \
 	daemon/main/$(DEPDIR)/$(am__dirstamp)
+daemon/main/WorkState.$(OBJEXT): daemon/main/$(am__dirstamp) \
+	daemon/main/$(DEPDIR)/$(am__dirstamp)
 daemon/main/Scheduler.$(OBJEXT): daemon/main/$(am__dirstamp) \
 	daemon/main/$(DEPDIR)/$(am__dirstamp)
 daemon/main/StackTrace.$(OBJEXT): daemon/main/$(am__dirstamp) \
@@ -1544,6 +1558,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/main/$(DEPDIR)/Options.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/main/$(DEPDIR)/Scheduler.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/main/$(DEPDIR)/StackTrace.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@daemon/main/$(DEPDIR)/WorkState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/main/$(DEPDIR)/nzbget.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/nntp/$(DEPDIR)/ArticleDownloader.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@daemon/nntp/$(DEPDIR)/ArticleWriter.Po@am__quote@

README.md

@@ -1,18 +1,19 @@
 # NZBGet #
 [![License](https://img.shields.io/badge/license-GPL-blue.svg)](http://www.gnu.org/licenses/)
 [![Build Status](https://img.shields.io/travis/nzbget/nzbget/develop.svg)](https://travis-ci.org/nzbget/nzbget)
+[![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/nzbget/nzbget.svg?label=code%20quality:%20c%2b%2b)](https://lgtm.com/projects/g/nzbget/nzbget/context:cpp)
+[![Code Quality: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/nzbget/nzbget.svg?label=code%20quality:%20js)](https://lgtm.com/projects/g/nzbget/nzbget/context:javascript)
+[![Total Alerts](https://img.shields.io/lgtm/alerts/g/nzbget/nzbget.svg)](https://lgtm.com/projects/g/nzbget/nzbget/alerts)
+
+[![Total downloads](https://img.shields.io/github/downloads/nzbget/nzbget/total.svg)](https://github.com/nzbget/nzbget/releases)
+[![Downloads (latest release)](https://img.shields.io/github/downloads/nzbget/nzbget/latest/total.svg?label=latest%20release)](https://github.com/nzbget/nzbget/releases/latest)
 
 NZBGet is a binary downloader, which downloads files from Usenet
 based on information given in nzb-files. 
 
-NZBGet is written in C++ and is known for its extraordinary performance and efficiency.
+NZBGet is written in C++ and is known for its performance and efficiency.
 
-NZBGet can be run at almost every platform - classic PCs, NAS, media players, SAT-receivers, WLAN-routers, etc.
-The download area provides precompiled binaries
-for Windows, Mac OS X and Linux (compatible with many CPUs and platform variants). For other platforms
-the program can be compiled from sources.
-
-- [Home page (nzbget.net)](http://nzbget.net) - for first time visitors, learn more about NZBGet;
-- [Downloads](http://nzbget.net/download) - get the binaries and sources;
-- [Documentation](http://nzbget.net/documentation) - installation manuals, HOW-TOs, API;
-- [Forum](http://forum.nzbget.net) - get support, share your ideas, scripts, add-ons.
+NZBGet can run on almost any device - classic PC, NAS, media player, SAT-receiver, WLAN-router, etc.
+The download area provides precompiled binaries for Windows, macOS, Linux (compatible with
+many CPUs and platform variants), FreeBSD and Android. For other platforms
+the program can be compiled from sources.

config.h.in

@@ -3,10 +3,6 @@
 /* Define to 1 to include debug-code */
 #undef DEBUG
 
-/* Define to 1 if deleting of files during reading of directory is not
-   properly supported by OS */
-#undef DIRBROWSER_SNAPSHOT
-
 /* Define to 1 to not use curses */
 #undef DISABLE_CURSES
 
@@ -86,6 +82,9 @@
 /* Define to 1 to use GnuTLS library for TLS/SSL-support. */
 #undef HAVE_LIBGNUTLS
 
+/* Define to 1 if lockf is supported */
+#undef HAVE_LOCKF
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
@@ -101,6 +100,9 @@
 /* Define to 1 to use OpenSSL library for TLS/SSL-support and decryption. */
 #undef HAVE_OPENSSL
 
+/* Define to 1 if pthread_cancel is supported */
+#undef HAVE_PTHREAD_CANCEL
+
 /* Define to 1 if you have the <regex.h> header file. */
 #undef HAVE_REGEX_H
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for nzbget 20.0.
+# Generated by GNU Autoconf 2.69 for nzbget 21.2-testing.
 #
 # Report bugs to <hugbug@users.sourceforge.net>.
 #
@@ -580,8 +580,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='nzbget'
 PACKAGE_TARNAME='nzbget'
-PACKAGE_VERSION='20.0'
-PACKAGE_STRING='nzbget 20.0'
+PACKAGE_VERSION='21.2-testing'
+PACKAGE_STRING='nzbget 21.2-testing'
 PACKAGE_BUGREPORT='hugbug@users.sourceforge.net'
 PACKAGE_URL=''
 
@@ -1348,7 +1348,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures nzbget 20.0 to adapt to many kinds of systems.
+\`configure' configures nzbget 21.2-testing to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1419,7 +1419,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of nzbget 20.0:";;
+     short | recursive ) echo "Configuration of nzbget 21.2-testing:";;
    esac
   cat <<\_ACEOF
 
@@ -1584,7 +1584,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-nzbget configure 20.0
+nzbget configure 21.2-testing
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2053,7 +2053,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by nzbget $as_me 20.0, which was
+It was created by nzbget $as_me 21.2-testing, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3026,7 +3026,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='nzbget'
- VERSION='20.0'
+ VERSION='21.2-testing'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -6005,6 +6005,32 @@ fi
 
 
 
+ac_fn_cxx_check_func "$LINENO" "lockf" "ac_cv_func_lockf"
+if test "x$ac_cv_func_lockf" = xyes; then :
+  ac_fn_cxx_check_decl "$LINENO" "lockf" "ac_cv_have_decl_lockf" "#include <unistd.h>
+"
+if test "x$ac_cv_have_decl_lockf" = xyes; then :
+
+$as_echo "#define HAVE_LOCKF 1" >>confdefs.h
+
+fi
+
+fi
+
+ac_fn_cxx_check_func "$LINENO" "pthread_cancel" "ac_cv_func_pthread_cancel"
+if test "x$ac_cv_func_pthread_cancel" = xyes; then :
+  ac_fn_cxx_check_decl "$LINENO" "pthread_cancel" "ac_cv_have_decl_pthread_cancel" "#include <pthread.h>
+"
+if test "x$ac_cv_have_decl_pthread_cancel" = xyes; then :
+
+$as_echo "#define HAVE_PTHREAD_CANCEL 1" >>confdefs.h
+
+fi
+
+fi
+
+
+
 ac_fn_cxx_check_func "$LINENO" "getopt_long" "ac_cv_func_getopt_long"
 if test "x$ac_cv_func_getopt_long" = xyes; then :
 
@@ -6585,20 +6611,6 @@ _ACEOF
 
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether dir-browser snapshot workaround is needed" >&5
-$as_echo_n "checking whether dir-browser snapshot workaround is needed... " >&6; }
-if test "$target_vendor" == "apple"; then
-	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-
-$as_echo "#define DIRBROWSER_SNAPSHOT 1" >>confdefs.h
-
-else
-	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for cpu cores via sysconf" >&5
 $as_echo_n "checking for cpu cores via sysconf... " >&6; }
 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -9151,7 +9163,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by nzbget $as_me 20.0, which was
+This file was extended by nzbget $as_me 21.2-testing, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -9217,7 +9229,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-nzbget config.status 20.0
+nzbget config.status 21.2-testing
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,7 +1,7 @@
 #
 #  This file is part of nzbget. See <http://nzbget.net>.
 #
-# Copyright (C) 2008-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
+# Copyright (C) 2008-2021 Andrey Prygunkov <hugbug@users.sourceforge.net>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,7 +21,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.65)
-AC_INIT(nzbget, 20.0, hugbug@users.sourceforge.net)
+AC_INIT(nzbget, 21.2-testing, hugbug@users.sourceforge.net)
 AC_CONFIG_AUX_DIR(posix)
 AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE([foreign subdir-objects])
@@ -77,6 +77,19 @@ AC_SEARCH_LIBS([inet_addr], [nsl])
 AC_SEARCH_LIBS([hstrerror], [resolv])
 
 
+dnl
+dnl Android NDK restrictions
+dnl
+AC_CHECK_FUNC(lockf,
+	[AC_CHECK_DECL(lockf,
+		[AC_DEFINE([HAVE_LOCKF], 1, [Define to 1 if lockf is supported])],,
+		[#include <unistd.h>])])
+AC_CHECK_FUNC(pthread_cancel,
+	[AC_CHECK_DECL(pthread_cancel,
+		[AC_DEFINE([HAVE_PTHREAD_CANCEL], 1, [Define to 1 if pthread_cancel is supported])],,
+		[#include <pthread.h>])])
+
+
 dnl
 dnl Getopt
 dnl
@@ -209,18 +222,6 @@ AC_TRY_COMPILE([
 AC_DEFINE_UNQUOTED(SOCKLEN_T, $SOCKLEN_T, [Determine what socket length (socklen_t) data type is])
 
 
-dnl
-dnl Dir-browser's snapshot
-dnl
-AC_MSG_CHECKING(whether dir-browser snapshot workaround is needed)
-if test "$target_vendor" == "apple"; then
-	AC_MSG_RESULT([[yes]])
-	AC_DEFINE([DIRBROWSER_SNAPSHOT], 1, [Define to 1 if deleting of files during reading of directory is not properly supported by OS])
-else
-	AC_MSG_RESULT([[no]])
-fi
-
-
 dnl
 dnl check cpu cores via sysconf
 dnl

daemon/connect/Connection.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -31,16 +31,12 @@ std::unique_ptr<Mutex> Connection::m_getHostByNameMutex;
 #endif
 #endif
 
-class ConnectionFinalizer
-{
-public:
-	~ConnectionFinalizer()
-	{
-		Connection::Final();
-	}
-};
-
-std::unique_ptr<ConnectionFinalizer> m_connectionFinalizer;
+#if defined(__linux__) && !defined(__ANDROID__)
+// Activate DNS resolving workaround for Android:
+//  - this is only necessary in general Linux build if we want it to run on Android.
+//  - the workaround isn't needed when targeting specifically Android using Android NDK.
+#define ANDROID_RESOLVE
+#endif
 
 void closesocket_gracefully(SOCKET socket)
 {
@@ -80,7 +76,7 @@ void closesocket_gracefully(SOCKET socket)
 	closesocket(socket);
 }
 
-#ifdef __linux__
+#ifdef ANDROID_RESOLVE
 CString ResolveAndroidHost(const char* host);
 #endif
 
@@ -109,8 +105,6 @@ void Connection::Init()
 	m_getHostByNameMutex = std::make_unique<Mutex>();
 #endif
 #endif
-
-	m_connectionFinalizer = std::make_unique<ConnectionFinalizer>();
 }
 
 void Connection::Final()
@@ -151,7 +145,7 @@ Connection::~Connection()
 {
 	debug("Destroying Connection");
 
-	Disconnect();
+	Connection::Disconnect();
 }
 
 void Connection::SetSuppressErrors(bool suppressErrors)
@@ -215,6 +209,8 @@ bool Connection::Bind()
 		return true;
 	}
 
+	int errcode = 0;
+
 #ifndef WIN32
 	if (m_host && m_host[0] == '/')
 	{
@@ -286,6 +282,7 @@ bool Connection::Bind()
 					break;
 				}
 				// Connection failed
+				errcode = GetLastNetworkError();
 				closesocket(m_socket);
 				m_socket = INVALID_SOCKET;
 			}
@@ -326,6 +323,7 @@ bool Connection::Bind()
 		if (res == -1)
 		{
 			// Connection failed
+			errcode = GetLastNetworkError();
 			closesocket(m_socket);
 			m_socket = INVALID_SOCKET;
 		}
@@ -334,7 +332,7 @@ bool Connection::Bind()
 
 	if (m_socket == INVALID_SOCKET)
 	{
-		ReportError("Binding socket failed for %s", m_host, true);
+		ReportError("Binding socket failed for %s", m_host, true, errcode);
 		return false;
 	}
 
@@ -600,7 +598,7 @@ bool Connection::DoConnect()
 		int res = getaddrinfo(m_host, portStr, &addr_hints, &addr_list);
 		debug("getaddrinfo for %s: %i", *m_host, res);
 
-#ifdef __linux__
+#ifdef ANDROID_RESOLVE
 		if (res != 0)
 		{
 			CString resolvedHost = ResolveAndroidHost(m_host);
@@ -791,18 +789,15 @@ bool Connection::ConnectWithTimeout(void* address, int address_len)
 	ret = connect(m_socket, (struct sockaddr*)address, address_len);
 	if (ret < 0)
 	{
+		int err = GetLastNetworkError();
 #ifdef WIN32
-		int err = WSAGetLastError();
 		if (err != WSAEWOULDBLOCK)
-		{
-			return false;
-		}
 #else
-		if (errno != EINPROGRESS)
+		if (err != EINPROGRESS)
+#endif
 		{
 			return false;
 		}
-#endif
 	}
 
 	//connect succeeded right away?
@@ -922,7 +917,16 @@ void Connection::Cancel()
 	}
 }
 
-void Connection::ReportError(const char* msgPrefix, const char* msgArg, bool PrintErrCode, int herrno, const char* herrMsg)
+int Connection::GetLastNetworkError()
+{
+#ifdef WIN32
+	return WSAGetLastError();
+#else
+	return errno;
+#endif
+}
+
+void Connection::ReportError(const char* msgPrefix, const char* msgArg, bool printErrCode, int errCode, const char* errMsg)
 {
 #ifndef DISABLE_TLS
 	if (m_tlsError)
@@ -935,34 +939,34 @@ void Connection::ReportError(const char* msgPrefix, const char* msgArg, bool Pri
 
 	BString<1024> errPrefix(msgPrefix, msgArg);
 
-	if (PrintErrCode)
+	if (printErrCode)
 	{
-#ifdef WIN32
-		int ErrCode = WSAGetLastError();
-		char errMsg[1024];
-		errMsg[0] = '\0';
-		FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, ErrCode, 0, errMsg, 1024, nullptr);
-		errMsg[1024-1] = '\0';
-#else
-		const char* errMsg = herrMsg;
-		int ErrCode = herrno;
-		if (herrno == 0)
+		BString<1024> printErrMsg;
+		if (errCode == 0)
 		{
-			ErrCode = errno;
-			errMsg = strerror(ErrCode);
+			errCode = GetLastNetworkError();
 		}
-		else if (!herrMsg)
+		if (errMsg)
 		{
-			errMsg = hstrerror(ErrCode);
-		}
-#endif
-		if (m_suppressErrors)
-		{
-			debug("%s: ErrNo %i, %s", *errPrefix, ErrCode, errMsg);
+			printErrMsg = errMsg;
 		}
 		else
 		{
-			PrintError(BString<1024>("%s: ErrNo %i, %s", *errPrefix, ErrCode, errMsg));
+#ifdef WIN32
+			FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, errCode, 0, printErrMsg, printErrMsg.Capacity(), nullptr);
+			printErrMsg[1024-1] = '\0';
+#else
+			printErrMsg = strerror(errCode);
+#endif
+		}
+
+		if (m_suppressErrors)
+		{
+			debug("%s: Error %i - %s", *errPrefix, errCode, (const char*)printErrMsg);
+		}
+		else
+		{
+			PrintError(BString<1024>("%s: Error %i - %s", *errPrefix, errCode, (const char*)printErrMsg));
 		}
 	}
 	else
@@ -1080,7 +1084,7 @@ in_addr_t Connection::ResolveHostAddr(const char* host)
 #endif
 		if (err)
 		{
-			ReportError("Could not resolve hostname %s", host, true, h_errnop);
+			ReportError("Could not resolve hostname %s", host, true, h_errnop, hstrerror(h_errnop));
 			return INADDR_NONE;
 		}
 
@@ -1146,7 +1150,7 @@ int Connection::FetchTotalBytesRead()
 }
 
 
-#ifdef __linux__
+#ifdef ANDROID_RESOLVE
 
 //******************************************************************************
 // Android resolver proxy from AOSP (reworked):

daemon/connect/Connection.h

@@ -56,6 +56,7 @@ public:
 	Connection(SOCKET socket, bool tls);
 	virtual ~Connection();
 	static void Init();
+	static void Final();
 	virtual bool Connect();
 	virtual bool Disconnect();
 	bool Bind();
@@ -135,9 +136,10 @@ protected:
 #endif
 #endif
 
-	void ReportError(const char* msgPrefix, const char* msgArg, bool PrintErrCode, int herrno = 0,
-		const char* herrMsg = nullptr);
+	void ReportError(const char* msgPrefix, const char* msgArg, bool printErrCode, int errCode = 0,
+		const char* errMsg = nullptr);
 	virtual void PrintError(const char* errMsg);
+	int GetLastNetworkError();
 	bool DoConnect();
 	bool DoDisconnect();
 	bool InitSocketOpts(SOCKET socket);
@@ -150,10 +152,6 @@ protected:
 	int send(SOCKET s, const char* buf, int len, int flags);
 	void CloseTls();
 #endif
-
-private:
-	static void Final();
-	friend class ConnectionFinalizer;
 };
 
 #endif

daemon/connect/TlsSocket.cpp

@@ -28,16 +28,6 @@
 #include "Util.h"
 #include "FileSystem.h"
 
-class TlsSocketFinalizer
-{
-public:
-	~TlsSocketFinalizer()
-	{
-		TlsSocket::Final();
-	}
-};
-
-std::unique_ptr<TlsSocketFinalizer> m_tlsSocketFinalizer;
 CString TlsSocket::m_certStore;
 
 #ifdef HAVE_LIBGNUTLS
@@ -189,8 +179,6 @@ void TlsSocket::Init()
 	OpenSSL_add_all_algorithms();
 
 #endif /* HAVE_OPENSSL */
-
-	m_tlsSocketFinalizer = std::make_unique<TlsSocketFinalizer>();
 }
 
 void TlsSocket::Final()
@@ -207,14 +195,18 @@ void TlsSocket::Final()
 	CRYPTO_set_locking_callback(nullptr);
 	CRYPTO_set_id_callback(nullptr);
 #endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	ERR_remove_state(0);
+#endif
 #if	OPENSSL_VERSION_NUMBER >= 0x10002000L && ! defined (LIBRESSL_VERSION_NUMBER)
 	SSL_COMP_free_compression_methods();
 #endif
 	//ENGINE_cleanup();
 	CONF_modules_free();
 	CONF_modules_unload(1);
+#ifndef OPENSSL_NO_COMP
 	COMP_zlib_cleanup();
+#endif
 	ERR_free_strings();
 	EVP_cleanup();
 	CRYPTO_cleanup_all_ex_data();
@@ -226,8 +218,10 @@ TlsSocket::~TlsSocket()
 	Close();
 
 #ifdef HAVE_OPENSSL
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	ERR_remove_state(0);
 #endif
+#endif
 }
 
 void TlsSocket::ReportError(const char* errMsg, bool suppressable)
@@ -571,7 +565,7 @@ bool TlsSocket::ValidateCert()
 	// hostname verification
 	if (!m_host.Empty() && X509_check_host(cert, m_host, m_host.Length(), 0, nullptr) != 1)
 	{
-		char* certHost = nullptr;
+		const unsigned char* certHost = nullptr;
         // Find the position of the CN field in the Subject field of the certificate
         int common_name_loc = X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1);
         if (common_name_loc >= 0)
@@ -584,7 +578,11 @@ bool TlsSocket::ValidateCert()
 				ASN1_STRING* common_name_asn1 = X509_NAME_ENTRY_get_data(common_name_entry);
 				if (common_name_asn1 != nullptr)
 				{
-					certHost = (char*)ASN1_STRING_data(common_name_asn1);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+					certHost = ASN1_STRING_get0_data(common_name_asn1);
+#else
+					certHost = ASN1_STRING_data(common_name_asn1);
+#endif
 				}
 			}
         }

daemon/connect/TlsSocket.h

@@ -34,6 +34,7 @@ public:
 	virtual ~TlsSocket();
 	static void Init();
 	static void InitOptions(const char* certStore) { m_certStore = certStore; }
+	static void Final();
 	bool Start();
 	void Close();
 	int Send(const char* buffer, int size);
@@ -44,12 +45,12 @@ protected:
 	virtual void PrintError(const char* errMsg);
 
 private:
+	SOCKET m_socket;
 	bool m_isClient;
 	CString m_host;
 	CString m_certFile;
 	CString m_keyFile;
 	CString m_cipher;
-	SOCKET m_socket;
 	bool m_suppressErrors = false;
 	bool m_initialized = false;
 	bool m_connected = false;
@@ -62,9 +63,6 @@ private:
 
 	void ReportError(const char* errMsg, bool suppressable = true);
 	bool ValidateCert();
-
-	static void Final();
-	friend class TlsSocketFinalizer;
 };
 
 #endif

daemon/connect/WebDownloader.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "WebDownloader.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Util.h"
 #include "FileSystem.h"
 
@@ -72,19 +73,19 @@ void WebDownloader::Run()
 
 		if ((((Status == adFailed) && (remainedDownloadRetries > 1)) ||
 			((Status == adConnectError) && (remainedConnectRetries > 1)))
-			&& !IsStopped() && !(!m_force && g_Options->GetPauseDownload()))
+			&& !IsStopped() && !(!m_force && g_WorkState->GetPauseDownload()))
 		{
 			detail("Waiting %i sec to retry", g_Options->GetUrlInterval());
 			int msec = 0;
 			while (!IsStopped() && (msec < g_Options->GetUrlInterval() * 1000) &&
-				!(!m_force && g_Options->GetPauseDownload()))
+				!(!m_force && g_WorkState->GetPauseDownload()))
 			{
-				usleep(100 * 1000);
+				Util::Sleep(100);
 				msec += 100;
 			}
 		}
 
-		if (IsStopped() || (!m_force && g_Options->GetPauseDownload()))
+		if (IsStopped() || (!m_force && g_WorkState->GetPauseDownload()))
 		{
 			Status = adRetry;
 			break;

daemon/extension/PostScript.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@
 #include "Log.h"
 #include "Util.h"
 #include "Options.h"
+#include "WorkState.h"
 
 static const int POSTPROCESS_PARCHECK = 92;
 static const int POSTPROCESS_SUCCESS = 93;
@@ -272,16 +273,16 @@ void PostScriptController::AddMessage(Message::EKind kind, const char* text)
 		m_postInfo->SetProgressLabel(text);
 	}
 
-	if (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = m_postInfo->GetStageTime();
 		time_t startTime = m_postInfo->GetStartTime();
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
+		while (g_WorkState->GetPausePostProcess() && !m_postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 
 			// update time stamps
 

daemon/feed/FeedCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "FeedCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "WebDownloader.h"
 #include "Util.h"
 #include "FileSystem.h"
@@ -29,6 +30,7 @@
 #include "FeedScript.h"
 #include "DiskState.h"
 #include "DupeCoordinator.h"
+#include "UrlCoordinator.h"
 
 std::unique_ptr<RegEx>& FeedCoordinator::FilterHelper::GetRegEx(int id)
 {
@@ -65,6 +67,9 @@ FeedCoordinator::FeedCoordinator()
 
 	m_downloadQueueObserver.m_owner = this;
 	DownloadQueue::Guard()->Attach(&m_downloadQueueObserver);
+
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
 }
 
 FeedCoordinator::~FeedCoordinator()
@@ -84,7 +89,7 @@ void FeedCoordinator::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
 	if (g_Options->GetServerMode())
@@ -93,60 +98,68 @@ void FeedCoordinator::Run()
 		g_DiskState->LoadFeeds(&m_feeds, &m_feedHistory);
 	}
 
-	int sleepInterval = 100;
-	int updateCounter = 0;
-	int cleanupCounter = 60000;
-
+	time_t lastCleanup = 0;
 	while (!IsStopped())
 	{
-		usleep(sleepInterval * 1000);
-
-		updateCounter += sleepInterval;
-		if (updateCounter >= 1000)
+		// this code should not be called too often, once per second is OK
+		if (!g_WorkState->GetPauseDownload() || m_force || g_Options->GetUrlForce())
 		{
-			// this code should not be called too often, once per second is OK
+			Guard guard(m_downloadsMutex);
 
-			if (!g_Options->GetPauseDownload() || m_force || g_Options->GetUrlForce())
+			time_t current = Util::CurrentTime();
+			if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
 			{
-				Guard guard(m_downloadsMutex);
-
-				time_t current = Util::CurrentTime();
-				if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
+				m_force = false;
+				// check feed list and update feeds
+				for (FeedInfo* feedInfo : &m_feeds)
 				{
-					m_force = false;
-					// check feed list and update feeds
-					for (FeedInfo* feedInfo : &m_feeds)
+					if (((feedInfo->GetInterval() > 0 &&
+						 (feedInfo->GetNextUpdate() == 0 ||
+						  current >= feedInfo->GetNextUpdate() ||
+						  current < feedInfo->GetNextUpdate() - feedInfo->GetInterval() * 60)) ||
+						feedInfo->GetFetch()) &&
+						feedInfo->GetStatus() != FeedInfo::fsRunning)
 					{
-						if (((feedInfo->GetInterval() > 0 &&
-							 (feedInfo->GetNextUpdate() == 0 ||
-							  current >= feedInfo->GetNextUpdate() ||
-							  current < feedInfo->GetNextUpdate() - feedInfo->GetInterval() * 60)) ||
-							feedInfo->GetFetch()) &&
-							feedInfo->GetStatus() != FeedInfo::fsRunning)
-						{
-							StartFeedDownload(feedInfo, feedInfo->GetFetch());
-						}
-						else if (feedInfo->GetFetch())
-						{
-							m_force = true;
-						}
+						StartFeedDownload(feedInfo, feedInfo->GetFetch());
+					}
+					else if (feedInfo->GetFetch())
+					{
+						m_force = true;
 					}
 				}
 			}
-
-			CheckSaveFeeds();
-			ResetHangingDownloads();
-			updateCounter = 0;
 		}
 
-		cleanupCounter += sleepInterval;
-		if (cleanupCounter >= 60000)
+		CheckSaveFeeds();
+		ResetHangingDownloads();
+
+		if (std::abs(Util::CurrentTime() - lastCleanup) >= 60)
 		{
 			// clean up feed history once a minute
 			CleanupHistory();
 			CleanupCache();
 			CheckSaveFeeds();
-			cleanupCounter = 0;
+			lastCleanup = Util::CurrentTime();
+		}
+
+		Guard guard(m_downloadsMutex);
+		if (m_force)
+		{
+			// don't sleep too long if there active feeds scheduled for redownload
+			m_waitCond.WaitFor(m_downloadsMutex, 1000, [&]{ return IsStopped(); });
+		}
+		else
+		{
+			// no active jobs, we can sleep longer:
+			//  - if option "UrlForce" is active or if the feed list is empty we need to wake up
+			//    only when a new feed preview is requested. We could wait indefinitely for that
+			//    but we need to do some job every now and then and therefore we sleep only 60 seconds.
+			//  - if option "UrlForce" is disabled we need also to wake up when state "DownloadPaused"
+			//    is changed. We detect this via notification from 'WorkState'. However such
+			//    notifications are not 100% reliable due to possible race conditions. Therefore
+			//    we sleep for max. 5 seconds.
+			int waitInterval = g_Options->GetUrlForce() || m_feeds.empty() ? 60000 : 5000;
+			m_waitCond.WaitFor(m_downloadsMutex, waitInterval, [&]{ return m_force || IsStopped(); });
 		}
 	}
 
@@ -160,7 +173,7 @@ void FeedCoordinator::Run()
 			completed = m_activeDownloads.size() == 0;
 		}
 		CheckSaveFeeds();
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 	debug("FeedCoordinator: Downloads are completed");
@@ -179,6 +192,15 @@ void FeedCoordinator::Stop()
 		feedDownloader->Stop();
 	}
 	debug("UrlDownloads are notified");
+
+	// Resume Run() to exit it
+	m_waitCond.NotifyAll();
+}
+
+void FeedCoordinator::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_force = true;
+	m_waitCond.NotifyAll();
 }
 
 void FeedCoordinator::ResetHangingDownloads()
@@ -308,12 +330,10 @@ void FeedCoordinator::FeedCompleted(FeedDownloader* feedDownloader)
 				m_save = true;
 			}
 
-			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 			for (std::unique_ptr<NzbInfo>& nzbInfo : addedNzbs)
 			{
-				downloadQueue->GetQueue()->Add(std::move(nzbInfo));
+				g_UrlCoordinator->AddUrlToQueue(std::move(nzbInfo), false);
 			}
-			downloadQueue->Save();
 		}
 		feedInfo->SetStatus(FeedInfo::fsFinished);
 	}
@@ -455,6 +475,9 @@ std::unique_ptr<NzbInfo> FeedCoordinator::CreateNzbInfo(FeedInfo* feedInfo, Feed
 	nzbInfo->SetDupeKey(feedItemInfo.GetDupeKey());
 	nzbInfo->SetDupeScore(feedItemInfo.GetDupeScore());
 	nzbInfo->SetDupeMode(feedItemInfo.GetDupeMode());
+	nzbInfo->SetSize(feedItemInfo.GetSize());
+	nzbInfo->SetMinTime(feedItemInfo.GetTime());
+	nzbInfo->SetMaxTime(feedItemInfo.GetTime());
 
 	return nzbInfo;
 }
@@ -520,12 +543,15 @@ std::shared_ptr<FeedItemList> FeedCoordinator::PreviewFeed(int id,
 			}
 
 			StartFeedDownload(feedInfo.get(), true);
+
+			m_force = true;
+			m_waitCond.NotifyAll();
 		}
 
 		// wait until the download in a separate thread completes
 		while (feedInfo->GetStatus() == FeedInfo::fsRunning)
 		{
-			usleep(100 * 1000);
+			Util::Sleep(100);
 		}
 
 		// now can process the feed
@@ -584,6 +610,8 @@ void FeedCoordinator::FetchFeed(int id)
 			m_force = true;
 		}
 	}
+
+	m_waitCond.NotifyAll();
 }
 
 std::unique_ptr<FeedFile> FeedCoordinator::parseFeed(FeedInfo* feedInfo)

daemon/feed/FeedCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -67,6 +67,13 @@ private:
 		virtual void Update(Subject* caller, void* aspect) { m_owner->DownloadQueueUpdate(caller, aspect); }
 	};
 
+	class WorkStateObserver: public Observer
+	{
+	public:
+		FeedCoordinator* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	class FeedCacheItem
 	{
 	public:
@@ -106,9 +113,12 @@ private:
 	FeedHistory m_feedHistory;
 	Mutex m_downloadsMutex;
 	DownloadQueueObserver m_downloadQueueObserver;
+	WorkStateObserver m_workStateObserver;
 	bool m_force = false;
 	bool m_save = false;
 	FeedCache m_feedCache;
+	ConditionVar m_waitCond;
+	bool m_wokenUp = false;
 
 	void StartFeedDownload(FeedInfo* feedInfo, bool force);
 	void FeedCompleted(FeedDownloader* feedDownloader);
@@ -122,6 +132,7 @@ private:
 	void CheckSaveFeeds();
 	std::unique_ptr<FeedFile> parseFeed(FeedInfo* feedInfo);
 	void SchedulerNextUpdate(FeedInfo* feedInfo, bool success);
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 extern FeedCoordinator* g_FeedCoordinator;

daemon/feed/FeedFilter.cpp

@@ -51,7 +51,7 @@ bool FeedFilter::Term::MatchValue(const char* strValue, int64 intValue)
 
 	if (m_command < fcEqual && !strValue)
 	{
-		intBuf.Format("%lld", intValue);
+		intBuf.Format("%" PRId64, intValue);
 		strValue = intBuf;
 	}
 

daemon/feed/FeedInfo.h

@@ -72,6 +72,7 @@ private:
 	int m_id;
 	CString m_name;
 	CString m_url;
+	bool m_backlog;
 	int m_interval;
 	CString m_filter;
 	uint32 m_filterHash;
@@ -87,7 +88,6 @@ private:
 	CString m_outputFilename;
 	bool m_fetch = false;
 	bool m_force = false;
-	bool m_backlog;
 };
 
 typedef std::deque<std::unique_ptr<FeedInfo>> Feeds;

daemon/frontend/Frontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 
 #include "nzbget.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Frontend.h"
 #include "Log.h"
 #include "Connection.h"
@@ -33,9 +34,24 @@ Frontend::Frontend()
 {
 	debug("Creating Frontend");
 
+	m_workStateObserver.m_owner = this;
+	g_WorkState->Attach(&m_workStateObserver);
+
 	m_updateInterval = g_Options->GetUpdateInterval();
 }
 
+void Frontend::Stop()
+{
+	Thread::Stop();
+
+	m_waitCond.NotifyAll();
+}
+
+void Frontend::WorkStateUpdate(Subject* caller, void* aspect)
+{
+	m_waitCond.NotifyAll();
+}
+
 bool Frontend::PrepareData()
 {
 	if (IsRemoteMode())
@@ -57,8 +73,8 @@ bool Frontend::PrepareData()
 		if (m_summary)
 		{
 			m_currentDownloadSpeed = g_StatMeter->CalcCurrentDownloadSpeed();
-			m_pauseDownload = g_Options->GetPauseDownload();
-			m_downloadLimit = g_Options->GetDownloadRate();
+			m_pauseDownload = g_WorkState->GetPauseDownload();
+			m_downloadLimit = g_WorkState->GetSpeedLimit();
 			m_threadCount = Thread::GetThreadCount();
 			g_StatMeter->CalcTotalStat(&m_upTimeSec, &m_dnTimeSec, &m_allBytes, &m_standBy);
 
@@ -108,8 +124,8 @@ void Frontend::ServerPauseUnpause(bool pause)
 	}
 	else
 	{
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(pause);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(pause);
 	}
 }
 
@@ -121,7 +137,7 @@ void Frontend::ServerSetDownloadRate(int rate)
 	}
 	else
 	{
-		g_Options->SetDownloadRate(rate);
+		g_WorkState->SetSpeedLimit(rate);
 	}
 }
 
@@ -307,3 +323,16 @@ bool Frontend::RequestEditQueue(DownloadQueue::EEditAction action, int offset, i
 	IdList ids = { id };
 	return client.RequestServerEditQueue(action, offset, nullptr, &ids, nullptr, rmId);
 }
+
+void Frontend::Wait(int milliseconds)
+{
+	if (g_WorkState->GetPauseFrontend())
+	{
+		Guard guard(m_waitMutex);
+		m_waitCond.WaitFor(m_waitMutex, 2000);
+	}
+	else
+	{
+		Util::Sleep(milliseconds);
+	}
+}

daemon/frontend/Frontend.h

@@ -27,6 +27,7 @@
 #include "DownloadInfo.h"
 #include "MessageBase.h"
 #include "QueueEditor.h"
+#include "Observer.h"
 
 class Frontend : public Thread
 {
@@ -51,7 +52,10 @@ protected:
 	int m_dnTimeSec = 0;
 	int64 m_allBytes = 0;
 	bool m_standBy = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
+	virtual void Stop();
 	bool PrepareData();
 	void FreeData();
 	GuardedMessageList GuardMessages();
@@ -63,12 +67,22 @@ protected:
 	bool RequestSetDownloadRate(int rate);
 	bool ServerEditQueue(DownloadQueue::EEditAction action, int offset, int entry);
 	bool RequestEditQueue(DownloadQueue::EEditAction action, int offset, int id);
+	void Wait(int milliseconds);
 
 private:
+	class WorkStateObserver : public Observer
+	{
+	public:
+		Frontend* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->WorkStateUpdate(caller, aspect); }
+	};
+
 	MessageList m_remoteMessages;
+	WorkStateObserver m_workStateObserver;
 
 	bool RequestMessages();
 	bool RequestFileList();
+	void WorkStateUpdate(Subject* caller, void* aspect);
 };
 
 #endif

daemon/frontend/LoggableFrontend.cpp

@@ -20,6 +20,7 @@
 
 
 #include "nzbget.h"
+#include "Util.h"
 #include "LoggableFrontend.h"
 #include "Log.h"
 
@@ -30,7 +31,7 @@ void LoggableFrontend::Run()
 	while (!IsStopped())
 	{
 		Update();
-		usleep(m_updateInterval * 1000);
+		Wait(m_updateInterval);
 	}
 	// Printing the last messages
 	Update();

daemon/frontend/NCursesFrontend.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -213,8 +213,10 @@ void NCursesFrontend::Run()
 			m_dataUpdatePos = m_updateInterval;
 		}
 
-		usleep(10 * 1000);
-		m_dataUpdatePos -= 10;
+		// update more often (sleep shorter) if need faster reaction on user input
+		int sleepInterval = m_inputMode == normal ? 100 : 10;
+		Wait(sleepInterval);
+		m_dataUpdatePos -= sleepInterval;
 	}
 
 	FreeData();

daemon/main/CommandLineParser.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -915,25 +915,18 @@ void CommandLineParser::ParseFileIdList(int argc, const char* argv[], int optind
 			}
 
 			int editQueueIdCount = 0;
-			if (editQueueIdTo != 0)
+			if (editQueueIdFrom < editQueueIdTo)
 			{
-				if (editQueueIdFrom < editQueueIdTo)
-				{
-					editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
-				}
-				else
-				{
-					editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
-				}
+				editQueueIdCount = editQueueIdTo - editQueueIdFrom + 1;
 			}
 			else
 			{
-				editQueueIdCount = 1;
+				editQueueIdCount = editQueueIdFrom - editQueueIdTo + 1;
 			}
 
 			for (int i = 0; i < editQueueIdCount; i++)
 			{
-				if (editQueueIdFrom < editQueueIdTo || editQueueIdTo == 0)
+				if (editQueueIdFrom < editQueueIdTo)
 				{
 					m_editQueueIdList.push_back(editQueueIdFrom + i);
 				}

daemon/main/CommandLineParser.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by

daemon/main/DiskService.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,23 +21,39 @@
 #include "nzbget.h"
 #include "DiskService.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "StatMeter.h"
 #include "Log.h"
 #include "Util.h"
 #include "FileSystem.h"
 
+DiskService::DiskService()
+{
+	g_WorkState->Attach(this);
+}
+
+void DiskService::Update(Subject* caller, void* aspect)
+{
+	WakeUp();
+}
+
+int DiskService::ServiceInterval()
+{
+	return m_waitingRequiredDir ? 1 :
+		g_Options->GetDiskSpace() <= 0 ? Service::Sleep :
+		// notifications from 'WorkState' are not 100% reliable due to race conditions
+		!g_WorkState->GetDownloading() ? 10 :
+		1;
+}
+
 void DiskService::ServiceWork()
 {
-	m_interval++;
-	if (m_interval == 5)
+	debug("Disk service work");
+
+	if (g_Options->GetDiskSpace() > 0 && g_WorkState->GetDownloading())
 	{
-		if (!g_Options->GetPauseDownload() &&
-			g_Options->GetDiskSpace() > 0 && !g_StatMeter->GetStandBy())
-		{
-			// check free disk space every 1 second
-			CheckDiskSpace();
-		}
-		m_interval = 0;
+		// check free disk space every 1 second
+		CheckDiskSpace();
 	}
 
 	if (m_waitingRequiredDir)
@@ -48,11 +64,13 @@ void DiskService::ServiceWork()
 
 void DiskService::CheckDiskSpace()
 {
+	debug("Disk service work: check disk space");
+
 	int64 freeSpace = FileSystem::FreeDiskSize(g_Options->GetDestDir());
 	if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 	{
 		warn("Low disk space on %s. Pausing download", g_Options->GetDestDir());
-		g_Options->SetPauseDownload(true);
+		g_WorkState->SetPauseDownload(true);
 	}
 
 	if (!Util::EmptyStr(g_Options->GetInterDir()))
@@ -61,13 +79,15 @@ void DiskService::CheckDiskSpace()
 		if (freeSpace > -1 && freeSpace / 1024 / 1024 < g_Options->GetDiskSpace())
 		{
 			warn("Low disk space on %s. Pausing download", g_Options->GetInterDir());
-			g_Options->SetPauseDownload(true);
+			g_WorkState->SetPauseDownload(true);
 		}
 	}
 }
 
 void DiskService::CheckRequiredDir()
 {
+	debug("Disk service work: check required dir");
+
 	if (!Util::EmptyStr(g_Options->GetRequiredDir()))
 	{
 		bool allExist = true;
@@ -97,7 +117,7 @@ void DiskService::CheckRequiredDir()
 		info("All required directories available");
 	}
 
-	g_Options->SetTempPauseDownload(false);
-	g_Options->SetTempPausePostprocess(false);
+	g_WorkState->SetTempPauseDownload(false);
+	g_WorkState->SetTempPausePostprocess(false);
 	m_waitingRequiredDir = false;
 }

daemon/main/DiskService.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,15 +22,19 @@
 #define DISKSERVICE_H
 
 #include "Service.h"
+#include "Observer.h"
 
-class DiskService : public Service
+class DiskService : public Service, public Observer
 {
+public:
+	DiskService();
+
 protected:
-	virtual int ServiceInterval() { return 200; }
+	virtual int ServiceInterval();
 	virtual void ServiceWork();
+	virtual void Update(Subject* caller, void* aspect);
 
 private:
-	int m_interval = 0;
 	bool m_waitingRequiredDir = true;
 	bool m_waitingReported = false;
 

daemon/main/Maintenance.cpp

@@ -68,7 +68,7 @@ Maintenance::~Maintenance()
 	{
 		while (m_updateScriptController)
 		{
-			usleep(20*1000);
+			Util::Sleep(20);
 		}
 	}
 }
@@ -179,7 +179,7 @@ bool Maintenance::ReadPackageInfoStr(const char* key, CString& value)
 		return false;
 	}
 
-	int len = (int)(pend - p);
+	size_t len = pend - p;
 	if (len >= sizeof(fileName))
 	{
 		error("Could not parse file %s", *fileName);

daemon/main/Options.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -642,8 +642,6 @@ void Options::CheckDir(CString& dir, const char* optionName,
 
 		FileSystem::NormalizePathSeparators((char*)usedir2);
 		dir = usedir2;
-
-		usedir2[usedir2.Length() - 1] = '\0';
 		SetOption(optionName, usedir2);
 	}
 
@@ -1906,10 +1904,10 @@ void Options::MergeOldScriptOption(OptEntries* optEntries, const char* optname,
 		{
 			for (OptEntry& opt : optEntries)
 			{
-				const char* optname = opt.GetName();
-				if (!strncasecmp(optname, "category", 8))
+				const char* catoptname = opt.GetName();
+				if (!strncasecmp(catoptname, "category", 8))
 				{
-					char* p = (char*)optname + 8;
+					char* p = (char*)catoptname + 8;
 					while (*p >= '0' && *p <= '9') p++;
 					if (p && (!strcasecmp(p, ".extensions")))
 					{

daemon/main/Options.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -300,6 +300,7 @@ public:
 	bool GetDirectRename() { return m_directRename; }
 	bool GetReorderFiles() { return m_reorderFiles; }
 	EFileNaming GetFileNaming() { return m_fileNaming; }
+	int GetDownloadRate() const { return m_downloadRate; }
 
 	Categories* GetCategories() { return &m_categories; }
 	Category* FindCategory(const char* name, bool searchAliases) { return m_categories.FindCategory(name, searchAliases); }
@@ -311,24 +312,6 @@ public:
 	bool GetDaemonMode() { return m_daemonMode; }
 	void SetRemoteClientMode(bool remoteClientMode) { m_remoteClientMode = remoteClientMode; }
 	bool GetRemoteClientMode() { return m_remoteClientMode; }
-	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; }
-	bool GetPauseDownload() const { return m_pauseDownload; }
-	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; }
-	bool GetPausePostProcess() const { return m_pausePostProcess; }
-	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; }
-	bool GetPauseScan() const { return m_pauseScan; }
-	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; }
-	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
-	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
-	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; }
-	void SetDownloadRate(int rate) { m_downloadRate = rate; }
-	int GetDownloadRate() const { return m_downloadRate; }
-	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; }
-	time_t GetResumeTime() const { return m_resumeTime; }
-	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; }
-	int GetLocalTimeOffset() { return m_localTimeOffset; }
-	void SetQuotaReached(bool quotaReached) { m_quotaReached = quotaReached; }
-	bool GetQuotaReached() { return m_quotaReached; }
 
 private:
 	OptEntries m_optEntries;
@@ -450,20 +433,12 @@ private:
 	int m_dailyQuota = 0;
 	bool m_reorderFiles = false;
 	EFileNaming m_fileNaming = nfArticle;
+	int m_downloadRate = 0;
 
-	// Current state
+	// Application mode
 	bool m_serverMode = false;
 	bool m_daemonMode = false;
 	bool m_remoteClientMode = false;
-	bool m_pauseDownload = false;
-	bool m_pausePostProcess = false;
-	bool m_pauseScan = false;
-	bool m_tempPauseDownload = true;
-	bool m_tempPausePostprocess = true;
-	int m_downloadRate = 0;
-	time_t m_resumeTime = 0;
-	int m_localTimeOffset = 0;
-	bool m_quotaReached = false;
 
 	void Init(const char* exeName, const char* configFilename, bool noConfig,
 		CmdOptList* commandLineOptions, bool noDiskAccess, Extender* extender);

daemon/main/Scheduler.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Scheduler.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "NewsServer.h"
 #include "ServerPool.h"
@@ -51,13 +52,33 @@ void Scheduler::FirstCheck()
 	CheckTasks();
 }
 
+void Scheduler::ScheduleNextWork()
+{
+	// Ideally we should calculate wait time until next scheduler task or until resume time.
+	// The first isn't trivial and the second requires watching/reaction on changed scheduled resume time.
+	// We do it simpler instead: check once per minute, when seconds are changing from 59 to 00.
+
+	time_t curTime = Util::CurrentTime();
+	tm sched;
+	gmtime_r(&curTime, &sched);
+	sched.tm_min++;
+	sched.tm_sec = 0;
+	time_t nextMinute = Util::Timegm(&sched);
+
+	m_serviceInterval = nextMinute - curTime;
+}
+
 void Scheduler::ServiceWork()
 {
+	debug("Scheduler service work");
+
 	if (!DownloadQueue::IsLoaded())
 	{
 		return;
 	}
 
+	debug("Scheduler service work: doing work");
+
 	if (!m_firstChecked)
 	{
 		FirstCheck();
@@ -68,6 +89,7 @@ void Scheduler::ServiceWork()
 	m_executeProcess = true;
 	CheckTasks();
 	CheckScheduledResume();
+	ScheduleNextWork();
 }
 
 void Scheduler::CheckTasks()
@@ -100,8 +122,8 @@ void Scheduler::CheckTasks()
 				}
 			}
 
-			time_t localCurrent = current + g_Options->GetLocalTimeOffset();
-			time_t localLastCheck = m_lastCheck + g_Options->GetLocalTimeOffset();
+			time_t localCurrent = current + g_WorkState->GetLocalTimeOffset();
+			time_t localLastCheck = m_lastCheck + g_WorkState->GetLocalTimeOffset();
 
 			tm tmCurrent;
 			gmtime_r(&localCurrent, &tmCurrent);
@@ -159,10 +181,12 @@ void Scheduler::CheckTasks()
 
 void Scheduler::ExecuteTask(Task* task)
 {
+#ifdef DEBUG
 	const char* commandName[] = { "Pause", "Unpause", "Pause Post-processing", "Unpause Post-processing",
 		"Set download rate", "Execute process", "Execute script",
 		"Pause Scan", "Unpause Scan", "Enable Server", "Disable Server", "Fetch Feed" };
 	debug("Executing scheduled command: %s", commandName[task->m_command]);
+#endif
 
 	bool executeProcess = m_executeProcess || task->m_hours == Task::STARTUP_TASK;
 
@@ -171,26 +195,26 @@ void Scheduler::ExecuteTask(Task* task)
 		case scDownloadRate:
 			if (!task->m_param.Empty())
 			{
-				g_Options->SetDownloadRate(atoi(task->m_param) * 1024);
+				g_WorkState->SetSpeedLimit(atoi(task->m_param) * 1024);
 				m_downloadRateChanged = true;
 			}
 			break;
 
 		case scPauseDownload:
 		case scUnpauseDownload:
-			g_Options->SetPauseDownload(task->m_command == scPauseDownload);
+			g_WorkState->SetPauseDownload(task->m_command == scPauseDownload);
 			m_pauseDownloadChanged = true;
 			break;
 
 		case scPausePostProcess:
 		case scUnpausePostProcess:
-			g_Options->SetPausePostProcess(task->m_command == scPausePostProcess);
+			g_WorkState->SetPausePostProcess(task->m_command == scPausePostProcess);
 			m_pausePostProcessChanged = true;
 			break;
 
 		case scPauseScan:
 		case scUnpauseScan:
-			g_Options->SetPauseScan(task->m_command == scPauseScan);
+			g_WorkState->SetPauseScan(task->m_command == scPauseScan);
 			m_pauseScanChanged = true;
 			break;
 
@@ -229,19 +253,19 @@ void Scheduler::PrintLog()
 {
 	if (m_downloadRateChanged)
 	{
-		info("Scheduler: setting download rate to %i KB/s", g_Options->GetDownloadRate() / 1024);
+		info("Scheduler: setting download rate to %i KB/s", g_WorkState->GetSpeedLimit() / 1024);
 	}
 	if (m_pauseDownloadChanged)
 	{
-		info("Scheduler: %s download", g_Options->GetPauseDownload() ? "pausing" : "unpausing");
+		info("Scheduler: %s download", g_WorkState->GetPauseDownload() ? "pausing" : "unpausing");
 	}
 	if (m_pausePostProcessChanged)
 	{
-		info("Scheduler: %s post-processing", g_Options->GetPausePostProcess() ? "pausing" : "unpausing");
+		info("Scheduler: %s post-processing", g_WorkState->GetPausePostProcess() ? "pausing" : "unpausing");
 	}
 	if (m_pauseScanChanged)
 	{
-		info("Scheduler: %s scan", g_Options->GetPauseScan() ? "pausing" : "unpausing");
+		info("Scheduler: %s scan", g_WorkState->GetPauseScan() ? "pausing" : "unpausing");
 	}
 	if (m_serverChanged)
 	{
@@ -308,14 +332,14 @@ void Scheduler::FetchFeed(const char* feedList)
 
 void Scheduler::CheckScheduledResume()
 {
-	time_t resumeTime = g_Options->GetResumeTime();
+	time_t resumeTime = g_WorkState->GetResumeTime();
 	time_t currentTime = Util::CurrentTime();
 	if (resumeTime > 0 && currentTime >= resumeTime)
 	{
 		info("Autoresume");
-		g_Options->SetResumeTime(0);
-		g_Options->SetPauseDownload(false);
-		g_Options->SetPausePostProcess(false);
-		g_Options->SetPauseScan(false);
+		g_WorkState->SetResumeTime(0);
+		g_WorkState->SetPauseDownload(false);
+		g_WorkState->SetPausePostProcess(false);
+		g_WorkState->SetPauseScan(false);
 	}
 }

daemon/main/Scheduler.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2008-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2008-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -66,7 +66,7 @@ public:
 	void AddTask(std::unique_ptr<Task> task);
 
 protected:
-	virtual int ServiceInterval() { return 1000; }
+	virtual int ServiceInterval() { return m_serviceInterval; }
 	virtual void ServiceWork();
 
 private:
@@ -84,6 +84,7 @@ private:
 	bool m_serverChanged;
 	ServerStatusList m_serverStatusList;
 	bool m_firstChecked = false;
+	int m_serviceInterval = 1;
 
 	void ExecuteTask(Task* task);
 	void CheckTasks();
@@ -93,6 +94,7 @@ private:
 	void FetchFeed(const char* feedList);
 	void CheckScheduledResume();
 	void FirstCheck();
+	void ScheduleNextWork();
 };
 
 #endif

daemon/main/StackTrace.cpp

@@ -282,7 +282,7 @@ public:
 	void DoSegFault()
 	{
 		char* N = nullptr;
-		strcpy(N, "");
+		*N = '\0';
 	}
 };
 

daemon/main/WorkState.cpp

@@ -0,0 +1,27 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "nzbget.h"
+#include "WorkState.h"
+
+void WorkState::Changed()
+{
+	Notify(nullptr);
+}

daemon/main/WorkState.h

@@ -0,0 +1,76 @@
+/*
+ *  This file is part of nzbget. See <http://nzbget.net>.
+ *
+ *  Copyright (C) 2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef WORKSTATE_H
+#define WORKSTATE_H
+
+#include "Observer.h"
+
+// WorkState is observable but notifications are not 100% reliable.
+// The changes via Set-methods and readings via Get-methods are not synchronized throughout the program.
+// As result race conditions may occur and some changes may go unnoticed.
+// When waiting for changes don't wait too long to avoid lock ups.
+
+class WorkState : public Subject
+{
+public:
+	void SetPauseDownload(bool pauseDownload) { m_pauseDownload = pauseDownload; Changed(); }
+	bool GetPauseDownload() const { return m_pauseDownload; }
+	void SetPausePostProcess(bool pausePostProcess) { m_pausePostProcess = pausePostProcess; Changed(); }
+	bool GetPausePostProcess() const { return m_pausePostProcess; }
+	void SetPauseScan(bool pauseScan) { m_pauseScan = pauseScan; Changed(); }
+	bool GetPauseScan() const { return m_pauseScan; }
+	void SetTempPauseDownload(bool tempPauseDownload) { m_tempPauseDownload = tempPauseDownload; Changed(); }
+	bool GetTempPauseDownload() const { return m_tempPauseDownload; }
+	void SetTempPausePostprocess(bool tempPausePostprocess) { m_tempPausePostprocess = tempPausePostprocess; Changed(); }
+	bool GetTempPausePostprocess() const { return m_tempPausePostprocess; }
+	void SetPauseFrontend(bool pauseFrontend) { m_pauseFrontend = pauseFrontend; Changed(); }
+	bool GetPauseFrontend() const { return m_pauseFrontend; }
+	void SetSpeedLimit(int speedLimit) { m_speedLimit = speedLimit; Changed(); }
+	int GetSpeedLimit() const { return m_speedLimit; }
+	void SetResumeTime(time_t resumeTime) { m_resumeTime = resumeTime; Changed(); }
+	time_t GetResumeTime() const { return m_resumeTime; }
+	void SetLocalTimeOffset(int localTimeOffset) { m_localTimeOffset = localTimeOffset; Changed(); }
+	int GetLocalTimeOffset() { return m_localTimeOffset; }
+	void SetQuotaReached(bool quotaReached) { m_quotaReached = quotaReached; Changed(); }
+	bool GetQuotaReached() { return m_quotaReached; }
+	void SetDownloading(bool downloading) { m_downloading = downloading; Changed(); }
+	bool GetDownloading() { return m_downloading; }
+
+private:
+	bool m_pauseDownload = false;
+	bool m_pausePostProcess = false;
+	bool m_pauseScan = false;
+	bool m_tempPauseDownload = true;
+	bool m_tempPausePostprocess = true;
+	bool m_pauseFrontend = false;
+	int m_downloadRate = 0;
+	time_t m_resumeTime = 0;
+	int m_localTimeOffset = 0;
+	bool m_quotaReached = false;
+	int m_speedLimit = 0;
+	bool m_downloading = false;
+
+	void Changed();
+};
+
+extern WorkState* g_WorkState;
+
+#endif

daemon/main/nzbget.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "Log.h"
 #include "NzbFile.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "CommandLineParser.h"
 #include "ScriptConfig.h"
 #include "Thread.h"
@@ -71,6 +72,7 @@ void RunMain();
 // Globals
 Log* g_Log;
 Options* g_Options;
+WorkState* g_WorkState;
 ServerPool* g_ServerPool;
 QueueCoordinator* g_QueueCoordinator;
 UrlCoordinator* g_UrlCoordinator;
@@ -193,6 +195,7 @@ private:
 	// globals
 	std::unique_ptr<Log> m_log;
 	std::unique_ptr<Options> m_options;
+	std::unique_ptr<WorkState> m_workState;
 	std::unique_ptr<ServerPool> m_serverPool;
 	std::unique_ptr<QueueCoordinator> m_queueCoordinator;
 	std::unique_ptr<UrlCoordinator> m_urlCoordinator;
@@ -223,13 +226,17 @@ private:
 
 	bool m_reloading = false;
 	bool m_daemonized = false;
+	bool m_stopped = false;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	void Init();
+	void Final();
 	void BootConfig();
 	void CreateGlobals();
 	void Cleanup();
 	void PrintOptions();
-	bool ProcessDirect();
+	void ProcessDirect();
 	void ProcessClientRequest();
 	void ProcessWebGet();
 	void ProcessSigVerify();
@@ -320,6 +327,17 @@ void NZBGet::Init()
 	InstallErrorHandler();
 }
 
+void NZBGet::Final()
+{
+	if (!m_reloading)
+	{
+#ifndef DISABLE_TLS
+		TlsSocket::Final();
+#endif
+		Connection::Final();
+	}
+}
+
 void NZBGet::CreateGlobals()
 {
 #ifdef WIN32
@@ -327,6 +345,9 @@ void NZBGet::CreateGlobals()
 	g_WinConsole = m_winConsole.get();
 #endif
 
+	m_workState = std::make_unique<WorkState>();
+	g_WorkState = m_workState.get();
+
 	m_serviceCoordinator = std::make_unique<ServiceCoordinator>();
 	g_ServiceCoordinator = m_serviceCoordinator.get();
 
@@ -400,7 +421,8 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetNoConfig(), (Options::CmdOptList*)m_commandLineParser->GetOptionList(), this);
 	m_options->SetRemoteClientMode(m_commandLineParser->GetRemoteClientMode());
 	m_options->SetServerMode(m_commandLineParser->GetServerMode());
-	m_options->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetPauseDownload(m_commandLineParser->GetPauseDownload());
+	m_workState->SetSpeedLimit(g_Options->GetDownloadRate());
 
 	m_log->InitOptions();
 
@@ -412,9 +434,9 @@ void NZBGet::BootConfig()
 		m_commandLineParser->GetClientOperation() == CommandLineParser::opClientNoOperation)
 	{
 		info("Pausing all activities due to errors in configuration");
-		m_options->SetPauseDownload(true);
-		m_options->SetPausePostProcess(true);
-		m_options->SetPauseScan(true);
+		m_workState->SetPauseDownload(true);
+		m_workState->SetPausePostProcess(true);
+		m_workState->SetPauseScan(true);
 	}
 
 	m_serverPool->SetTimeout(m_options->GetArticleTimeout());
@@ -453,41 +475,35 @@ void NZBGet::Cleanup()
 #endif
 }
 
-bool NZBGet::ProcessDirect()
+void NZBGet::ProcessDirect()
 {
 #ifdef DEBUG
 	if (m_commandLineParser->GetTestBacktrace())
 	{
-		TestSegFault();
+		TestSegFault(); // never returns
 	}
 #endif
 
 	if (m_commandLineParser->GetWebGet())
 	{
-		ProcessWebGet();
-		return true;
+		ProcessWebGet(); // never returns
 	}
 
 	if (m_commandLineParser->GetSigVerify())
 	{
-		ProcessSigVerify();
-		return true;
+		ProcessSigVerify(); // never returns
 	}
 
 	// client request
 	if (m_commandLineParser->GetClientOperation() != CommandLineParser::opClientNoOperation)
 	{
-		ProcessClientRequest();
-		return true;
+		ProcessClientRequest(); // never returns
 	}
 
 	if (m_commandLineParser->GetPrintOptions())
 	{
-		PrintOptions();
-		return true;
+		PrintOptions(); // never returns
 	}
-
-	return false;
 }
 
 void NZBGet::StartRemoteServer()
@@ -531,7 +547,7 @@ void NZBGet::StopRemoteServer()
 		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
 		maxWaitMSec > 0)
 	{
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		maxWaitMSec -= 100;
 	}
 
@@ -550,7 +566,7 @@ void NZBGet::StopRemoteServer()
 		(m_remoteSecureServer && m_remoteSecureServer->IsRunning())) &&
 		maxWaitMSec > 0)
 	{
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		maxWaitMSec -= 100;
 	}
 
@@ -608,7 +624,7 @@ void NZBGet::StopFrontend()
 		}
 		while (m_frontend->IsRunning())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 		}
 		debug("Frontend stopped");
 	}
@@ -688,7 +704,14 @@ void NZBGet::DoMainLoop()
 				m_serviceCoordinator->Stop();
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
+
+		if (m_options->GetServerMode() && !m_stopped)
+		{
+			// wait for stop signal
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&]{ return m_stopped; });
+		}
 	}
 
 	debug("Main program loop terminated");
@@ -700,10 +723,7 @@ void NZBGet::Run(bool reload)
 
 	Init();
 
-	if (ProcessDirect())
-	{
-		return;
-	}
+	ProcessDirect();
 
 	StartRemoteServer();
 	StartFrontend();
@@ -722,6 +742,8 @@ void NZBGet::Run(bool reload)
 
 	StopRemoteServer();
 	StopFrontend();
+
+	Final();
 }
 
 void NZBGet::ProcessClientRequest()
@@ -892,6 +914,11 @@ void NZBGet::Stop(bool reload)
 #endif
 		}
 	}
+
+	// trigger stop/reload signal
+	Guard guard(m_waitMutex);
+	m_stopped = true;
+	m_waitCond.NotifyAll();
 }
 
 void NZBGet::PrintOptions()
@@ -900,6 +927,7 @@ void NZBGet::PrintOptions()
 	{
 		printf("%s = \"%s\"\n", optEntry.GetName(), optEntry.GetValue());
 	}
+	exit(0);
 }
 
 #ifndef WIN32
@@ -932,7 +960,12 @@ void NZBGet::Daemonize()
 			error("Starting daemon failed: could not create lock-file %s", m_options->GetLockFile());
 			exit(1);
 		}
+
+#ifdef HAVE_LOCKF
 		if (lockf(lfp, F_TLOCK, 0) < 0)
+#else
+		if (flock(lfp, LOCK_EX) < 0)
+#endif
 		{
 			error("Starting daemon failed: could not acquire lock on lock-file %s", m_options->GetLockFile());
 			exit(1);

daemon/main/nzbget.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -115,7 +115,7 @@ compiled */
 
 // WINDOWS INCLUDES
 
-// Using "WIN32_LEAN_AND_MEAN" to disable including on many unneeded headers
+// Using "WIN32_LEAN_AND_MEAN" to disable including of many unneeded headers
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 
@@ -169,6 +169,7 @@ using namespace MSXML;
 #include <sys/statvfs.h>
 #include <sys/wait.h>
 #include <sys/un.h>
+#include <sys/file.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <stdint.h>
@@ -208,6 +209,7 @@ using namespace MSXML;
 #include <stdarg.h>
 #include <time.h>
 #include <ctype.h>
+#include <inttypes.h>
 
 #include <string>
 #include <vector>
@@ -221,6 +223,10 @@ using namespace MSXML;
 #include <fstream>
 #include <memory>
 #include <functional>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <chrono>
 
 // NOTE: do not include <iostream> in "nzbget.h". <iostream> contains objects requiring
 // intialization, causing every unit in nzbget to have initialization routine. This in particular
@@ -272,9 +278,6 @@ typedef int pid_t;
 #ifdef HAVE_MEMORY_H
 # include <memory.h>
 #endif
-#ifdef HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
 #endif /* NOT DISABLE_PARCHECK */
 
 
@@ -301,7 +304,6 @@ typedef int pid_t;
 #define S_ISDIR(mode) __S_ISTYPE((mode), _S_IFDIR)
 #define S_ISREG(mode) __S_ISTYPE((mode), _S_IFREG)
 #define S_DIRMODE nullptr
-#define usleep(usec) Sleep((usec) / 1000)
 #define socklen_t int
 #define SHUT_WR 0x01
 #define SHUT_RDWR 0x02
@@ -382,6 +384,16 @@ typedef signed long long int64;
 typedef unsigned long long uint64;
 #endif
 
+#ifndef PRId64
+#define PRId64 "lld"
+#endif
+#ifndef PRIi64
+#define PRIi64 "lli"
+#endif
+#ifndef PRIu64
+#define PRIu64 "llu"
+#endif
+
 typedef unsigned char uchar;
 
 // Assume little endian if byte order is not defined

daemon/nntp/ArticleDownloader.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@
 #include "Decoder.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "StatMeter.h"
 #include "Util.h"
@@ -97,13 +98,13 @@ void ArticleDownloader::Run()
 		while (!m_connection && !(IsStopped() || serverConfigGeneration != g_ServerPool->GetGeneration()))
 		{
 			m_connection = g_ServerPool->GetConnection(level, wantServer, &failedServers);
-			usleep(5 * 1000);
+			Util::Sleep(5);
 		}
 		SetLastUpdateTimeNow();
 		SetStatus(adRunning);
 
-		if (IsStopped() || ((g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
@@ -194,8 +195,8 @@ void ArticleDownloader::Run()
 			break;
 		}
 
-		if (IsStopped() || ((g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) && !force) ||
-			(g_Options->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
+		if (IsStopped() || ((g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) && !force) ||
+			(g_WorkState->GetTempPauseDownload() && !m_fileInfo->GetExtraPriority()) ||
 			serverConfigGeneration != g_ServerPool->GetGeneration())
 		{
 			status = adRetry;
@@ -339,12 +340,12 @@ ArticleDownloader::EStatus ArticleDownloader::Download()
 	while (!IsStopped() && !m_decoder.GetEof())
 	{
 		// throttle the bandwidth
-		while (!IsStopped() && (g_Options->GetDownloadRate() > 0.0f) &&
-			(g_StatMeter->CalcCurrentDownloadSpeed() > g_Options->GetDownloadRate() ||
-			g_StatMeter->CalcMomentaryDownloadSpeed() > g_Options->GetDownloadRate()))
+		while (!IsStopped() && (g_WorkState->GetSpeedLimit() > 0.0f) &&
+			(g_StatMeter->CalcCurrentDownloadSpeed() > g_WorkState->GetSpeedLimit() ||
+			g_StatMeter->CalcMomentaryDownloadSpeed() > g_WorkState->GetSpeedLimit()))
 		{
 			SetLastUpdateTimeNow();
-			usleep(10 * 1000);
+			Util::Sleep(10);
 		}
 
 		char* buffer;

daemon/nntp/ArticleWriter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -113,7 +113,7 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 
 		while (!m_articleData.GetData() && g_ArticleCache->GetFlushing())
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			m_articleData = g_ArticleCache->Alloc(m_articleSize);
 		}
 
@@ -126,11 +126,11 @@ bool ArticleWriter::Start(Decoder::EFormat format, const char* filename, int64 f
 	if (!m_articleData.GetData())
 	{
 		bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_format == Decoder::efYenc;
-		const char* filename = directWrite ? m_outputFilename : m_tempFilename;
-		if (!m_outFile.Open(filename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
+		const char* outFilename = directWrite ? m_outputFilename : m_tempFilename;
+		if (!m_outFile.Open(outFilename, directWrite ? DiskFile::omReadWrite : DiskFile::omWrite))
 		{
 			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
-				"Could not %s file %s: %s", directWrite ? "open" : "create", filename,
+				"Could not %s file %s: %s", directWrite ? "open" : "create", outFilename,
 				*FileSystem::GetLastErrorMessage());
 			return false;
 		}
@@ -775,6 +775,11 @@ CachedSegmentData ArticleCache::Alloc(int size)
 			{
 				g_DiskState->WriteCacheFlag();
 			}
+			if (!m_allocated)
+			{
+				// Resume Run(), the notification arrives later, after releasing m_allocMutex
+				m_allocCond.NotifyAll();
+			}
 			m_allocated += size;
 		}
 	}
@@ -821,21 +826,36 @@ void ArticleCache::Run()
 	bool justFlushed = false;
 	while (!IsStopped() || m_allocated > 0)
 	{
-		if ((justFlushed || resetCounter >= 1000  || IsStopped() ||
+		if ((justFlushed || resetCounter >= 1000 || IsStopped() ||
 			 (g_Options->GetDirectWrite() && m_allocated >= fillThreshold)) &&
 			m_allocated > 0)
 		{
 			justFlushed = CheckFlush(m_allocated >= fillThreshold);
 			resetCounter = 0;
 		}
+		else if (!m_allocated)
+		{
+			Guard guard(m_allocMutex);
+			m_allocCond.Wait(m_allocMutex, [&]{ return IsStopped() || m_allocated > 0; });
+			resetCounter = 0;
+		}
 		else
 		{
-			usleep(5 * 1000);
+			Util::Sleep(5);
 			resetCounter += 5;
 		}
 	}
 }
 
+void ArticleCache::Stop()
+{
+	Thread::Stop();
+
+	// Resume Run() to exit it
+	Guard guard(m_allocMutex);
+	m_allocCond.NotifyAll();
+}
+
 bool ArticleCache::CheckFlush(bool flushEverything)
 {
 	debug("Checking cache, Allocated: %i, FlushEverything: %i", (int)m_allocated, (int)flushEverything);

daemon/nntp/ArticleWriter.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -95,6 +95,7 @@ public:
 	};
 
 	virtual void Run();
+	virtual void Stop();
 	CachedSegmentData Alloc(int size);
 	bool Realloc(CachedSegmentData* segment, int newSize);
 	void Free(CachedSegmentData* segment);
@@ -111,6 +112,7 @@ private:
 	Mutex m_flushMutex;
 	Mutex m_contentMutex;
 	FileInfo* m_fileInfo = nullptr;
+	ConditionVar m_allocCond;
 
 	bool CheckFlush(bool flushEverything);
 };

daemon/nntp/Decoder.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -374,10 +374,7 @@ int Decoder::DecodeUx(char* buffer, int len)
 			}
 			else
 			{
-				if (effLen >= 1)
-				{
-					*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
-				}
+				*optr++ = UU_DECODE_CHAR (iptr[0]) << 2 | UU_DECODE_CHAR (iptr[1]) >> 4;
 				if (effLen >= 2)
 				{
 					*optr++ = UU_DECODE_CHAR (iptr[1]) << 4 | UU_DECODE_CHAR (iptr[2]) >> 2;

daemon/nntp/NewsServer.cpp

@@ -25,10 +25,10 @@
 NewsServer::NewsServer(int id, bool active, const char* name, const char* host, int port, int ipVersion,
 	const char* user, const char* pass, bool joinGroup, bool tls, const char* cipher,
 	int maxConnections, int retention, int level, int group, bool optional) :
-		m_id(id), m_active(active), m_port(port), m_ipVersion(ipVersion), m_level(level), m_normLevel(level),
-		m_group(group), m_maxConnections(maxConnections), m_joinGroup(joinGroup), m_tls(tls),
-		m_name(name), m_host(host ? host : ""), m_user(user ? user : ""), m_password(pass ? pass : ""),
-		m_cipher(cipher ? cipher : ""), m_retention(retention), m_optional(optional)
+		m_id(id), m_active(active), m_name(name), m_host(host ? host : ""), m_port(port), m_ipVersion(ipVersion),
+		m_user(user ? user : ""), m_password(pass ? pass : ""), m_joinGroup(joinGroup), m_tls(tls),
+		m_cipher(cipher ? cipher : ""), m_maxConnections(maxConnections), m_retention(retention),
+		m_level(level), m_normLevel(level), m_group(group), m_optional(optional)
 {
 	if (m_name.Empty())
 	{

daemon/nntp/NewsServer.h

@@ -65,19 +65,19 @@ private:
 	int m_stateId = 0;
 	bool m_active;
 	CString m_name;
-	int m_group;
 	CString m_host;
 	int m_port;
 	int m_ipVersion;
 	CString m_user;
 	CString m_password;
-	int m_maxConnections;
-	int m_level;
-	int m_normLevel;
 	bool m_joinGroup;
 	bool m_tls;
 	CString m_cipher;
+	int m_maxConnections;
 	int m_retention;
+	int m_level;
+	int m_normLevel;
+	int m_group;
 	bool m_optional = false;
 	time_t m_blockTime = 0;
 };

daemon/nntp/StatMeter.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "StatMeter.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "DiskState.h"
 #include "Util.h"
@@ -57,8 +58,8 @@ void ServerVolume::CalcSlots(time_t locCurTime)
 void ServerVolume::AddData(int bytes)
 {
 	time_t curTime = Util::CurrentTime();
-	time_t locCurTime = curTime + g_Options->GetLocalTimeOffset();
-	time_t locDataTime = m_dataTime + g_Options->GetLocalTimeOffset();
+	time_t locCurTime = curTime + g_WorkState->GetLocalTimeOffset();
+	time_t locDataTime = m_dataTime + g_WorkState->GetLocalTimeOffset();
 
 	int lastMinSlot = m_minSlot;
 	int lastHourSlot = m_hourSlot;
@@ -138,28 +139,28 @@ void ServerVolume::LogDebugInfo()
 
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerSeconds[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerSeconds[i]);
 	}
 	info("Secs: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 60; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerMinutes[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerMinutes[i]);
 	}
 	info("Mins: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < 24; i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", i, m_bytesPerHours[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", i, m_bytesPerHours[i]);
 	}
 	info("Hours: %s", *msg);
 
 	msg.Clear();
 	for (int i = 0; i < (int)m_bytesPerDays.size(); i++)
 	{
-		msg.AppendFmt("[%i]=%lli ", m_firstDay + i, m_bytesPerDays[i]);
+		msg.AppendFmt("[%i]=%" PRIi64 " ", m_firstDay + i, m_bytesPerDays[i]);
 	}
 	info("Days: %s", *msg);
 }
@@ -188,10 +189,10 @@ void StatMeter::AdjustTimeOffset()
 	tmSplittedTime.tm_isdst = -1;
 	time_t locTime = mktime(&tmSplittedTime);
 	time_t localTimeDelta = utcTime - locTime;
-	g_Options->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
+	g_WorkState->SetLocalTimeOffset((int)localTimeDelta + g_Options->GetTimeCorrection());
 	m_lastTimeOffset = utcTime;
 
-	debug("UTC delta: %i (%i+%i)", g_Options->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
+	debug("UTC delta: %i (%i+%i)", g_WorkState->GetLocalTimeOffset(), (int)localTimeDelta, g_Options->GetTimeCorrection());
 }
 
 /*
@@ -380,9 +381,9 @@ void StatMeter::LogDebugInfo()
 	int timeDiff = (int)Util::CurrentTime() - m_speedStartTime * SPEEDMETER_SLOTSIZE;
 	info("      Speed: %i", speed);
 	info("      SpeedStartTime: %i", m_speedStartTime);
-	info("      SpeedTotalBytes: %lli", m_speedTotalBytes);
+	info("      SpeedTotalBytes: %" PRIi64, m_speedTotalBytes);
 	info("      SpeedBytesIndex: %i", m_speedBytesIndex);
-	info("      AllBytes: %lli", m_allBytes);
+	info("      AllBytes: %" PRIi64, m_allBytes);
 	info("      Time: %i", (int)Util::CurrentTime());
 	info("      TimeDiff: %i", timeDiff);
 	for (int i=0; i < SPEEDMETER_SLOTS; i++)
@@ -446,7 +447,7 @@ bool StatMeter::Load(bool* perfectServerMatch)
 
 	for (ServerVolume& serverVolume : m_serverVolumes)
 	{
-		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_Options->GetLocalTimeOffset());
+		serverVolume.CalcSlots(serverVolume.GetDataTime() + g_WorkState->GetLocalTimeOffset());
 	}
 
 	return ok;
@@ -465,20 +466,20 @@ void StatMeter::CheckQuota()
 	bool monthlyQuotaReached = g_Options->GetMonthlyQuota() > 0 && monthBytes >= (int64)g_Options->GetMonthlyQuota() * 1024 * 1024;
 	bool dailyQuotaReached = g_Options->GetDailyQuota() > 0 && dayBytes >= (int64)g_Options->GetDailyQuota() * 1024 * 1024;
 
-	if (monthlyQuotaReached && !g_Options->GetQuotaReached())
+	if (monthlyQuotaReached && !g_WorkState->GetQuotaReached())
 	{
 		warn("Monthly quota reached at %s", *Util::FormatSize(monthBytes));
 	}
-	else if (dailyQuotaReached && !g_Options->GetQuotaReached())
+	else if (dailyQuotaReached && !g_WorkState->GetQuotaReached())
 	{
 		warn("Daily quota reached at %s", *Util::FormatSize(dayBytes));
 	}
-	else if (!monthlyQuotaReached && !dailyQuotaReached && g_Options->GetQuotaReached())
+	else if (!monthlyQuotaReached && !dailyQuotaReached && g_WorkState->GetQuotaReached())
 	{
 		info("Quota lifted");
 	}
 
-	g_Options->SetQuotaReached(monthlyQuotaReached || dailyQuotaReached);
+	g_WorkState->SetQuotaReached(monthlyQuotaReached || dailyQuotaReached);
 }
 
 void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
@@ -487,7 +488,7 @@ void StatMeter::CalcQuotaUsage(int64& monthBytes, int64& dayBytes)
 
 	ServerVolume totalVolume = m_serverVolumes[0];
 
-	time_t locTime = Util::CurrentTime() + g_Options->GetLocalTimeOffset();
+	time_t locTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
 	int daySlot = (int)(locTime / 86400) - totalVolume.GetFirstDay();
 
 	dayBytes = 0;
@@ -515,7 +516,7 @@ int StatMeter::CalcMonthSlots(ServerVolume& volume)
 {
 	int elapsedDays;
 
-	time_t locCurTime = Util::CurrentTime() + g_Options->GetLocalTimeOffset();
+	time_t locCurTime = Util::CurrentTime() + g_WorkState->GetLocalTimeOffset();
 	tm dayparts;
 	gmtime_r(&locCurTime, &dayparts);
 

daemon/nntp/StatMeter.h

@@ -84,7 +84,6 @@ public:
 	void AddServerData(int bytes, int serverId);
 	void CalcTotalStat(int* upTimeSec, int* dnTimeSec, int64* allBytes, bool* standBy);
 	void CalcQuotaUsage(int64& monthBytes, int64& dayBytes);
-	bool GetStandBy() { return m_standBy; }
 	void IntervalCheck();
 	void EnterLeaveStandBy(bool enter);
 	GuardedServerVolumes GuardServerVolumes();

daemon/nserv/NServFrontend.cpp

@@ -34,7 +34,7 @@ void NServFrontend::Run()
 	while (!IsStopped())
 	{
 		Update();
-		usleep(100 * 1000);
+		Util::Sleep(100);
 	}
 	// Printing the last messages
 	Update();

daemon/nserv/NServMain.cpp

@@ -124,7 +124,7 @@ int NServMain(int argc, char* argv[])
 	}
 
 	info("Press Ctrl+C to quit");
-	while (getchar()) usleep(1000*200);
+	while (getchar()) Util::Sleep(200);
 
 	for (std::unique_ptr<NntpServer>& serv: instances)
 	{
@@ -140,7 +140,7 @@ int NServMain(int argc, char* argv[])
 		{
 			hasRunning |= serv->IsRunning();
 		}
-		usleep(50 * 1000);
+		Util::Sleep(50);
 	} while (hasRunning);
 
 	return 0;

daemon/nserv/NntpServer.cpp

@@ -40,12 +40,12 @@ private:
 	int m_id;
 	int m_serverId;
 	std::unique_ptr<Connection> m_connection;
-	int m_latency;
-	int m_speed;
 	const char* m_dataDir;
 	const char* m_cacheDir;
 	const char* m_secureCert;
 	const char* m_secureKey;
+	int m_latency;
+	int m_speed;
 	const char* m_messageid;
 	CString m_filename;
 	int m_part;
@@ -103,7 +103,7 @@ void NntpServer::Run()
 				break;
 			}
 			m_connection.reset();
-			usleep(500 * 1000);
+			Util::Sleep(500);
 			continue;
 		}
 		
@@ -172,7 +172,7 @@ void NntpProcessor::Run()
 		}
 		else if (!strncasecmp(line, "GROUP ", 6))
 		{
-			m_connection->WriteLine(CString::FormatStr("211 0 0 0 %s\r\n", line + 7));
+			m_connection->WriteLine(CString::FormatStr("211 0 0 0 %s\r\n", line + 6));
 		}
 		else if (!strncasecmp(line, "AUTHINFO ", 9))
 		{
@@ -216,7 +216,7 @@ void NntpProcessor::ServArticle()
 
 	if (m_latency)
 	{
-		usleep(1000 * m_latency);
+		Util::Sleep(m_latency);
 	}
 
 	bool ok = false;
@@ -268,16 +268,16 @@ bool NntpProcessor::ServerInList(const char* servList)
 
 void NntpProcessor::SendSegment()
 {
-	detail("[%i] Sending segment %s (%i=%lli:%i)", m_id, *m_filename, m_part, (long long)m_offset, m_size);
+	detail("[%i] Sending segment %s (%i=%" PRIi64 ":%i)", m_id, *m_filename, m_part, m_offset, m_size);
 
 	if (m_speed > 0)
 	{
-		m_start = Util::GetCurrentTicks();
+		m_start = Util::CurrentTicks();
 	}
 
 	BString<1024> fullFilename("%s/%s", m_dataDir, *m_filename);
 	BString<1024> cacheFileDir("%s/%s", m_cacheDir, *m_filename);
-	BString<1024> cacheFileName("%i=%lli-%i", m_part, (long long)m_offset, m_size);
+	BString<1024> cacheFileName("%i=%" PRIi64 "-%i", m_part, m_offset, m_size);
 	BString<1024> cacheFullFilename("%s/%s", *cacheFileDir, *cacheFileName);
 	BString<1024> cacheKey("%s/%s", *m_filename, *cacheFileName);
 
@@ -383,7 +383,7 @@ void NntpProcessor::SendData(const char* buffer, int size)
 		return;
 	}
 
-	int64 expectedTime = (int64)1000 * size / (m_speed * 1024) - (Util::GetCurrentTicks() - m_start) / 1000;
+	int64 expectedTime = (int64)1000 * size / (m_speed * 1024) - (Util::CurrentTicks() - m_start) / 1000;
 
 	int chunkNum = 21;
 	int chunkSize = size;
@@ -407,10 +407,10 @@ void NntpProcessor::SendData(const char* buffer, int size)
 		}
 		
 		m_connection->Send(buffer + sent, len);
-		int64 now = Util::GetCurrentTicks();
+		int64 now = Util::CurrentTicks();
 		if (now + pause * 1000 < m_start + expectedTime * 1000)
 		{
-			usleep(pause * 1000);
+			Util::Sleep(pause);
 		}
 		sent += len;
 	}

daemon/nserv/NntpServer.h

@@ -65,13 +65,13 @@ private:
 	int m_id;
 	CString m_host;
 	int m_port;
-	int m_latency;
-	int m_speed;
-	CString m_dataDir;
-	CString m_cacheDir;
 	CString m_secureCert;
 	CString m_secureKey;
 	std::unique_ptr<Connection> m_connection;
+	CString m_dataDir;
+	CString m_cacheDir;
+	int m_latency;
+	int m_speed;
 	NntpCache* m_cache;
 };
 

daemon/nserv/NzbGenerator.cpp

@@ -118,11 +118,11 @@ void NzbGenerator::AppendFile(DiskFile& outfile, const char* filename, const cha
 	for (int segno = 1; segno <= segmentCount; segno++)
 	{
 		int segSize = (int)(segOffset + m_segmentSize < fileSize ? m_segmentSize : fileSize - segOffset);
-		outfile.Print("<segment bytes=\"%i\" number=\"%i\">%s%s%s?%i=%lli:%i</segment>\n",
-			m_segmentSize, segno, 
+		outfile.Print("<segment bytes=\"%i\" number=\"%i\">%s%s%s?%i=%" PRIi64 ":%i</segment>\n",
+			m_segmentSize, segno,
 			relativePath ? relativePath : "",
 			relativePath ? "/" : "",
-			FileSystem::BaseFileName(filename), segno, (long long)segOffset, (int)segSize);
+			FileSystem::BaseFileName(filename), segno, segOffset, segSize);
 		segOffset += segSize;
 	}
 

daemon/nserv/YEncoder.cpp

@@ -64,8 +64,8 @@ void YEncoder::WriteSegment()
 	StringBuilder outbuf;
 	outbuf.Reserve(std::max(2048, std::min((int)(m_size * 1.1), 16 * 1024 * 1024)));
 
-	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%lli name=%s\r\n", m_part, (long long)m_fileSize, FileSystem::BaseFileName(m_filename)));
-	outbuf.Append(CString::FormatStr("=ypart begin=%lli end=%lli\r\n", (long long)(m_offset + 1), (long long)(m_offset + m_size)));
+	outbuf.Append(CString::FormatStr("=ybegin part=%i line=128 size=%" PRIi64 " name=%s\r\n", m_part, m_fileSize, FileSystem::BaseFileName(m_filename)));
+	outbuf.Append(CString::FormatStr("=ypart begin=%" PRIi64 " end=%" PRIi64 "\r\n", m_offset + 1, m_offset + m_size));
 
 	Crc32 crc;
 	CharBuffer inbuf(std::min(m_size, 16 * 1024 * 1024));

daemon/postprocess/DirectUnpack.cpp

@@ -105,7 +105,7 @@ void DirectUnpack::Run()
 			{
 				break;
 			}
-			usleep(100 * 1000);
+			Util::Sleep(100);
 		}
 	}
 

daemon/postprocess/DupeMatcher.cpp

@@ -80,7 +80,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 		curTime + timeoutSec > Util::CurrentTime() &&
 		curTime >= Util::CurrentTime())					// in a case clock was changed
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	if (unrar.IsRunning())
@@ -91,7 +91,7 @@ bool RarLister::FindLargestFile(DupeMatcher* owner, const char* directory,
 	// wait until terminated or killed
 	while (unrar.IsRunning())
 	{
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	*maxSize = unrar.m_maxSize;
@@ -161,7 +161,7 @@ bool DupeMatcher::Prepare()
 	char filename[1024];
 	FindLargestFile(m_destDir, filename, sizeof(filename), &m_maxSize, &m_compressed);
 	bool sizeOK = SizeDiffOK(m_maxSize, m_expectedSize, 20);
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, sizeOK ? "" : ", size mismatch");
 	return sizeOK;
 }
@@ -173,7 +173,7 @@ bool DupeMatcher::MatchDupeContent(const char* dupeDir)
 	char filename[1024];
 	FindLargestFile(dupeDir, filename, sizeof(filename), &dupeMaxSize, &dupeCompressed);
 	bool ok = dupeMaxSize == m_maxSize && dupeCompressed == m_compressed;
-	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+	PrintMessage(Message::mkDetail, "Found main file %s with size %" PRIi64 " bytes%s",
 		filename, m_maxSize, ok ? "" : ", size mismatch");
 	return ok;
 }

daemon/postprocess/ParChecker.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -42,14 +42,6 @@ const char* Par2CmdLineErrStr[] = { "OK",
 	"internal error occurred",
 	"out of memory" };
 
-// Sleep interval for synchronisation (microseconds)
-#ifdef WIN32
-// Windows doesn't allow sleep intervals less than one millisecond
-#define SYNC_SLEEP_INTERVAL 1000
-#else
-#define SYNC_SLEEP_INTERVAL 100
-#endif
-
 class RepairThread;
 
 class Repairer : public Par2::Par2Repairer, public ParChecker::AbstractRepairer
@@ -74,8 +66,8 @@ protected:
 private:
 	typedef vector<Thread*> Threads;
 
-	Par2::CommandLine commandLine;
 	ParChecker* m_owner;
+	Par2::CommandLine commandLine;
 	Threads m_threads;
 	bool m_parallel;
 	Mutex progresslock;
@@ -83,6 +75,7 @@ private:
 	virtual void BeginRepair();
 	virtual void EndRepair();
 	void RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength);
+	static void SyncSleep();
 
 	friend class ParChecker;
 	friend class RepairThread;
@@ -253,7 +246,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 
 		if (!jobAdded)
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			SyncSleep();
 		}
 	}
 
@@ -268,7 +261,7 @@ bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength)
 			if (repairThread->IsWorking())
 			{
 				working = true;
-				usleep(SYNC_SLEEP_INTERVAL);
+				SyncSleep();
 				break;
 			}
 		}
@@ -305,6 +298,17 @@ void Repairer::RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t b
 	}
 }
 
+// Sleep for synchronisation
+void Repairer::SyncSleep()
+{
+#ifdef WIN32
+	// Windows doesn't allow sleep intervals less than one millisecond
+	Sleep(1);
+#else
+	usleep(100);
+#endif
+}
+
 void RepairThread::Run()
 {
 	while (!IsStopped())
@@ -316,7 +320,7 @@ void RepairThread::Run()
 		}
 		else
 		{
-			usleep(SYNC_SLEEP_INTERVAL);
+			Repairer::SyncSleep();
 		}
 	}
 }
@@ -690,7 +694,7 @@ bool ParChecker::LoadMainParBak()
 					Guard guard(m_queuedParFilesMutex);
 					queuedParFilesChanged = m_queuedParFilesChanged;
 				}
-				usleep(100 * 1000);
+				Util::Sleep(100);
 			}
 		}
 	}
@@ -756,7 +760,7 @@ int ParChecker::ProcessMorePars()
 						Guard guard(m_queuedParFilesMutex);
 						queuedParFilesChanged = m_queuedParFilesChanged;
 					}
-					usleep(100 * 1000);
+					Util::Sleep(100);
 				}
 			}
 		}
@@ -822,29 +826,25 @@ bool ParChecker::AddSplittedFragments()
 	DirBrowser dir(m_destDir);
 	while (const char* filename = dir.Next())
 	{
-		if (IsParredFile(filename) && !IsProcessedFile(filename))
+		if (!IsParredFile(filename) && !IsProcessedFile(filename))
 		{
 			for (Par2::Par2RepairerSourceFile *sourcefile : GetRepairer()->sourcefiles)
 			{
 				std::string target = sourcefile->TargetFileName();
-				const char* filename2 = target.c_str();
-				const char* basename2 = FileSystem::BaseFileName(filename2);
-				int baseLen = strlen(basename2);
+				const char* current = FileSystem::BaseFileName(target.c_str());
 
-				if (!strncasecmp(filename, basename2, baseLen))
+				// if file was renamed by par-renamer we also check the original filename
+				const char* original = FindFileOrigname(current);
+
+				if (MaybeSplittedFragement(filename, current) ||
+					(!Util::EmptyStr(original) && strcasecmp(original, current) &&
+					MaybeSplittedFragement(filename, original)))
 				{
-					const char* p = filename + baseLen;
-					if (*p == '.')
-					{
-						for (p++; *p && strchr("0123456789", *p); p++) ;
-						if (!*p)
-						{
-							debug("Found splitted fragment %s", filename);
-							BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
-							Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
-							extrafiles.push_back(extrafile);
-						}
-					}
+					detail("Found splitted fragment %s", filename);
+					BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename);
+					Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename));
+					extrafiles.push_back(extrafile);
+					break;
 				}
 			}
 		}
@@ -865,6 +865,40 @@ bool ParChecker::AddSplittedFragments()
 	return fragmentsAdded;
 }
 
+bool ParChecker::MaybeSplittedFragement(const char* filename1, const char* filename2)
+{
+	// check if name is same but the first name has additional numerical extension
+	int len = strlen(filename2);
+	if (!strncasecmp(filename1, filename2, len))
+	{
+		const char* p = filename1 + len;
+		if (*p == '.')
+		{
+			for (p++; *p && strchr("0123456789", *p); p++) ;
+			if (!*p)
+			{
+				return true;
+			}
+		}
+	}
+
+	// check if same name (without extension) and extensions are numerical and exactly 3 characters long
+	const char* ext1 = strrchr(filename1, '.');
+	const char* ext2 = strrchr(filename2, '.');
+	if (ext1 && ext2 && (strlen(ext1) == 4) && (strlen(ext2) == 4) &&
+		!strncasecmp(filename1, filename2, ext1 - filename1))
+	{
+		for (ext1++; *ext1 && strchr("0123456789", *ext1); ext1++) ;
+		for (ext2++; *ext2 && strchr("0123456789", *ext2); ext2++) ;
+		if (!*ext1 && !*ext2)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
 bool ParChecker::AddMissingFiles()
 {
 	return AddExtraFiles(true, false, m_destDir);
@@ -1395,7 +1429,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment
 		int64 blockEnd = blockStart + blocksize < fileSize - 1 ? blockStart + blocksize : fileSize - 1;
 		bool blockOK = false;
 		bool blockEndFound = false;
-		Par2::u64 curOffset = 0;
+		int64 curOffset = 0;
 		for (Segment& segment : segments)
 		{
 			if (!blockOK && segment.GetSuccess() && segment.GetOffset() <= blockStart &&

daemon/postprocess/ParChecker.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -132,6 +132,7 @@ protected:
 	virtual void RegisterParredFile(const char* filename) {}
 	virtual bool IsParredFile(const char* filename) { return false; }
 	virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) { return fsUnknown; }
+	virtual const char* FindFileOrigname(const char* filename) { return nullptr; }
 	virtual void RequestDupeSources(DupeSourceList* dupeSourceList) {}
 	virtual void StatDupeSources(DupeSourceList* dupeSourceList) {}
 	EStage GetStage() { return m_stage; }
@@ -221,6 +222,7 @@ private:
 	bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrc);
 	void CheckEmptyFiles();
 	CString GetPacketCreator();
+	bool MaybeSplittedFragement(const char* filename1, const char* filename2);
 
 	friend class Repairer;
 };

daemon/postprocess/PrePostProcessor.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "PrePostProcessor.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "HistoryCoordinator.h"
 #include "DupeCoordinator.h"
@@ -49,7 +50,7 @@ void PrePostProcessor::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
 	if (g_Options->GetServerMode())
@@ -59,13 +60,25 @@ void PrePostProcessor::Run()
 
 	while (!IsStopped())
 	{
-		if (!g_Options->GetTempPausePostprocess() && m_queuedJobs)
+		if (g_WorkState->GetTempPausePostprocess())
+		{
+			// Postprocess is paused: just wait and loop
+			Util::Sleep(200);
+			continue;
+		}
+
+		if (m_queuedJobs)
 		{
 			// check post-queue every 200 msec
 			CheckPostQueue();
+			Util::Sleep(200);
+		}
+		else
+		{
+			// Wait until we get the stop signal or more jobs in the queue
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&]{ return m_queuedJobs || IsStopped(); });
 		}
-
-		usleep(200 * 1000);
 	}
 
 	WaitJobs();
@@ -89,7 +102,7 @@ void PrePostProcessor::WaitJobs()
 			}
 		}
 		CheckPostQueue();
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	// kill remaining post-processing jobs; not safe but we can't wait any longer
@@ -124,7 +137,7 @@ void PrePostProcessor::WaitJobs()
 				break;
 			}
 		}
-		usleep(200 * 1000);
+		Util::Sleep(200);
 	}
 
 	// disconnect remaining direct unpack jobs
@@ -142,23 +155,30 @@ void PrePostProcessor::WaitJobs()
 void PrePostProcessor::Stop()
 {
 	Thread::Stop();
-	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
-	for (NzbInfo* postJob : m_activeJobs)
 	{
-		if (postJob->GetPostInfo() && postJob->GetPostInfo()->GetPostThread())
+		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+		for (NzbInfo* postJob : m_activeJobs)
 		{
-			postJob->GetPostInfo()->GetPostThread()->Stop();
+			if (postJob->GetPostInfo() && postJob->GetPostInfo()->GetPostThread())
+			{
+				postJob->GetPostInfo()->GetPostThread()->Stop();
+			}
+		}
+
+		for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+		{
+			if (nzbInfo->GetUnpackThread())
+			{
+				((DirectUnpack*)nzbInfo->GetUnpackThread())->Stop(downloadQueue, nzbInfo);
+			}
 		}
 	}
 
-	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
-	{
-		if (nzbInfo->GetUnpackThread())
-		{
-			((DirectUnpack*)nzbInfo->GetUnpackThread())->Stop(downloadQueue, nzbInfo);
-		}
-	}
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
 }
 
 /**
@@ -197,11 +217,13 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 	}
 
 	DownloadQueue::Aspect* queueAspect = (DownloadQueue::Aspect*)aspect;
-	if (queueAspect->action == DownloadQueue::eaNzbFound)
+	if (queueAspect->action == DownloadQueue::eaNzbFound ||
+		queueAspect->action == DownloadQueue::eaUrlFound)
 	{
 		NzbFound(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
-	else if (queueAspect->action == DownloadQueue::eaNzbAdded)
+	else if (queueAspect->action == DownloadQueue::eaNzbAdded ||
+		queueAspect->action == DownloadQueue::eaUrlAdded)
 	{
 		NzbAdded(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
@@ -220,6 +242,14 @@ void PrePostProcessor::DownloadQueueUpdate(void* aspect)
 			"Collection %s deleted from queue", queueAspect->nzbInfo->GetName());
 		NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo);
 	}
+	else if (queueAspect->action == DownloadQueue::eaUrlDeleted)
+	{
+		NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo);
+	}
+	else if (queueAspect->action == DownloadQueue::eaUrlFailed)
+	{
+		NzbCompleted(queueAspect->downloadQueue, queueAspect->nzbInfo, true);
+	}
 	else if ((queueAspect->action == DownloadQueue::eaFileCompleted ||
 		queueAspect->action == DownloadQueue::eaFileDeleted))
 	{
@@ -314,7 +344,6 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 		nzbInfo->PrintMessage(Message::mkInfo, "Queueing %s for post-processing", nzbInfo->GetName());
 
 		nzbInfo->EnterPostProcess();
-		m_queuedJobs++;
 
 		if (nzbInfo->GetParStatus() == NzbInfo::psNone &&
 			g_Options->GetParCheck() != Options::pcAlways &&
@@ -330,7 +359,13 @@ void PrePostProcessor::NzbDownloaded(DownloadQueue* downloadQueue, NzbInfo* nzbI
 			((DirectUnpack*)nzbInfo->GetUnpackThread())->NzbDownloaded(downloadQueue, nzbInfo);
 		}
 
-		downloadQueue->Save();
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
+
+		// We have more jobs in the queue, notify Run()
+		Guard guard(m_waitMutex);
+		m_queuedJobs++;
+		m_waitCond.NotifyAll();
 	}
 	else
 	{
@@ -371,6 +406,7 @@ void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo
 
 void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool saveQueue)
 {
+	bool downloadDupe = nzbInfo->GetDupeHint() == NzbInfo::dhRedownloadAuto;
 	bool addToHistory = g_Options->GetKeepHistory() > 0 && !nzbInfo->GetAvoidHistory();
 	if (addToHistory)
 	{
@@ -384,7 +420,8 @@ void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbIn
 		(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsHealth ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsBad ||
-		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan))
+		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan ||
+		 (nzbInfo->GetDeleteStatus() == NzbInfo::dsCopy && downloadDupe)))
 	{
 		g_DupeCoordinator->NzbCompleted(downloadQueue, nzbInfo);
 		needSave = true;
@@ -574,7 +611,7 @@ NzbInfo* PrePostProcessor::PickNextJob(DownloadQueue* downloadQueue, bool allowP
 			!g_QueueScriptCoordinator->HasJob(nzbInfo1->GetId(), nullptr) &&
 			nzbInfo1->GetDirectUnpackStatus() != NzbInfo::nsRunning &&
 			(!nzbInfo || nzbInfo1->GetPriority() > nzbInfo->GetPriority()) &&
-			(!g_Options->GetPausePostProcess() || nzbInfo1->GetForcePriority()) &&
+			(!g_WorkState->GetPausePostProcess() || nzbInfo1->GetForcePriority()) &&
 			(allowPar || !nzbInfo1->GetPostInfo()->GetNeedParCheck()) &&
 			(std::find(m_activeJobs.begin(), m_activeJobs.end(), nzbInfo1) == m_activeJobs.end()) &&
 			nzbInfo1->IsDownloadCompleted(true))
@@ -608,7 +645,7 @@ void PrePostProcessor::CheckPostQueue()
 
 		PostInfo* postInfo = postJob->GetPostInfo();
 		if (postInfo->GetStage() == PostInfo::ptQueued &&
-			(!g_Options->GetPausePostProcess() || postInfo->GetNzbInfo()->GetForcePriority()))
+			(!g_WorkState->GetPausePostProcess() || postInfo->GetNzbInfo()->GetForcePriority()))
 		{
 			StartJob(downloadQueue, postInfo, allowPar);
 			CheckRequestPar(downloadQueue);
@@ -628,6 +665,8 @@ void PrePostProcessor::CheckPostQueue()
 
 void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo, bool allowPar)
 {
+	NzbInfo* nzbInfo = postInfo->GetNzbInfo();
+
 	if (!postInfo->GetStartTime())
 	{
 		postInfo->SetStartTime(Util::CurrentTime());
@@ -637,8 +676,8 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 	postInfo->SetFileProgress(0);
 	postInfo->SetProgressLabel("");
 
-	if (postInfo->GetNzbInfo()->GetParRenameStatus() == NzbInfo::rsNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone &&
+	if (nzbInfo->GetParRenameStatus() == NzbInfo::rsNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
 		g_Options->GetParRename())
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptParRenaming);
@@ -647,10 +686,10 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 	}
 
 #ifndef DISABLE_PARCHECK
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone)
+	if (nzbInfo->GetParStatus() == NzbInfo::psNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone)
 	{
-		if (ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+		if (ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 		{
 			if (!allowPar)
 			{
@@ -664,54 +703,54 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		}
 		else
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
-				"Nothing to par-check for %s", postInfo->GetNzbInfo()->GetName());
-			postInfo->GetNzbInfo()->SetParStatus(NzbInfo::psSkipped);
+			nzbInfo->PrintMessage(Message::mkInfo,
+				"Nothing to par-check for %s", nzbInfo->GetName());
+			nzbInfo->SetParStatus(NzbInfo::psSkipped);
 		}
 		return;
 	}
 	
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped &&
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
 		((g_Options->GetParScan() != Options::psDupe &&
-		  postInfo->GetNzbInfo()->CalcHealth() < postInfo->GetNzbInfo()->CalcCriticalHealth(false) &&
-		  postInfo->GetNzbInfo()->CalcCriticalHealth(false) < 1000) ||
-		  postInfo->GetNzbInfo()->CalcHealth() == 0) &&
-		ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+		  nzbInfo->CalcHealth() < nzbInfo->CalcCriticalHealth(false) &&
+		  nzbInfo->CalcCriticalHealth(false) < 1000) ||
+		  nzbInfo->CalcHealth() == 0) &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 	{
-		if (postInfo->GetNzbInfo()->CalcHealth() == 0)
+		if (nzbInfo->CalcHealth() == 0)
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
-				"Skipping par-check for %s due to health 0%%", postInfo->GetNzbInfo()->GetName());
+			nzbInfo->PrintMessage(Message::mkWarning,
+				"Skipping par-check for %s due to health 0%%", nzbInfo->GetName());
 		}
 		else
 		{
-			postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
+			nzbInfo->PrintMessage(Message::mkWarning,
 				"Skipping par-check for %s due to health %.1f%% below critical %.1f%%",
-				postInfo->GetNzbInfo()->GetName(),
-				postInfo->GetNzbInfo()->CalcHealth() / 10.0, postInfo->GetNzbInfo()->CalcCriticalHealth(false) / 10.0);
+				nzbInfo->GetName(),
+				nzbInfo->CalcHealth() / 10.0, nzbInfo->CalcCriticalHealth(false) / 10.0);
 		}
-		postInfo->GetNzbInfo()->SetParStatus(NzbInfo::psFailure);
+		nzbInfo->SetParStatus(NzbInfo::psFailure);
 		return;
 	}
 	
-	if (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped &&
-		postInfo->GetNzbInfo()->GetFailedSize() - postInfo->GetNzbInfo()->GetParFailedSize() > 0 &&
-		ParParser::FindMainPars(postInfo->GetNzbInfo()->GetDestDir(), nullptr))
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
+		nzbInfo->GetFailedSize() - nzbInfo->GetParFailedSize() > 0 &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
 	{
-		postInfo->GetNzbInfo()->PrintMessage(Message::mkInfo,
+		nzbInfo->PrintMessage(Message::mkInfo,
 			"Collection %s with health %.1f%% needs par-check",
-			postInfo->GetNzbInfo()->GetName(), postInfo->GetNzbInfo()->CalcHealth() / 10.0);
+			nzbInfo->GetName(), nzbInfo->CalcHealth() / 10.0);
 		postInfo->SetRequestParCheck(true);
 		return;
 	}
 #endif
 
-	NzbParameter* unpackParameter = postInfo->GetNzbInfo()->GetParameters()->Find("*Unpack:");
+	NzbParameter* unpackParameter = nzbInfo->GetParameters()->Find("*Unpack:");
 	bool wantUnpack = !(unpackParameter && !strcasecmp(unpackParameter->GetValue(), "no"));
-	bool unpack = wantUnpack && postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone;
+	bool unpack = wantUnpack && nzbInfo->GetUnpackStatus() == NzbInfo::usNone &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone;
 
-	if (postInfo->GetNzbInfo()->GetRarRenameStatus() == NzbInfo::rsNone &&
+	if (nzbInfo->GetRarRenameStatus() == NzbInfo::rsNone &&
 		unpack && g_Options->GetRarRename())
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptRarRenaming);
@@ -719,43 +758,63 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		return;
 	}
 
-	bool parFailed = postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psFailure ||
-		postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psRepairPossible ||
-		postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psManual;
+#ifndef DISABLE_PARCHECK
+	if (nzbInfo->GetParStatus() == NzbInfo::psSkipped &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+		g_Options->GetParCheck() == Options::pcAuto &&
+		!UnpackController::HasCompletedArchiveFiles(nzbInfo) &&
+		ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr))
+	{
+		nzbInfo->PrintMessage(Message::mkInfo,
+			"Requesting par-check for collection %s without archive files",
+			nzbInfo->GetName());
+		postInfo->SetRequestParCheck(true);
+		return;
+	}
+#endif
+
+	bool parFailed = nzbInfo->GetParStatus() == NzbInfo::psFailure ||
+		nzbInfo->GetParStatus() == NzbInfo::psRepairPossible ||
+		nzbInfo->GetParStatus() == NzbInfo::psManual;
 
 	bool cleanup = !unpack && wantUnpack &&
-		postInfo->GetNzbInfo()->GetCleanupStatus() == NzbInfo::csNone &&
+		nzbInfo->GetCleanupStatus() == NzbInfo::csNone &&
 		!Util::EmptyStr(g_Options->GetExtCleanupDisk()) &&
-		((postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSuccess &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usFailure &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usSpace &&
-		  postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usPassword) ||
-		 (postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSuccess &&
-		  postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psFailure) ||
-		 ((postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usNone ||
-		   postInfo->GetNzbInfo()->GetUnpackStatus() == NzbInfo::usSkipped) &&
-		  (postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psNone ||
-		   postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psSkipped) &&
-		  postInfo->GetNzbInfo()->CalcHealth() == 1000));
+		((nzbInfo->GetParStatus() == NzbInfo::psSuccess &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
+		  nzbInfo->GetUnpackStatus() != NzbInfo::usPassword) ||
+		 (nzbInfo->GetUnpackStatus() == NzbInfo::usSuccess &&
+		  nzbInfo->GetParStatus() != NzbInfo::psFailure) ||
+		 ((nzbInfo->GetUnpackStatus() == NzbInfo::usNone ||
+		   nzbInfo->GetUnpackStatus() == NzbInfo::usSkipped) &&
+		  (nzbInfo->GetParStatus() == NzbInfo::psNone ||
+		   nzbInfo->GetParStatus() == NzbInfo::psSkipped) &&
+		  nzbInfo->CalcHealth() == 1000));
 
 	bool moveInter = !unpack &&
-		postInfo->GetNzbInfo()->GetMoveStatus() == NzbInfo::msNone &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usFailure &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usSpace &&
-		postInfo->GetNzbInfo()->GetUnpackStatus() != NzbInfo::usPassword &&
-		postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psFailure &&
-		postInfo->GetNzbInfo()->GetParStatus() != NzbInfo::psManual &&
-		postInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsNone &&
+		nzbInfo->GetMoveStatus() == NzbInfo::msNone &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usPassword &&
+		nzbInfo->GetParStatus() != NzbInfo::psFailure &&
+		nzbInfo->GetParStatus() != NzbInfo::psManual &&
+		nzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+		!(((nzbInfo->GetUnpackStatus() == NzbInfo::usNone ||
+		   nzbInfo->GetUnpackStatus() == NzbInfo::usSkipped) &&
+		  (nzbInfo->GetParStatus() == NzbInfo::psNone ||
+		   nzbInfo->GetParStatus() == NzbInfo::psSkipped) &&
+		  nzbInfo->CalcHealth() < 1000)) &&
 		!Util::EmptyStr(g_Options->GetInterDir()) &&
-		!strncmp(postInfo->GetNzbInfo()->GetDestDir(), g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
-		postInfo->GetNzbInfo()->GetDestDir()[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
+		!strncmp(nzbInfo->GetDestDir(), g_Options->GetInterDir(), strlen(g_Options->GetInterDir())) &&
+		nzbInfo->GetDestDir()[strlen(g_Options->GetInterDir())] == PATH_SEPARATOR;
 
 	if (unpack && parFailed)
 	{
-		postInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
-			"Skipping unpack for %s due to %s", postInfo->GetNzbInfo()->GetName(),
-			postInfo->GetNzbInfo()->GetParStatus() == NzbInfo::psManual ? "required par-repair" : "par-failure");
-		postInfo->GetNzbInfo()->SetUnpackStatus(NzbInfo::usSkipped);
+		nzbInfo->PrintMessage(Message::mkWarning,
+			"Skipping unpack for %s due to %s", nzbInfo->GetName(),
+			nzbInfo->GetParStatus() == NzbInfo::psManual ? "required par-repair" : "par-failure");
+		nzbInfo->SetUnpackStatus(NzbInfo::usSkipped);
 		unpack = false;
 	}
 
@@ -798,6 +857,7 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post
 		NzbCompleted(downloadQueue, nzbInfo, false);
 	}
 
+	Guard guard(m_waitMutex);
 	m_queuedJobs--;
 }
 
@@ -833,16 +893,16 @@ void PrePostProcessor::UpdatePauseState()
 		}
 	}
 
-	if (needPause && !g_Options->GetTempPauseDownload())
+	if (needPause && !g_WorkState->GetTempPauseDownload())
 	{
 		info("Pausing download before post-processing");
 	}
-	else if (!needPause && g_Options->GetTempPauseDownload())
+	else if (!needPause && g_WorkState->GetTempPauseDownload())
 	{
 		info("Unpausing download after post-processing");
 	}
 
-	g_Options->SetTempPauseDownload(needPause);
+	g_WorkState->SetTempPauseDownload(needPause);
 }
 
 bool PrePostProcessor::EditList(DownloadQueue* downloadQueue, IdList* idList,

daemon/postprocess/PrePostProcessor.h

@@ -44,6 +44,8 @@ protected:
 private:
 	int m_queuedJobs = 0;
 	RawNzbList m_activeJobs;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	void CheckPostQueue();
 	void CheckRequestPar(DownloadQueue* downloadQueue);

daemon/postprocess/RarReader.cpp

@@ -173,7 +173,7 @@ bool RarVolume::Skip(DiskFile& file, RarBlock* block, int64 size)
 	uint8 buf[256];
 	while (size > 0)
 	{
-		int64 len = size <= sizeof(buf) ? size : sizeof(buf);
+		int64 len = size <= (int64)sizeof(buf) ? size : (int64)sizeof(buf);
 		if (!Read(file, block, buf, len)) return false;
 		size -= len;
 	}
@@ -289,8 +289,10 @@ RarVolume::RarBlock RarVolume::ReadRar3Block(DiskFile& file)
 		block.trailsize = blocksize - sizeof(buf) - 4;
 	}
 
+#ifdef DEBUG
 	static int num = 0;
-	debug("%i) %llu, %i, %i, %i, %u, %llu", ++num, (long long)block.crc, (int)block.type, (int)block.flags, (int)size, (int)block.addsize, (long long)block.trailsize);
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
 
 	return block;
 }
@@ -450,8 +452,10 @@ RarVolume::RarBlock RarVolume::ReadRar5Block(DiskFile& file)
 	if ((block.flags & RAR5_BLOCK_DATAAREA) && !ReadV(file, &block, &datasize)) return {0};
 	block.trailsize += datasize;
 
+#ifdef DEBUG
 	static int num = 0;
-	debug("%i) %llu, %i, %i, %i, %u, %llu", ++num, (long long)block.crc, (int)block.type, (int)block.flags, (int)size, (int)block.addsize, (long long)block.trailsize);
+	debug("%i) %u, %i, %i, %i, %" PRIu64 ", %" PRIu64, ++num, block.crc, block.type, block.flags, size, block.addsize, block.trailsize);
+#endif
 
 	return block;
 }
@@ -527,23 +531,25 @@ bool RarVolume::ReadRar5File(DiskFile& file, RarBlock& block, RarFile& innerFile
 		}
 	}
 
-	debug("%llu, %i, %s", (long long)block.trailsize, (int)namelen, (const char*)name);
+	debug("%" PRIu64 ", %" PRIu64 ", %s", block.trailsize, namelen, (const char*)name);
 
 	return true;
 }
 
 void RarVolume::LogDebugInfo()
 {
+#ifdef DEBUG
 	debug("Volume: version:%i, multi:%i, vol-no:%i, new-naming:%i, has-next:%i, encrypted:%i, file-count:%i, [%s]",
 		(int)m_version, (int)m_multiVolume, m_volumeNo, (int)m_newNaming, (int)m_hasNextVolume,
 		(int)m_encrypted, (int)m_files.size(), FileSystem::BaseFileName(m_filename));
 
 	for (RarFile& file : m_files)
 	{
-		debug("  time:%i, size:%lli, attr:%i, split-before:%i, split-after:%i, [%s]",
-			(int)file.m_time, (long long)file.m_size, (int)file.m_attr,
-			(int)file.m_splitBefore, (int)file.m_splitAfter, *file.m_filename);
+		debug("  time:%i, size:%" PRIi64 ", attr:%i, split-before:%i, split-after:%i, [%s]",
+			file.m_time, file.m_size, file.m_attr,
+			file.m_splitBefore, file.m_splitAfter, *file.m_filename);
 	}
+#endif
 }
 
 bool RarVolume::DecryptRar3Prepare(const uint8 salt[8])

daemon/postprocess/RarRenamer.cpp

@@ -273,6 +273,7 @@ void RarRenamer::MakeSets()
 		}),
 		m_sets.end());
 
+#ifdef DEBUG
 	// debug log
 	for (RarVolumeSet& set : m_sets)
 	{
@@ -282,6 +283,7 @@ void RarRenamer::MakeSets()
 			debug("   %s", FileSystem::BaseFileName(volume->GetFilename()));
 		}
 	}
+#endif
 }
 
 bool RarRenamer::SameArchiveName(const char* filename1, const char* filename2, bool newNaming)
@@ -346,7 +348,7 @@ bool RarRenamer::IsSetProperlyNamed(RarVolumeSet& set)
 			{
 				setPartLen = partNo.Length();
 			}
-			bool ok = atoi(partNo) == volume->GetVolumeNo() + 1 &&
+			bool ok = (uint32)atoi(partNo) == volume->GetVolumeNo() + 1 &&
 				partNo.Length() == setPartLen &&
 				!strncmp(setBasename, filename, regExPart.GetMatchStart(1));
 			if (!ok)

daemon/postprocess/Rename.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2016-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -213,6 +213,10 @@ void RenameController::RegisterRenamedFile(const char* oldFilename, const char*
 	{
 		if (!strcasecmp(completedFile.GetFilename(), oldFilename))
 		{
+			if (Util::EmptyStr(completedFile.GetOrigname()))
+			{
+				completedFile.SetOrigname(completedFile.GetFilename());
+			}
 			completedFile.SetFilename(newFilename);
 			break;
 		}

daemon/postprocess/Repair.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@
 #include "DupeCoordinator.h"
 #include "ParParser.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "DiskState.h"
 #include "Log.h"
 #include "FileSystem.h"
@@ -114,6 +115,19 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char
 		ParChecker::fsUnknown;
 }
 
+const char* RepairController::PostParChecker::FindFileOrigname(const char* filename)
+{
+	for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles())
+	{
+		if (!strcasecmp(completedFile.GetFilename(), filename))
+		{
+			return completedFile.GetOrigname();
+		}
+	}
+
+	return nullptr;
+}
+
 void RepairController::PostParChecker::RequestDupeSources(DupeSourceList* dupeSourceList)
 {
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
@@ -519,7 +533,7 @@ void RepairController::UpdateParCheckProgress()
 
 void RepairController::CheckPauseState(PostInfo* postInfo)
 {
-	if (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
+	if (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority())
 	{
 		time_t stageTime = postInfo->GetStageTime();
 		time_t startTime = postInfo->GetStartTime();
@@ -528,9 +542,9 @@ void RepairController::CheckPauseState(PostInfo* postInfo)
 		time_t waitTime = Util::CurrentTime();
 
 		// wait until Post-processor is unpaused
-		while (g_Options->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
+		while (g_WorkState->GetPausePostProcess() && !postInfo->GetNzbInfo()->GetForcePriority() && !IsStopped())
 		{
-			usleep(50 * 1000);
+			Util::Sleep(50);
 
 			// update time stamps
 

daemon/postprocess/Repair.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -68,6 +68,7 @@ private:
 		virtual void RegisterParredFile(const char* filename);
 		virtual bool IsParredFile(const char* filename);
 		virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments);
+		virtual const char* FindFileOrigname(const char* filename);
 		virtual void RequestDupeSources(DupeSourceList* dupeSourceList);
 		virtual void StatDupeSources(DupeSourceList* dupeSourceList);
 	private:

daemon/postprocess/Unpack.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -874,8 +874,10 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 
 	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: Extracting from ", 23))
 	{
+#ifdef DEBUG
 		const char *filename = text + 23;
 		debug("Filename: %s", filename);
+#endif
 		SetProgressLabel(text + 7);
 	}
 
@@ -886,7 +888,8 @@ void UnpackController::AddMessage(Message::EKind kind, const char* text)
 		m_unpackDecryptError = true;
 	}
 
-	if (m_unpacker == upUnrar && !strncmp(text, "Unrar: The specified password is incorrect.", 43))
+	if (m_unpacker == upUnrar && (!strncmp(text, "Unrar: The specified password is incorrect.", 43) ||
+		!strncmp(text, "Unrar: Incorrect password for", 29)))
 	{
 		m_unpackPasswordError = true;
 	}
@@ -929,3 +932,23 @@ void UnpackController::SetProgressLabel(const char* progressLabel)
 	GuardedDownloadQueue guard = DownloadQueue::Guard();
 	m_postInfo->SetProgressLabel(progressLabel);
 }
+
+bool UnpackController::HasCompletedArchiveFiles(NzbInfo* nzbInfo)
+{
+	RegEx regExRar(".*\\.rar$");
+	RegEx regExSevenZip(".*\\.7z$");
+	RegEx regExSevenZipMulti(".*\\.7z\\.[0-9]+$");
+
+	for (CompletedFile& completedFile: nzbInfo->GetCompletedFiles())
+	{
+		const char* filename = completedFile.GetFilename();
+		if (regExRar.Match(filename) ||
+			regExSevenZip.Match(filename) ||
+			regExSevenZipMulti.Match(filename))
+		{
+			return true;
+		}
+	}
+
+	return false;
+}

daemon/postprocess/Unpack.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2013-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2013-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -32,6 +32,7 @@ public:
 	virtual void Run();
 	virtual void Stop();
 	static void StartJob(PostInfo* postInfo);
+	static bool HasCompletedArchiveFiles(NzbInfo* nzbInfo);
 
 protected:
 	virtual bool ReadLine(char* buf, int bufSize, FILE* stream);

daemon/queue/DirectRenamer.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2017-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -208,7 +208,8 @@ void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fi
 			fileInfo->SetPaused(false);
 		}
 
-		downloadQueue->Save();
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
 	}
 
 	if (fileInfo->GetParFile())
@@ -258,7 +259,8 @@ void DirectRenamer::CheckState(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		// all first articles downloaded
 		UnpausePars(nzbInfo);
 		nzbInfo->SetWaitingPar(true);
-		downloadQueue->Save();
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
 	}
 
 	if (nzbInfo->GetWaitingPar() && !nzbInfo->GetLoadingPar())
@@ -376,12 +378,20 @@ void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo,
 			{
 				nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s",
 					fileInfo->GetFilename(), *newName);
+				if (Util::EmptyStr(fileInfo->GetOrigname()))
+				{
+					fileInfo->SetOrigname(fileInfo->GetFilename());
+				}
 				fileInfo->SetFilename(newName);
 				fileInfo->SetFilenameConfirmed(true);
 				renamedCount++;
 			}
 			else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName))
 			{
+				if (Util::EmptyStr(fileInfo->GetOrigname()))
+				{
+					fileInfo->SetOrigname(fileInfo->GetFilename());
+				}
 				fileInfo->SetFilename(newName);
 				fileInfo->SetFilenameConfirmed(true);
 				renamedCount++;
@@ -404,6 +414,10 @@ void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo,
 
 		if (newName && RenameCompletedFile(nzbInfo, completedFile.GetFilename(), newName))
 		{
+			if (Util::EmptyStr(completedFile.GetOrigname()))
+			{
+				completedFile.SetOrigname(completedFile.GetFilename());
+			}
 			completedFile.SetFilename(newName);
 			renamedCount++;
 		}
@@ -523,12 +537,12 @@ void RenameContentAnalyzer::Reset()
 void RenameContentAnalyzer::Append(const void* buffer, int len)
 {
 #ifndef DISABLE_PARCHECK
-	if (m_dataSize < sizeof(m_signature))
+	if ((size_t)m_dataSize < sizeof(m_signature))
 	{
 		memcpy(m_signature + m_dataSize, buffer, std::min((size_t)len, sizeof(m_signature) - m_dataSize));
 	}
 
-	if (m_dataSize >= sizeof(m_signature) && (*(Par2::MAGIC*)m_signature) == Par2::packet_magic)
+	if ((size_t)m_dataSize >= sizeof(m_signature) && (*(Par2::MAGIC*)m_signature) == Par2::packet_magic)
 	{
 		m_parFile = true;
 		m_parSetId = ((Par2::PACKET_HEADER*)m_signature)->setid.print().c_str();

daemon/queue/DiskState.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@
 #include "FileSystem.h"
 
 static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version ";
-const int DISKSTATE_QUEUE_VERSION = 60;
-const int DISKSTATE_FILE_VERSION = 5;
+const int DISKSTATE_QUEUE_VERSION = 62;
+const int DISKSTATE_FILE_VERSION = 6;
 const int DISKSTATE_STATS_VERSION = 3;
 const int DISKSTATE_FEEDS_VERSION = 3;
 
@@ -45,12 +45,10 @@ int64 StateDiskFile::PrintLine(const char* format, ...)
 {
 	va_list ap;
 	va_start(ap, format);
-	CString str;
-	str.FormatV(format, ap);
+	BString<1024> str;
+	int len = str.FormatV(format, ap);
 	va_end(ap);
 
-	int len = str.Length();
-
 	// replacing terminating <NULL> with <LF>
 	str[len++] = '\n';
 
@@ -117,8 +115,8 @@ public:
 private:
 	BString<1024> m_destFilename;
 	BString<1024> m_tempFilename;
-	bool m_transactional;
 	int m_formatVersion;
+	bool m_transactional;
 	int m_fileVersion;
 	StateDiskFile m_file;
 
@@ -313,6 +311,10 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
 		}
 	}
 
+	// progress-file isn't needed after saving of full queue data
+	StateFile progressStateFile("progress", DISKSTATE_QUEUE_VERSION, true);
+	progressStateFile.Discard();
+
 	return ok;
 }
 
@@ -335,7 +337,12 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 
 			formatVersion = stateFile.GetFileVersion();
 
-			if (formatVersion < 47)
+			if (formatVersion <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+			else if (formatVersion < 47)
 			{
 				error("Failed to read queue and history data. Only queue and history from NZBGet v13 or newer can be converted by this NZBGet version. "
 					"Old queue and history data still can be converted using NZBGet v16 as an intermediate version.");
@@ -351,6 +358,26 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 		}
 	}
 
+	{
+		StateFile stateFile("progress", DISKSTATE_QUEUE_VERSION, true);
+		if (stateFile.FileExists())
+		{
+			StateDiskFile* infile = stateFile.BeginRead();
+			if (!infile)
+			{
+				return false;
+			}
+
+			if (stateFile.GetFileVersion() <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+
+			if (!LoadProgress(downloadQueue->GetQueue(), servers, *infile, stateFile.GetFileVersion())) goto error;
+		}
+	}
+
 	if (formatVersion == 0 || formatVersion >= 57)
 	{
 		StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
@@ -361,6 +388,13 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
 			{
 				return false;
 			}
+
+			if (stateFile.GetFileVersion() <= 0)
+			{
+				error("Failed to read queue: diskstate file is corrupted");
+				goto error;
+			}
+
 			if (!LoadHistory(downloadQueue->GetHistory(), servers, *infile, stateFile.GetFileVersion())) goto error;
 		}
 	}
@@ -388,6 +422,42 @@ error:
 	return ok;
 }
 
+bool DiskState::SaveDownloadProgress(DownloadQueue* downloadQueue)
+{
+	int count = 0;
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		count += nzbInfo->GetChanged() ? 1 : 0;
+	}
+
+	debug("Saving queue progress to disk");
+
+	bool ok = true;
+
+	{
+		StateFile stateFile("progress", DISKSTATE_QUEUE_VERSION, true);
+		if (count > 0)
+		{
+			StateDiskFile* outfile = stateFile.BeginWrite();
+			if (!outfile)
+			{
+				return false;
+			}
+
+			SaveProgress(downloadQueue->GetQueue(), *outfile, count);
+
+			// now rename to dest file name
+			ok = stateFile.FinishWrite();
+		}
+		else
+		{
+			stateFile.Discard();
+		}
+	}
+
+	return ok;
+}
+
 void DiskState::SaveQueue(NzbList* queue, StateDiskFile& outfile)
 {
 	debug("Saving nzb list to disk");
@@ -420,6 +490,50 @@ error:
 	return false;
 }
 
+void DiskState::SaveProgress(NzbList* queue, StateDiskFile& outfile, int changedCount)
+{
+	debug("Saving nzb progress to disk");
+
+	outfile.PrintLine("%i", changedCount);
+	for (NzbInfo* nzbInfo : queue)
+	{
+		if (nzbInfo->GetChanged())
+		{
+			outfile.PrintLine("%i", nzbInfo->GetId());
+			SaveNzbInfo(nzbInfo, outfile);
+		}
+	}
+}
+
+bool DiskState::LoadProgress(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion)
+{
+	debug("Loading nzb progress from disk");
+
+	// load nzb-infos
+	int size;
+	if (infile.ScanLine("%i", &size) != 1) goto error;
+	for (int i = 0; i < size; i++)
+	{
+		int id;
+		if (infile.ScanLine("%i", &id) != 1) goto error;
+
+		NzbInfo* nzbInfo = queue->Find(id);
+		if (!nzbInfo)
+		{
+			error("NZB with id %i could not be found", id);
+			goto error;
+		}
+
+		if (!LoadNzbInfo(nzbInfo, servers, infile, formatVersion)) goto error;
+	}
+
+	return true;
+
+error:
+	error("Error reading nzb progress from disk");
+	return false;
+}
+
 void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 {
 	outfile.PrintLine("%i", nzbInfo->GetId());
@@ -464,7 +578,7 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i,%i,%i", nzbInfo->GetTotalArticles(), nzbInfo->GetSuccessArticles(), nzbInfo->GetFailedArticles());
 
 	outfile.PrintLine("%s", nzbInfo->GetDupeKey());
-	outfile.PrintLine("%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore());
+	outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore(), (int)nzbInfo->GetDupeHint());
 
 	Util::SplitInt64(nzbInfo->GetDownloadedSize(), &High1, &Low1);
 	outfile.PrintLine("%u,%u,%i,%i,%i,%i,%i", High1, Low1, nzbInfo->GetDownloadSec(), nzbInfo->GetPostTotalSec(),
@@ -473,11 +587,12 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i", (int)nzbInfo->GetCompletedFiles()->size());
 	for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles())
 	{
-		outfile.PrintLine("%i,%i,%u,%i,%s,%s,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
+		outfile.PrintLine("%i,%i,%u,%i,%s,%s", completedFile.GetId(), (int)completedFile.GetStatus(),
 			completedFile.GetCrc(), (int)completedFile.GetParFile(),
 			completedFile.GetHash16k() ? completedFile.GetHash16k() : "",
-			completedFile.GetParSetId() ? completedFile.GetParSetId() : "",
-			completedFile.GetFilename());
+			completedFile.GetParSetId() ? completedFile.GetParSetId() : "");
+		outfile.PrintLine("%s", completedFile.GetFilename());
+		outfile.PrintLine("%s", completedFile.GetOrigname() ? completedFile.GetOrigname() : "");
 	}
 
 	outfile.PrintLine("%i", (int)nzbInfo->GetParameters()->size());
@@ -690,10 +805,19 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	nzbInfo->SetDupeKey(buf);
 
-	int dupeMode, dupeScore;
-	if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	int dupeMode, dupeScore, dupeHint;
+	dupeHint = 0; //clang requires initialization in a separate line (due to goto statements)
+	if (formatVersion >= 61)
+	{
+		if (infile.ScanLine("%i,%i,%i", &dupeMode, &dupeScore, &dupeHint) != 3) goto error;
+	}
+	else
+	{
+		if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	}
 	nzbInfo->SetDupeMode((EDupeMode)dupeMode);
 	nzbInfo->SetDupeScore(dupeScore);
+	nzbInfo->SetDupeMode((EDupeMode)dupeHint);
 
 	if (formatVersion >= 48)
 	{
@@ -707,6 +831,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		nzbInfo->SetUnpackSec(unpackSec);
 	}
 
+	nzbInfo->GetCompletedFiles()->clear();
 	if (infile.ScanLine("%i", &fileCount) != 1) goto error;
 	for (int i = 0; i < fileCount; i++)
 	{
@@ -719,6 +844,8 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		int parFile = 0;
 		char* hash16k = nullptr;
 		char* parSetId = nullptr;
+		char filenameBuf[1024];
+		char origName[1024];
 
 		if (formatVersion >= 49)
 		{
@@ -756,14 +883,22 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 			{
 				fileName++;
 			}
+			if (formatVersion >= 62)
+			{
+				if (!infile.ReadLine(filenameBuf, sizeof(filenameBuf))) goto error;
+				fileName = filenameBuf;
+				if (!infile.ReadLine(origName, sizeof(origName))) goto error;
+			}
 		}
 
 		nzbInfo->GetCompletedFiles()->emplace_back(id, fileName,
+			Util::EmptyStr(origName) ? nullptr : origName,
 			(CompletedFile::EStatus)status, crc, (bool)parFile,
 			Util::EmptyStr(hash16k) ? nullptr : hash16k,
 			Util::EmptyStr(parSetId) ? nullptr : parSetId);
 	}
 
+	nzbInfo->GetParameters()->clear();
 	int parameterCount;
 	if (infile.ScanLine("%i", &parameterCount) != 1) goto error;
 	for (int i = 0; i < parameterCount; i++)
@@ -779,6 +914,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		}
 	}
 
+	nzbInfo->GetScriptStatuses()->clear();
 	int scriptCount;
 	if (infile.ScanLine("%i", &scriptCount) != 1) goto error;
 	for (int i = 0; i < scriptCount; i++)
@@ -808,6 +944,7 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 		}
 	}
 
+	nzbInfo->GetFileList()->clear();
 	if (infile.ScanLine("%i", &fileCount) != 1) goto error;
 	for (int i = 0; i < fileCount; i++)
 	{
@@ -900,6 +1037,7 @@ bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool ar
 {
 	outfile.PrintLine("%s", fileInfo->GetSubject());
 	outfile.PrintLine("%s", fileInfo->GetFilename());
+	outfile.PrintLine("%s", fileInfo->GetOrigname() ? fileInfo->GetOrigname() : "");
 
 	outfile.PrintLine("%i,%i", (int)fileInfo->GetFilenameConfirmed(), (int)fileInfo->GetTime());
 
@@ -963,6 +1101,12 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	if (fileSummary) fileInfo->SetFilename(buf);
 
+	if (formatVersion >= 6)
+	{
+		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+		if (fileSummary) fileInfo->SetOrigname(Util::EmptyStr(buf) ? nullptr : buf);
+	}
+
 	if (formatVersion >= 5)
 	{
 		int time, filenameConfirmed;
@@ -1055,7 +1199,8 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool c
 	outfile.PrintLine("%u,%u,%u,%u,%u,%u", High1, Low1, High2, Low2, High3, Low3);
 
 	outfile.PrintLine("%s", fileInfo->GetFilename());
-	outfile.PrintLine("%s", fileInfo->GetHash16k());
+	outfile.PrintLine("%s", fileInfo->GetHash16k() ? fileInfo->GetHash16k() : "");
+	outfile.PrintLine("%s", fileInfo->GetParSetId() ? fileInfo->GetParSetId() : "");
 	outfile.PrintLine("%i", (int)fileInfo->GetParFile());
 
 	SaveServerStats(fileInfo->GetServerStats(), outfile);
@@ -1114,6 +1259,11 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil
 	{
 		if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 		fileInfo->SetHash16k(*buf ? buf : nullptr);
+		if (formatVersion >= 6)
+		{
+			if (!infile.ReadLine(buf, sizeof(buf))) goto error;
+			fileInfo->SetParSetId(*buf ? buf : nullptr);
+		}
 		int parFile = 0;
 		if (infile.ScanLine("%i", &parFile) != 1) goto error;
 		fileInfo->SetParFile((bool)parFile);
@@ -1451,7 +1601,8 @@ void DiskState::CleanupQueueDir(DownloadQueue* downloadQueue)
 
 	for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
 	{
-		if (historyInfo->GetKind() == HistoryInfo::hkNzb)
+		if (historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			historyInfo->GetKind() == HistoryInfo::hkUrl)
 		{
 			NzbInfo* nzbInfo = historyInfo->GetNzbInfo();
 			nzbIdList.push_back(nzbInfo->GetId());

daemon/queue/DiskState.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -36,6 +36,7 @@ public:
 	bool DownloadQueueExists();
 	bool SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory);
 	bool LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers);
+	bool SaveDownloadProgress(DownloadQueue* downloadQueue);
 	bool SaveFile(FileInfo* fileInfo);
 	bool LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles);
 	bool SaveAllFileInfos(DownloadQueue* downloadQueue);
@@ -63,6 +64,8 @@ private:
 	bool LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFile& infile, int formatVersion, bool completed);
 	void SaveQueue(NzbList* queue, StateDiskFile& outfile);
 	bool LoadQueue(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion);
+	void SaveProgress(NzbList* queue, StateDiskFile& outfile, int changedCount);
+	bool LoadProgress(NzbList* queue, Servers* servers, StateDiskFile& infile, int formatVersion);
 	void SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile);
 	bool LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& infile, int formatVersion);
 	void SaveDupInfo(DupInfo* dupInfo, StateDiskFile& outfile);

daemon/queue/DownloadInfo.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -346,10 +346,12 @@ void NzbInfo::UpdateMinMaxTime()
 	}
 }
 
-void NzbInfo::AddMessage(Message::EKind kind, const char * text)
+void NzbInfo::AddMessage(Message::EKind kind, const char * text, bool print)
 {
-	switch (kind)
+	if (print)
 	{
+		switch (kind)
+		{
 		case Message::mkDetail:
 			detail("%s", text);
 			break;
@@ -369,6 +371,7 @@ void NzbInfo::AddMessage(Message::EKind kind, const char * text)
 		case Message::mkDebug:
 			debug("%s", text);
 			break;
+		}
 	}
 
 	Guard guard(m_logMutex);
@@ -496,6 +499,9 @@ bool NzbInfo::IsDupeSuccess()
 		m_parStatus == NzbInfo::psFailure ||
 		m_unpackStatus == NzbInfo::usFailure ||
 		m_unpackStatus == NzbInfo::usPassword ||
+		m_urlStatus == NzbInfo::lsFailed ||
+		m_urlStatus == NzbInfo::lsScanSkipped ||
+		m_urlStatus == NzbInfo::lsScanFailed ||
 		(m_parStatus == NzbInfo::psSkipped &&
 		 m_unpackStatus == NzbInfo::usSkipped &&
 		 CalcHealth() < CalcCriticalHealth(true)));
@@ -628,6 +634,10 @@ const char* NzbInfo::MakeTextStatus(bool ignoreScriptStatus)
 		{
 			status = "DELETED/DUPE";
 		}
+		else if (m_deleteStatus == NzbInfo::dsGood)
+		{
+			status = "DELETED/GOOD";
+		}
 		else
 		{
 			const char* urlStatusName[] = { "FAILURE/INTERNAL_ERROR", "FAILURE/INTERNAL_ERROR", "FAILURE/INTERNAL_ERROR",
@@ -828,10 +838,10 @@ void FileInfo::SetActiveDownloads(int activeDownloads)
 }
 
 
-CompletedFile::CompletedFile(int id, const char* filename, EStatus status, uint32 crc,
-	bool parFile, const char* hash16k, const char* parSetId) :
-	m_id(id), m_filename(filename), m_status(status), m_crc(crc), m_parFile(parFile),
-	m_hash16k(hash16k), m_parSetId(parSetId)
+CompletedFile::CompletedFile(int id, const char* filename, const char* origname, EStatus status,
+	uint32 crc, bool parFile, const char* hash16k, const char* parSetId) :
+	m_id(id), m_filename(filename), m_origname(origname), m_status(status),
+	m_crc(crc), m_parFile(parFile), m_hash16k(hash16k), m_parSetId(parSetId)
 {
 	if (FileInfo::m_idMax < m_id)
 	{

daemon/queue/DownloadInfo.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -140,6 +140,8 @@ public:
 	void SetSubject(const char* subject) { m_subject = subject; }
 	const char* GetFilename() { return m_filename; }
 	void SetFilename(const char* filename) { m_filename = filename; }
+	void SetOrigname(const char* origname) { m_origname = origname; }
+	const char* GetOrigname() { return m_origname; }
 	void MakeValidFilename();
 	bool GetFilenameConfirmed() { return m_filenameConfirmed; }
 	void SetFilenameConfirmed(bool filenameConfirmed) { m_filenameConfirmed = filenameConfirmed; }
@@ -209,6 +211,7 @@ private:
 	ServerStatList m_serverStats;
 	CString m_subject;
 	CString m_filename;
+	CString m_origname;
 	int64 m_size = 0;
 	int64 m_remainingSize = 0;
 	int64 m_successSize = 0;
@@ -259,11 +262,13 @@ public:
 		cfFailure
 	};
 
-	CompletedFile(int id, const char* filename, EStatus status, uint32 crc, 
-		bool parFile, const char* hash16k, const char* parSetId);
+	CompletedFile(int id, const char* filename, const char* oldname, EStatus status,
+		uint32 crc, bool parFile, const char* hash16k, const char* parSetId);
 	int GetId() { return m_id; }
 	void SetFilename(const char* filename) { m_filename = filename; }
 	const char* GetFilename() { return m_filename; }
+	void SetOrigname(const char* origname) { m_origname = origname; }
+	const char* GetOrigname() { return m_origname; }
 	bool GetParFile() { return m_parFile; }
 	EStatus GetStatus() { return m_status; }
 	uint32 GetCrc() { return m_crc; }
@@ -275,6 +280,7 @@ public:
 private:
 	int m_id;
 	CString m_filename;
+	CString m_origname;
 	EStatus m_status;
 	uint32 m_crc;
 	bool m_parFile;
@@ -446,6 +452,13 @@ public:
 		nkUrl
 	};
 
+	enum EDupeHint
+	{
+		dhNone,
+		dhRedownloadManual,
+		dhRedownloadAuto
+	};
+
 	int GetId() { return m_id; }
 	void SetId(int id);
 	static void ResetGenId(bool max);
@@ -580,6 +593,8 @@ public:
 	void SetDupeScore(int dupeScore) { m_dupeScore = dupeScore; }
 	EDupeMode GetDupeMode() { return m_dupeMode; }
 	void SetDupeMode(EDupeMode dupeMode) { m_dupeMode = dupeMode; }
+	EDupeHint GetDupeHint() { return m_dupeHint; }
+	void SetDupeHint(EDupeHint dupeHint) { m_dupeHint = dupeHint; }
 	uint32 GetFullContentHash() { return m_fullContentHash; }
 	void SetFullContentHash(uint32 fullContentHash) { m_fullContentHash = fullContentHash; }
 	uint32 GetFilteredContentHash() { return m_filteredContentHash; }
@@ -615,7 +630,7 @@ public:
 	void LeavePostProcess();
 	bool IsDupeSuccess();
 	const char* MakeTextStatus(bool ignoreScriptStatus);
-	void AddMessage(Message::EKind kind, const char* text);
+	void AddMessage(Message::EKind kind, const char* text, bool print = true);
 	void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3);
 	int GetMessageCount() { return m_messageCount; }
 	void SetMessageCount(int messageCount) { m_messageCount = messageCount; }
@@ -698,6 +713,7 @@ private:
 	CString m_dupeKey = "";
 	int m_dupeScore = 0;
 	EDupeMode m_dupeMode = dmScore;
+	EDupeHint m_dupeHint = dhNone;
 	uint32 m_fullContentHash = 0;
 	uint32 m_filteredContentHash = 0;
 	FileList m_fileList;
@@ -885,7 +901,7 @@ public:
 	};
 
 	HistoryInfo(std::unique_ptr<NzbInfo> nzbInfo) : m_info(nzbInfo.release()),
-		m_kind(nzbInfo->GetKind() == NzbInfo::nkNzb ? hkNzb : hkUrl) {}
+		m_kind(GetNzbInfo()->GetKind() == NzbInfo::nkNzb ? hkNzb : hkUrl) {}
 	HistoryInfo(std::unique_ptr<DupInfo> dupInfo) : m_info(dupInfo.release()), m_kind(hkDup) {}
 	~HistoryInfo();
 	EKind GetKind() { return m_kind; }
@@ -898,8 +914,8 @@ public:
 	const char* GetName();
 
 private:
-	EKind m_kind;
 	void* m_info;
+	EKind m_kind;
 	time_t m_time = 0;
 };
 
@@ -916,9 +932,15 @@ public:
 		eaNzbAdded,
 		eaNzbDeleted,
 		eaNzbNamed,
+		eaNzbReturned,
 		eaFileCompleted,
 		eaFileDeleted,
-		eaUrlCompleted
+		eaUrlFound,
+		eaUrlAdded,
+		eaUrlDeleted,
+		eaUrlCompleted,
+		eaUrlFailed,
+		eaUrlReturned
 	};
 
 	struct Aspect
@@ -999,6 +1021,7 @@ public:
 	virtual bool EditList(IdList* idList, NameList* nameList, EMatchMode matchMode, EEditAction action, const char* args) = 0;
 	virtual void HistoryChanged() = 0;
 	virtual void Save() = 0;
+	virtual void SaveChanged() = 0;
 	void CalcRemainingSize(int64* remaining, int64* remainingForced);
 
 protected:

daemon/queue/DupeCoordinator.cpp

@@ -202,12 +202,19 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		}
 	}
 
+	if (!sameContent && nzbInfo->GetDupeHint() != NzbInfo::dhNone)
+	{
+		// dupe check when "download again" URLs: checking same content only
+		return;
+	}
+
 	if (!sameContent && !good && nzbInfo->GetDupeMode() == dmScore)
 	{
 		// nzb-files having success-duplicates in recent history (with different content) are added to history for backup
 		for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
 		{
-			if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+			if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+				 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 				historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 				SameNameOrKey(historyInfo->GetNzbInfo()->GetName(), historyInfo->GetNzbInfo()->GetDupeKey(),
 					nzbInfo->GetName(), nzbInfo->GetDupeKey()) &&
@@ -237,7 +244,7 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 				sameContent ? "exactly same content" : good ? "good status" : "success status");
 		}
 
-		if (nzbInfo->GetFeedId())
+		if (nzbInfo->GetFeedId() && nzbInfo->GetDupeHint() == NzbInfo::dhNone)
 		{
 			warn("%s", *message);
 			// Flag saying QueueCoordinator to skip nzb-file
@@ -263,7 +270,9 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		{
 			NzbInfo* queuedNzbInfo = (*it++).get();
 			if (queuedNzbInfo != nzbInfo &&
-				queuedNzbInfo->GetKind() == NzbInfo::nkNzb &&
+				queuedNzbInfo->GetDeleteStatus() == NzbInfo::dsNone &&
+				(queuedNzbInfo->GetKind() == NzbInfo::nkNzb ||
+				 (queuedNzbInfo->GetKind() == NzbInfo::nkUrl && nzbInfo->GetKind() == NzbInfo::nkUrl)) &&
 				queuedNzbInfo->GetDupeMode() != dmForce &&
 				SameNameOrKey(queuedNzbInfo->GetName(), queuedNzbInfo->GetDupeKey(),
 					nzbInfo->GetName(), nzbInfo->GetDupeKey()))
@@ -286,9 +295,13 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 					// the existing queue item is moved to history as dupe-backup
 					info("Moving collection %s with lower duplicate score to history", queuedNzbInfo->GetName());
 					queuedNzbInfo->SetDeleteStatus(NzbInfo::dsDupe);
+					int oldSize = downloadQueue->GetQueue()->size();
 					downloadQueue->EditEntry(queuedNzbInfo->GetId(),
 						DownloadQueue::eaGroupDelete, nullptr);
+					int newSize = downloadQueue->GetQueue()->size();
+					index += oldSize == newSize ? 1 : 0;
 					it = downloadQueue->GetQueue()->begin() + index;
+					index--;
 				}
 			}
 		}
@@ -377,7 +390,8 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI
 	HistoryInfo* historyDupe = nullptr;
 	for (HistoryInfo* historyInfo : downloadQueue->GetHistory())
 	{
-		if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+		if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 			historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 			historyInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsDupe &&
 			historyInfo->GetNzbInfo()->CalcHealth() >= historyInfo->GetNzbInfo()->CalcCriticalHealth(true) &&
@@ -395,6 +409,7 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI
 	if (historyDupe)
 	{
 		info("Found duplicate %s for %s", historyDupe->GetNzbInfo()->GetName(), nzbName);
+		historyDupe->GetNzbInfo()->SetDupeHint(NzbInfo::dhRedownloadAuto);
 		g_HistoryCoordinator->Redownload(downloadQueue, historyDupe);
 	}
 }
@@ -465,7 +480,8 @@ void DupeCoordinator::HistoryCleanup(DownloadQueue* downloadQueue, HistoryInfo*
 	{
 		HistoryInfo* historyInfo = (*it).get();
 
-		if (historyInfo->GetKind() == HistoryInfo::hkNzb &&
+		if ((historyInfo->GetKind() == HistoryInfo::hkNzb ||
+			 historyInfo->GetKind() == HistoryInfo::hkUrl) &&
 			historyInfo->GetNzbInfo()->GetDupeMode() != dmForce &&
 			historyInfo->GetNzbInfo()->GetDeleteStatus() == NzbInfo::dsDupe &&
 			historyInfo != markHistoryInfo &&

daemon/queue/HistoryCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -131,7 +131,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 	{
 		nzbInfo->UpdateCompletedStats(fileInfo);
 		nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(),
-			CompletedFile::cfNone, 0, fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId());
+			fileInfo->GetOrigname(), CompletedFile::cfNone, 0, fileInfo->GetParFile(),
+			fileInfo->GetHash16k(), fileInfo->GetParSetId());
 	}
 
 	// Cleaning up parked files if par-check was successful or unpack was successful or
@@ -183,6 +184,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 		nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure);
 	}
 
+	nzbInfo->SetDupeHint(NzbInfo::dhNone);
+
 	nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName());
 }
 
@@ -421,6 +424,8 @@ void HistoryCoordinator::MoveToQueue(DownloadQueue* downloadQueue, HistoryList::
 		// start postprocessing
 		debug("Restarting postprocessing for %s", *nicename);
 		g_PrePostProcessor->NzbDownloaded(downloadQueue, nzbInfo);
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaNzbReturned, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
 	}
 }
 
@@ -433,8 +438,13 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 		historyInfo->DiscardNzbInfo();
 		nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 		nzbInfo->SetDeleteStatus(NzbInfo::dsNone);
+		nzbInfo->SetDupeHint(nzbInfo->GetDupeHint() == NzbInfo::dhNone ? NzbInfo::dhRedownloadManual : nzbInfo->GetDupeHint());
 		downloadQueue->GetQueue()->Add(std::unique_ptr<NzbInfo>(nzbInfo), true);
 		downloadQueue->GetHistory()->erase(itHistory);
+
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaUrlReturned, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
+
 		return;
 	}
 
@@ -522,6 +532,9 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 	MoveToQueue(downloadQueue, itHistory, historyInfo, false);
 
 	g_PrePostProcessor->NzbAdded(downloadQueue, nzbInfo);
+
+	DownloadQueue::Aspect aspect = {DownloadQueue::eaNzbReturned, downloadQueue, nzbInfo, nullptr};
+	downloadQueue->Notify(&aspect);
 }
 
 void HistoryCoordinator::HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory,

daemon/queue/HistoryCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ public:
 	void Redownload(DownloadQueue* downloadQueue, HistoryInfo* historyInfo);
 
 protected:
-	virtual int ServiceInterval() { return 600000; }
+	virtual int ServiceInterval() { return 60 * 60; }
 	virtual void ServiceWork();
 
 private:

daemon/queue/NzbFile.cpp

@@ -117,6 +117,12 @@ void NzbFile::ParseSubject(FileInfo* fileInfo, bool TryQuotes)
 {
 	// Example subject: some garbage "title" yEnc (10/99)
 
+	if (!fileInfo->GetSubject())
+	{
+		// Malformed file element without subject. We generate subject using internal element id.
+		fileInfo->SetSubject(CString::FormatStr("%d", fileInfo->GetId()));
+	}
+
 	// strip the "yEnc (10/99)"-suffix
 	BString<1024> subject = fileInfo->GetSubject();
 	char* end = subject + strlen(subject) - 1;

daemon/queue/QueueCoordinator.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "nzbget.h"
 #include "QueueCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "ServerPool.h"
 #include "ArticleDownloader.h"
 #include "ArticleWriter.h"
@@ -71,6 +72,18 @@ void QueueCoordinator::CoordinatorDownloadQueue::Save()
 
 	m_wantSave = false;
 	m_historyChanged = false;
+
+	// queue has changed, time to wake up if in standby
+	m_owner->WakeUp();
+}
+
+void QueueCoordinator::CoordinatorDownloadQueue::SaveChanged()
+{
+	if (g_Options->GetServerMode())
+	{
+		g_DiskState->SaveDownloadProgress(this);
+		m_stateChanged = true;
+	}
 }
 
 QueueCoordinator::QueueCoordinator()
@@ -78,6 +91,7 @@ QueueCoordinator::QueueCoordinator()
 	debug("Creating QueueCoordinator");
 
 	CoordinatorDownloadQueue::Init(&m_downloadQueue);
+	g_WorkState->Attach(this);
 }
 
 QueueCoordinator::~QueueCoordinator()
@@ -177,8 +191,9 @@ void QueueCoordinator::Run()
 	AdjustDownloadsLimit();
 	bool wasStandBy = true;
 	bool articeDownloadsRunning = false;
-	int resetCounter = 0;
+	time_t lastReset = 0;
 	g_StatMeter->IntervalCheck();
+	int waitInterval = 100;
 
 	while (!IsStopped())
 	{
@@ -199,7 +214,7 @@ void QueueCoordinator::Run()
 				downloadsChecked = true;
 				m_hasMoreJobs = hasMoreArticles || articeDownloadsRunning;
 				if (hasMoreArticles && !IsStopped() && (int)m_activeDownloads.size() < m_downloadsLimit &&
-					(!g_Options->GetTempPauseDownload() || fileInfo->GetExtraPriority()))
+					(!g_WorkState->GetTempPauseDownload() || fileInfo->GetExtraPriority()))
 				{
 					StartArticleDownload(fileInfo, articleInfo, connection);
 					articeDownloadsRunning = true;
@@ -227,6 +242,7 @@ void QueueCoordinator::Run()
 		if (standBy != wasStandBy)
 		{
 			g_StatMeter->EnterLeaveStandBy(standBy);
+			g_WorkState->SetDownloading(!standBy);
 			wasStandBy = standBy;
 			if (standBy)
 			{
@@ -235,18 +251,23 @@ void QueueCoordinator::Run()
 		}
 
 		// sleep longer in StandBy
-		int sleepInterval = downloadStarted ? 0 : standBy ? 100 : 5;
-		usleep(sleepInterval * 1000);
-
-		if (!standBy)
+		if (standBy)
 		{
+			Guard guard(m_waitMutex);
+			// sleeping max. 2 seconds; can't sleep much longer because we can't rely on
+			// notifications from 'WorkState' and we also have periodical work to do here
+			waitInterval = std::min(waitInterval * 2, 2000);
+			m_waitCond.WaitFor(m_waitMutex, waitInterval, [&]{ return m_hasMoreJobs || IsStopped(); });
+		}
+		else
+		{
+			int sleepInterval = downloadStarted ? 0 : 5;
+			Util::Sleep(sleepInterval);
 			g_StatMeter->AddSpeedReading(0);
+			waitInterval = 100;
 		}
 
-		Util::SetStandByMode(standBy);
-
-		resetCounter += sleepInterval;
-		if (resetCounter >= 1000)
+		if (lastReset != Util::CurrentTime())
 		{
 			// this code should not be called too often, once per second is OK
 			g_ServerPool->CloseUnusedConnections();
@@ -255,20 +276,31 @@ void QueueCoordinator::Run()
 			{
 				SaveAllPartialState();
 			}
-			resetCounter = 0;
 			g_StatMeter->IntervalCheck();
 			g_Log->IntervalCheck();
 			AdjustDownloadsLimit();
+			Util::SetStandByMode(standBy);
+			lastReset = Util::CurrentTime();
 		}
 	}
 
 	WaitJobs();
 	SaveAllPartialState();
+	SaveQueueIfChanged();
 	SaveAllFileState();
 
 	debug("Exiting QueueCoordinator-loop");
 }
 
+void QueueCoordinator::WakeUp()
+{
+	debug("Waking up QueueCoordinator");
+	// Resume Run()
+	Guard guard(m_waitMutex);
+	m_hasMoreJobs = true;
+	m_waitCond.NotifyAll();
+}
+
 void QueueCoordinator::WaitJobs()
 {
 	// waiting for downloads
@@ -283,7 +315,7 @@ void QueueCoordinator::WaitJobs()
 				break;
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 
@@ -419,7 +451,7 @@ void QueueCoordinator::CheckDupeFileInfos(NzbInfo* nzbInfo)
 			{
 				// If more than two files have same filename we don't filter them out since that
 				// naming might be intentional and correct filenames must be read from article bodies.
-				int dupeCount = std::count_if(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
+				int dupeCount = (int)std::count_if(nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(),
 					[fileInfo2](std::unique_ptr<FileInfo>& fileInfo3)
 					{
 						return !strcmp(fileInfo3->GetFilename(), fileInfo2->GetFilename());
@@ -455,12 +487,18 @@ void QueueCoordinator::Stop()
 	Thread::Stop();
 
 	debug("Stopping ArticleDownloads");
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
-	for (ArticleDownloader* articleDownloader : m_activeDownloads)
 	{
-		articleDownloader->Stop();
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		for (ArticleDownloader* articleDownloader : m_activeDownloads)
+		{
+			articleDownloader->Stop();
+		}
 	}
 	debug("ArticleDownloads are notified");
+
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
 }
 
 /*
@@ -495,7 +533,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
 			bool nzbPaused = nzbInfo->GetFileList()->size() - nzbInfo->GetPausedFileCount() <= 0;
 
 			if ((!fileInfo || nzbHigherPriority) && !nzbPaused &&
-				(!(g_Options->GetPauseDownload() || g_Options->GetQuotaReached()) || nzbInfo->GetForcePriority()))
+				(!(g_WorkState->GetPauseDownload() || g_WorkState->GetQuotaReached()) || nzbInfo->GetForcePriority()))
 			{
 				for (FileInfo* fileInfo1 : nzbInfo->GetFileList())
 				{
@@ -619,11 +657,18 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art
 	articleDownloader->Start();
 }
 
-void QueueCoordinator::Update(Subject* Caller, void* Aspect)
+void QueueCoordinator::Update(Subject* caller, void* aspect)
 {
+	if (caller == g_WorkState)
+	{
+		debug("Notification from WorkState received");
+		WakeUp();
+		return;
+	}
+
 	debug("Notification from ArticleDownloader received");
 
-	ArticleDownloader* articleDownloader = (ArticleDownloader*)Caller;
+	ArticleDownloader* articleDownloader = (ArticleDownloader*)caller;
 	if ((articleDownloader->GetStatus() == ArticleDownloader::adFinished) ||
 		(articleDownloader->GetStatus() == ArticleDownloader::adFailed) ||
 		(articleDownloader->GetStatus() == ArticleDownloader::adRetry))
@@ -768,7 +813,8 @@ void QueueCoordinator::DeleteDownloader(DownloadQueue* downloadQueue,
 	if (deleteFileObj)
 	{
 		DeleteFileInfo(downloadQueue, fileInfo, fileCompleted);
-		downloadQueue->Save();
+		nzbInfo->SetChanged(true);
+		downloadQueue->SaveChanged();
 	}
 }
 
@@ -776,7 +822,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 {
 	while (g_ArticleCache->FileBusy(fileInfo))
 	{
-		usleep(5*1000);
+		Util::Sleep(5);
 	}
 
 	NzbInfo* nzbInfo = fileInfo->GetNzbInfo();
@@ -820,7 +866,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi
 			fileInfo->GetId(),
 			completed && fileInfo->GetOutputFilename() ?
 			FileSystem::BaseFileName(fileInfo->GetOutputFilename()) : fileInfo->GetFilename(),
-			fileStatus,
+			fileInfo->GetOrigname(), fileStatus,
 			fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0,
 			fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId());
 	}
@@ -866,33 +912,45 @@ void QueueCoordinator::DiscardTempFiles(FileInfo* fileInfo)
 	}
 }
 
-void QueueCoordinator::SaveAllPartialState()
+void QueueCoordinator::SaveQueueIfChanged()
 {
 	if (!g_Options->GetServerMode())
 	{
 		return;
 	}
 
-	bool hasUnsavedData = false;
+	bool hasChanges = false;
 	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
 	{
-		if (g_Options->GetContinuePartial())
-		{
-			for (FileInfo* fileInfo : nzbInfo->GetFileList())
-			{
-				SavePartialState(fileInfo);
-			}
-		}
-		hasUnsavedData |= nzbInfo->GetChanged();
+		hasChanges |= nzbInfo->GetChanged();
 	}
 
-	if (hasUnsavedData)
+	if (hasChanges)
 	{
 		downloadQueue->Save();
 	}
 }
 
+void QueueCoordinator::SaveAllPartialState()
+{
+	if (!g_Options->GetServerMode() || !g_Options->GetContinuePartial())
+	{
+		return;
+	}
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+	for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
+	{
+		for (FileInfo* fileInfo : nzbInfo->GetFileList())
+		{
+			SavePartialState(fileInfo);
+		}
+	}
+
+	downloadQueue->SaveChanged();
+}
+
 void QueueCoordinator::SavePartialState(FileInfo* fileInfo)
 {
 	if (fileInfo->GetPartialChanged())
@@ -978,9 +1036,9 @@ void QueueCoordinator::LogDebugInfo()
 	downloadQueue->CalcRemainingSize(&remaining, &remainingForced);
 	info("     Remaining: %.1f MB, Forced: %.1f MB", remaining / 1024.0 / 1024.0, remainingForced / 1024.0 / 1024.0);
 	info("     Download: %s, Post-process: %s, Scan: %s",
-		 (g_Options->GetPauseDownload() ? "paused" : g_Options->GetTempPauseDownload() ? "temp-paused" : "active"),
-		 (g_Options->GetPausePostProcess() ? "paused" : "active"),
-		 (g_Options->GetPauseScan() ? "paused" : "active"));
+		 (g_WorkState->GetPauseDownload() ? "paused" : g_WorkState->GetTempPauseDownload() ? "temp-paused" : "active"),
+		 (g_WorkState->GetPausePostProcess() ? "paused" : "active"),
+		 (g_WorkState->GetPauseScan() ? "paused" : "active"));
 
 	info("   ---------- QueueCoordinator");
 	info("    Active Downloads: %i, Limit: %i", (int)m_activeDownloads.size(), m_downloadsLimit);
@@ -1315,7 +1373,8 @@ void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbIn
 		downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr);
 	}
 
-	downloadQueue->Save();
+	nzbInfo->SetChanged(true);
+	downloadQueue->SaveChanged();
 
 	DownloadQueue::Aspect namedAspect = { DownloadQueue::eaNzbNamed, downloadQueue, nzbInfo, nullptr };
 	downloadQueue->Notify(&namedAspect);

daemon/queue/QueueCoordinator.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -41,7 +41,7 @@ public:
 	virtual ~QueueCoordinator();
 	virtual void Run();
 	virtual void Stop();
-	void Update(Subject* Caller, void* Aspect);
+	void Update(Subject* caller, void* aspect);
 
 	// editing queue
 	NzbInfo* AddNzbFileToQueue(std::unique_ptr<NzbInfo> nzbInfo, NzbInfo* urlInfo, bool addFirst);
@@ -67,6 +67,7 @@ private:
 			EEditAction action, const char* args);
 		virtual void HistoryChanged() { m_historyChanged = true; }
 		virtual void Save();
+		virtual void SaveChanged();
 	private:
 		QueueCoordinator* m_owner;
 		bool m_massEdit = false;
@@ -94,6 +95,8 @@ private:
 	bool m_hasMoreJobs = true;
 	int m_downloadsLimit;
 	int m_serverConfigGeneration = 0;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
 
 	bool GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
 	bool GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo, ArticleInfo* &articleInfo);
@@ -108,11 +111,13 @@ private:
 	void ResetHangingDownloads();
 	void AdjustDownloadsLimit();
 	void Load();
+	void SaveQueueIfChanged();
 	void SaveAllPartialState();
 	void SavePartialState(FileInfo* fileInfo);
 	void LoadPartialState(FileInfo* fileInfo);
 	void SaveAllFileState();
 	void WaitJobs();
+	void WakeUp();
 };
 
 extern QueueCoordinator* g_QueueCoordinator;

daemon/queue/QueueEditor.h

@@ -33,9 +33,9 @@ private:
 	class EditItem
 	{
 	public:
-		int m_offset;
 		FileInfo* m_fileInfo;
 		NzbInfo* m_nzbInfo;
+		int m_offset;
 
 		EditItem(FileInfo* fileInfo, NzbInfo* nzbInfo, int offset) :
 			m_fileInfo(fileInfo), m_nzbInfo(nzbInfo), m_offset(offset) {}

daemon/queue/Scanner.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Scanner.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Log.h"
 #include "QueueCoordinator.h"
 #include "HistoryCoordinator.h"
@@ -73,60 +74,72 @@ void Scanner::QueueData::SetNzbId(int nzbId)
 
 void Scanner::InitOptions()
 {
-	m_nzbDirInterval = g_Options->GetNzbDirInterval() * 1000;
+	m_nzbDirInterval = 1;
 	m_scanScript = ScanScriptController::HasScripts();
 }
 
+int Scanner::ServiceInterval()
+{
+	return m_requestedNzbDirScan ? Service::Now :
+		g_Options->GetNzbDirInterval() <= 0 ? Service::Sleep :
+		// g_Options->GetPauseScan() ? Service::Sleep :   // for that to work we need to react on changing of pause-state
+		m_nzbDirInterval;
+}
+
 void Scanner::ServiceWork()
 {
+	debug("Scanner service work");
+
 	if (!DownloadQueue::IsLoaded())
 	{
 		return;
 	}
 
+	m_nzbDirInterval = g_Options->GetNzbDirInterval();
+
+	if (g_WorkState->GetPauseScan() && !m_requestedNzbDirScan)
+	{
+		return;
+	}
+
+	debug("Scanner service work: doing work");
+
 	Guard guard(m_scanMutex);
 
-	if (m_requestedNzbDirScan ||
-		(!g_Options->GetPauseScan() && g_Options->GetNzbDirInterval() > 0 &&
-		 m_nzbDirInterval >= g_Options->GetNzbDirInterval() * 1000))
+	// check nzbdir every g_pOptions->GetNzbDirInterval() seconds or if requested
+	bool checkStat = !m_requestedNzbDirScan;
+	m_requestedNzbDirScan = false;
+	m_scanning = true;
+	CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
+	if (!checkStat && m_scanScript)
 	{
-		// check nzbdir every g_pOptions->GetNzbDirInterval() seconds or if requested
-		bool checkStat = !m_requestedNzbDirScan;
-		m_requestedNzbDirScan = false;
-		m_scanning = true;
+		// if immediate scan requested, we need second scan to process files extracted by scan-scripts
 		CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
-		if (!checkStat && m_scanScript)
-		{
-			// if immediate scan requested, we need second scan to process files extracted by scan-scripts
-			CheckIncomingNzbs(g_Options->GetNzbDir(), "", checkStat);
-		}
-		m_scanning = false;
-		m_nzbDirInterval = 0;
-
-		// if NzbDirFileAge is less than NzbDirInterval (that can happen if NzbDirInterval
-		// is set for rare scans like once per hour) we make 4 scans:
-		//   - one additional scan is neccessary to check sizes of detected files;
-		//   - another scan is required to check files which were extracted by scan-scripts;
-		//   - third scan is needed to check sizes of extracted files.
-		if (g_Options->GetNzbDirInterval() > 0 && g_Options->GetNzbDirFileAge() < g_Options->GetNzbDirInterval())
-		{
-			int maxPass = m_scanScript ? 3 : 1;
-			if (m_pass < maxPass)
-			{
-				// scheduling another scan of incoming directory in NzbDirFileAge seconds.
-				m_nzbDirInterval = (g_Options->GetNzbDirInterval() - g_Options->GetNzbDirFileAge()) * 1000;
-				m_pass++;
-			}
-			else
-			{
-				m_pass = 0;
-			}
-		}
-
-		DropOldFiles();
-		m_queueList.clear();
 	}
-	m_nzbDirInterval += 200;
+	m_scanning = false;
+
+	// if NzbDirFileAge is less than NzbDirInterval (that can happen if NzbDirInterval
+	// is set for rare scans like once per hour) we make 4 scans:
+	//   - one additional scan is neccessary to check sizes of detected files;
+	//   - another scan is required to check files which were extracted by scan-scripts;
+	//   - third scan is needed to check sizes of extracted files.
+	if (g_Options->GetNzbDirInterval() > 0 && g_Options->GetNzbDirFileAge() < g_Options->GetNzbDirInterval())
+	{
+		int maxPass = m_scanScript ? 3 : 1;
+		if (m_pass < maxPass)
+		{
+			// scheduling another scan of incoming directory in NzbDirFileAge seconds.
+			m_nzbDirInterval = g_Options->GetNzbDirFileAge();
+			m_pass++;
+		}
+		else
+		{
+			m_pass = 0;
+		}
+	}
+
+	DropOldFiles();
+	m_queueList.clear();
 }
 
 /**
@@ -438,6 +451,7 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch
 		nzbInfo->SetUrl(urlInfo->GetUrl());
 		nzbInfo->SetUrlStatus(urlInfo->GetUrlStatus());
 		nzbInfo->SetFeedId(urlInfo->GetFeedId());
+		nzbInfo->SetDupeHint(urlInfo->GetDupeHint());
 	}
 
 	if (nzbFile.GetPassword())
@@ -458,7 +472,14 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch
 	{
 		addedNzb = g_QueueCoordinator->AddNzbFileToQueue(std::move(nzbInfo), std::move(urlInfo), addTop);
 	}
-	else if (!urlInfo)
+	else if (urlInfo)
+	{
+		for (Message& message : nzbInfo->GuardCachedMessages())
+		{
+			urlInfo->AddMessage(message.GetKind(), message.GetText(), false);
+		}
+	}
+	else
 	{
 		nzbInfo->SetDeleteStatus(NzbInfo::dsScan);
 		addedNzb = g_QueueCoordinator->AddNzbFileToQueue(std::move(nzbInfo), std::move(urlInfo), addTop);
@@ -478,16 +499,17 @@ void Scanner::ScanNzbDir(bool syncMode)
 		Guard guard(m_scanMutex);
 		m_scanning = true;
 		m_requestedNzbDirScan = true;
+		WakeUp();
 	}
 
 	while (syncMode && (m_scanning || m_requestedNzbDirScan))
 	{
-		usleep(100 * 1000);
+		Util::Sleep(100);
 	}
 }
 
 Scanner::EAddStatus Scanner::AddExternalFile(const char* nzbName, const char* category,
-	int priority, const char* dupeKey, int dupeScore,  EDupeMode dupeMode,
+	int priority, const char* dupeKey, int dupeScore, EDupeMode dupeMode,
 	NzbParameterList* parameters, bool addTop, bool addPaused, NzbInfo* urlInfo,
 	const char* fileName, const char* buffer, int bufSize, int* nzbId)
 {

daemon/queue/Scanner.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -46,7 +46,7 @@ public:
 	void InitPPParameters(const char* category, NzbParameterList* parameters, bool reset);
 
 protected:
-	virtual int ServiceInterval() { return 200; }
+	virtual int ServiceInterval();
 	virtual void ServiceWork();
 
 private:

daemon/queue/UrlCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "UrlCoordinator.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "WebDownloader.h"
 #include "Util.h"
 #include "FileSystem.h"
@@ -60,6 +61,12 @@ void UrlDownloader::ProcessHeader(const char* line)
 	}
 }
 
+UrlCoordinator::UrlCoordinator()
+{
+	m_downloadQueueObserver.m_owner = this;
+	DownloadQueue::Guard()->Attach(&m_downloadQueueObserver);
+}
+
 UrlCoordinator::~UrlCoordinator()
 {
 	debug("Destroying UrlCoordinator");
@@ -79,41 +86,45 @@ void UrlCoordinator::Run()
 
 	while (!DownloadQueue::IsLoaded())
 	{
-		usleep(20 * 1000);
+		Util::Sleep(20);
 	}
 
-	int resetCounter = 0;
-
 	while (!IsStopped())
 	{
+		time_t lastReset = 0;
 		bool downloadStarted = false;
-		if (!g_Options->GetPauseDownload() || g_Options->GetUrlForce())
+
 		{
-			// start download for next URL
+			NzbInfo* nzbInfo = nullptr;
 			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 			if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
 			{
-				NzbInfo* nzbInfo = GetNextUrl(downloadQueue);
-				bool hasMoreUrls = nzbInfo != nullptr;
-				bool urlDownloadsRunning = !m_activeDownloads.empty();
-				m_hasMoreJobs = hasMoreUrls || urlDownloadsRunning;
-				if (hasMoreUrls && !IsStopped())
+				nzbInfo = GetNextUrl(downloadQueue);
+				if (nzbInfo && (!g_WorkState->GetPauseDownload() || g_Options->GetUrlForce()))
 				{
 					StartUrlDownload(nzbInfo);
 					downloadStarted = true;
 				}
 			}
+			m_hasMoreJobs = !m_activeDownloads.empty() || nzbInfo;
 		}
 
-		int sleepInterval = downloadStarted ? 0 : 100;
-		usleep(sleepInterval * 1000);
-
-		resetCounter += sleepInterval;
-		if (resetCounter >= 1000)
+		if (lastReset != Util::CurrentTime())
 		{
 			// this code should not be called too often, once per second is OK
 			ResetHangingDownloads();
-			resetCounter = 0;
+			lastReset = Util::CurrentTime();
+		}
+
+		if (!m_hasMoreJobs && !IsStopped())
+		{
+			Guard guard(m_waitMutex);
+			m_waitCond.Wait(m_waitMutex, [&] { return m_hasMoreJobs || IsStopped(); });
+		}
+		else
+		{
+			int sleepInterval = downloadStarted ? 0 : 100;
+			Util::Sleep(sleepInterval);
 		}
 	}
 
@@ -136,7 +147,7 @@ void UrlCoordinator::WaitJobs()
 				break;
 			}
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
 		ResetHangingDownloads();
 	}
 
@@ -148,12 +159,33 @@ void UrlCoordinator::Stop()
 	Thread::Stop();
 
 	debug("Stopping UrlDownloads");
-	GuardedDownloadQueue guard = DownloadQueue::Guard();
-	for (UrlDownloader* urlDownloader : m_activeDownloads)
 	{
-		urlDownloader->Stop();
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		for (UrlDownloader* urlDownloader : m_activeDownloads)
+		{
+			urlDownloader->Stop();
+		}
 	}
 	debug("UrlDownloads are notified");
+
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
+}
+
+void UrlCoordinator::DownloadQueueUpdate(Subject* caller, void* aspect)
+{
+	debug("Notification from download queue received");
+
+	DownloadQueue::Aspect* queueAspect = (DownloadQueue::Aspect*)aspect;
+	if (queueAspect->action == DownloadQueue::eaUrlAdded ||
+		queueAspect->action == DownloadQueue::eaUrlReturned)
+	{
+		// Resume Run()
+		Guard guard(m_waitMutex);
+		m_hasMoreJobs = true;
+		m_waitCond.NotifyAll();
+	}
 }
 
 void UrlCoordinator::ResetHangingDownloads()
@@ -194,8 +226,6 @@ void UrlCoordinator::LogDebugInfo()
  */
 NzbInfo* UrlCoordinator::GetNextUrl(DownloadQueue* downloadQueue)
 {
-	bool pauseDownload = g_Options->GetPauseDownload();
-
 	NzbInfo* nzbInfo = nullptr;
 
 	for (NzbInfo* nzbInfo1 : downloadQueue->GetQueue())
@@ -203,7 +233,6 @@ NzbInfo* UrlCoordinator::GetNextUrl(DownloadQueue* downloadQueue)
 		if (nzbInfo1->GetKind() == NzbInfo::nkUrl &&
 			nzbInfo1->GetUrlStatus() == NzbInfo::lsNone &&
 			nzbInfo1->GetDeleteStatus() == NzbInfo::dsNone &&
-			(!pauseDownload || g_Options->GetUrlForce()) &&
 			(!nzbInfo || nzbInfo1->GetPriority() > nzbInfo->GetPriority()))
 		{
 			nzbInfo = nzbInfo1;
@@ -277,13 +306,11 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 		// remove downloader from downloader list
 		m_activeDownloads.erase(std::find(m_activeDownloads.begin(), m_activeDownloads.end(), urlDownloader));
 
-		nzbInfo->SetActiveDownloads(0);
-
 		retry = urlDownloader->GetStatus() == WebDownloader::adRetry && !nzbInfo->GetDeleting();
 
 		if (nzbInfo->GetDeleting())
 		{
-			nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
+			nzbInfo->SetDeleteStatus(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ? NzbInfo::dsManual : nzbInfo->GetDeleteStatus());
 			nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 			nzbInfo->SetDeleting(false);
 		}
@@ -309,6 +336,7 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 	if (retry)
 	{
+		nzbInfo->SetActiveDownloads(0);
 		return;
 	}
 
@@ -324,7 +352,7 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 		if (addStatus == Scanner::asSuccess)
 		{
-			// if scanner has successfully added nzb-file to queue, our pNZBInfo is
+			// if scanner has successfully added nzb-file to queue, our nzbInfo is
 			// already removed from queue and destroyed
 			return;
 		}
@@ -336,31 +364,13 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* urlDownloader)
 
 	g_QueueScriptCoordinator->EnqueueScript(nzbInfo, QueueScriptCoordinator::qeUrlCompleted);
 
-	std::unique_ptr<NzbInfo> oldNzbInfo;
-
 	{
 		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
 
-		// delete URL from queue
-		oldNzbInfo = downloadQueue->GetQueue()->Remove(nzbInfo);
+		nzbInfo->SetActiveDownloads(0);
 
-		// add failed URL to history
-		if (g_Options->GetKeepHistory() > 0 &&
-			nzbInfo->GetUrlStatus() != NzbInfo::lsFinished &&
-			!nzbInfo->GetAvoidHistory())
-		{
-			std::unique_ptr<HistoryInfo> historyInfo = std::make_unique<HistoryInfo>(std::move(oldNzbInfo));
-			historyInfo->SetTime(Util::CurrentTime());
-			downloadQueue->GetHistory()->Add(std::move(historyInfo), true);
-			downloadQueue->HistoryChanged();
-		}
-
-		downloadQueue->Save();
-	}
-
-	if (oldNzbInfo)
-	{
-		g_DiskState->DiscardFiles(oldNzbInfo.get());
+		DownloadQueue::Aspect aspect = {DownloadQueue::eaUrlFailed, downloadQueue, nzbInfo, nullptr};
+		downloadQueue->Notify(&aspect);
 	}
 }
 
@@ -380,26 +390,40 @@ bool UrlCoordinator::DeleteQueueEntry(DownloadQueue* downloadQueue, NzbInfo* nzb
 				return true;
 			}
 		}
+
+		return false;
 	}
 
 	info("Deleting URL %s", nzbInfo->GetName());
 
-	nzbInfo->SetDeleteStatus(NzbInfo::dsManual);
+	nzbInfo->SetDeleteStatus(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ? NzbInfo::dsManual : nzbInfo->GetDeleteStatus());
 	nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 
-	std::unique_ptr<NzbInfo> oldNzbInfo = downloadQueue->GetQueue()->Remove(nzbInfo);
-
-	if (g_Options->GetKeepHistory() > 0 && !avoidHistory)
-	{
-		std::unique_ptr<HistoryInfo> historyInfo = std::make_unique<HistoryInfo>(std::move(oldNzbInfo));
-		historyInfo->SetTime(Util::CurrentTime());
-		downloadQueue->GetHistory()->Add(std::move(historyInfo), true);
-		downloadQueue->HistoryChanged();
-	}
-	else
-	{
-		g_DiskState->DiscardFiles(oldNzbInfo.get());
-	}
+	DownloadQueue::Aspect deletedAspect = {DownloadQueue::eaUrlDeleted, downloadQueue, nzbInfo, nullptr};
+	downloadQueue->Notify(&deletedAspect);
 
 	return true;
 }
+
+void UrlCoordinator::AddUrlToQueue(std::unique_ptr<NzbInfo> nzbInfo, bool addFirst)
+{
+	debug("Adding URL to queue");
+										
+	NzbInfo* addedNzb = nzbInfo.get();
+
+	GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
+
+	DownloadQueue::Aspect foundAspect = {DownloadQueue::eaUrlFound, downloadQueue, addedNzb, nullptr};
+	downloadQueue->Notify(&foundAspect);
+
+	if (addedNzb->GetDeleteStatus() != NzbInfo::dsManual)
+	{
+		downloadQueue->GetQueue()->Add(std::move(nzbInfo), addFirst);
+
+		DownloadQueue::Aspect addedAspect = {DownloadQueue::eaUrlAdded, downloadQueue, addedNzb, nullptr};
+		downloadQueue->Notify(&addedAspect);
+	}
+
+	downloadQueue->Save();
+}
+

daemon/queue/UrlCoordinator.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2012-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2012-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,12 +33,14 @@ class UrlDownloader;
 class UrlCoordinator : public Thread, public Observer, public Debuggable
 {
 public:
+	UrlCoordinator();
 	virtual ~UrlCoordinator();
 	virtual void Run();
 	virtual void Stop();
 	void Update(Subject* caller, void* aspect);
 
 	// Editing the queue
+	void AddUrlToQueue(std::unique_ptr<NzbInfo> nzbInfo, bool addFirst);
 	bool HasMoreJobs() { return m_hasMoreJobs; }
 	bool DeleteQueueEntry(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool avoidHistory);
 
@@ -48,15 +50,26 @@ protected:
 private:
 	typedef std::list<UrlDownloader*> ActiveDownloads;
 
+	class DownloadQueueObserver: public Observer
+	{
+	public:
+		UrlCoordinator* m_owner;
+		virtual void Update(Subject* caller, void* aspect) { m_owner->DownloadQueueUpdate(caller, aspect); }
+	};
+
 	ActiveDownloads m_activeDownloads;
 	bool m_hasMoreJobs = true;
 	bool m_force;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
+	DownloadQueueObserver m_downloadQueueObserver;
 
 	NzbInfo* GetNextUrl(DownloadQueue* downloadQueue);
 	void StartUrlDownload(NzbInfo* nzbInfo);
 	void UrlCompleted(UrlDownloader* urlDownloader);
 	void ResetHangingDownloads();
 	void WaitJobs();
+	void DownloadQueueUpdate(Subject* caller, void* aspect);
 };
 
 extern UrlCoordinator* g_UrlCoordinator;

daemon/remote/BinRpc.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -23,12 +23,14 @@
 #include "BinRpc.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "QueueEditor.h"
 #include "Util.h"
 #include "FileSystem.h"
 #include "DownloadInfo.h"
 #include "Scanner.h"
 #include "StatMeter.h"
+#include "UrlCoordinator.h"
 
 extern void ExitProc();
 extern void Reload();
@@ -312,20 +314,20 @@ void PauseUnpauseBinCommand::Execute()
 		return;
 	}
 
-	g_Options->SetResumeTime(0);
+	g_WorkState->SetResumeTime(0);
 
 	switch (ntohl(PauseUnpauseRequest.m_action))
 	{
 		case rpDownload:
-			g_Options->SetPauseDownload(ntohl(PauseUnpauseRequest.m_pause));
+			g_WorkState->SetPauseDownload(ntohl(PauseUnpauseRequest.m_pause));
 			break;
 
 		case rpPostProcess:
-			g_Options->SetPausePostProcess(ntohl(PauseUnpauseRequest.m_pause));
+			g_WorkState->SetPausePostProcess(ntohl(PauseUnpauseRequest.m_pause));
 			break;
 
 		case rpScan:
-			g_Options->SetPauseScan(ntohl(PauseUnpauseRequest.m_pause));
+			g_WorkState->SetPauseScan(ntohl(PauseUnpauseRequest.m_pause));
 			break;
 	}
 
@@ -340,7 +342,7 @@ void SetDownloadRateBinCommand::Execute()
 		return;
 	}
 
-	g_Options->SetDownloadRate(ntohl(SetDownloadRequest.m_downloadRate));
+	g_WorkState->SetSpeedLimit(ntohl(SetDownloadRequest.m_downloadRate));
 	SendBoolResponse(true, "Rate-Command completed successfully");
 }
 
@@ -430,9 +432,7 @@ void DownloadBinCommand::Execute()
 		nzbInfo->SetDupeScore(dupeScore);
 		nzbInfo->SetDupeMode((EDupeMode)dupeMode);
 
-		GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
-		downloadQueue->GetQueue()->Add(std::move(nzbInfo), addTop);
-		downloadQueue->Save();
+		g_UrlCoordinator->AddUrlToQueue(std::move(nzbInfo), addTop);
 
 		ok = true;
 	}
@@ -675,10 +675,10 @@ void ListBinCommand::Execute()
 		Util::SplitInt64(remainingSize, &sizeHi, &sizeLo);
 		ListResponse.m_remainingSizeHi = htonl(sizeHi);
 		ListResponse.m_remainingSizeLo = htonl(sizeLo);
-		ListResponse.m_downloadLimit = htonl(g_Options->GetDownloadRate());
-		ListResponse.m_downloadPaused = htonl(g_Options->GetPauseDownload());
-		ListResponse.m_postPaused = htonl(g_Options->GetPausePostProcess());
-		ListResponse.m_scanPaused = htonl(g_Options->GetPauseScan());
+		ListResponse.m_downloadLimit = htonl(g_WorkState->GetSpeedLimit());
+		ListResponse.m_downloadPaused = htonl(g_WorkState->GetPauseDownload());
+		ListResponse.m_postPaused = htonl(g_WorkState->GetPausePostProcess());
+		ListResponse.m_scanPaused = htonl(g_WorkState->GetPauseScan());
 		ListResponse.m_threadCount = htonl(Thread::GetThreadCount() - 1); // not counting itself
 		ListResponse.m_postJobCount = htonl(postJobCount);
 

daemon/remote/RemoteClient.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2005 Bo Cordes Petersen <placebodk@sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -497,7 +497,7 @@ bool RemoteClient::RequestServerList(bool files, bool groups, const char* patter
 			}
 
 			printf("-----------------------------------\n");
-			printf("Groups: %i\n", downloadQueue->GetQueue()->size());
+			printf("Groups: %i\n", (int)downloadQueue->GetQueue()->size());
 			if (pattern)
 			{
 				printf("Matches: %i\n", matches);
@@ -797,7 +797,7 @@ bool RemoteClient::RequestServerEditQueue(DownloadQueue::EEditAction action, int
 
 	if (textLen > 0)
 	{
-		strcpy(trailingData, text);
+		strncpy(trailingData, text, textLen);
 	}
 
 	int32* ids = (int32*)(trailingData + textLen);
@@ -812,7 +812,8 @@ bool RemoteClient::RequestServerEditQueue(DownloadQueue::EEditAction action, int
 		char *names = trailingData + textLen + idLength;
 		for (CString& name : nameList)
 		{
-			int len = strlen(name);
+			// "len" must be less or equal than: "buffer size" - "already used buffer" - "ending \0"
+			size_t len = strnlen(name, length - (names - trailingData) - 1);
 			strncpy(names, name, len + 1);
 			names += len + 1;
 		}

daemon/remote/RemoteServer.cpp

@@ -79,7 +79,7 @@ void RemoteServer::Run()
 				break;
 			}
 			m_connection.reset();
-			usleep(500 * 1000);
+			Util::Sleep(500);
 			continue;
 		}
 
@@ -112,7 +112,7 @@ void RemoteServer::Run()
 			Guard guard(m_processorsMutex);
 			completed = m_activeProcessors.size() == 0;
 		}
-		usleep(100 * 1000);
+		Util::Sleep(100);
 	}
 	debug("RemoteServer: request processor are completed");
 

daemon/remote/WebServer.cpp

@@ -53,7 +53,7 @@ void WebProcessor::Init()
 
 	for (int j = uaControl; j <= uaAdd; j++)
 	{
-		for (int i = 0; i < sizeof(m_serverAuthToken[j]) - 1; i++)
+		for (size_t i = 0; i < sizeof(m_serverAuthToken[j]) - 1; i++)
 		{
 			int ch = rand() % (10 + 26 + 26);
 			if (0 <= ch && ch < 10)
@@ -222,18 +222,11 @@ void WebProcessor::ParseUrl()
 	if (pauth1 && pauth1 < pauth2)
 	{
 		char* pstart = m_url + 1;
-		int len = 0;
-		char* pend = strchr(pstart + 1, '/');
-		if (pend)
-		{
-			len = (int)(pend - pstart < (int)sizeof(m_authInfo) - 1 ? pend - pstart : (int)sizeof(m_authInfo) - 1);
-		}
-		else
-		{
-			len = strlen(pstart);
-		}
+		char* pend = pauth2;
+		int len = std::min((int)(pend - pstart), (int)sizeof(m_authInfo) - 1);
 		strncpy(m_authInfo, pstart, len);
 		m_authInfo[len] = '\0';
+		WebUtil::UrlDecode(m_authInfo);
 		m_url = CString(pend);
 	}
 
@@ -301,9 +294,10 @@ bool WebProcessor::IsAuthorizedIp(const char* remoteAddr)
 	// split option AuthorizedIP into tokens and check each token
 	bool authorized = false;
 	Tokenizer tok(g_Options->GetAuthorizedIp(), ",;");
-	while (const char* iP = tok.Next())
+	while (const char* ip = tok.Next())
 	{
-		if (!strcmp(iP, remoteIp))
+		WildMask mask(ip);
+		if (mask.Match(remoteIp))
 		{
 			authorized = true;
 			break;
@@ -465,8 +459,8 @@ void WebProcessor::SendBodyResponse(const char* body, int bodyLen, const char* c
 		"Access-Control-Allow-Credentials: true\r\n"
 		"Access-Control-Max-Age: 86400\r\n"
 		"Access-Control-Allow-Headers: Content-Type, Authorization\r\n"
-		"Set-Cookie: Auth-Type=%s\r\n"
-		"Set-Cookie: Auth-Token=%s; HttpOnly\r\n"
+		"Set-Cookie: Auth-Type=%s; SameSite=Lax\r\n"
+		"Set-Cookie: Auth-Token=%s; HttpOnly; SameSite=Lax\r\n"
 		"Content-Length: %i\r\n"
 		"%s"					// Content-Type: xxx
 		"%s"					// Content-Encoding: gzip

daemon/remote/XmlRpc.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include "XmlRpc.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "Scanner.h"
 #include "FeedCoordinator.h"
 #include "ServerPool.h"
@@ -34,6 +35,7 @@
 #include "ScriptConfig.h"
 #include "QueueScript.h"
 #include "CommandScript.h"
+#include "UrlCoordinator.h"
 
 extern void ExitProc();
 extern void Reload();
@@ -403,12 +405,12 @@ void XmlRpcProcessor::Dispatch()
 		int valueLen = 0;
 		if (const char* methodPtr = WebUtil::JsonFindField(m_request, "method", &valueLen))
 		{
-			valueLen = valueLen >= sizeof(methodName) ? sizeof(methodName) - 1 : valueLen;
+			valueLen = valueLen >= (int)sizeof(methodName) ? (int)sizeof(methodName) - 1 : valueLen;
 			methodName.Set(methodPtr + 1, valueLen - 2);
 		}
 		if (const char* requestIdPtr = WebUtil::JsonFindField(m_request, "id", &valueLen))
 		{
-			valueLen = valueLen >= sizeof(requestId) ? sizeof(requestId) - 1 : valueLen;
+			valueLen = valueLen >= (int)sizeof(requestId) ? (int)sizeof(requestId) - 1 : valueLen;
 			requestId.Set(requestIdPtr, valueLen);
 		}
 	}
@@ -559,9 +561,7 @@ void XmlRpcProcessor::BuildResponse(const char* response, const char* callbackFu
 void XmlRpcProcessor::BuildErrorResponse(int errCode, const char* errText)
 {
 	ErrorXmlCommand command(errCode, errText);
-	command.SetRequest(m_request);
 	command.SetProtocol(m_protocol);
-	command.PrepareParams();
 	command.Execute();
 	BuildResponse(command.GetResponse(), "", command.GetFault(), nullptr);
 }
@@ -892,7 +892,7 @@ bool XmlCommand::NextParamAsInt(int* value)
 		}
 		*value = atoi(param + 1);
 		m_requestPtr = param + 1;
-		while (strchr("-+0123456789&", *m_requestPtr))
+		while (*m_requestPtr && strchr("-+0123456789&", *m_requestPtr))
 		{
 			m_requestPtr++;
 		}
@@ -1101,20 +1101,20 @@ void PauseUnpauseXmlCommand::Execute()
 {
 	bool ok = true;
 
-	g_Options->SetResumeTime(0);
+	g_WorkState->SetResumeTime(0);
 
 	switch (m_pauseAction)
 	{
 		case paDownload:
-			g_Options->SetPauseDownload(m_pause);
+			g_WorkState->SetPauseDownload(m_pause);
 			break;
 
 		case paPostProcess:
-			g_Options->SetPausePostProcess(m_pause);
+			g_WorkState->SetPausePostProcess(m_pause);
 			break;
 
 		case paScan:
-			g_Options->SetPauseScan(m_pause);
+			g_WorkState->SetPauseScan(m_pause);
 			break;
 
 		default:
@@ -1136,7 +1136,7 @@ void ScheduleResumeXmlCommand::Execute()
 
 	time_t curTime = Util::CurrentTime();
 
-	g_Options->SetResumeTime(curTime + seconds);
+	g_WorkState->SetResumeTime(curTime + seconds);
 
 	BuildBoolResponse(true);
 }
@@ -1178,7 +1178,7 @@ void SetDownloadRateXmlCommand::Execute()
 		return;
 	}
 
-	g_Options->SetDownloadRate(rate * 1024);
+	g_WorkState->SetSpeedLimit(rate * 1024);
 	BuildBoolResponse(true);
 }
 
@@ -1321,11 +1321,11 @@ void StatusXmlCommand::Execute()
 	int articleCacheMBytes = (int)(articleCache / 1024 / 1024);
 
 	int downloadRate = (int)(g_StatMeter->CalcCurrentDownloadSpeed());
-	int downloadLimit = (int)(g_Options->GetDownloadRate());
-	bool downloadPaused = g_Options->GetPauseDownload();
-	bool postPaused = g_Options->GetPausePostProcess();
-	bool scanPaused = g_Options->GetPauseScan();
-	bool quotaReached = g_Options->GetQuotaReached();
+	int downloadLimit = (int)(g_WorkState->GetSpeedLimit());
+	bool downloadPaused = g_WorkState->GetPauseDownload();
+	bool postPaused = g_WorkState->GetPausePostProcess();
+	bool scanPaused = g_WorkState->GetPauseScan();
+	bool quotaReached = g_WorkState->GetQuotaReached();
 	int threadCount = Thread::GetThreadCount() - 1; // not counting itself
 
 	uint32 downloadedSizeHi, downloadedSizeLo;
@@ -1353,7 +1353,7 @@ void StatusXmlCommand::Execute()
 	int freeDiskSpaceMB = (int)(freeDiskSpace / 1024 / 1024);
 
 	int serverTime = (int)Util::CurrentTime();
-	int resumeTime = (int)g_Options->GetResumeTime();
+	int resumeTime = (int)g_WorkState->GetResumeTime();
 	bool feedActive = g_FeedCoordinator->HasActiveDownloads();
 	int queuedScripts = g_QueueScriptCoordinator->GetQueueSize();
 
@@ -1443,7 +1443,7 @@ void LogXmlCommand::Execute()
 
 		AppendCondResponse(",\n", IsJson() && index++ > 0);
 		AppendFmtResponse(IsJson() ? JSON_LOG_ITEM : XML_LOG_ITEM,
-			message.GetId(), messageType[message.GetKind()], message.GetTime(),
+			message.GetId(), messageType[message.GetKind()], (int)message.GetTime(),
 			*EncodeStr(message.GetText()));
 	}
 
@@ -1550,7 +1550,7 @@ void ListFilesXmlCommand::Execute()
 				AppendCondResponse(",\n", IsJson() && index++ > 0);
 				AppendFmtResponse(IsJson() ? JSON_LIST_ITEM : XML_LIST_ITEM,
 					fileInfo->GetId(), fileSizeLo, fileSizeHi, remainingSizeLo, remainingSizeHi,
-					fileInfo->GetTime(), BoolToStr(fileInfo->GetFilenameConfirmed()),
+					(int)fileInfo->GetTime(), BoolToStr(fileInfo->GetFilenameConfirmed()),
 					BoolToStr(fileInfo->GetPaused()), fileInfo->GetNzbInfo()->GetId(),
 					*xmlNzbNicename, *xmlNzbNicename, *EncodeStr(fileInfo->GetNzbInfo()->GetFilename()),
 					*EncodeStr(fileInfo->GetSubject()), *EncodeStr(fileInfo->GetFilename()),
@@ -1748,14 +1748,14 @@ void NzbInfoXmlCommand::AppendNzbInfoFields(NzbInfo* nzbInfo)
 			deleteStatusName[nzbInfo->GetDeleteStatus()], markStatusName[nzbInfo->GetMarkStatus()],
 			urlStatusName[nzbInfo->GetUrlStatus()],
 			fileSizeLo, fileSizeHi, fileSizeMB, nzbInfo->GetFileCount(),
-			nzbInfo->GetMinTime(), nzbInfo->GetMaxTime(),
+			(int)nzbInfo->GetMinTime(), (int)nzbInfo->GetMaxTime(),
 			nzbInfo->GetTotalArticles(), nzbInfo->GetCurrentSuccessArticles(), nzbInfo->GetCurrentFailedArticles(),
 			nzbInfo->CalcHealth(), nzbInfo->CalcCriticalHealth(false),
 			*EncodeStr(nzbInfo->GetDupeKey()), nzbInfo->GetDupeScore(), dupeModeName[nzbInfo->GetDupeMode()],
 			BoolToStr(nzbInfo->GetDeleteStatus() != NzbInfo::dsNone),
 			downloadedSizeLo, downloadedSizeHi, downloadedSizeMB, nzbInfo->GetDownloadSec(),
-			nzbInfo->GetPostTotalSec() + (nzbInfo->GetPostInfo() && nzbInfo->GetPostInfo()->GetStartTime() ?
-				Util::CurrentTime() - nzbInfo->GetPostInfo()->GetStartTime() : 0),
+			(int)(nzbInfo->GetPostTotalSec() + (nzbInfo->GetPostInfo() && nzbInfo->GetPostInfo()->GetStartTime() ?
+				Util::CurrentTime() - nzbInfo->GetPostInfo()->GetStartTime() : 0)),
 			nzbInfo->GetParSec(), nzbInfo->GetRepairSec(), nzbInfo->GetUnpackSec(), messageCount, nzbInfo->GetExtraParBlocks());
 
 	// Post-processing parameters
@@ -1856,8 +1856,8 @@ void NzbInfoXmlCommand::AppendPostInfoFields(PostInfo* postInfo, int logEntries,
 
 		AppendFmtResponse(itemStart, *EncodeStr(postInfo->GetProgressLabel()),
 			postInfo->GetStageProgress(),
-			postInfo->GetStageTime() ? curTime - postInfo->GetStageTime() : 0,
-			postInfo->GetStartTime() ? curTime - postInfo->GetStartTime() : 0);
+			(int)(postInfo->GetStageTime() ? curTime - postInfo->GetStageTime() : 0),
+			(int)(postInfo->GetStartTime() ? curTime - postInfo->GetStartTime() : 0));
 	}
 	else
 	{
@@ -1884,7 +1884,7 @@ void NzbInfoXmlCommand::AppendPostInfoFields(PostInfo* postInfo, int logEntries,
 
 				AppendCondResponse(",\n", IsJson() && index++ > 0);
 				AppendFmtResponse(IsJson() ? JSON_LOG_ITEM : XML_LOG_ITEM,
-					message.GetId(), messageType[message.GetKind()], message.GetTime(),
+					message.GetId(), messageType[message.GetKind()], (int)message.GetTime(),
 					*EncodeStr(message.GetText()));
 			}
 		}
@@ -2238,7 +2238,7 @@ void DownloadXmlCommand::Execute()
 		}
 	}
 
-	if (!strncasecmp(nzbContent, "http://", 6) || !strncasecmp(nzbContent, "https://", 7))
+	if (!strncasecmp(nzbContent, "http://", 7) || !strncasecmp(nzbContent, "https://", 8))
 	{
 		// add url
 		std::unique_ptr<NzbInfo> nzbInfo = std::make_unique<NzbInfo>();
@@ -2256,11 +2256,7 @@ void DownloadXmlCommand::Execute()
 
 		info("Queue %s", *nzbInfo->MakeNiceUrlName(nzbContent, nzbFilename));
 
-		{
-			GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
-			downloadQueue->GetQueue()->Add(std::move(nzbInfo), addTop);
-			downloadQueue->Save();
-		}
+		g_UrlCoordinator->AddUrlToQueue(std::move(nzbInfo), addTop);
 
 		if (v13)
 		{
@@ -2468,8 +2464,8 @@ void HistoryXmlCommand::Execute()
 		"\"Kind\" : \"%s\",\n"
 		"\"Name\" : \"%s\",\n"
 		"\"HistoryTime\" : %i,\n"
-		"\"FileSizeLo\" : %i,\n"
-		"\"FileSizeHi\" : %i,\n"
+		"\"FileSizeLo\" : %u,\n"
+		"\"FileSizeHi\" : %u,\n"
 		"\"FileSizeMB\" : %i,\n"
 		"\"DupeKey\" : \"%s\",\n"
 		"\"DupeScore\" : %i,\n"
@@ -2506,7 +2502,7 @@ void HistoryXmlCommand::Execute()
 
 			AppendFmtResponse(IsJson() ? JSON_HISTORY_ITEM_START : XML_HISTORY_ITEM_START,
 				historyInfo->GetId(), *EncodeStr(historyInfo->GetName()), nzbInfo->GetParkedFileCount(),
-				BoolToStr(nzbInfo->GetCompletedFiles()->size()), historyInfo->GetTime(), status);
+				BoolToStr(nzbInfo->GetCompletedFiles()->size()), (int)historyInfo->GetTime(), status);
 		}
 		else if (historyInfo->GetKind() == HistoryInfo::hkDup)
 		{
@@ -2518,7 +2514,7 @@ void HistoryXmlCommand::Execute()
 
 			AppendFmtResponse(IsJson() ? JSON_HISTORY_DUP_ITEM : XML_HISTORY_DUP_ITEM,
 				historyInfo->GetId(), historyInfo->GetId(), "DUP", *EncodeStr(historyInfo->GetName()),
-				historyInfo->GetTime(), fileSizeLo, fileSizeHi, fileSizeMB,
+				(int)historyInfo->GetTime(), fileSizeLo, fileSizeHi, fileSizeMB,
 				*EncodeStr(dupInfo->GetDupeKey()), dupInfo->GetDupeScore(),
 				dupeModeName[dupInfo->GetDupeMode()], dupStatusName[dupInfo->GetStatus()],
 				status);
@@ -2840,8 +2836,8 @@ void ViewFeedXmlCommand::Execute()
 		"<member><name>Title</name><value><string>%s</string></value></member>\n"
 		"<member><name>Filename</name><value><string>%s</string></value></member>\n"
 		"<member><name>URL</name><value><string>%s</string></value></member>\n"
-		"<member><name>SizeLo</name><value><i4>%i</i4></value></member>\n"
-		"<member><name>SizeHi</name><value><i4>%i</i4></value></member>\n"
+		"<member><name>SizeLo</name><value><i4>%u</i4></value></member>\n"
+		"<member><name>SizeHi</name><value><i4>%u</i4></value></member>\n"
 		"<member><name>SizeMB</name><value><i4>%i</i4></value></member>\n"
 		"<member><name>Category</name><value><string>%s</string></value></member>\n"
 		"<member><name>AddCategory</name><value><string>%s</string></value></member>\n"
@@ -2861,8 +2857,8 @@ void ViewFeedXmlCommand::Execute()
 		"\"Title\" : \"%s\",\n"
 		"\"Filename\" : \"%s\",\n"
 		"\"URL\" : \"%s\",\n"
-		"\"SizeLo\" : %i,\n"
-		"\"SizeHi\" : %i,\n"
+		"\"SizeLo\" : %u,\n"
+		"\"SizeHi\" : %u,\n"
 		"\"SizeMB\" : %i,\n"
 		"\"Category\" : \"%s\",\n"
 		"\"AddCategory\" : \"%s\",\n"
@@ -2897,7 +2893,7 @@ void ViewFeedXmlCommand::Execute()
 				*EncodeStr(feedItemInfo.GetTitle()), *EncodeStr(feedItemInfo.GetFilename()),
 				*EncodeStr(feedItemInfo.GetUrl()), sizeLo, sizeHi, sizeMB,
 				*EncodeStr(feedItemInfo.GetCategory()), *EncodeStr(feedItemInfo.GetAddCategory()),
-				BoolToStr(feedItemInfo.GetPauseNzb()), feedItemInfo.GetPriority(), feedItemInfo.GetTime(),
+				BoolToStr(feedItemInfo.GetPauseNzb()), feedItemInfo.GetPriority(), (int)feedItemInfo.GetTime(),
 				matchStatusType[feedItemInfo.GetMatchStatus()], feedItemInfo.GetMatchRule(),
 				*EncodeStr(feedItemInfo.GetDupeKey()), feedItemInfo.GetDupeScore(),
 				dupeModeType[feedItemInfo.GetDupeMode()], statusType[feedItemInfo.GetStatus()]);
@@ -3123,11 +3119,11 @@ void ServerVolumesXmlCommand::Execute()
 	"\"ServerID\" : %i,\n"
 	"\"DataTime\" : %i,\n"
 	"\"FirstDay\" : %i,\n"
-	"\"TotalSizeLo\" : %i,\n"
-	"\"TotalSizeHi\" : %i,\n"
+	"\"TotalSizeLo\" : %u,\n"
+	"\"TotalSizeHi\" : %u,\n"
 	"\"TotalSizeMB\" : %i,\n"
-	"\"CustomSizeLo\" : %i,\n"
-	"\"CustomSizeHi\" : %i,\n"
+	"\"CustomSizeLo\" : %u,\n"
+	"\"CustomSizeHi\" : %u,\n"
 	"\"CustomSizeMB\" : %i,\n"
 	"\"CustomTime\" : %i,\n"
 	"\"SecSlot\" : %i,\n"

daemon/util/FileSystem.cpp

@@ -304,10 +304,29 @@ bool FileSystem::AllocateFile(const char* filename, int64 size, bool sparse, CSt
 			errmsg = GetLastErrorMessage();
 			return false;
 		}
-		char c = '0';
-		fwrite(&c, 1, size, file);
+
+		// write zeros in 16K chunks
+		CharBuffer zeros(16 * 1024);
+		memset(zeros, 0, zeros.Size());
+		for (int64 remaining = size; remaining > 0;)
+		{
+			int64 needbytes = std::min(remaining, (int64)zeros.Size());
+			int64 written = fwrite(zeros, 1, needbytes, file);
+			if (written != needbytes)
+			{
+				errmsg = GetLastErrorMessage();
+				fclose(file);
+				return false;
+			}
+			remaining -= written;
+		}
 		fclose(file);
+
 		ok = FileSize(filename) == size;
+		if (!ok)
+		{
+			errmsg = "created file has wrong size";
+		}
 	}
 #endif
 	return ok;
@@ -1020,52 +1039,9 @@ CString FileSystem::WidePathToUtfPath(const wchar_t* wpath)
 #endif
 
 
-#ifdef WIN32
-DirBrowser::DirBrowser(const char* path)
-{
-	BString<1024> mask("%s%c*.*", path, PATH_SEPARATOR);
-	m_file = FindFirstFileW(FileSystem::UtfPathToWidePath(mask), &m_findData);
-	m_first = true;
-}
-
-DirBrowser::~DirBrowser()
-{
-	if (m_file != INVALID_HANDLE_VALUE)
-	{
-		FindClose(m_file);
-	}
-}
-
-const char* DirBrowser::InternNext()
-{
-	bool ok = false;
-	if (m_first)
-	{
-		ok = m_file != INVALID_HANDLE_VALUE;
-		m_first = false;
-	}
-	else
-	{
-		ok = FindNextFileW(m_file, &m_findData) != 0;
-	}
-	if (ok)
-	{
-		m_filename = FileSystem::WidePathToUtfPath(m_findData.cFileName);
-		return m_filename;
-	}
-	return nullptr;
-}
-
-#else
-
-#ifdef DIRBROWSER_SNAPSHOT
 DirBrowser::DirBrowser(const char* path, bool snapshot) :
 	m_snapshot(snapshot)
-#else
-DirBrowser::DirBrowser(const char* path)
-#endif
 {
-#ifdef DIRBROWSER_SNAPSHOT
 	if (m_snapshot)
 	{
 		DirBrowser dir(path, false);
@@ -1076,35 +1052,57 @@ DirBrowser::DirBrowser(const char* path)
 		m_snapshotIter = m_snapshotFiles.begin();
 	}
 	else
-#endif
 	{
+#ifdef WIN32
+		BString<1024> mask("%s%c*.*", path, PATH_SEPARATOR);
+		m_file = FindFirstFileW(FileSystem::UtfPathToWidePath(mask), &m_findData);
+		m_first = true;
+#else
 		m_dir = opendir(path);
+#endif
 	}
 }
 
 DirBrowser::~DirBrowser()
 {
-#ifdef DIRBROWSER_SNAPSHOT
-	if (!m_snapshot)
-#endif
+#ifdef WIN32
+	if (m_file != INVALID_HANDLE_VALUE)
 	{
-		if (m_dir)
-		{
-			closedir(m_dir);
-		}
+		FindClose(m_file);
 	}
+#else
+	if (m_dir)
+	{
+		closedir(m_dir);
+	}
+#endif
 }
 
 const char* DirBrowser::InternNext()
 {
-#ifdef DIRBROWSER_SNAPSHOT
 	if (m_snapshot)
 	{
 		return m_snapshotIter == m_snapshotFiles.end() ? nullptr : **m_snapshotIter++;
 	}
 	else
-#endif
 	{
+#ifdef WIN32
+		bool ok = false;
+		if (m_first)
+		{
+			ok = m_file != INVALID_HANDLE_VALUE;
+			m_first = false;
+		}
+		else
+		{
+			ok = FindNextFileW(m_file, &m_findData) != 0;
+		}
+		if (ok)
+		{
+			m_filename = FileSystem::WidePathToUtfPath(m_findData.cFileName);
+			return m_filename;
+		}
+#else
 		if (m_dir)
 		{
 			m_findData = readdir(m_dir);
@@ -1113,10 +1111,10 @@ const char* DirBrowser::InternNext()
 				return m_findData->d_name;
 			}
 		}
+#endif
 		return nullptr;
 	}
 }
-#endif
 
 const char* DirBrowser::Next()
 {

daemon/util/FileSystem.h

@@ -84,18 +84,14 @@ public:
 class DirBrowser
 {
 public:
-#ifdef DIRBROWSER_SNAPSHOT
 	DirBrowser(const char* path, bool snapshot = true);
-#else
-	DirBrowser(const char* path);
-#endif
 	~DirBrowser();
 	const char* Next();
 
 private:
 #ifdef WIN32
 	WIN32_FIND_DATAW m_findData;
-	HANDLE m_file;
+	HANDLE m_file = INVALID_HANDLE_VALUE;
 	bool m_first;
 	CString m_filename;
 #else
@@ -103,12 +99,10 @@ private:
 	struct dirent* m_findData;
 #endif
 
-#ifdef DIRBROWSER_SNAPSHOT
 	bool m_snapshot;
 	typedef std::deque<CString> FileList;
 	FileList m_snapshotFiles;
 	FileList::iterator m_snapshotIter;
-#endif
 
 	const char* InternNext();
 };

daemon/util/Log.cpp

@@ -97,6 +97,7 @@ void Log::Filelog(const char* msg, ...)
 
 	m_logFile->Seek(0, DiskFile::soEnd);
 
+#ifdef DEBUG
 #ifdef WIN32
 	uint64 processId = GetCurrentProcessId();
 	uint64 threadId = GetCurrentThreadId();
@@ -104,8 +105,7 @@ void Log::Filelog(const char* msg, ...)
 	uint64 processId = (uint64)getpid();
 	uint64 threadId = (uint64)pthread_self();
 #endif
-#ifdef DEBUG
-	m_logFile->Print("%s\t%llu\t%llu\t%s%s", time, processId, threadId, tmp2, LINE_ENDING);
+	m_logFile->Print("%s\t%" PRIu64 "\t%" PRIu64 "\t%s%s", time, processId, threadId, tmp2, LINE_ENDING);
 #else
 	m_logFile->Print("%s\t%s%s", time, tmp2, LINE_ENDING);
 #endif

daemon/util/NString.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -76,18 +76,21 @@ void BString<size>::AppendFmtV(const char* format, va_list ap)
 }
 
 template <int size>
-void BString<size>::Format(const char* format, ...)
+int BString<size>::Format(const char* format, ...)
 {
 	va_list ap;
 	va_start(ap, format);
-	FormatV(format, ap);
+	int len = FormatV(format, ap);
 	va_end(ap);
+	return len;
 }
 
 template <int size>
-void BString<size>::FormatV(const char* format, va_list ap)
+int BString<size>::FormatV(const char* format, va_list ap)
 {
-	vsnprintf(m_data, size, format, ap);
+	// ensure result isn't negative (in case of bad argument)
+	int len = std::max(vsnprintf(m_data, size, format, ap), 0);
+	return len;
 }
 
 bool CString::operator==(const CString& other)
@@ -159,26 +162,28 @@ void CString::AppendFmtV(const char* format, va_list ap)
 	va_end(ap2);
 }
 
-void CString::Format(const char* format, ...)
+int CString::Format(const char* format, ...)
 {
 	va_list ap;
 	va_start(ap, format);
-	FormatV(format, ap);
+	int len = FormatV(format, ap);
 	va_end(ap);
+	return len;
 }
 
-void CString::FormatV(const char* format, va_list ap)
+int CString::FormatV(const char* format, va_list ap)
 {
 	va_list ap2;
 	va_copy(ap2, ap);
 
-	int newLen = vsnprintf(nullptr, 0, format, ap);
-	if (newLen < 0) return; // bad argument
+	// "std::max" to ensure result isn't negative (in case of bad argument)
+	int newLen = std::max(vsnprintf(nullptr, 0, format, ap), 0);
 
 	m_data = (char*)realloc(m_data, newLen + 1);
-	vsnprintf(m_data, newLen + 1, format, ap2);
+	newLen = vsnprintf(m_data, newLen + 1, format, ap2);
 
 	va_end(ap2);
+	return newLen;
 }
 
 int CString::Find(const char* str, int pos)
@@ -281,7 +286,7 @@ WString::WString(const char* utfstr)
 	m_data = (wchar_t*)malloc((strlen(utfstr) * 2 + 1) * sizeof(wchar_t));
 
 	wchar_t* out = m_data;
-	unsigned int codepoint;
+	unsigned int codepoint = 0;
 	while (*utfstr != 0)
 	{
 		unsigned char ch = (unsigned char)*utfstr;

daemon/util/NString.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -46,8 +46,8 @@ public:
 	void Append(const char* str, int len = 0);
 	void AppendFmt(const char* format, ...) PRINTF_SYNTAX(2);
 	void AppendFmtV(const char* format, va_list ap);
-	void Format(const char* format, ...) PRINTF_SYNTAX(2);
-	void FormatV(const char* format, va_list ap);
+	int Format(const char* format, ...) PRINTF_SYNTAX(2);
+	int FormatV(const char* format, va_list ap);
 
 protected:
 	char m_data[size];
@@ -83,8 +83,8 @@ public:
 	void Append(const char* str, int len = 0);
 	void AppendFmt(const char* format, ...) PRINTF_SYNTAX(2);
 	void AppendFmtV(const char* format, va_list ap);
-	void Format(const char* format, ...) PRINTF_SYNTAX(2);
-	void FormatV(const char* format, va_list ap);
+	int Format(const char* format, ...) PRINTF_SYNTAX(2);
+	int FormatV(const char* format, va_list ap);
 	int Find(const char* str, int pos = 0);
 	void Replace(int pos, int len, const char* str, int strLen = 0);
 	void Replace(const char* from, const char* to);
@@ -150,6 +150,7 @@ public:
 	CharBuffer(int size) : m_size(size) { m_data = (char*)malloc(size); }
 	CharBuffer(CharBuffer& other) : m_data(other.m_data), m_size(other.m_size) { other.m_data = nullptr; other.m_size = 0; }
 	~CharBuffer() { free(m_data); }
+	CharBuffer& operator=(CharBuffer&& other) = delete;
 	int Size() { return m_size; }
 	void Reserve(int size) { m_data = (char*)realloc(m_data, size); m_size = size; }
 	void Clear() { free(m_data); m_data = nullptr; m_size = 0; }

daemon/util/Script.cpp

@@ -66,7 +66,7 @@ void ChildWatchDog::Run()
 	time_t start = Util::CurrentTime();
 	while (!IsStopped() && (Util::CurrentTime() - start) < WAIT_SECONDS)
 	{
-		usleep(10 * 1000);
+		Util::Sleep(10);
 	}
 
 	if (!IsStopped())
@@ -394,7 +394,7 @@ int ScriptController::Execute()
 	}
 	while (watchDog.IsRunning())
 	{
-		usleep(5 * 1000);
+		Util::Sleep(5);
 	}
 #endif
 
@@ -584,11 +584,13 @@ void ScriptController::StartProcess(int* pipein, int* pipeout)
 	args.emplace_back(nullptr);
 	char* const* argdata = (char* const*)args.data();
 
+#ifdef DEBUG
 	debug("Starting  process: %s", script);
 	for (const char* arg : m_args)
 	{
 		debug("arg: %s", arg);
 	}
+#endif
 
 	debug("forking");
 	pid_t pid = fork();
@@ -739,7 +741,7 @@ void ScriptController::TerminateAll()
 				time_t curtime = Util::CurrentTime();
 				while (!script->m_completed && std::abs(curtime - Util::CurrentTime()) <= 10)
 				{
-					usleep(100 * 1000);
+					Util::Sleep(100);
 				}
 			}
 			script->Terminate();

daemon/util/Service.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -30,6 +30,11 @@ Service::Service()
 	g_ServiceCoordinator->RegisterService(this);
 }
 
+void Service::WakeUp()
+{
+	g_ServiceCoordinator->WakeUp();
+};
+
 
 ServiceCoordinator::ServiceCoordinator()
 {
@@ -46,29 +51,65 @@ void ServiceCoordinator::Run()
 {
 	debug("Entering ServiceCoordinator-loop");
 
-	const int stepMSec = 100;
-	int curTick = 0;
-
 	while (!IsStopped())
 	{
+		// perform service work
 		for (Service* service : m_services)
 		{
-			if (curTick >= service->m_lastTick + service->ServiceInterval() ||	// interval expired
-				curTick == 0 ||													// first start
-				curTick + 10000 < service->m_lastTick)							// int overflow
+			int serviceInterval = service->ServiceInterval();
+			if (serviceInterval >= Service::Now &&
+				abs(Util::CurrentTime() - service->m_lastWork) >= serviceInterval)
 			{
 				service->ServiceWork();
-				service->m_lastTick = curTick;
+				service->m_lastWork = Util::CurrentTime();
 			}
 		}
 
-		curTick += stepMSec;
-		usleep(stepMSec * 1000);
+		// calculate wait interval
+		int waitInterval = 60 * 60 * 24; // something very large
+		time_t curTime = Util::CurrentTime();
+		for (Service* service : m_services)
+		{
+			int serviceInterval = service->ServiceInterval();
+			int remaining = Service::Sleep;
+			if (serviceInterval >= Service::Now)
+			{
+				remaining = serviceInterval - (curTime - service->m_lastWork);
+				waitInterval = std::min(waitInterval, remaining);
+			}
+			debug("serviceInterval: %i, remaining: %i", serviceInterval, remaining);
+		}
+
+		debug("Waiting in ServiceCoordinator: %i", waitInterval);
+		if (waitInterval > 0)
+		{
+			Guard guard(m_waitMutex);
+			m_waitCond.WaitFor(m_waitMutex, waitInterval * 1000, [&] { return m_workenUp || IsStopped(); });
+			m_workenUp = false;
+		}
 	}
 
 	debug("Exiting ServiceCoordinator-loop");
 }
 
+void ServiceCoordinator::Stop()
+{
+	Thread::Stop();
+	
+	// Resume Run() to exit it
+	Guard guard(m_waitMutex);
+	m_waitCond.NotifyAll();
+}
+
+void ServiceCoordinator::WakeUp()
+{
+	debug("Waking up ServiceCoordinator");
+	// Resume Run()
+	Guard guard(m_waitMutex);
+	m_workenUp = true;
+	m_waitCond.NotifyAll();
+}
+
 void ServiceCoordinator::RegisterService(Service* service)
 {
 	m_services.push_back(service);

daemon/util/Service.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2015-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -28,12 +28,16 @@ class Service
 public:
 	Service();
 
+	static const int Now = 0;
+	static const int Sleep = -1;
+
 protected:
 	virtual int ServiceInterval() = 0;
 	virtual void ServiceWork() = 0;
+	void WakeUp();
 
 private:
-	int m_lastTick = 0;
+	time_t m_lastWork = 0;
 
 	friend class ServiceCoordinator;
 };
@@ -46,11 +50,16 @@ public:
 	ServiceCoordinator();
 	virtual ~ServiceCoordinator();
 	virtual void Run();
+	virtual void Stop();
 
 private:
 	ServiceList m_services;
+	Mutex m_waitMutex;
+	ConditionVar m_waitCond;
+	bool m_workenUp = false;
 
 	void RegisterService(Service* service);
+	void WakeUp();
 
 	friend class Service;
 };

daemon/util/Thread.cpp

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -26,51 +26,6 @@ int Thread::m_threadCount = 1; // take the main program thread into account
 std::unique_ptr<Mutex> Thread::m_threadMutex;
 
 
-Mutex::Mutex()
-{
-#ifdef WIN32
-	InitializeCriticalSection(&m_mutexObj);
-#else
-	pthread_mutex_init(&m_mutexObj, nullptr);
-#endif
-}
-
-Mutex::~Mutex()
-{
-#ifdef WIN32
-	DeleteCriticalSection(&m_mutexObj);
-#else
-	pthread_mutex_destroy(&m_mutexObj);
-#endif
-}
-
-void Mutex::Lock()
-{
-#ifdef WIN32
-	EnterCriticalSection(&m_mutexObj);
-#ifdef DEBUG
-	// CriticalSections on Windows can be locked many times from the same thread,
-	// but we do not want this and must treat such situations as errors and detect them.
-	if (m_mutexObj.RecursionCount > 1)
-	{
-		error("Internal program error: inconsistent thread-lock detected");
-	}
-#endif
-#else
-	pthread_mutex_lock(&m_mutexObj);
-#endif
-}
-
-void Mutex::Unlock()
-{
-#ifdef WIN32
-	LeaveCriticalSection(&m_mutexObj);
-#else
-	pthread_mutex_unlock(&m_mutexObj);
-#endif
-}
-
-
 void Thread::Init()
 {
 	debug("Initializing global thread data");
@@ -94,27 +49,23 @@ void Thread::Start()
 
 	m_running = true;
 
-	// NOTE: we must guarantee, that in a time we set m_running
-	// to value returned from pthread_create, the thread-object still exists.
-	// This is not obvious!
-	// pthread_create could wait long enough before returning result
-	// back to allow the started thread to complete its job and terminate.
-	// We lock mutex m_threadMutex on calling pthread_create; the started thread
-	// then also try to lock the mutex (see thread_handler) and therefore
-	// must wait until we unlock it
+	// NOTE: "m_threadMutex" ensures that "t" lives until the very end of the function
 	Guard guard(m_threadMutex);
 
-#ifdef WIN32
-	m_threadObj = (HANDLE)_beginthread(Thread::thread_handler, 0, (void*)this);
-	m_running = m_threadObj != 0;
-#else
-	pthread_attr_t m_attr;
-	pthread_attr_init(&m_attr);
-	pthread_attr_setdetachstate(&m_attr, PTHREAD_CREATE_DETACHED);
-	pthread_attr_setinheritsched(&m_attr, PTHREAD_INHERIT_SCHED);
-	m_running = !pthread_create(&m_threadObj, &m_attr, Thread::thread_handler, (void *) this);
-	pthread_attr_destroy(&m_attr);
-#endif
+	// start the new thread
+	std::thread t([&]{
+		{
+			// trying to lock "m_threadMutex", this will wait until function "Start()" is completed
+			// and "t" is detached.
+			Guard guard(m_threadMutex);
+		}
+
+		thread_handler();
+	});
+
+	// save the native handle to be able to cancel (Kill) the thread and then detach
+	m_threadObj = t.native_handle();
+	t.detach();
 }
 
 void Thread::Stop()
@@ -140,7 +91,12 @@ bool Thread::Kill()
 #ifdef WIN32
 	bool terminated = TerminateThread(m_threadObj, 0) != 0;
 #else
+#ifdef HAVE_PTHREAD_CANCEL
 	bool terminated = pthread_cancel(m_threadObj) == 0;
+#else
+	bool terminated = false;
+	warn("Could not kill thread: thread cancelling isn't supported on this platform");
+#endif
 #endif
 
 	if (terminated)
@@ -150,43 +106,25 @@ bool Thread::Kill()
 	return terminated;
 }
 
-#ifdef WIN32
-void __cdecl Thread::thread_handler(void* object)
-#else
-void* Thread::thread_handler(void* object)
-#endif
+void Thread::thread_handler()
 {
-	{
-		Guard guard(m_threadMutex);
-		m_threadCount++;
-	}
+	m_threadCount++;
 
 	debug("Entering Thread-func");
 
-	Thread* thread = (Thread*)object;
-
-	thread->Run();
+	Run();
 
 	debug("Thread-func exited");
 
-	if (thread->m_autoDestroy)
+	m_running = false;
+
+	m_threadCount--;
+
+	if (m_autoDestroy)
 	{
 		debug("Autodestroying Thread-object");
-		delete thread;
+		delete this;
 	}
-	else
-	{
-		thread->m_running = false;
-	}
-
-	{
-		Guard guard(m_threadMutex);
-		m_threadCount--;
-	}
-
-#ifndef WIN32
-	return nullptr;
-#endif
 }
 
 int Thread::GetThreadCount()

daemon/util/Thread.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,18 +25,15 @@
 class Mutex
 {
 public:
-	Mutex();
+	Mutex() {};
 	Mutex(const Mutex&) = delete;
-	~Mutex();
-	void Lock();
-	void Unlock();
+	void Lock() { m_mutexObj.lock(); }
+	void Unlock() { m_mutexObj.unlock(); }
 
 private:
-#ifdef WIN32
-	CRITICAL_SECTION m_mutexObj;
-#else
-	pthread_mutex_t m_mutexObj;
-#endif
+	std::mutex m_mutexObj;
+
+	friend class ConditionVar;
 };
 
 class Guard
@@ -80,6 +77,29 @@ private:
 	void Unlock() { if (m_mutex) { m_mutex->Unlock(); m_mutex = nullptr; } }
 };
 
+class ConditionVar
+{
+public:
+	ConditionVar() {}
+	ConditionVar(const ConditionVar& other) = delete;
+	void Wait(Mutex& mutex) { Lock l(mutex); m_condObj.wait(l); }
+	template <typename Pred> void Wait(Mutex& mutex, Pred pred) { Lock l(mutex); m_condObj.wait(l, pred); }
+	void WaitFor(Mutex& mutex, int msec) { Lock l(mutex); m_condObj.wait_for(l, std::chrono::milliseconds(msec)); }
+	template <typename Pred> void WaitFor(Mutex& mutex, int msec, Pred pred)
+		{ Lock l(mutex); m_condObj.wait_for(l, std::chrono::milliseconds(msec), pred); }
+	void NotifyOne() { m_condObj.notify_one(); }
+	void NotifyAll() { m_condObj.notify_all(); }
+
+private:
+	std::condition_variable m_condObj;
+
+	struct Lock : public std::unique_lock<std::mutex>
+	{
+		Lock(Mutex& mutex) : std::unique_lock<std::mutex>(mutex.m_mutexObj, std::adopt_lock) {}
+		~Lock() { release(); }
+	};
+};
+
 class Thread
 {
 public:
@@ -105,20 +125,12 @@ protected:
 private:
 	static std::unique_ptr<Mutex> m_threadMutex;
 	static int m_threadCount;
-#ifdef WIN32
-	HANDLE m_threadObj = 0;
-#else
-	pthread_t m_threadObj = 0;
-#endif
+	std::thread::native_handle_type m_threadObj = 0;
 	bool m_running = false;
 	bool m_stopped = false;
 	bool m_autoDestroy = false;
 
-#ifdef WIN32
-	static void __cdecl thread_handler(void* object);
-#else
-	static void *thread_handler(void* object);
-#endif
+	void thread_handler();
 };
 
 #endif

daemon/util/Util.cpp

@@ -122,7 +122,7 @@ void Util::Init()
 	}
 
 	// init static vars there
-	GetCurrentTicks();
+	CurrentTicks();
 }
 
 int64 Util::JoinInt64(uint32 Hi, uint32 Lo)
@@ -493,7 +493,7 @@ b -= c; b -= a; b ^= (a<<10); \
 c -= a; c -= b; c ^= (b>>15); \
 }
 
-uint32 hash(register uint8 *k, register uint32  length, register uint32  initval)
+uint32 hash(uint8 *k, uint32 length, uint32 initval)
 // register uint8 *k;        /* the key */
 // register uint32  length;   /* the length of the key */
 // register uint32  initval;    /* the previous hash, or an arbitrary value */
@@ -663,7 +663,7 @@ int Util::NumberOfCpuCores()
 	return -1;
 }
 
-int64 Util::GetCurrentTicks()
+int64 Util::CurrentTicks()
 {
 #ifdef WIN32
 	static int64 hz=0, hzo=0;
@@ -682,6 +682,15 @@ int64 Util::GetCurrentTicks()
 #endif
 }
 
+void Util::Sleep(int milliseconds)
+{
+#ifdef WIN32
+	::Sleep(milliseconds);
+#else
+	usleep(milliseconds * 1000);
+#endif
+}
+
 uint32 WebUtil::DecodeBase64(char* inputBuffer, int inputBufferLength, char* outputBuffer)
 {
 	uint32 InputBufferIndex  = 0;

daemon/util/Util.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -33,7 +33,6 @@ class Util
 {
 public:
 	static bool MatchFileExt(const char* filename, const char* extensionList, const char* listSeparator);
-	static int64 GetCurrentTicks();
 
 	/*
 	* Split command line into arguments.
@@ -65,6 +64,8 @@ public:
 	static void SetStandByMode(bool standBy);
 
 	static time_t CurrentTime();
+	static int64 CurrentTicks();
+	static void Sleep(int milliseconds);
 
 	/* cross platform version of GNU timegm, which is similar to mktime but takes an UTC time as parameter */
 	static time_t Timegm(tm const *t);

daemon/windows/WinConsole.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2014-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2014-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "nzbget.h"
 #include "Log.h"
 #include "Options.h"
+#include "WorkState.h"
 #include "FeedCoordinator.h"
 #include "StatMeter.h"
 #include "WinConsole.h"
@@ -29,14 +30,11 @@
 #include "Util.h"
 #include "resource.h"
 
-extern Options* g_Options;
 extern char* (*g_Arguments)[];
 extern int g_ArgumentCount;
 extern void ExitProc();
 extern void Reload();
 extern WinConsole* g_WinConsole;
-extern FeedCoordinator* g_FeedCoordinator;
-extern StatMeter* g_StatMeter;
 
 #define UM_TRAYICON (WM_USER + 1)
 #define UM_QUIT (WM_USER + 2)
@@ -62,7 +60,7 @@ BOOL WINAPI WinConsole::ConsoleCtrlHandler(DWORD dwCtrlType)
 			ExitProc();
 			while (g_WinConsole)
 			{
-				usleep(20 * 1000);
+				Util::Sleep(20);
 			}
 			return TRUE;
 
@@ -198,7 +196,7 @@ void WinConsole::Run()
 		}
 		else
 		{
-			usleep(20 * 1000);
+			Util::Sleep(20);
 			counter += 20;
 			if (counter >= 200)
 			{
@@ -290,10 +288,10 @@ LRESULT WinConsole::TrayWndProc(HWND hwndWin, UINT uMsg, WPARAM wParam, LPARAM l
 		case UM_TRAYICON:
 			if (lParam == WM_LBUTTONUP && !m_doubleClick)
 			{
-				g_Options->SetPauseDownload(!g_Options->GetPauseDownload());
-				g_Options->SetPausePostProcess(g_Options->GetPauseDownload());
-				g_Options->SetPauseScan(g_Options->GetPauseDownload());
-				g_Options->SetResumeTime(0);
+				g_WorkState->SetPauseDownload(!g_WorkState->GetPauseDownload());
+				g_WorkState->SetPausePostProcess(g_WorkState->GetPauseDownload());
+				g_WorkState->SetPauseScan(g_WorkState->GetPauseDownload());
+				g_WorkState->SetResumeTime(0);
 				UpdateTrayIcon();
 			}
 			else if (lParam == WM_LBUTTONDBLCLK && m_doubleClick)
@@ -662,6 +660,7 @@ void WinConsole::LoadPrefs()
 void WinConsole::ApplyPrefs()
 {
 	ShowWindow(GetConsoleWindow(), m_showConsole ? SW_SHOW : SW_HIDE);
+	g_WorkState->SetPauseFrontend(!m_showConsole);
 	if (m_showTrayIcon)
 	{
 		UpdateTrayIcon();
@@ -761,12 +760,12 @@ void WinConsole::UpdateTrayIcon()
 	strncpy(oldTip, m_iconData->szTip, sizeof(m_iconData->szTip));
 	oldTip[200-1] = '\0';
 
-	if (g_Options->GetPauseDownload())
+	if (g_WorkState->GetPauseDownload())
 	{
 		m_iconData->hIcon = m_pausedIcon;
 		strncpy(m_iconData->szTip, "NZBGet - paused", sizeof(m_iconData->szTip));
 	}
-	else if (!g_StatMeter->GetStandBy())
+	else if (g_WorkState->GetDownloading())
 	{
 		m_iconData->hIcon = m_workingIcon;
 		BString<100> tip("NZBGet - downloading at %s", *Util::FormatSpeed(g_StatMeter->CalcCurrentDownloadSpeed()));
@@ -956,8 +955,8 @@ void WinConsole::ResetFactoryDefaults()
 	BString<1024> path;
 	CString errmsg;
 
-	g_Options->SetPauseDownload(true);
-	g_Options->SetPausePostProcess(true);
+	g_WorkState->SetPauseDownload(true);
+	g_WorkState->SetPausePostProcess(true);
 
 	char commonAppDataPath[MAX_PATH];
 	SHGetFolderPath(nullptr, CSIDL_COMMON_APPDATA, nullptr, 0, commonAppDataPath);
@@ -973,7 +972,7 @@ void WinConsole::ResetFactoryDefaults()
 		while (retry > 0 && FileSystem::DirectoryExists(path) &&
 			!FileSystem::DeleteDirectoryWithContent(path, errmsg))
 		{
-			usleep(200 * 1000);
+			Util::Sleep(200);
 			retry--;
 		}
 

lib/catch/catch.h

@@ -2913,7 +2913,7 @@ namespace Catch {
                 for( std::vector<Ptr<Pattern> >::const_iterator it = m_patterns.begin(), itEnd = m_patterns.end(); it != itEnd; ++it )
                     if( !(*it)->matches( testCase ) )
                         return false;
-                    return true;
+                return true;
             }
         };
 

lib/par2/par2repairer.h

@@ -141,6 +141,9 @@ protected:
   virtual bool RepairData(u32 inputindex, size_t blocklength) { return false; }
 
 protected:
+  std::ostream&             cout;
+  std::ostream&             cerr;
+
   ParHeaders* headers;                                 // Headers
   bool alreadyloaded;                                // Already loaded ?
   CommandLine::NoiseLevel   noiselevel;              // OnScreen display
@@ -196,9 +199,6 @@ protected:
   u64                       totalsize;               // Total data size
 
   bool                      cancelled;               // repair cancelled
-
-  std::ostream&             cout;
-  std::ostream&             cerr;
 };
 
 } // end namespace Par2