Set version to 2.3.3

Merge branch 'develop'
Update text files for 2.3.3
2026-01-08 07:28:30 -05:00 · 2018-04-22 19:15:41 +02:00 · 2018-04-22 19:14:52 +02:00 · 2018-04-22 18:56:38 +02:00 · 2018-04-21 18:57:45 +00:00 · 2018-04-21 16:41:28 +02:00
30 changed files with 8457 additions and 8423 deletions
--- a/4
+++ b/4
@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 2.3.3Beta1
-Summary: SABnzbd-2.3.3Beta1
+Version: 2.3.3
+Summary: SABnzbd-2.3.3
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org
--- a/README.mkd
+++ b/README.mkd
@@ -1,15 +1,18 @@
-Release Notes - SABnzbd 2.3.3 Beta 1
+Release Notes - SABnzbd 2.3.3
 =========================================================

 ## Changes since 2.3.2
 - Introduce and enforce "host_whitelist" to reject DNS Rebinding attacks.
  If you access SABnzbd from a non-standard URL, you will need to add
  the hostname. More info: https://sabnzbd.org/hostname-check
- SABYenc updated to 3.3.3 to fix false-positive CRC errors
+- SABYenc updated to 3.3.5 to fix false-positive CRC errors
 - SSL-Ciphers can now be set per-server
+- Failed URL fetches also trigger post-processing script (if configured)
+- Added "max_url_retries" to set maximum retries of URL fetching
+- Added "ignore_empty_files" option to not warn on empty files in NZB
 - Added "X-Frame-Options" HTTP-header to prevent click-jacking
 - Added "httpOnly" HTTP-header to prevent script cookie access
- Added "ignore_empty_files" option to not warn on empty files in NZB
+- Extended SAN list of newly generated self-signed certificates
 - Indicate that SMPL-skin is no longer supported
 - Removed undocumented "callback" option from API calls
 - macOS: 64bit version of unrar instead of 32bit
@@ -18,9 +21,12 @@ Release Notes - SABnzbd 2.3.3 Beta 1

 ## Bugfixes since 2.3.2
 - NZB's can be added via command-line but this was not shown in "--help"
- Language-change via API would not only change language after restart
- Correctly indicate that 99 is maximum server priority
- Only show SSL-warning if it was actually tested
+- Only show bad-SSL-warning if it was actually tested
+- Language-change via API did not directly change display language
+- Cheetah 3 is also accepted as template engine
+- Correctly indicate that 99 is the maximum server priority
+- Results of unpacked zip/7zip files were not sorted
+- Joining of split files was limited to 999 files
 - Media files with "password" in filename were detected as encrypted
 - Servers that could not be DNS-resolved could block the queue
 - Detect '502 Byte limit exceeded' as payment problem
--- a/SABnzbd.py
+++ b/SABnzbd.py
@@ -44,7 +44,7 @@ import re

 try:
    import Cheetah
-    if Cheetah.Version[0] != '2':
+    if Cheetah.Version[0] < '2':
        raise ValueError
 except ValueError:
    print "Sorry, requires Python module Cheetah 2.0rc7 or higher."
@@ -779,7 +779,7 @@ def commandline_handler(frozen=True):
    if not service:
        # Get and remove any NZB file names
        for entry in args:
-            if get_ext(entry) in ('.nzb', '.zip', '.rar', '.gz', '.bz2'):
+            if get_ext(entry) in VALID_NZB_FILES + VALID_ARCHIVES:
                upload_nzbs.append(os.path.abspath(entry))

    for opt, arg in opts:
@@ -1344,8 +1344,6 @@ def main():
                            'tools.encode.on': True,
                            'tools.gzip.on': True,
                            'tools.gzip.mime_types': mime_gzip,
-                            'tools.response_headers.on': True,
-                            'tools.response_headers.headers': [('X-Frame-Options', 'SameOrigin')],
                            'request.show_tracebacks': True,
                            'error_page.401': sabnzbd.panic.error_page_401,
                            'error_page.404': sabnzbd.panic.error_page_404
@@ -1516,9 +1514,7 @@ def main():
        # Or special restart cases like Mac and WindowsService
        if sabnzbd.TRIGGER_RESTART:
            # Shutdown
-            cherrypy.engine.exit()
-            sabnzbd.halt()
-            sabnzbd.SABSTOP = True
+            sabnzbd.shutdown_program()

            if sabnzbd.downloader.Downloader.do.paused:
                sabnzbd.RESTART_ARGS.append('-p')
@@ -1725,9 +1721,7 @@ if __name__ == '__main__':

                def stop(self):
                    logging.info('[osx] sabApp Quit - stopping main thread ')
-                    sabnzbd.halt()
-                    cherrypy.engine.exit()
-                    sabnzbd.SABSTOP = True
+                    sabnzbd.shutdown_program()
                    logging.info('[osx] sabApp Quit - main thread stopped')

            sabApp = startApp()
--- a/interfaces/Config/templates/config_server.tmpl
+++ b/interfaces/Config/templates/config_server.tmpl
@@ -415,6 +415,8 @@
        // Exception when change of priority, reload
        \$('input[name="priority"], input[name="displayname"]').on('change', function() {
            \$('.fullform').submit(function() {
+                // No ajax this time
+                \$('input[name="ajax"]').val('')
                // Skip the fancy stuff, just submit
                this.submit()
            })
--- a/interfaces/smpl/templates/main.tmpl
+++ b/interfaces/smpl/templates/main.tmpl
@@ -1255,7 +1255,7 @@ function loadingJSON(){
    <option value="0" >$T("none")</option>
  </select></p>

-  <p><strong>This skin is no longer supported</strong></p>
+  <p><strong>This skin is no longer supported and may lose functionality in future releases.</strong></p>

    <!--#if $new_release#-->
    <!--#set $msg=$T('ft-newRelease@1')%($new_release)#-->
--- a/interfaces/smpl/templates/static/MochiKit/MochiKit.js
+++ b/interfaces/smpl/templates/static/MochiKit/MochiKit.js
--- a/interfaces/smpl/templates/static/MochiKit/package.js
+++ b/interfaces/smpl/templates/static/MochiKit/package.js
@@ -1,2 +1,2 @@
-dojo.hostenv.conditionalLoadModule({"common": ["MochiKit.MochiKit"]});
-dojo.hostenv.moduleLoaded("MochiKit.*");
+dojo.hostenv.conditionalLoadModule({"common": ["MochiKit.MochiKit"]});
+dojo.hostenv.moduleLoaded("MochiKit.*");
--- a/licenses/License-pynewsleecher.txt
+++ b/licenses/License-pynewsleecher.txt
@@ -1,11 +1,11 @@
-The original author of SABnzbd based his work on Pynewsleecher by Freddy@madcowdesease.org.
-
-Few parts of Pynewsleecher have survived the generations of SABnzbd in a
-recognizable form.
-Still, we wish to thank Freddy for his inspiration.
-
-The home of the Pynewsleecher project:
-http://www.madcowdisease.org/mcd/pynewsleecher
-
-The software does not carry any license information.
-
+The original author of SABnzbd based his work on Pynewsleecher by Freddy@madcowdesease.org.
+
+Few parts of Pynewsleecher have survived the generations of SABnzbd in a
+recognizable form.
+Still, we wish to thank Freddy for his inspiration.
+
+The home of the Pynewsleecher project:
+http://www.madcowdisease.org/mcd/pynewsleecher
+
+The software does not carry any license information.
+
--- a/licenses/License-systrayicon.txt
+++ b/licenses/License-systrayicon.txt
@@ -1,8 +1,8 @@
-On http://www.brunningonline.net/simon/blog/archives/001835.html,
-the author licensed SysTrayIcon.py under a variant of the WTFPL:
-
-> Any road up, help yourself. Consider SysTrayIcon.py to be under an
-> "Aleister Crowley" style license - "Do what thou wilt shall be the
-> only law".
-> 
-> Err, but don't sue me if it doesn't work. ;-)
+On http://www.brunningonline.net/simon/blog/archives/001835.html,
+the author licensed SysTrayIcon.py under a variant of the WTFPL:
+
+> Any road up, help yourself. Consider SysTrayIcon.py to be under an
+> "Aleister Crowley" style license - "Do what thou wilt shall be the
+> only law".
+> 
+> Err, but don't sue me if it doesn't work. ;-)
--- a/po/main/de.po
+++ b/po/main/de.po
@@ -8,14 +8,14 @@ msgstr ""
 "Project-Id-Version: sabnzbd\n"
 "Report-Msgid-Bugs-To: FULL NAME <EMAIL@ADDRESS>\n"
 "POT-Creation-Date: 2018-03-15 13:08+0000\n"
-"PO-Revision-Date: 2018-02-14 14:17+0000\n"
-"Last-Translator: Safihre <safihre@sabnzbd.org>\n"
+"PO-Revision-Date: 2018-04-15 21:22+0000\n"
+"Last-Translator: ciho <Unknown>\n"
 "Language-Team: German <de@li.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2018-03-16 05:37+0000\n"
-"X-Generator: Launchpad (build 18571)\n"
+"X-Launchpad-Export-Date: 2018-04-16 05:40+0000\n"
+"X-Generator: Launchpad (build 18610)\n"

 #: SABnzbd.py [Error message]
 msgid "MultiPar binary... NOT found!"
@@ -306,7 +306,7 @@ msgstr "Abgebrochen, unerwünschte Dateieindung gefunden"

 #: sabnzbd/assembler.py [Warning message]
 msgid "WARNING: Paused job \"%s\" because of rating (%s)"
-msgstr "WARNUNG: Job \"%s\" aufgrund der Bewertung (%s) pausiert."
+msgstr "WARNUNG: Aufgabe \"%s\" aufgrund der Bewertung (%s) pausiert."

 #: sabnzbd/assembler.py [Warning message]
 msgid "WARNING: Aborted job \"%s\" because of rating (%s)"
@@ -650,7 +650,7 @@ msgstr "Server-Adresse \"%s:%s\" ist ungültig."

 #: sabnzbd/interface.py
 msgid "Refused connection with hostname \"%s\" from:"
-msgstr ""
+msgstr "Verbindung vom Host \"%s\" abgelehnt von:"

 #: sabnzbd/interface.py
 msgid "User logged in to the web interface"
@@ -774,6 +774,8 @@ msgstr "Fehlerhafter Parameter"
 msgid ""
 "Category folder cannot be a subfolder of the Temporary Download Folder."
 msgstr ""
+"Der Category-Ordner darf kein Unterordner des Temporärer Download-Ordners "
+"sein."

 #: sabnzbd/interface.py # sabnzbd/interface.py
 msgid "Back"
@@ -2539,6 +2541,9 @@ msgid ""
 "Disconnect all active connections to usenet servers. Connections will be "
 "reopened after a few seconds if there are items in the queue."
 msgstr ""
+"Alle aktiven Verbindungen zu Usenet-Servern trennen. Verbindungen werden "
+"nach ein paar Sekunden wiederhergestellt, falls sich noch Artikel in der "
+"Warteschlange befinden."

 #: sabnzbd/skintext.py
 msgid "This will send a test email to your account."
@@ -3287,7 +3292,8 @@ msgstr "Aktion wenn eine verschlüsselte RAR Datei geladen wird"
 msgid ""
 "In case of \"Pause\", you'll need to set a password and resume the job."
 msgstr ""
-"Im Fall von \"Pause\" müssen Sie ein Kennwort setzen und den Job fortsetzen."
+"Im Fall von \"Pause\" müssen Sie ein Kennwort setzen und den Aufgabe "
+"fortsetzen."

 #: sabnzbd/skintext.py
 msgid "Detect Duplicate Downloads"
@@ -3437,7 +3443,7 @@ msgstr "IONice-Parameter"

 #: sabnzbd/skintext.py
 msgid "External process priority"
-msgstr ""
+msgstr "Priorität von externem Prozess"

 #: sabnzbd/skintext.py
 msgid "Disconnect on Empty Queue"
@@ -4108,7 +4114,7 @@ msgstr "Geräte, welche die Nachrichten empfangen sollen"

 #: sabnzbd/skintext.py [Pushover settings]
 msgid "Emergency retry"
-msgstr ""
+msgstr "Notfall Wiederanlauf"

 #: sabnzbd/skintext.py [Pushover settings]
 msgid "How often (in seconds) the same notification will be sent"
@@ -4116,11 +4122,12 @@ msgstr "Wie oft die selbe benachrichtigung (in Sekunden) geschickt wird."

 #: sabnzbd/skintext.py [Pushover settings]
 msgid "Emergency expire"
-msgstr ""
+msgstr "Notfall Verfall"

 #: sabnzbd/skintext.py [Pushover settings]
 msgid "How many seconds your notification will continue to be retried"
 msgstr ""
+"Wieviele Sekunden soll versucht werden deine Nachricht erneut zu versenden"

 #: sabnzbd/skintext.py [Header for Pushbullet notification section]
 msgid "Pushbullet"
@@ -4277,7 +4284,7 @@ msgstr "S01E05 Episoden-Ordner"

 #: sabnzbd/skintext.py
 msgid "Job Name as Filename"
-msgstr ""
+msgstr "Aufgabe Name als Ordnername"

 #: sabnzbd/skintext.py
 msgid "Title"
@@ -4349,7 +4356,7 @@ msgstr "Ursprünglicher Dateiname"

 #: sabnzbd/skintext.py
 msgid "Original Job Name"
-msgstr ""
+msgstr "Ursprünglicher Aufgabe Name"

 #: sabnzbd/skintext.py
 msgid "Lower Case"
@@ -4721,6 +4728,8 @@ msgid ""
 "All usernames, passwords and API-keys are automatically removed from the log "
 "and the included copy of your settings."
 msgstr ""
+"Alle Benutzernamen, Passwörter und API-Schlüssel werden automatisch aus dem "
+"Log und der darin enthaltenen Kopie deiner Einstellungen entfernt."

 #: sabnzbd/skintext.py
 msgid "Sort by Age <small>Oldest&rarr;Newest</small>"
--- a/po/main/fr.po
+++ b/po/main/fr.po
@@ -8,13 +8,13 @@ msgstr ""
 "Project-Id-Version: sabnzbd\n"
 "Report-Msgid-Bugs-To: FULL NAME <EMAIL@ADDRESS>\n"
 "POT-Creation-Date: 2018-03-15 13:08+0000\n"
-"PO-Revision-Date: 2018-02-14 14:17+0000\n"
-"Last-Translator: Safihre <safihre@sabnzbd.org>\n"
+"PO-Revision-Date: 2018-03-16 21:07+0000\n"
+"Last-Translator: Fred <88com88@gmail.com>\n"
 "Language-Team: French <fr@li.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2018-03-16 05:37+0000\n"
+"X-Launchpad-Export-Date: 2018-03-17 05:36+0000\n"
 "X-Generator: Launchpad (build 18571)\n"

 #: SABnzbd.py [Error message]
@@ -653,7 +653,7 @@ msgstr "L' adresse du serveur \"%s:%s\" n'est pas valide."

 #: sabnzbd/interface.py
 msgid "Refused connection with hostname \"%s\" from:"
-msgstr ""
+msgstr "Connexion refusée avec le nom d'hôte \"%s\" à partir de :"

 #: sabnzbd/interface.py
 msgid "User logged in to the web interface"
@@ -2547,6 +2547,9 @@ msgid ""
 "Disconnect all active connections to usenet servers. Connections will be "
 "reopened after a few seconds if there are items in the queue."
 msgstr ""
+"Déconnecte toutes les connexions actives aux serveurs Usenet. Les connexions "
+"seront rouvertes après quelques secondes si des éléments sont présents dans "
+"la file d'attente."

 #: sabnzbd/skintext.py
 msgid "This will send a test email to your account."
@@ -3449,7 +3452,7 @@ msgstr "Paramètres 'IONice'"

 #: sabnzbd/skintext.py
 msgid "External process priority"
-msgstr ""
+msgstr "Priorité de processus externe"

 #: sabnzbd/skintext.py
 msgid "Disconnect on Empty Queue"
--- a/po/main/nl.po
+++ b/po/main/nl.po
@@ -8,13 +8,13 @@ msgstr ""
 "Project-Id-Version: sabnzbd\n"
 "Report-Msgid-Bugs-To: FULL NAME <EMAIL@ADDRESS>\n"
 "POT-Creation-Date: 2018-03-15 13:08+0000\n"
-"PO-Revision-Date: 2018-02-14 14:17+0000\n"
+"PO-Revision-Date: 2018-03-16 12:21+0000\n"
 "Last-Translator: Safihre <safihre@sabnzbd.org>\n"
 "Language-Team: Dutch <nl@li.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2018-03-16 05:37+0000\n"
+"X-Launchpad-Export-Date: 2018-03-17 05:36+0000\n"
 "X-Generator: Launchpad (build 18571)\n"

 #: SABnzbd.py [Error message]
@@ -642,7 +642,7 @@ msgstr "Serveradres \"%s:%s\" is niet geldig."

 #: sabnzbd/interface.py
 msgid "Refused connection with hostname \"%s\" from:"
-msgstr ""
+msgstr "Verbinding met hostnaam \"%s\" geweigerd van:"

 #: sabnzbd/interface.py
 msgid "User logged in to the web interface"
@@ -2514,6 +2514,8 @@ msgid ""
 "Disconnect all active connections to usenet servers. Connections will be "
 "reopened after a few seconds if there are items in the queue."
 msgstr ""
+"Verbreek alle actieve verbindingen naar usenet servers. Verbindingen worden "
+"na een paar seconden weer geopend als er nog downloads in de wachtrij staan."

 #: sabnzbd/skintext.py
 msgid "This will send a test email to your account."
@@ -3400,7 +3402,7 @@ msgstr "\"IONice\" parameters"

 #: sabnzbd/skintext.py
 msgid "External process priority"
-msgstr ""
+msgstr "Externe process prioriteit"

 #: sabnzbd/skintext.py
 msgid "Disconnect on Empty Queue"
--- a/sabnzbd/init.py
+++ b/sabnzbd/init.py
@@ -769,7 +769,7 @@ def system_standby():

 def shutdown_program():
    """ Stop program after halting and saving """
-    logging.info("Performing sabnzbd shutdown")
+    logging.info("[%s] Performing SABnzbd shutdown", misc.caller_name())
    sabnzbd.halt()
    cherrypy.engine.exit()
    sabnzbd.SABSTOP = True
--- a/sabnzbd/api.py
+++ b/sabnzbd/api.py
@@ -41,7 +41,7 @@ except ImportError:
    pass

 import sabnzbd
-from sabnzbd.constants import VALID_ARCHIVES, Status, \
+from sabnzbd.constants import VALID_ARCHIVES, VALID_NZB_FILES, Status, \
     TOP_PRIORITY, REPAIR_PRIORITY, HIGH_PRIORITY, NORMAL_PRIORITY, LOW_PRIORITY, \
     KIBI, MEBI, GIGI, JOB_ADMIN
 import sabnzbd.config as config
@@ -398,7 +398,7 @@ def _api_addlocalfile(name, output, kwargs):
                if get_ext(name) in VALID_ARCHIVES:
                    res = sabnzbd.dirscanner.ProcessArchiveFile(
                        fn, name, pp=pp, script=script, cat=cat, priority=priority, keep=True, nzbname=nzbname)
-                elif get_ext(name) in ('.nzb', '.gz', '.bz2'):
+                elif get_ext(name) in VALID_NZB_FILES:
                    res = sabnzbd.dirscanner.ProcessSingleFile(
                        fn, name, pp=pp, script=script, cat=cat, priority=priority, keep=True, nzbname=nzbname)
            else:
@@ -589,10 +589,7 @@ def _api_resume(name, output, kwargs):

 def _api_shutdown(name, output, kwargs):
    """ API: accepts output """
-    logging.info('Shutdown requested by API')
-    sabnzbd.halt()
-    cherrypy.engine.exit()
-    sabnzbd.SABSTOP = True
+    sabnzbd.shutdown_program()
    return report(output)


--- a/sabnzbd/cfg.py
+++ b/sabnzbd/cfg.py
@@ -264,6 +264,7 @@ disable_key = OptionBool('misc', 'disable_api_key', False, protect=True)
 no_penalties = OptionBool('misc', 'no_penalties', False)
 debug_log_decoding = OptionBool('misc', 'debug_log_decoding', False)
 ignore_empty_files = OptionBool('misc', 'ignore_empty_files', False)
+x_frame_options = OptionBool('misc', 'x_frame_options', True)

 # Text values
 rss_odd_titles = OptionList('misc', 'rss_odd_titles', ['nzbindex.nl/', 'nzbindex.com/', 'nzbclub.com/'])
@@ -280,6 +281,7 @@ marker_file = OptionStr('misc', 'nomedia_marker', '')
 ipv6_servers = OptionNumber('misc', 'ipv6_servers', 1, 0, 2)
 url_base = OptionStr('misc', 'url_base', '/sabnzbd')
 host_whitelist = OptionList('misc', 'host_whitelist')
+max_url_retries = OptionNumber('misc', 'max_url_retries', 10, 1)

 ##############################################################################
 # Config - Notifications
--- a/sabnzbd/constants.py
+++ b/sabnzbd/constants.py
@@ -51,7 +51,7 @@ RENAMES_FILE = '__renames__'
 ATTRIB_FILE = 'SABnzbd_attrib'
 REPAIR_REQUEST = 'repair-all.sab'

-SABYENC_VERSION_REQUIRED = '3.3.3'
+SABYENC_VERSION_REQUIRED = '3.3.5'

 DB_HISTORY_VERSION = 1
 DB_HISTORY_NAME = 'history%s.db' % DB_HISTORY_VERSION
@@ -78,7 +78,6 @@ DEF_ARTICLE_CACHE_DEFAULT = '500M'
 DEF_ARTICLE_CACHE_MAX = '1G'
 DEF_TIMEOUT = 60
 DEF_SCANRATE = 5
-MAX_URL_RETRIES = 10
 MAX_DECODE_QUEUE = 10
 LIMIT_DECODE_QUEUE = 100
 MAX_WARNINGS = 20
@@ -98,6 +97,7 @@ STOP_PRIORITY = -4
 STAGES = {'Source': 0, 'Download': 1, 'Servers': 2, 'Repair': 3, 'Filejoin': 4, 'Unpack': 5, 'Script': 6}

 VALID_ARCHIVES = ('.zip', '.rar', '.7z')
+VALID_NZB_FILES = ('.nzb', '.gz', '.bz2')

 IGNORED_FOLDERS = ('@eaDir', '.appleDouble')

--- a/sabnzbd/dirscanner.py
+++ b/sabnzbd/dirscanner.py
@@ -28,7 +28,7 @@ import bz2
 import threading

 import sabnzbd
-from sabnzbd.constants import SCAN_FILE_NAME, VALID_ARCHIVES
+from sabnzbd.constants import SCAN_FILE_NAME, VALID_ARCHIVES, VALID_NZB_FILES
 import sabnzbd.utils.rarfile as rarfile
 from sabnzbd.encoding import platform_encode
 from sabnzbd.decorators import NzbQueueLocker
@@ -358,7 +358,7 @@ class DirScanner(threading.Thread):
                    continue

                ext = os.path.splitext(path)[1].lower()
-                candidate = ext in ('.nzb', '.gz', '.bz2') or ext in VALID_ARCHIVES
+                candidate = ext in VALID_NZB_FILES + VALID_ARCHIVES
                if candidate:
                    try:
                        stat_tuple = os.stat(path)
--- a/sabnzbd/interface.py
+++ b/sabnzbd/interface.py
@@ -42,7 +42,7 @@ import sabnzbd.scheduler as scheduler
 from Cheetah.Template import Template
 from sabnzbd.misc import real_path, to_units, from_units, time_format, \
    long_path, calc_age, same_file, probablyipv4, probablyipv6, \
-    cat_to_opts, int_conv, globber, globber_full, remove_all, get_base_url
+    int_conv, globber, globber_full, remove_all, get_base_url
 from sabnzbd.newswrapper import GetServerParms
 from sabnzbd.rating import Rating
 from sabnzbd.bpsmeter import BPSMeter
@@ -70,6 +70,7 @@ from sabnzbd.api import list_scripts, list_cats, del_from_section, \
    format_bytes, std_time, report, del_hist_job, Ttemplate, build_queue_header, \
    _api_test_email, _api_test_notif

+
 ##############################################################################
 # Global constants
 ##############################################################################
@@ -81,17 +82,9 @@ DIRECTIVES = {
 FILTER = LatinFilter


-def check_server(host, port, ajax):
-    """ Check if server address resolves properly """
-    if host.lower() == 'localhost' and sabnzbd.AMBI_LOCALHOST:
-        return badParameterResponse(T('Warning: LOCALHOST is ambiguous, use numerical IP-address.'), ajax)
-
-    if GetServerParms(host, int_conv(port)):
-        return ""
-    else:
-        return badParameterResponse(T('Server address "%s:%s" is not valid.') % (host, port), ajax)
-
-
+##############################################################################
+# Security functions
+##############################################################################
 def secured_expose(wrap_func=None, check_configlock=False, check_session_key=False):
    """ Wrapper for both cherrypy.expose and login/access check """
    if not wrap_func:
@@ -102,6 +95,10 @@ def secured_expose(wrap_func=None, check_configlock=False, check_session_key=Fal

    @functools.wraps(wrap_func)
    def internal_wrap(*args, **kwargs):
+        # Add X-Frame-Headers headers to page-requests
+        if cfg.x_frame_options():
+            cherrypy.response.headers['X-Frame-Options'] = 'SameOrigin'
+
        # Check if config is locked
        if check_configlock and cfg.configlock():
            cherrypy.response.status = 403
@@ -165,8 +162,9 @@ def check_hostname():
    if not host:
        return False

-    # Remove the port-part
-    host = host.split(':')[0].lower()
+    # Remove the port-part (like ':8080'), if it is there, always on the right hand side. 
+    # Not to be confused with IPv6 colons (within square brackets)
+    host = re.sub(':[0123456789]+$', '', host).lower()

    # Fine if localhost or IP
    if host == 'localhost' or probablyipv4(host) or probablyipv6(host):
@@ -181,67 +179,6 @@ def check_hostname():
    return False


-def ConvertSpecials(p):
-    """ Convert None to 'None' and 'Default' to '' """
-    if p is None:
-        p = 'None'
-    elif p.lower() == T('Default').lower():
-        p = ''
-    return p
-
-
-def Raiser(root='', **kwargs):
-    args = {}
-    for key in kwargs:
-        val = kwargs.get(key)
-        if val:
-            args[key] = val
-    # Add extras
-    if args:
-        root = '%s?%s' % (root, urllib.urlencode(args))
-    # Optionally add the leading /sabnzbd/ (or what the user set)
-    if not root.startswith(cfg.url_base()):
-        root = cherrypy.request.script_name + root
-    # Send the redirect
-    return cherrypy.HTTPRedirect(root)
-
-
-def queueRaiser(root, kwargs):
-    return Raiser(root, start=kwargs.get('start'),
-                  limit=kwargs.get('limit'),
-                  search=kwargs.get('search'))
-
-
-def rssRaiser(root, kwargs):
-    return Raiser(root, feed=kwargs.get('feed'))
-
-
-def IsNone(value):
-    """ Return True if either None, 'None' or '' """
-    return value is None or value == "" or value.lower() == 'none'
-
-
-def Strip(txt):
-    """ Return stripped string, can handle None """
-    try:
-        return txt.strip()
-    except:
-        return None
-
-
-##############################################################################
-# Web login support
-##############################################################################
-def get_users():
-    users = {}
-    users[cfg.username()] = cfg.password()
-    return users
-
-
-def encrypt_pwd(pwd):
-    return pwd
-
-
 # Create a more unique ID for each instance
 COOKIE_SECRET = str(randint(1000,100000)*os.getpid())

@@ -294,6 +231,16 @@ def check_login():
    return check_login_cookie()


+def get_users():
+    users = {}
+    users[cfg.username()] = cfg.password()
+    return users
+
+
+def encrypt_pwd(pwd):
+    return pwd
+
+
 def set_auth(conf):
    """ Set the authentication for CherryPy """
    if cfg.username() and cfg.password() and not cfg.html_login():
@@ -379,6 +326,37 @@ def log_warning_and_ip(txt):
    logging.warning('%s', txt)


+##############################################################################
+# Helper raiser functions
+##############################################################################
+def Raiser(root='', **kwargs):
+    args = {}
+    for key in kwargs:
+        val = kwargs.get(key)
+        if val:
+            args[key] = val
+    # Add extras
+    if args:
+        root = '%s?%s' % (root, urllib.urlencode(args))
+    # Optionally add the leading /sabnzbd/ (or what the user set)
+    if not root.startswith(cfg.url_base()):
+        root = cherrypy.request.script_name + root
+    # Send the redirect
+    return cherrypy.HTTPRedirect(root)
+
+
+def queueRaiser(root, kwargs):
+    return Raiser(root, start=kwargs.get('start'),
+                  limit=kwargs.get('limit'),
+                  search=kwargs.get('search'))
+
+
+def rssRaiser(root, kwargs):
+    return Raiser(root, feed=kwargs.get('feed'))
+
+
+##############################################################################
+# Page definitions
 ##############################################################################
 class MainPage(object):

@@ -463,10 +441,7 @@ class MainPage(object):
        if pid_in and int(pid_in) != os.getpid():
            return "Incorrect PID for this instance, remove PID from URL to initiate shutdown."

-        logging.info('Shutdown requested by interface')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()
        return T('SABnzbd shutdown finished')

    @secured_expose(check_session_key=True)
@@ -636,10 +611,7 @@ class Wizard(object):
    @secured_expose
    def exit(self, **kwargs):
        """ Stop SABnzbd """
-        logging.info('Shutdown requested by wizard')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()
        return T('SABnzbd shutdown finished')

 def get_access_info():
@@ -1025,19 +997,12 @@ class QueuePage(object):
            if cat == 'None':
                cat = None
            NzbQueue.do.change_cat(nzo_id, cat)
-            cat, pp, script, priority = cat_to_opts(cat)
-            NzbQueue.do.change_script(nzo_id, script)
-            NzbQueue.do.change_opts(nzo_id, pp)
-            NzbQueue.do.set_priority(nzo_id, priority)

        raise queueRaiser(self.__root, kwargs)

    @secured_expose(check_session_key=True)
    def shutdown(self, **kwargs):
-        logging.info('Shutdown requested by interface')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()
        return T('SABnzbd shutdown finished')

    @secured_expose(check_session_key=True)
@@ -1403,11 +1368,11 @@ SPECIAL_BOOL_LIST = \
              'rss_filenames', 'ipv6_hosting', 'keep_awake', 'empty_postproc', 'html_login', 'wait_for_dfolder',
              'max_art_opt', 'warn_empty_nzb', 'enable_bonjour', 'reject_duplicate_files', 'warn_dupl_jobs',
              'replace_illegal', 'backup_for_duplicates', 'disable_api_key', 'api_logging',
-              'ignore_empty_files'
+              'ignore_empty_files', 'x_frame_options'
     )
 SPECIAL_VALUE_LIST = \
    ('size_limit', 'folder_max_length', 'fsys_type', 'movie_rename_limit', 'nomedia_marker',
-              'req_completion_rate', 'wait_ext_drive', 'show_sysload', 'url_base',
+              'max_url_retries', 'req_completion_rate', 'wait_ext_drive', 'show_sysload', 'url_base',
              'direct_unpack_threads', 'ipv6_servers', 'selftest_host', 'rating_host'
     )
 SPECIAL_LIST_LIST = ('rss_odd_titles', 'quick_check_ext_ignore', 'host_whitelist')
@@ -1488,15 +1453,16 @@ class ConfigGeneral(object):

        wlist = []
        interfaces = globber_full(sabnzbd.DIR_INTERFACES)
-        for k in interfaces:
-            if k.endswith(DEF_STDINTF):
-                interfaces.remove(k)
-                interfaces.insert(0, k)
-                break
        for k in interfaces:
            if k.endswith(DEF_STDCONFIG):
                interfaces.remove(k)
-                break
+                continue
+
+            # TEMPORARY: Remove when smpl is really depricated
+            # Do not show smpl unless it's selected one
+            if k.endswith('smpl') and 'smpl' not in cfg.web_dir():
+                interfaces.remove(k)
+
        for web in interfaces:
            rweb = os.path.basename(web)
            if os.access(web + '/' + DEF_MAIN_TMPL, os.R_OK):
@@ -1662,6 +1628,17 @@ def unique_svr_name(server):
    return new_name


+def check_server(host, port, ajax):
+    """ Check if server address resolves properly """
+    if host.lower() == 'localhost' and sabnzbd.AMBI_LOCALHOST:
+        return badParameterResponse(T('Warning: LOCALHOST is ambiguous, use numerical IP-address.'), ajax)
+
+    if GetServerParms(host, int_conv(port)):
+        return ""
+    else:
+        return badParameterResponse(T('Server address "%s:%s" is not valid.') % (host, port), ajax)
+
+
 def handle_server(kwargs, root=None, new_svr=False):
    """ Internal server handler """
    ajax = kwargs.get('ajax')
@@ -2020,6 +1997,28 @@ class ConfigRss(object):
        raise rssRaiser(self.__root, kwargs)


+def ConvertSpecials(p):
+    """ Convert None to 'None' and 'Default' to '' """
+    if p is None:
+        p = 'None'
+    elif p.lower() == T('Default').lower():
+        p = ''
+    return p
+
+
+def IsNone(value):
+    """ Return True if either None, 'None' or '' """
+    return value is None or value == "" or value.lower() == 'none'
+
+
+def Strip(txt):
+    """ Return stripped string, can handle None """
+    try:
+        return txt.strip()
+    except:
+        return None
+
+
 ##############################################################################
 _SCHED_ACTIONS = ('resume', 'pause', 'pause_all', 'shutdown', 'restart', 'speedlimit',
                  'pause_post', 'resume_post', 'scan_folder', 'rss_scan', 'remove_failed',
--- a/sabnzbd/misc.py
+++ b/sabnzbd/misc.py
@@ -174,7 +174,7 @@ def cat_to_opts(cat, pp=None, script=None, priority=None):
        if priority == DEFAULT_PRIORITY:
            priority = def_cat.priority()

-    # logging.debug('Cat->Attrib cat=%s pp=%s script=%s prio=%s', cat, pp, script, priority)
+    logging.debug('Cat->Attrib cat=%s pp=%s script=%s prio=%s', cat, pp, script, priority)
    return cat, pp, script, priority


@@ -412,6 +412,7 @@ def sanitize_files_in_folder(folder):
            new_path = os.path.join(root, sanitize_filename(file_))
            if path != new_path:
                try:
+                    logging.debug('Filename-sanitizer will rename %s to %s', path, new_path)
                    os.rename(path, new_path)
                    path = new_path
                except:
@@ -995,6 +996,22 @@ def create_dirs(dirpath):
    return dirpath


+@synchronized(DIR_LOCK)
+def recursive_listdir(dir):
+    """ List all files in dirs and sub-dirs """
+    filelist = []
+    for root, dirs, files in os.walk(dir):
+        for file in files:
+            if '.AppleDouble' not in root and '.DS_Store' not in root:
+                try:
+                    p = os.path.join(root, file)
+                    filelist.append(p)
+                except UnicodeDecodeError:
+                    # Just skip failing names
+                    pass
+    return filelist
+
+
@synchronized(DIR_LOCK)
 def move_to_path(path, new_path):
    """ Move a file to a new path, optionally give unique filename
@@ -1515,7 +1532,9 @@ def probablyipv4(ip):


 def probablyipv6(ip):
-    if ip.count(':') >= 2 and re.sub('[0123456789abcdefABCDEF:]', '', ip) == '':
+    # Returns True if the given input is probably an IPv6 address
+    # Square Brackets like '[2001::1]' are OK
+    if ip.count(':') >= 2 and re.sub('[0123456789abcdefABCDEF:\[\]]', '', ip) == '':
        return True
    else:
        return False
--- a/sabnzbd/newsunpack.py
+++ b/sabnzbd/newsunpack.py
@@ -33,7 +33,7 @@ from sabnzbd.encoding import TRANS, unicoder, platform_encode, deunicode
 import sabnzbd.utils.rarfile as rarfile
 from sabnzbd.misc import format_time_string, find_on_path, make_script_path, int_conv, \
    real_path, globber, globber_full, get_all_passwords, renamer, clip_path, \
-    has_win_device, calc_age, long_path, remove_file
+    has_win_device, calc_age, long_path, remove_file, recursive_listdir
 from sabnzbd.sorting import SeriesSorter
 import sabnzbd.cfg as cfg
 from sabnzbd.constants import Status
@@ -65,14 +65,14 @@ else:
 from subprocess import Popen

 # Regex globals
-RAR_RE = re.compile(r'\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d)$', re.I)
+RAR_RE = re.compile(r'\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$', re.I)
 RAR_RE_V3 = re.compile(r'\.(?P<ext>part\d*)$', re.I)

 LOADING_RE = re.compile(r'^Loading "(.+)"')
 TARGET_RE = re.compile(r'^(?:File|Target): "(.+)" -')
 EXTRACTFROM_RE = re.compile(r'^Extracting\sfrom\s(.+)')
 EXTRACTED_RE = re.compile(r'^(Extracting|Creating|...)\s+(.*?)\s+OK\s*$')
-SPLITFILE_RE = re.compile(r'\.(\d\d\d$)', re.I)
+SPLITFILE_RE = re.compile(r'\.(\d\d\d?\d$)', re.I)
 ZIP_RE = re.compile(r'\.(zip$)', re.I)
 SEVENZIP_RE = re.compile(r'\.7z$', re.I)
 SEVENMULTI_RE = re.compile(r'\.7z\.\d+$', re.I)
@@ -246,7 +246,7 @@ def unpack_magic(nzo, workdir, workdir_complete, dele, one_folder, joinables, zi
    rerun = False
    force_rerun = False
    newfiles = []
-    error = 0
+    error = None
    new_joins = new_rars = new_zips = new_ts = None

    if cfg.enable_filejoin():
@@ -257,8 +257,6 @@ def unpack_magic(nzo, workdir, workdir_complete, dele, one_folder, joinables, zi
            if newf:
                newfiles.extend(newf)
            logging.info('Filejoin finished on %s', workdir)
-            nzo.set_action_line()
-            rerun = not error

    if cfg.enable_unrar():
        new_rars = [rar for rar in xrars if rar not in rars]
@@ -268,32 +266,27 @@ def unpack_magic(nzo, workdir, workdir_complete, dele, one_folder, joinables, zi
            if newf:
                newfiles.extend(newf)
            logging.info('Unrar finished on %s', workdir)
-            nzo.set_action_line()
-            rerun = not error

    if cfg.enable_7zip():
        new_sevens = [seven for seven in xsevens if seven not in sevens]
        if new_sevens:
            logging.info('7za starting on %s', workdir)
-            if unseven(nzo, workdir, workdir_complete, dele, one_folder, new_sevens):
-                error = True
+            error, newf = unseven(nzo, workdir, workdir_complete, dele, one_folder, new_sevens)
+            if newf:
+                newfiles.extend(newf)
            logging.info('7za finished on %s', workdir)
-            nzo.set_action_line()
-            rerun = not error

    if cfg.enable_unzip():
        new_zips = [zip for zip in xzips if zip not in zips]
        if new_zips:
            logging.info('Unzip starting on %s', workdir)
            if SEVEN_COMMAND:
-                if unseven(nzo, workdir, workdir_complete, dele, one_folder, new_zips):
-                    error = True
+                error, newf = unseven(nzo, workdir, workdir_complete, dele, one_folder, new_zips)
            else:
-                if unzip(nzo, workdir, workdir_complete, dele, one_folder, new_zips):
-                    error = True
+                error, newf = unzip(nzo, workdir, workdir_complete, dele, one_folder, new_zips)
+            if newf:
+                newfiles.extend(newf)
            logging.info('Unzip finished on %s', workdir)
-            nzo.set_action_line()
-            rerun = not error

    if cfg.enable_tsjoin():
        new_ts = [_ts for _ts in xts if _ts not in ts]
@@ -303,8 +296,12 @@ def unpack_magic(nzo, workdir, workdir_complete, dele, one_folder, joinables, zi
            if newf:
                newfiles.extend(newf)
            logging.info('TS Joining finished on %s', workdir)
-            nzo.set_action_line()
-            rerun = not error
+
+    # Refresh history and set output
+    nzo.set_action_line()
+
+    # Only re-run if something was unpacked and it was success
+    rerun = error in (False, 0)

    # During a Retry we might miss files that failed during recursive unpack
    if nzo.reuse and depth == 1 and any(build_filelists(workdir, workdir_complete)):
@@ -853,6 +850,9 @@ def unzip(nzo, workdir, workdir_complete, delete, one_folder, zips):
        unzip_failed = False
        tms = time.time()

+        # For file-bookkeeping
+        orig_dir_content = recursive_listdir(workdir_complete)
+
        for _zip in zips:
            logging.info("Starting extract on zipfile: %s ", _zip)
            nzo.set_action_line(T('Unpacking'), '%s' % unicoder(os.path.basename(_zip)))
@@ -870,6 +870,9 @@ def unzip(nzo, workdir, workdir_complete, delete, one_folder, zips):
        msg = T('%s files in %s') % (str(i), format_time_string(time.time() - tms))
        nzo.set_unpack_info('Unpack', msg)

+        # What's new?
+        new_files = list(set(orig_dir_content + recursive_listdir(workdir_complete)))
+
        # Delete the old files if we have to
        if delete and not unzip_failed:
            i = 0
@@ -890,12 +893,12 @@ def unzip(nzo, workdir, workdir_complete, delete, one_folder, zips):
                    except OSError:
                        logging.warning(T('Deleting %s failed!'), brokenzip)

-        return unzip_failed
+        return unzip_failed, new_files
    except:
        msg = sys.exc_info()[1]
        nzo.fail_msg = T('Unpacking failed, %s') % msg
        logging.error(T('Error "%s" while running unzip() on %s'), msg, nzo.final_name)
-        return True
+        return True, []


 def ZIP_Extract(zipfile, extraction_path, one_folder):
@@ -929,6 +932,7 @@ def unseven(nzo, workdir, workdir_complete, delete, one_folder, sevens):
    """
    i = 0
    unseven_failed = False
+    new_files = []
    tms = time.time()

    # Find multi-volume sets, because 7zip will not provide actual set members
@@ -955,18 +959,19 @@ def unseven(nzo, workdir, workdir_complete, delete, one_folder, sevens):
        else:
            extraction_path = os.path.split(seven)[0]

-        res, msg = seven_extract(nzo, seven, extensions, extraction_path, one_folder, delete)
+        res, new_files_set, msg = seven_extract(nzo, seven, extensions, extraction_path, one_folder, delete)
        if res:
            unseven_failed = True
            nzo.set_unpack_info('Unpack', msg)
        else:
            i += 1
+        new_files.extend(new_files_set)

    if not unseven_failed:
        msg = T('%s files in %s') % (str(i), format_time_string(time.time() - tms))
        nzo.set_unpack_info('Unpack', msg)

-    return unseven_failed
+    return unseven_failed, new_files


 def seven_extract(nzo, sevenset, extensions, extraction_path, one_folder, delete):
@@ -982,7 +987,7 @@ def seven_extract(nzo, sevenset, extensions, extraction_path, one_folder, delete
            msg = T('Trying 7zip with password "%s"') % unicoder(password)
            nzo.fail_msg = msg
            nzo.set_unpack_info('Unpack', msg)
-        fail, msg = seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete, password)
+        fail, new_files, msg = seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete, password)
        if fail != 2:
            break

@@ -991,7 +996,7 @@ def seven_extract(nzo, sevenset, extensions, extraction_path, one_folder, delete
        msg = '%s (%s)' % (T('Unpacking failed, archive requires a password'), os.path.basename(sevenset))
        nzo.fail_msg = msg
        logging.error(msg)
-    return fail, msg
+    return fail, new_files, msg


 def seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete, password):
@@ -1025,6 +1030,9 @@ def seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete
    if not os.path.exists(name):
        return 1, T('7ZIP set "%s" is incomplete, cannot unpack') % unicoder(sevenset)

+    # For file-bookkeeping
+    orig_dir_content = recursive_listdir(extraction_path)
+
    command = [SEVEN_COMMAND, method, '-y', overwrite, parm, case, password,
               '-o%s' % extraction_path, name]

@@ -1039,6 +1047,9 @@ def seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete

    ret = p.wait()

+    # What's new?
+    new_files = list(set(orig_dir_content + recursive_listdir(extraction_path)))
+
    if ret == 0 and delete:
        if extensions:
            for ext in extensions:
@@ -1054,7 +1065,7 @@ def seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete
                logging.warning(T('Deleting %s failed!'), sevenset)

    # Always return an error message, even when return code is 0
-    return ret, T('Could not unpack %s') % unicoder(sevenset)
+    return ret, new_files, T('Could not unpack %s') % unicoder(sevenset)


 ##############################################################################
@@ -2079,26 +2090,10 @@ def build_filelists(workdir, workdir_complete=None, check_both=False, check_rar=
    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])

    if workdir_complete:
-        for root, dirs, files in os.walk(workdir_complete):
-            for _file in files:
-                if '.AppleDouble' not in root and '.DS_Store' not in root:
-                    try:
-                        p = os.path.join(root, _file)
-                        filelist.append(p)
-                    except UnicodeDecodeError:
-                        # Just skip failing names
-                        pass
+        filelist.extend(recursive_listdir(workdir_complete))

    if workdir and (not filelist or check_both):
-        for root, dirs, files in os.walk(workdir):
-            for _file in files:
-                if '.AppleDouble' not in root and '.DS_Store' not in root:
-                    try:
-                        p = os.path.join(root, _file)
-                        filelist.append(p)
-                    except UnicodeDecodeError:
-                        # Just skip failing names
-                        pass
+        filelist.extend(recursive_listdir(workdir))

    for file in filelist:
        # Extra check for rar (takes CPU/disk)
--- a/sabnzbd/nzbstuff.py
+++ b/sabnzbd/nzbstuff.py
@@ -596,11 +596,6 @@ class NzbObject(TryList):
        filename = platform_encode(filename)
        nzbname = platform_encode(nzbname)

-        if pp is None:
-            r = u = d = None
-        else:
-            r, u, d = sabnzbd.pp_to_opts(pp)
-
        self.filename = filename    # Original filename
        if nzbname and nzb:
            work_name = nzbname     # Use nzbname if set and only for non-future slot
@@ -629,6 +624,26 @@ class NzbObject(TryList):
        self.work_name = create_work_name(work_name)
        self.final_name = create_work_name(work_name)

+        # Determine category and find pp/script values based on input
+        # Later will be re-evaluated based on import steps
+        if pp is None:
+            r = u = d = None
+        else:
+            r, u, d = sabnzbd.pp_to_opts(pp)
+        self.set_priority(priority) # Parse priority of input
+        self.repair = r             # True if we want to repair this set
+        self.unpack = u             # True if we want to unpack this set
+        self.delete = d             # True if we want to delete this set
+        self.script = script        # External script for this set
+        self.cat = cat              # User-set category
+
+        # Information fields
+        self.url = url or filename
+        self.groups = []
+        self.avg_date = datetime.datetime.fromtimestamp(0.0)
+        self.avg_stamp = 0.0        # Avg age in seconds (calculated from avg_age)
+
+        # Bookkeeping values
        self.meta = {}
        self.servercount = {}       # Dict to keep bytes per server
        self.created = False        # dirprefixes + work_name created
@@ -638,19 +653,6 @@ class NzbObject(TryList):
        self.bytes_tried = 0        # Which bytes did we try
        self.bytes_missing = 0      # Bytes missing
        self.bad_articles = 0       # How many bad (non-recoverable) articles
-        self.set_priority(priority) # Parse priority of input
-        self.repair = r             # True if we want to repair this set
-        self.unpack = u             # True if we want to unpack this set
-        self.delete = d             # True if we want to delete this set
-        self.script = script        # External script for this set
-        self.cat = cat              # Indexer category
-        if url:
-            self.url = str(url)     # Source URL
-        else:
-            self.url = filename
-        self.groups = []
-        self.avg_date = datetime.datetime.fromtimestamp(0.0)
-        self.avg_stamp = 0.0        # Avg age in seconds (calculated from avg_age)

        self.partable = {}          # Holds one parfile-name for each set
        self.extrapars = {}         # Holds the extra parfile names for all sets
@@ -663,7 +665,7 @@ class NzbObject(TryList):

        self.finished_files = []    # List of all finished NZFs

-        # the current status of the nzo eg:
+        # The current status of the nzo eg:
        # Queued, Downloading, Repairing, Unpacking, Failed, Complete
        self.status = status
        self.avg_bps_freq = 0
@@ -697,10 +699,7 @@ class NzbObject(TryList):
        # Stores one line containing the last failure
        self.fail_msg = ''
        # Stores various info about the nzo to be
-        if nzo_info:
-            self.nzo_info = nzo_info
-        else:
-            self.nzo_info = {}
+        self.nzo_info = nzo_info or {}

        # Temporary store for custom foldername - needs to be stored because of url fetching
        self.custom_name = nzbname
--- a/sabnzbd/osxmenu.py
+++ b/sabnzbd/osxmenu.py
@@ -36,7 +36,7 @@ import logging
 import sabnzbd
 import sabnzbd.cfg

-from sabnzbd.constants import VALID_ARCHIVES, MEBI, Status
+from sabnzbd.constants import VALID_ARCHIVES, VALID_NZB_FILES, MEBI, Status
 from sabnzbd.misc import get_filename, get_ext, diskspace, to_units
 from sabnzbd.panic import launch_a_browser
 import sabnzbd.notifier as notifier
@@ -754,7 +754,7 @@ class SABnzbdDelegate(NSObject):
                    if get_ext(name) in VALID_ARCHIVES:
                        # logging.info('[osx] archive')
                        dirscanner.ProcessArchiveFile(fn, name, keep=True)
-                    elif get_ext(name) in ('.nzb', '.gz', '.bz2'):
+                    elif get_ext(name) in VALID_NZB_FILES:
                        # logging.info('[osx] nzb')
                        dirscanner.ProcessSingleFile(fn, name, keep=True)
        # logging.info('opening done')
@@ -764,10 +764,7 @@ class SABnzbdDelegate(NSObject):
        self.setMenuTitle_("\n\n%s\n" % (T('Stopping...')))
        self.status_item.setHighlightMode_(NO)
        self.osx_icon = False
-        logging.info('[osx] application stopping daemon')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()
        try:
            notifier.send_notification('SABnzbd', T('SABnzbd shutdown finished'), notifier.NOTIFICATION['other'])
        except AttributeError:
--- a/sabnzbd/postproc.py
+++ b/sabnzbd/postproc.py
@@ -31,7 +31,7 @@ from sabnzbd.newsunpack import unpack_magic, par2_repair, external_processing, \
    sfv_check, build_filelists, rar_sort
 from threading import Thread
 from sabnzbd.misc import real_path, get_unique_path, create_dirs, move_to_path, \
-    make_script_path, long_path, clip_path, \
+    make_script_path, long_path, clip_path, recursive_listdir, \
    on_cleanup_list, renamer, remove_dir, remove_all, globber, globber_full, \
    set_permissions, cleanup_empty_directories, fix_unix_encoding, \
    sanitize_and_trim_path, sanitize_files_in_folder, remove_file
@@ -385,10 +385,12 @@ def process_job(nzo):
                    nzo.status = Status.EXTRACTING
                    logging.info("Running unpack_magic on %s", filename)
                    unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), (), ())
+                    logging.info("Unpacked files %s", newfiles)
+
                    if sabnzbd.WIN32:
                        # Sanitize the resulting files
                        newfiles = sanitize_files_in_folder(tmp_workdir_complete)
-                    logging.info("unpack_magic finished on %s", filename)
+                    logging.info("Finished unpack_magic on %s", filename)
                else:
                    nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))

@@ -866,10 +868,7 @@ def nzb_redirect(wdir, nzbname, pp, script, cat, priority):
        if so send to queue and remove if on CleanList
        Returns list of processed NZB's
    """
-    files = []
-    for root, _dirs, names in os.walk(wdir):
-        for name in names:
-            files.append(os.path.join(root, name))
+    files = recursive_listdir(wdir)

    for file_ in files:
        if os.path.splitext(file_)[1].lower() != '.nzb':
--- a/sabnzbd/sabtray.py
+++ b/sabnzbd/sabtray.py
@@ -166,10 +166,7 @@ class SABTrayThread(SysTrayIconThread):
    # menu handler - adapted from interface.py
    def shutdown(self, icon):
        self.hover_text = self.txt_shutdown
-        logging.info('Shutdown requested by tray')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()

    # adapted from interface.py
    def pause(self):
--- a/sabnzbd/sabtraylinux.py
+++ b/sabnzbd/sabtraylinux.py
@@ -169,9 +169,7 @@ class StatusIcon(Thread):

    def shutdown(self, icon):
        self.hover_text = T('Shutdown')
-        sabnzbd.halt()
-        cherrypy.engine.exit()
-        sabnzbd.SABSTOP = True
+        sabnzbd.shutdown_program()

    def pause(self):
        scheduler.plan_resume(0)
--- a/sabnzbd/urlgrabber.py
+++ b/sabnzbd/urlgrabber.py
@@ -30,11 +30,12 @@ from httplib import IncompleteRead
 from threading import Thread

 import sabnzbd
-from sabnzbd.constants import DEF_TIMEOUT, MAX_URL_RETRIES, FUTURE_Q_FOLDER, Status
+from sabnzbd.constants import DEF_TIMEOUT, FUTURE_Q_FOLDER, VALID_NZB_FILES, Status
 from sabnzbd.encoding import unicoder
 import sabnzbd.misc as misc
 import sabnzbd.dirscanner as dirscanner
 from sabnzbd.nzbqueue import NzbQueue
+from sabnzbd.postproc import PostProcessor
 import sabnzbd.cfg as cfg
 import sabnzbd.emailer as emailer
 import sabnzbd.notifier as notifier
@@ -64,8 +65,8 @@ class URLGrabber(Thread):
            future_nzo.url_tries += 1

            # Too many tries? Cancel
-            if future_nzo.url_tries > MAX_URL_RETRIES:
-                bad_fetch(future_nzo, url, T('Maximum retries'))
+            if future_nzo.url_tries > cfg.max_url_retries():
+                self.fail_to_history(future_nzo, url, T('Maximum retries'))
                return

            future_nzo.url_wait = time.time() + when
@@ -119,11 +120,11 @@ class URLGrabber(Thread):
                nzo_info = {}
                wait = 0
                retry = True
-                fn = None
+                fetch_request = None

                logging.info('Grabbing URL %s', url)
                try:
-                    fn = _build_request(url)
+                    fetch_request = _build_request(url)
                except Exception, e:
                    # Cannot list exceptions here, because of unpredictability over platforms
                    error0 = str(sys.exc_info()[0]).lower()
@@ -146,16 +147,17 @@ class URLGrabber(Thread):
                        # Catch if the server send retry (e.headers is case-INsensitive)
                        wait = misc.int_conv(e.headers['retry-after'])

-                new_url = dereferring(url, fn)
+                # Check if dereference is used
+                new_url = dereferring(url, fetch_request)
                if new_url:
                    self.add(new_url, future_nzo)
                    continue

-                if fn:
-                    for hdr in fn.headers:
+                if fetch_request:
+                    for hdr in fetch_request.headers:
                        try:
                            item = hdr.lower()
-                            value = fn.headers[hdr]
+                            value = fetch_request.headers[hdr]
                        except:
                            continue
                        if item in ('content-encoding',) and value == 'gzip':
@@ -194,16 +196,16 @@ class URLGrabber(Thread):
                    # For sites that have a rate-limiting attribute
                    msg = ''
                    retry = True
-                    fn = None
+                    fetch_request = None
                elif retry:
-                    fn, msg, retry, wait, data = _analyse(fn, url, future_nzo)
+                    fetch_request, msg, retry, wait, data = _analyse(fetch_request, url, future_nzo)

-                if not fn:
+                if not fetch_request:
                    if retry:
                        logging.info('Retry URL %s', url)
                        self.add(url, future_nzo, wait)
                    else:
-                        bad_fetch(future_nzo, url, msg)
+                        self.fail_to_history(future_nzo, url, msg)
                    continue

                if not filename:
@@ -225,12 +227,12 @@ class URLGrabber(Thread):
                    filename += '.gz'
                if not data:
                    try:
-                        data = fn.read()
+                        data = fetch_request.read()
                    except (IncompleteRead, IOError):
-                        bad_fetch(future_nzo, url, T('Server could not complete request'))
-                        fn.close()
+                        self.fail_to_history(future_nzo, url, T('Server could not complete request'))
+                        fetch_request.close()
                        continue
-                fn.close()
+                fetch_request.close()

                if '<nzb' in data and misc.get_ext(filename) != '.nzb':
                    filename += '.nzb'
@@ -247,7 +249,7 @@ class URLGrabber(Thread):
                del data

                # Check if nzb file
-                if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'):
+                if misc.get_ext(filename) in VALID_NZB_FILES:
                    res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
                                                       nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
                                                       nzo_id=future_nzo.nzo_id)[0]
@@ -263,8 +265,9 @@ class URLGrabber(Thread):
                            logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
                            when = 120
                        self.add(url, future_nzo, when)
-                # Check if a supported archive
+
                else:
+                    # Check if a supported archive
                    status, zf, exp_ext = dirscanner.is_archive(path)
                    if status == 0:
                        if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
@@ -275,9 +278,10 @@ class URLGrabber(Thread):
                        dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
                                                     nzbname=nzbname, url=future_nzo.url, keep=False,
                                                     nzo_id=future_nzo.nzo_id)
+                    else:
                        # Not a supported filetype, not an nzb (text/html ect)
                        try:
-                            os.remove(fn)
+                            os.remove(fetch_request)
                        except:
                            pass
                        logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
@@ -286,6 +290,38 @@ class URLGrabber(Thread):
                logging.error(T('URLGRABBER CRASHED'), exc_info=True)
                logging.debug("URLGRABBER Traceback: ", exc_info=True)

+    def fail_to_history(self, nzo, url, msg='', content=False):
+        """ Create History entry for failed URL Fetch
+            msg: message to be logged
+            content: report in history that cause is a bad NZB file
+        """
+        # Remove the "Trying to fetch" part
+        if url:
+            nzo.filename = url
+            nzo.final_name = url.strip()
+
+        if content:
+            # Bad content
+            msg = T('Unusable NZB file')
+        else:
+            # Failed fetch
+            msg = T('URL Fetching failed; %s') % unicoder(msg)
+
+        # Mark as failed
+        nzo.status = Status.FAILED
+        nzo.fail_msg = msg
+
+        notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat)
+        if cfg.email_endjob() > 0:
+            emailer.badfetch_mail(msg, url)
+
+        # Parse category to make sure script is set correctly after a grab
+        nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat, script=nzo.script)
+
+        # Add to history and run script if desired
+        NzbQueue.do.remove(nzo.nzo_id, add_to_history=False)
+        PostProcessor.do.process(nzo)
+

 def _build_request(url):
    # Detect basic auth
@@ -309,14 +345,14 @@ def _build_request(url):
    return urllib2.urlopen(req)


-def _analyse(fn, url, future_nzo):
+def _analyse(fetch_request, url, future_nzo):
    """ Analyze response of indexer
-        returns fn|None, error-message|None, retry, wait-seconds, data
+        returns fetch_request|None, error-message|None, retry, wait-seconds, data
    """
    data = None
-    if not fn or fn.code != 200:
-        if fn:
-            msg = fn.msg
+    if not fetch_request or fetch_request.code != 200:
+        if fetch_request:
+            msg = fetch_request.msg
        else:
            msg = ''

@@ -326,22 +362,22 @@ def _analyse(fn, url, future_nzo):
        return None, msg, True, when, data

    # Check for an error response
-    if not fn or fn.msg != 'OK':
+    if not fetch_request or fetch_request.msg != 'OK':
        # Increasing wait-time in steps for standard errors
        when = DEF_TIMEOUT * (future_nzo.url_tries + 1)
        logging.debug('Received nothing from indexer, retry after %s sec', when)
-        return None, fn.msg, True, when, data
+        return None, fetch_request.msg, True, when, data

-    return fn, fn.msg, False, 0, data
+    return fetch_request, fetch_request.msg, False, 0, data


-def dereferring(url, fn):
+def dereferring(url, fetch_request):
    """ Find out if we're being diverted to another location.
        If so, return new url else None
    """
    if 'derefer.me' in url:
        _RE_DEREFER = re.compile(r'content=".*url=([^"]+)">')
-        data = fn.read()
+        data = fetch_request.read()
        for line in data.split('\n'):
            if '<meta' in line:
                m = _RE_DEREFER.search(data)
@@ -350,34 +386,3 @@ def dereferring(url, fn):
    return None


-def bad_fetch(nzo, url, msg='', content=False):
-    """ Create History entry for failed URL Fetch
-        msg : message to be logged
-        retry : make retry link in history
-        content : report in history that cause is a bad NZB file
-    """
-    if msg:
-        msg = unicoder(msg)
-    else:
-        msg = ''
-
-    nzo.status = Status.FAILED
-
-    if url:
-        nzo.filename = url
-        nzo.final_name = url.strip()
-
-    if content:
-        # Bad content
-        msg = T('Unusable NZB file')
-    else:
-        # Failed fetch
-        msg = T('URL Fetching failed; %s') % msg
-
-    nzo.fail_msg = msg
-
-    notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat)
-    if cfg.email_endjob() > 0:
-        emailer.badfetch_mail(msg, url)
-
-    NzbQueue.do.remove(nzo.nzo_id, add_to_history=True)
--- a/sabnzbd/utils/certgen.py
+++ b/sabnzbd/utils/certgen.py
@@ -12,8 +12,9 @@ from cryptography import x509
 from cryptography.x509.oid import NameOID
 import datetime
 import os
-from sabnzbd.getipaddress import localipv4
+import socket

+from sabnzbd.getipaddress import localipv4

 # Ported from cryptography/utils.py
 def int_from_bytes(data, byteorder, signed=False):
@@ -60,15 +61,25 @@ def generate_local_cert(private_key, days_valid=3560, output_file='cert.cert', L
        # x509.NameAttribute(NameOID.COMMON_NAME, CN),
    ])

-    # build SubjectAltName list since we are not using a common name
-    san_list = [
-        x509.DNSName(u"localhost"),
-        x509.DNSName(u"127.0.0.1"),
-        ]
-    # append local v4 ip (functions already has try/catch logic)
-    mylocalipv4 = localipv4()
-    if mylocalipv4:
-        san_list.append(x509.DNSName(u"" + mylocalipv4))
+
+    # build Subject Alternate Names (aka SAN) list
+    # First the host names, add with x509.DNSName():
+    san_list = [x509.DNSName(u"localhost")]
+    san_list.append(x509.DNSName(unicode(socket.gethostname())))
+
+    # Then the host IP addresses, add with x509.IPAddress()
+    # Inside a try-except, just to be sure
+    try:
+        import ipaddress
+        san_list.append(x509.IPAddress(ipaddress.IPv4Address(u"127.0.0.1")))
+        san_list.append(x509.IPAddress(ipaddress.IPv6Address(u"::1")))
+
+        # append local v4 ip
+        mylocalipv4 = localipv4()
+        if mylocalipv4:
+            san_list.append(x509.IPAddress(ipaddress.IPv4Address(unicode(mylocalipv4))))
+    except:
+        pass

    cert = x509.CertificateBuilder().subject_name(
        subject
--- a/sabnzbd/utils/upload.py
+++ b/sabnzbd/utils/upload.py
@@ -26,7 +26,7 @@ from sabnzbd.encoding import unicoder
 import sabnzbd.cfg as cfg
 from sabnzbd.misc import get_ext, get_filename, get_from_url
 import sabnzbd.newsunpack
-from sabnzbd.constants import VALID_ARCHIVES
+from sabnzbd.constants import VALID_ARCHIVES, VALID_NZB_FILES

 from sabnzbd.dirscanner import ProcessArchiveFile, ProcessSingleFile

@@ -60,7 +60,7 @@ def add_local(f):
        if fn:
            if get_ext(fn) in VALID_ARCHIVES:
                ProcessArchiveFile(fn, f, keep=True)
-            elif get_ext(fn) in ('.nzb', '.gz', '.bz2'):
+            elif get_ext(fn) in VALID_NZB_FILES:
                ProcessSingleFile(fn, f, keep=True)
        else:
            logging.error("Filename not found: %s", f)
--- a/sabnzbd/version.py
+++ b/sabnzbd/version.py
@@ -4,5 +4,5 @@

 # You MUST use double quotes (so " and not ')

-__version__ = "2.4.0-develop"
-__baseline__ = "unknown"
+__version__ = "2.3.3"
+__baseline__ = "1f04343a4d9453050d6c36087ee2e2fd6c5eee32"
--- a/scripts/Sample-PostProc.cmd
+++ b/scripts/Sample-PostProc.cmd
@@ -1,15 +1,15 @@
-@echo off
-rem Example of a post processing script for SABnzbd
-
-echo.
-echo Running in directory "%~d0%~p0"
-echo.
-echo The first parameter (result-dir) = %1
-echo The second parameter (nzb-name)  = %2
-echo The third parameter (nice name)  = %3
-echo The fourth parameter (newzbin #) = %4
-echo The fifth parameter (category)   = %5
-echo The sixth parameter (group)      = %6
-echo The seventh parameter (status)   = %7
-echo The eight parameter (failure_url)= %8
-echo.
+@echo off
+rem Example of a post processing script for SABnzbd
+
+echo.
+echo Running in directory "%~d0%~p0"
+echo.
+echo The first parameter (result-dir) = %1
+echo The second parameter (nzb-name)  = %2
+echo The third parameter (nice name)  = %3
+echo The fourth parameter (newzbin #) = %4
+echo The fifth parameter (category)   = %5
+echo The sixth parameter (group)      = %6
+echo The seventh parameter (status)   = %7
+echo The eight parameter (failure_url)= %8
+echo.