Correctly use dict.keys()

Solves https://forums.sabnzbd.org/viewtopic.php?f=2&t=25087

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.1.0
-Summary: SABnzbd-3.1.0
+Version: 3.1.1
+Summary: SABnzbd-3.1.1
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,15 @@
-Release Notes - SABnzbd 3.1.0
+Release Notes - SABnzbd 3.1.1
 =========================================================
 
+## Changes and bugfixes since 3.1.1
+- Enforce CRLF line endings on outgoing email messages.
+- Queue Repair would fail if Rating is enabled.
+- It was not possible to set directory-settings to empty values.
+- Deobfuscate-during-download was not triggered.
+- Failed to start on Python 3.5 with HTTPS enabled.
+- Could show traceback when formatting error/warnings messages.
+- Windows/macOS: improve handling of program restart.
+
 ## Changes since 3.0.2
 - Added option to automatically deobfuscate final filenames: after unpacking, 
   detect and rename obfuscated or meaningless filenames to the job name, 

SABnzbd.py

@@ -125,17 +125,23 @@ class GUIHandler(logging.Handler):
 
     def emit(self, record):
         """ Emit a record by adding it to our private queue """
+        # If % is part of the msg, this could fail
+        try:
+            record_msg = record.msg % record.args
+        except TypeError:
+            record_msg = record.msg + str(record.args)
+
         if record.levelname == "WARNING":
-            sabnzbd.LAST_WARNING = record.msg % record.args
+            sabnzbd.LAST_WARNING = record_msg
         else:
-            sabnzbd.LAST_ERROR = record.msg % record.args
+            sabnzbd.LAST_ERROR = record_msg
 
         if len(self.store) >= self.size:
             # Loose the oldest record
             self.store.pop(0)
         try:
             # Append traceback, if available
-            warning = {"type": record.levelname, "text": record.msg % record.args, "time": int(time.time())}
+            warning = {"type": record.levelname, "text": record_msg, "time": int(time.time())}
             if record.exc_info:
                 warning["text"] = "%s\n%s" % (warning["text"], traceback.format_exc())
             self.store.append(warning)
@@ -1287,7 +1293,7 @@ def main():
             sabnzbd.cfg.enable_https.set(False)
 
         # So the cert and key files do exist, now let's check if they are valid:
-        trialcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        trialcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
         try:
             trialcontext.load_cert_chain(https_cert, https_key)
             logging.info("HTTPS keys are OK")
@@ -1530,6 +1536,7 @@ def main():
         # Check for auto-restart request
         # Or special restart cases like Mac and WindowsService
         if sabnzbd.TRIGGER_RESTART:
+            logging.info("Performing triggered restart")
             # Shutdown
             sabnzbd.shutdown_program()
 
@@ -1548,7 +1555,7 @@ def main():
                 my_name = sabnzbd.MY_FULLNAME.replace("/Contents/MacOS/SABnzbd", "")
                 my_args = " ".join(sys.argv[1:])
                 cmd = 'kill -9 %s && open "%s" --args %s' % (my_pid, my_name, my_args)
-                logging.info("Launching: ", cmd)
+                logging.info("Launching: %s", cmd)
                 os.system(cmd)
             elif sabnzbd.WIN_SERVICE:
                 # Use external service handler to do the restart

sabnzbd/__init__.py

@@ -465,15 +465,6 @@ def trigger_restart(timeout=None):
     if timeout:
         time.sleep(timeout)
 
-    # Add extra arguments
-    if sabnzbd.downloader.Downloader.do.paused:
-        sabnzbd.RESTART_ARGS.append("-p")
-    sys.argv = sabnzbd.RESTART_ARGS
-
-    # Stop all services
-    sabnzbd.halt()
-    cherrypy.engine.exit()
-
     if sabnzbd.WIN32:
         # Remove connection info for faster restart
         del_connection_info()
@@ -482,6 +473,15 @@ def trigger_restart(timeout=None):
     if hasattr(sys, "frozen"):
         sabnzbd.TRIGGER_RESTART = True
     else:
+        # Add extra arguments
+        if sabnzbd.downloader.Downloader.do.paused:
+            sabnzbd.RESTART_ARGS.append("-p")
+        sys.argv = sabnzbd.RESTART_ARGS
+
+        # Stop all services
+        sabnzbd.halt()
+        cherrypy.engine.exit()
+
         # Do the restart right now
         cherrypy.engine._do_execv()
 

sabnzbd/api.py

@@ -1746,8 +1746,8 @@ def build_history(start=0, limit=0, search=None, failed_only=0, categories=None)
     # Un-reverse the queue
     items.reverse()
 
-    # Global check if rating is enabled
-    rating_enabled = cfg.rating_enable()
+    # Global check if rating is enabled and available (queue-repair)
+    rating_enabled = cfg.rating_enable() and Rating.do
 
     for item in items:
         item["size"] = to_units(item["bytes"], "B")

sabnzbd/config.py

@@ -236,7 +236,7 @@ class OptionDir(Option):
         'create' means try to create (but don't set permanent create flag)
         """
         error = None
-        if value and (value != self.get() or create):
+        if value is not None and (value != self.get() or create):
             value = value.strip()
             if self.__validation:
                 error, value = self.__validation(self.__root, value, super().default())

sabnzbd/dirscanner.py

@@ -46,7 +46,7 @@ def compare_stat_tuple(tup1, tup2):
 
 def clean_file_list(inp_list, folder, files):
     """ Remove elements of "inp_list" not found in "files" """
-    for path in sorted(inp_list.keys()):
+    for path in sorted(inp_list):
         fld, name = os.path.split(path)
         if fld == folder:
             present = False

sabnzbd/downloader.py

@@ -914,7 +914,8 @@ class Downloader(Thread):
         # Clean expired timers
         now = time.time()
         kicked = []
-        for server_id in self._timers.keys():
+        # Create a copy so we can remove during iteration
+        for server_id in list(self._timers):
             if not [stamp for stamp in self._timers[server_id] if stamp >= now]:
                 logging.debug("Forcing re-evaluation of server-id %s", server_id)
                 del self._timers[server_id]

sabnzbd/emailer.py

@@ -27,6 +27,7 @@ import glob
 
 from Cheetah.Template import Template
 from email.message import EmailMessage
+from email import policy
 
 from sabnzbd.constants import *
 import sabnzbd
@@ -296,4 +297,4 @@ def _prepare_message(txt):
                 msg[keyword] = value
 
     msg.set_content("\n".join(payload))
-    return msg.as_bytes()
+    return msg.as_bytes(policy=msg.policy.clone(linesep="\r\n"))

sabnzbd/interface.py

@@ -529,7 +529,7 @@ class Wizard:
         else:
             # Sort servers to get the first enabled one
             server_names = sorted(
-                servers.keys(),
+                servers,
                 key=lambda svr: "%d%02d%s"
                 % (int(not servers[svr].enable()), servers[svr].priority(), servers[svr].displayname().lower()),
             )
@@ -1580,7 +1580,7 @@ class ConfigServer:
         new = []
         servers = config.get_servers()
         server_names = sorted(
-            list(servers.keys()),
+            servers,
             key=lambda svr: "%d%02d%s"
             % (int(not servers[svr].enable()), servers[svr].priority(), servers[svr].displayname().lower()),
         )

sabnzbd/newsunpack.py

@@ -1350,7 +1350,7 @@ def PAR_Verify(parfile, nzo, setname, joinables, single=False):
                     block_table[nzf.blocks] = nzf
 
             if block_table:
-                nzf = block_table[min(block_table.keys())]
+                nzf = block_table[min(block_table)]
                 logging.info("Found new par2file %s", nzf.filename)
 
                 # Move from extrapar list to files to be downloaded
@@ -1650,7 +1650,7 @@ def MultiPar_Verify(parfile, nzo, setname, joinables, single=False):
                     block_table[nzf.blocks] = nzf
 
             if block_table:
-                nzf = block_table[min(block_table.keys())]
+                nzf = block_table[min(block_table)]
                 logging.info("Found new par2file %s", nzf.filename)
 
                 # Move from extrapar list to files to be downloaded

sabnzbd/nzbparser.py

@@ -118,7 +118,7 @@ def nzbfile_parser(raw_data, nzo):
                     pass
 
         # Sort the articles by part number, compatible with Python 3.5
-        raw_article_db_sorted = [raw_article_db[partnum] for partnum in sorted(raw_article_db.keys())]
+        raw_article_db_sorted = [raw_article_db[partnum] for partnum in sorted(raw_article_db)]
 
         # Create NZF
         nzf = sabnzbd.nzbstuff.NzbFile(file_date, file_name, raw_article_db_sorted, file_bytes, nzo)

sabnzbd/nzbstuff.py

@@ -357,15 +357,15 @@ class NzbFile(TryList):
         self.valid = bool(raw_article_db)
 
         if self.valid and self.nzf_id:
-            # Save first article separate so we can do duplicate file detection
+            # Save first article separate so we can do
+            # duplicate file detection and deobfuscate-during-download
             first_article = self.add_article(raw_article_db.pop(0))
             first_article.lowest_partnum = True
+            self.nzo.first_articles.append(first_article)
+            self.nzo.first_articles_count += 1
 
-            # For non-par2 files we also use it to do deobfuscate-during-download
-            # And we count how many bytes are available for repair
+            # Count how many bytes are available for repair
             if sabnzbd.par2file.is_parfile(self.filename):
-                self.nzo.first_articles.append(first_article)
-                self.nzo.first_articles_count += 1
                 self.nzo.bytes_par2 += self.bytes
 
             # Any articles left?
@@ -383,7 +383,7 @@ class NzbFile(TryList):
         if raw_article_db:
             # Convert 2.x.x jobs
             if isinstance(raw_article_db, dict):
-                raw_article_db = [raw_article_db[partnum] for partnum in sorted(raw_article_db.keys())]
+                raw_article_db = [raw_article_db[partnum] for partnum in sorted(raw_article_db)]
 
             for raw_article in raw_article_db:
                 self.add_article(raw_article)
@@ -473,7 +473,7 @@ class NzbFile(TryList):
 
         # Convert 2.x.x jobs
         if isinstance(self.decodetable, dict):
-            self.decodetable = [self.decodetable[partnum] for partnum in sorted(self.decodetable.keys())]
+            self.decodetable = [self.decodetable[partnum] for partnum in sorted(self.decodetable)]
 
         # Set non-transferable values
         self.md5 = None
@@ -1493,7 +1493,7 @@ class NzbObject(TryList):
             # Sort the servers first
             servers = config.get_servers()
             server_names = sorted(
-                servers.keys(),
+                servers,
                 key=lambda svr: "%d%02d%s"
                 % (int(not servers[svr].enable()), servers[svr].priority(), servers[svr].displayname().lower()),
             )
@@ -1609,7 +1609,7 @@ class NzbObject(TryList):
 
                 pos_nzf_table = self.build_pos_nzf_table(nzf_ids)
 
-                keys = list(pos_nzf_table.keys())
+                keys = list(pos_nzf_table)
                 keys.sort()
 
                 if target == keys:
@@ -1626,7 +1626,7 @@ class NzbObject(TryList):
 
                 pos_nzf_table = self.build_pos_nzf_table(nzf_ids)
 
-                keys = list(pos_nzf_table.keys())
+                keys = list(pos_nzf_table)
                 keys.sort()
 
                 if target == keys:
@@ -1702,8 +1702,11 @@ class NzbObject(TryList):
             self.renamed_file(yenc_filename, nzf.filename)
             nzf.filename = yenc_filename
 
+    @synchronized(NZO_LOCK)
     def verify_all_filenames_and_resort(self):
-        """ Verify all filenames based on par2 info and then re-sort files """
+        """Verify all filenames based on par2 info and then re-sort files.
+        Locked so all files are verified at once without interuptions.
+        """
         logging.info("Checking all filenames for %s", self.final_name)
         for nzf_verify in self.files:
             self.verify_nzf_filename(nzf_verify)

sabnzbd/osxmenu.py

@@ -234,7 +234,7 @@ class SABnzbdDelegate(NSObject):
             100: "100%",
         }
 
-        for speed in sorted(speeds.keys()):
+        for speed in sorted(speeds):
             menu_speed_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
                 "%s" % (speeds[speed]), "speedlimitAction:", ""
             )

sabnzbd/postproc.py

@@ -962,7 +962,7 @@ def rar_renamer(nzo, workdir):
         # So, all rar files with rarvolnr 1, find the contents (files inside the rar),
         # and match with rarfiles with rarvolnr 2, and put them in the correct rarset.
         # And so on, until the highest rarvolnr minus 1 matched against highest rarvolnr
-        for n in range(1, len(rarvolnr.keys())):
+        for n in range(1, len(rarvolnr)):
             logging.debug("Deobfuscate: Finding matches between rar sets %s and %s" % (n, n + 1))
             for base_obfuscated_filename in rarvolnr[n]:
                 matchcounter = 0

sabnzbd/rss.py

@@ -151,8 +151,7 @@ def remove_obsolete(jobs, new_jobs):
     """
     now = time.time()
     limit = now - 259200  # 3days (3x24x3600)
-    olds = list(jobs.keys())
-    for old in olds:
+    for old in list(jobs):
         tm = jobs[old]["time"]
         if old not in new_jobs:
             if jobs[old].get("status", " ")[0] in ("G", "B"):
@@ -178,7 +177,7 @@ class RSSQueue:
             self.jobs = sabnzbd.load_admin(RSS_FILE_NAME)
             if self.jobs:
                 for feed in self.jobs:
-                    remove_obsolete(self.jobs[feed], list(self.jobs[feed].keys()))
+                    remove_obsolete(self.jobs[feed], list(self.jobs[feed]))
         except:
             logging.warning(T("Cannot read %s"), RSS_FILE_NAME)
             logging.info("Traceback: ", exc_info=True)

sabnzbd/sorting.py

@@ -532,7 +532,7 @@ def check_for_sequence(regex, files):
                 prefix = name[: match1.start()]
 
     # Don't do anything if only one or no files matched
-    if len(list(matches.keys())) < 2:
+    if len(list(matches)) < 2:
         return {}
 
     key_prev = 0
@@ -540,7 +540,7 @@ def check_for_sequence(regex, files):
     alphabet = "abcdefghijklmnopqrstuvwxyz"
 
     # Check the dictionary to see if the keys are in a numeric or alphabetic sequence
-    for akey in sorted(matches.keys()):
+    for akey in sorted(matches):
         if akey.isdigit():
             key = int(akey)
         elif akey in alphabet: