Limit feedparser<6.0.0 for 3.0.x

Closes #1589

ABOUT.txt

@@ -1,5 +1,5 @@
 *******************************************
-*** This is SABnzbd 3.0.1               ***
+*** This is SABnzbd                     ***
 *******************************************
 
 SABnzbd is an open-source cross-platform binary newsreader.

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.0.1
-Summary: SABnzbd-3.0.1
+Version: 3.0.2
+Summary: SABnzbd-3.0.2
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,18 @@
-Release Notes - SABnzbd 3.0.1
+Release Notes - SABnzbd 3.0.2
 =========================================================
 
+## Bugfixes since 3.0.1
+- Priority was not parsed correctly if supplied as through the API.
+- API-call `addfile` could fail if `name` and `nzbfile` were used.
+- Permissions were still not set correctly when creating directories.
+- Propagation delay label was shown even if no delay was activated.
+- Reading RSS feed with no categories set could result in crash.
+- Jobs with numeric names could crash post-processing.
+- Jobs with missing articles could result in crash.
+- macOS: changed the power assertion to `NoIdleSleep`.
+- Windows: end-of-queue-script did not run on Windows.
+- Windows: crash if the virus scanner removed the certificate bundle.
+
 ## Bugfixes since 3.0.0
 - Basic Authentication resulted in crash. 
 - Permissions were not set correctly when creating directories.

SABnzbd.py

@@ -1168,9 +1168,14 @@ def main():
     if importlib.util.find_spec("certifi") is not None:
         import certifi
 
-        os.environ["SSL_CERT_FILE"] = certifi.where()
-        logging.info("Certifi version: %s", certifi.__version__)
-        logging.info("Loaded additional certificates from: %s", os.environ["SSL_CERT_FILE"])
+        try:
+            os.environ["SSL_CERT_FILE"] = certifi.where()
+            logging.info("Certifi version: %s", certifi.__version__)
+            logging.info("Loaded additional certificates from: %s", os.environ["SSL_CERT_FILE"])
+        except:
+            # Sometimes the certificate file is blocked
+            logging.warning(T("Could not load additional certificates from certifi package"))
+            logging.info("Traceback: ", exc_info=True)
 
     # Extra startup info
     if sabnzbd.cfg.log_level() > 1:
@@ -1698,9 +1703,7 @@ if __name__ == "__main__":
         # This code is made with trial-and-error, please improve!
         class startApp(Thread):
             def run(self):
-                logging.info("[osx] sabApp Starting - starting main thread")
                 main()
-                logging.info("[osx] sabApp Stopping - main thread quit ")
                 AppHelper.stopEventLoop()
 
         sabApp = startApp()

requirements.txt

@@ -1,7 +1,7 @@
 sabyenc3>=4.0.0
 cheetah3>=3.0.0
 cryptography
-feedparser
+feedparser<6.0.0
 configobj
 cheroot<8.4.3
 cherrypy

sabnzbd/__init__.py

@@ -848,10 +848,10 @@ def change_queue_complete_action(action, new=True):
 
 def run_script(script):
     """ Run a user script (queue complete only) """
-    command = [os.path.join(cfg.script_dir.get_path(), script)]
-    if os.path.exists(command[0]):
+    script_path = filesystem.make_script_path(script)
+    if script_path:
         try:
-            stup, need_shell, command, creationflags = sabnzbd.newsunpack.build_command(command)
+            stup, need_shell, command, creationflags = sabnzbd.newsunpack.build_command([script_path])
             logging.info("Spawning external command %s", command)
             subprocess.Popen(
                 command,

sabnzbd/api.py

@@ -350,7 +350,7 @@ def _api_translate(name, output, kwargs):
 def _api_addfile(name, output, kwargs):
     """ API: accepts name, output, pp, script, cat, priority, nzbname """
     # Normal upload will send the nzb in a kw arg called name or nzbfile
-    if not name:
+    if not name or isinstance(name, str):
         name = kwargs.get("nzbfile", None)
     if hasattr(name, "file") and hasattr(name, "filename") and name.filename:
         cat = kwargs.get("cat")

sabnzbd/filesystem.py

@@ -561,13 +561,10 @@ def create_all_dirs(path, apply_umask=False):
         else:
             # We need to build the directory recursively so we can
             # apply permissions to only the newly created folders
-            # We cannot use os.makedirs() to do this as it ignores the mode
-            try:
-                # Try the user permissions setting
-                umask = int(sabnzbd.cfg.umask(), 8) | int("0700", 8)
-            except:
-                # Use default
-                umask = int("0700", 8)
+            # We cannot use os.makedirs() as it could ignore the mode
+            umask = sabnzbd.cfg.umask()
+            if umask:
+                umask = int(umask, 8) | int("0700", 8)
 
             # Build path from root
             path_part_combined = "/"
@@ -578,7 +575,7 @@ def create_all_dirs(path, apply_umask=False):
                     if not os.path.exists(path_part_combined):
                         os.mkdir(path_part_combined)
                         # Try to set permissions if desired, ignore failures
-                        if apply_umask:
+                        if umask and apply_umask:
                             set_chmod(path_part_combined, umask, report=False)
         return path
     except OSError:

sabnzbd/nzbqueue.py

@@ -710,15 +710,15 @@ class NzbQueue:
             Not locked for performance, since it only reads the queue
         """
         # Pre-calculate propagation delay
-        propagtion_delay = float(cfg.propagation_delay() * 60)
+        propagation_delay = float(cfg.propagation_delay() * 60)
         for nzo in self.__nzo_list:
             # Not when queue paused and not a forced item
             if nzo.status not in (Status.PAUSED, Status.GRABBING) or nzo.priority == TOP_PRIORITY:
                 # Check if past propagation delay, or forced
                 if (
-                    not propagtion_delay
+                    not propagation_delay
                     or nzo.priority == TOP_PRIORITY
-                    or (nzo.avg_stamp + propagtion_delay) < time.time()
+                    or (nzo.avg_stamp + propagation_delay) < time.time()
                 ):
                     if not nzo.server_in_try_list(server):
                         article = nzo.get_article(server, servers)

sabnzbd/nzbstuff.py

@@ -20,6 +20,7 @@ sabnzbd.nzbstuff - misc
 """
 
 import os
+import pickle
 import time
 import re
 import logging
@@ -481,8 +482,15 @@ class NzbFile(TryList):
         self.md5 = None
 
     def __eq__(self, other):
-        """ Assume it's the same file if the bytes and first article are the same """
-        return self.bytes == other.bytes and self.decodetable[0] == other.decodetable[0]
+        """ Assume it's the same file if the numer bytes and first article
+            are the same or if there are no articles left, use the filenames
+        """
+        if self.bytes == other.bytes:
+            if self.decodetable and other.decodetable:
+                return self.decodetable[0] == other.decodetable[0]
+            # Fallback to filename comparison
+            return self.filename == other.filename
+        return False
 
     def __hash__(self):
         """ Required because we implement eq. The same file can be spread
@@ -556,6 +564,8 @@ NzbObjectSaver = (
     "rating_filtered",
 )
 
+NzoAttributeSaver = ("cat", "pp", "script", "priority", "final_name", "password", "url")
+
 # Lock to prevent errors when saving the NZO data
 NZO_LOCK = threading.RLock()
 
@@ -707,7 +717,7 @@ class NzbObject(TryList):
             self.final_name = self.final_name.replace(".", " ")
 
         # Check against identical checksum or series/season/episode
-        if (not reuse) and nzb and dup_check and priority != REPAIR_PRIORITY:
+        if (not reuse) and nzb and dup_check and self.priority != REPAIR_PRIORITY:
             duplicate, series = self.has_duplicates()
         else:
             duplicate = series = 0
@@ -779,14 +789,10 @@ class NzbObject(TryList):
 
         # Pickup backed-up attributes when re-using
         if reuse:
-            cat, pp, script, priority, name, password, self.url = get_attrib_file(self.workpath, 7)
-            if name:
-                self.final_name = name
-            if password:
-                self.password = password
+            cat, pp, script = self.load_attribs()
 
         # Determine category and find pp/script values
-        self.cat, pp_tmp, self.script, priority = cat_to_opts(cat, pp, script, priority)
+        self.cat, pp_tmp, self.script, priority = cat_to_opts(cat, pp, script, self.priority)
         self.set_priority(priority)
         self.repair, self.unpack, self.delete = pp_to_opts(pp_tmp)
 
@@ -1339,8 +1345,9 @@ class NzbObject(TryList):
                 labels.append(T("WAIT %s sec") % dif)
 
         # Propagation delay label
-        if (self.avg_stamp + float(cfg.propagation_delay() * 60)) > time.time() and self.priority != TOP_PRIORITY:
-            wait_time = int((self.avg_stamp + float(cfg.propagation_delay() * 60) - time.time()) / 60 + 0.5)
+        propagation_delay = float(cfg.propagation_delay() * 60)
+        if propagation_delay and self.avg_stamp + propagation_delay > time.time() and self.priority != TOP_PRIORITY:
+            wait_time = int((self.avg_stamp + propagation_delay - time.time()) / 60 + 0.5)
             labels.append(T("PROPAGATING %s min") % wait_time)  # Queue indicator while waiting for propagation of post
 
         return labels
@@ -1892,9 +1899,36 @@ class NzbObject(TryList):
             sabnzbd.save_data(self, self.nzo_id, self.workpath)
 
     def save_attribs(self):
-        set_attrib_file(
-            self.workpath, (self.cat, self.pp, self.script, self.priority, self.final_name, self.password, self.url)
-        )
+        """ Save specific attributes for Retry """
+        attribs = {}
+        for attrib in NzoAttributeSaver:
+            attribs[attrib] = getattr(self, attrib)
+        logging.debug("Saving attributes %s for %s", attribs, self.final_name)
+        sabnzbd.save_data(attribs, ATTRIB_FILE, self.workpath)
+
+    def load_attribs(self):
+        """ Load saved attributes and return them to be parsed """
+        attribs = sabnzbd.load_data(ATTRIB_FILE, self.workpath, remove=False)
+        logging.debug("Loaded attributes %s for %s", attribs, self.final_name)
+
+        # TODO: Remove fallback to old method in SABnzbd 3.2.0
+        if not attribs:
+            cat, pp, script, self.priority, name, password, self.url = get_attrib_file(self.workpath, 7)
+            if name:
+                # Could be converted to integer due to the logic in get_attrib_file
+                self.final_name = str(name)
+            if password:
+                self.password = password
+            return cat, pp, script
+
+        # Only a subset we want to apply directly to the NZO
+        for attrib in ("final_name", "priority", "password", "url"):
+            # Only set if it is present and has a value
+            if attribs.get(attrib):
+                setattr(self, attrib, attribs[attrib])
+
+        # Rest is to be used directly in the NZO-init flow
+        return attribs["cat"], attribs["pp"], attribs["script"]
 
     @synchronized(NZO_LOCK)
     def build_pos_nzf_table(self, nzf_ids):
@@ -2150,15 +2184,6 @@ def get_attrib_file(path, size):
         return [None for _ in range(size)]
 
 
-def set_attrib_file(path, attribs):
-    """ Write job's attributes to file """
-    logging.debug("Writing attributes %s to %s", attribs, path)
-    path = os.path.join(path, ATTRIB_FILE)
-    with open(path, "w", encoding="utf-8") as attr_file:
-        for item in attribs:
-            attr_file.write("%s\n" % item)
-
-
 def name_extractor(subject):
     """ Try to extract a file name from a subject line, return `subject` if in doubt """
     result = subject

sabnzbd/rss.py

@@ -791,7 +791,7 @@ def _get_link(entry):
             except AttributeError:
                 try:  # nzb.su
                     category = entry.tags[0]["term"]
-                except (AttributeError, KeyError):
+                except (AttributeError, IndexError, KeyError):
                     try:
                         category = entry.description
                     except AttributeError:

sabnzbd/utils/sleepless.py

@@ -45,12 +45,16 @@ def keep_awake(reason):
         Multiple calls allowed.
     """
     global assertion_id
-    kIOPMAssertionTypeNoIdleSleep = "PreventUserIdleSystemSleep"
-    kIOPMAssertionLevelOn = 255
-    errcode, assertion_id = IOPMAssertionCreateWithName(
-        kIOPMAssertionTypeNoIdleSleep, kIOPMAssertionLevelOn, reason, None
-    )
-    return errcode == 0
+
+    # Each assertion needs to be released, so make sure to only set it once
+    if not assertion_id:
+        kIOPMAssertionTypeNoIdleSleep = "NoIdleSleepAssertion"
+        kIOPMAssertionLevelOn = 255
+        errcode, assertion_id = IOPMAssertionCreateWithName(
+            kIOPMAssertionTypeNoIdleSleep, kIOPMAssertionLevelOn, reason, None
+        )
+        return errcode == 0
+    return True
 
 
 def allow_sleep():

tests/test_filesystem.py

@@ -768,22 +768,32 @@ class TestCreateAllDirs(ffs.TestCase, PermissionCheckerHelper):
 
     @set_config({"umask": "0777"})
     def test_permissions_777(self):
-        self._permissions_runner("/test_base777", "/test_base777/se 1/ep 1", "0700")
+        self._permissions_runner("/test_base777")
+        self._permissions_runner("/test_base777_nomask", apply_umask=False)
 
     @set_config({"umask": "0770"})
     def test_permissions_770(self):
-        self._permissions_runner("/test_base770", "/test_base770/se 1/ep 1", "0700")
+        self._permissions_runner("/test_base770")
+        self._permissions_runner("/test_base770_nomask", apply_umask=False)
 
     @set_config({"umask": "0600"})
     def test_permissions_600(self):
-        self._permissions_runner("/test_base600", "/test_base600/se 1/ep 1", "0700")
+        self._permissions_runner("/test_base600")
+        self._permissions_runner("/test_base600_nomask", apply_umask=False)
 
     @set_config({"umask": "0700"})
     def test_permissions_450(self):
         with pytest.raises(OSError):
-            self._permissions_runner("/test_base_450", "/test_base_450/se 1/ep 1", "0450")
+            self._permissions_runner("/test_base450", perms_base="0450")
 
-    def _permissions_runner(self, test_base, new_dir, perms_base):
+    @set_config({"umask": ""})
+    def test_no_umask(self):
+        self._permissions_runner("/test_base_perm700", perms_base="0700")
+        self._permissions_runner("/test_base_perm750", perms_base="0750")
+        self._permissions_runner("/test_base_perm777", perms_base="0777")
+        self._permissions_runner("/test_base_perm600", perms_base="0600")
+
+    def _permissions_runner(self, test_base, perms_base="0700", apply_umask=True):
         # Create base directory and set the base permissions
         perms_base_int = int(perms_base, 8)
         self.fs.create_dir(test_base, perms_base_int)
@@ -791,11 +801,18 @@ class TestCreateAllDirs(ffs.TestCase, PermissionCheckerHelper):
         self.assert_dir_perms(test_base, perms_base_int)
 
         # Create directories with permissions
-        filesystem.create_all_dirs(new_dir, apply_umask=True)
+        new_dir = os.path.join(test_base, "se 1", "ep1")
+        filesystem.create_all_dirs(new_dir, apply_umask=apply_umask)
 
         # If permissions needed to be set, verify the new folder has the
         # right permissions and verify the base didn't change
-        perms_test_int = int(cfg.umask(), 8) | int("0700", 8)
+        if apply_umask and cfg.umask():
+            perms_test_int = int(cfg.umask(), 8) | int("0700", 8)
+        else:
+            # Get the current umask, since os.mkdir masks that out
+            cur_umask = os.umask(0)
+            os.umask(cur_umask)
+            perms_test_int = int("0777", 8) & ~cur_umask
         self.assert_dir_perms(new_dir, perms_test_int)
         self.assert_dir_perms(test_base, perms_base_int)
 

tests/test_utils/test_sleepless.py

@@ -59,3 +59,17 @@ class TestSleepless:
         sleepless.allow_sleep()
         assert not self.check_msg_in_assertions()
         assert sleepless.assertion_id is None
+
+    def test_sleepless_multi_call(self):
+        # If we set it twice, is it still cleared with one call
+        assert not self.check_msg_in_assertions()
+        assert sleepless.assertion_id is None
+
+        sleepless.keep_awake(self.sleep_msg)
+        time.sleep(2)
+        sleepless.keep_awake(self.sleep_msg)
+        assert self.check_msg_in_assertions()
+
+        sleepless.allow_sleep()
+        assert not self.check_msg_in_assertions()
+        assert sleepless.assertion_id is None

tools/make_mo.py

@@ -281,26 +281,25 @@ if os.path.exists(tl):
         TOOL = [tl]
 
 result = True
-if len(sys.argv) > 1 and sys.argv[1] == "all":
+if len(sys.argv) > 1 and sys.argv[1] == "nsis":
     print("NSIS MO file")
     result = result and process_po_folder(DOMAIN_N, PON_DIR)
 
     print("Patch NSIS script")
     patch_nsis()
+else:
+    print("Email MO files")
+    result = result and process_po_folder(DOMAIN_E, POE_DIR)
 
-print("Email MO files")
-result = result and process_po_folder(DOMAIN_E, POE_DIR)
+    print("Create email templates from MO files")
+    make_templates()
 
-print("Create email templates from MO files")
-make_templates()
+    print("Main program MO files")
+    # -n option added to remove all newlines from the translations
+    result = result and process_po_folder(DOMAIN, PO_DIR, "-n")
 
-
-print("Main program MO files")
-# -n option added to remove all newlines from the translations
-result = result and process_po_folder(DOMAIN, PO_DIR, "-n")
-
-print("Remove temporary templates")
-remove_mo_files()
+    print("Remove temporary templates")
+    remove_mo_files()
 
 print()
 if result: