Correct: Compressed NZB filename could exceed OS limitation

See https://www.reddit.com/r/SABnzbd/comments/wrserd/importing_nzb_files/

.github/workflows/build_release.yml

@@ -88,7 +88,7 @@ jobs:
       # We need the official Python, because the GA ones only support newer macOS versions
       # The deployment target is picked up by the Python build tools automatically
       # If updated, make sure to also set LSMinimumSystemVersion in SABnzbd.spec
-      PYTHON_VERSION: "3.10.4"
+      PYTHON_VERSION: "3.10.6"
       MACOSX_DEPLOYMENT_TARGET: "10.9"
       # We need to force compile for universal2 support
       CFLAGS: -arch arm64 -arch x86_64

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.6.0RC2
-Summary: SABnzbd-3.6.0RC2
+Version: 3.6.1RC2
+Summary: SABnzbd-3.6.1RC2
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,13 @@
-Release Notes - SABnzbd 3.6.0 Release Candidate 1
+Release Notes - SABnzbd 3.6.1 Release Candidate 2
 =========================================================
 
+## Bugfixes since 3.6.0
+- Downloads in `Checking` status were not displayed correctly.
+- Prevent showing crash on Status window during shutdown.
+- Prevent scheduler crash if removed event is canceled.
+- Updated UnRar to 6.11 (Windows) and 6.12 (macOS).
+- macOS/Windows: Updated dependencies and Python versions.
+
 ## Changes since 3.5.3
 - Significantly increased performance by using the yEnc-decoding
   library of @animetosho. Usenet articles are now decoded using
@@ -19,7 +26,7 @@ Release Notes - SABnzbd 3.6.0 Release Candidate 1
 - Removed included `Deobfuscate.py` as it is outdated.
 - Show a warning when there are no valid news servers active.
 - Show a warning if the filesystem does not support special characters.
-- Allow multiple parameters to be passed to par2cmdline/Multipar
+- Allow multiple parameters to be passed to par2cmdline/Multipar.
 - Linux: Added AppStream metadata, desktop shortcut and MimeInfo.
 - Linux: Added support for bash completion.
 - macOS: Application and included tools fully native on M1 systems.
@@ -28,11 +35,12 @@ Release Notes - SABnzbd 3.6.0 Release Candidate 1
 
 # API changes since 3.5.3
 - Removed several (status) fields from the `queue` API call.
-- Remove unused and undocumented API calls: `addid`, `options`, `rescan`
+- Remove unused and undocumented API calls: `addid`, `options`, `rescan`,
   `osx_icon`, `set_speedlimit`, `get_speedlimit`, `set_colorscheme`.
+- Removed undocumented `xcat` parameter.
 - `None` values in XML API-output are left empty.
 - Adding NZB's would not always return `nzo_ids`.
-- Removed undocumented `xcat` parameter.
+- Prevent crash in `history` call during post-processing.
 
 # Bugfixes since 3.5.3
 - Extended timeout when measuring system performance.
@@ -44,7 +52,6 @@ Release Notes - SABnzbd 3.6.0 Release Candidate 1
 - Prevent traceback when status functions timeout.
 - Prevent crash if not enough repair blocks are available.
 - Prevent crash when there is an unknown `language` setting.
-- Prevent crash in API `history` call during post-processing.
 - Source release had Windows line-endings.
 - Windows: If a job password contained a double quote it
   would not be picked up by UnRar and unpack would fail.

builder/requirements.txt

@@ -1,17 +1,18 @@
 # Basic build requirements
 # Note that not all sub-dependencies are listed, but only ones we know could cause trouble
-pyinstaller==5.1
-pyinstaller-hooks-contrib==2022.5
+pyinstaller==5.3
+pyinstaller-hooks-contrib==2022.8
 altgraph==0.17.2
 wrapt==1.14.1
-setuptools==62.3.2
-pkginfo==1.8.2
+setuptools==63.3.0
+pkginfo==1.8.3
 PyGithub==1.55
-charset-normalizer==2.0.12
+charset-normalizer==2.1.0
 certifi
 
 # orjson does not support 32bit Windows, exclude it based on Python-version
-orjson==3.6.8; python_version > '3.8'
+# This way we also test ujson on Python 3.7 and 3.8 in the CI-tests
+orjson==3.7.11; python_version > '3.8'
 
 # For the macOS build
 dmgbuild==1.5.2; sys_platform == 'darwin'

requirements.txt

@@ -1,27 +1,27 @@
 # Main requirements
 # Note that not all sub-dependencies are listed, but only ones we know could cause trouble
-sabyenc3==5.4.1
+sabyenc3==5.4.3
 cheetah3==3.2.6 # pyup: ignore
-cffi==1.15
+cffi==1.15.1
 pycparser==2.21
 feedparser==6.0.10
 configobj==5.0.6
 cheroot==8.6.0
 six==1.16.0
-cherrypy==18.6.1
-jaraco.functools==3.5.0
-jaraco.collections==3.5.1
-jaraco.text==3.7.0
-jaraco.classes==3.2.1
-jaraco.context==4.1.1
+cherrypy==18.8.0
+jaraco.functools==3.5.1
+jaraco.collections==3.5.2
+jaraco.text==3.8.1
+jaraco.classes==3.2.2
+jaraco.context==4.1.2
 more-itertools==8.13.0
 zc.lockfile==2.0
 python-dateutil==2.8.2
-tempora==5.0.1
+tempora==5.0.2
 pytz==2022.1
 sgmllib3k==1.0.0
 portend==3.1.0
-chardet==4.0.0
+chardet==5.0.0
 PySocks==1.7.1
 puremagic==1.14
 guessit==3.4.3
@@ -30,11 +30,11 @@ rebulk==3.1.0
 
 # Recent cryptography versions require Rust. If you run into issues compiling this
 # SABnzbd will also work with older pre-Rust versions such as cryptography==3.3.2
-cryptography==37.0.2
+cryptography==37.0.4
 
 # We recommend using "orjson" as it is 2x as fast as "ujson". However, it requires
 # Rust so SABnzbd works just as well with "ujson" or the Python built in "json" module
-ujson==5.3.0
+ujson==5.4.0
 
 # Windows system integration
 pywin32==304; sys_platform == 'win32'

sabnzbd/__init__.py

@@ -317,28 +317,6 @@ def initialize(pause_downloader=False, clean_up=False, repair=0):
         cfg.cache_limit.set(misc.get_cache_limit())
     sabnzbd.ArticleCache.new_limit(cfg.cache_limit.get_int())
 
-    # Values we want to remove
-    deprecated_options = [
-        cfg.ampm,
-        cfg.enable_https_verification,
-        cfg.enable_meta,
-        cfg.replace_illegal,
-        cfg.html_login,
-        cfg.osx_speed,
-        cfg.allow_incomplete_nzb,
-        cfg.disable_key,
-        cfg.show_sysload,
-    ]
-    for deprecated_option in deprecated_options:
-        if deprecated_option() != deprecated_option.default:
-            sabnzbd.misc.helpful_warning(
-                T(
-                    "We are planning to remove the '%s' setting, which you have changed from the default value. "
-                    "Could you let us know why you made this change at: https://github.com/sabnzbd/sabnzbd/discussions"
-                ),
-                deprecated_option.keyword,
-            )
-
     logging.info("All processes started")
     sabnzbd.RESTART_REQ = False
     sabnzbd.__INITIALIZED__ = True

sabnzbd/api.py

@@ -1362,7 +1362,7 @@ def build_queue(start: int = 0, limit: int = 0, search: Optional[str] = None, nz
         if not sabnzbd.Downloader.paused and nzo.status not in (Status.PAUSED, Status.FETCHING, Status.GRABBING):
             if is_propagating:
                 slot["status"] = Status.PROP
-            elif status == Status.CHECKING:
+            elif nzo.status == Status.CHECKING:
                 slot["status"] = Status.CHECKING
             else:
                 slot["status"] = Status.DOWNLOADING

sabnzbd/constants.py

@@ -50,7 +50,7 @@ RENAMES_FILE = "__renames__"
 ATTRIB_FILE = "SABnzbd_attrib"
 REPAIR_REQUEST = "repair-all.sab"
 
-SABYENC_VERSION_REQUIRED = "5.4.1"
+SABYENC_VERSION_REQUIRED = "5.4.3"
 
 DB_HISTORY_VERSION = 1
 DB_HISTORY_NAME = "history%s.db" % DB_HISTORY_VERSION

sabnzbd/filesystem.py

@@ -1222,6 +1222,8 @@ def backup_nzb(nzb_path: str):
 
 def save_compressed(folder: str, filename: str, data_fp: BinaryIO) -> str:
     """Save compressed NZB file in folder, return path to saved nzb file"""
+    # Make sure it's a clean filename
+    filename = sanitize_filename(filename)
     if filename.endswith(".nzb"):
         filename += ".gz"
     else:

sabnzbd/getipaddress.py

@@ -44,9 +44,10 @@ def timeout(max_timeout: float):
         def func_wrapper(*args, **kwargs):
             """Closure for function."""
             # Raises a TimeoutError if execution exceeds max_timeout
+            # Raises a RuntimeError is SABnzbd is already shutting down when called
             try:
                 return sabnzbd.THREAD_POOL.submit(item, *args, **kwargs).result(max_timeout)
-            except TimeoutError:
+            except (TimeoutError, RuntimeError):
                 return None
 
         return func_wrapper

sabnzbd/newsunpack.py

@@ -44,7 +44,6 @@ from sabnzbd.misc import (
     run_command,
     build_and_run_command,
     format_time_left,
-    helpful_warning,
 )
 from sabnzbd.filesystem import (
     make_script_path,
@@ -64,7 +63,6 @@ from sabnzbd.filesystem import (
     SEVENMULTI_RE,
 )
 from sabnzbd.nzbstuff import NzbObject
-from sabnzbd.par2file import FilePar2Info
 from sabnzbd.sorting import SeriesSorter
 import sabnzbd.cfg as cfg
 from sabnzbd.constants import Status, JOB_ADMIN
@@ -1977,21 +1975,6 @@ def rar_sort(a: str, b: str) -> int:
 
 
 def quick_check_set(setname: str, nzo: NzbObject) -> bool:
-    new = quick_check_set_new(setname, nzo)
-    old = quick_check_set_old(setname, nzo)
-
-    if new != old:
-        helpful_warning(
-            "We are testing a new verification method and on the current job it produced unexpected results. It would "
-            "help us if you could share this NZB (%s) and the Info/Debug-logging at: bugs@sabnzbd.org or at "
-            "https://github.com/sabnzbd/sabnzbd/discussions/2160 "
-            "Disable this warning in Config - Specials - helpful_warnings",
-            nzo.final_name,
-        )
-    return old
-
-
-def quick_check_set_old(setname: str, nzo: NzbObject) -> bool:
     """Check all on-the-fly md5sums of a set"""
     par2pack = nzo.par2packs.get(setname)
     if par2pack is None:
@@ -2063,84 +2046,6 @@ def quick_check_set_old(setname: str, nzo: NzbObject) -> bool:
     return result
 
 
-def quick_check_set_new(setname: str, nzo: NzbObject) -> bool:
-    """Verify based on number of missing articles and md5of16k"""
-    # Use extracted par2 sets as a guide
-    par2pack = nzo.par2packs.get(setname)
-    if par2pack is None:
-        return False
-
-    # We use bitwise assigment (&=) so False always wins in case of failure
-    # This way the renames always get saved!
-    result = True
-    nzf_list = nzo.finished_files
-    renames = {}
-
-    # Files to ignore
-    ignore_ext = cfg.quick_check_ext_ignore()
-
-    for file in par2pack:
-        par2info = par2pack[file]
-        found = False
-        file_to_ignore = get_ext(file).replace(".", "") in ignore_ext
-        for nzf in nzf_list:
-            # Match based on md5of16k
-            if nzf.md5of16k == par2info.hash16k:
-                # Make sure it's unique
-                if par2info.has_duplicate:
-                    logging.info("Quick-check of file %s failed due to non-unique hash!", file)
-                    break
-
-                found = True
-                # Do a simple filename based check
-                if file == nzf.filename:
-                    logging.debug("Quick-check of file %s OK", file)
-                    result &= True
-                else:
-                    # Obfuscated, rename
-                    logging.debug("Quick-check will rename %s to %s", nzf.filename, file)
-                    try:
-                        # Note: file can and is allowed to be in a subdirectory.
-                        # Subdirectories in par2 always contain "/", not "\"
-                        new_path = renamer(
-                            nzf.filepath, os.path.join(nzo.download_path, file), create_local_directories=True
-                        )
-                        renames[file] = nzf.filename
-                        nzf.filepath = new_path
-                        nzf.filename = get_filename(new_path)
-                        result &= True
-                        found = True
-                    except IOError:
-                        # Renamed failed for some reason, probably already done
-                        pass
-
-                # Additional verification based on bad articles and expected file size
-                if file_to_ignore:
-                    logging.debug("Quick-check skipping additional checks for file %s", file)
-                elif nzf.has_bad_articles:
-                    logging.info("Quick-check of file %s failed due to bad article(s)!", file)
-                    result = False
-                elif os.path.getsize(nzf.filepath) != par2info.filesize:
-                    logging.info("Quick-check of file %s failed due to incorrect file size!", file)
-                    result = False
-                break
-
-        if not found:
-            if file_to_ignore:
-                # We don't care about these files
-                logging.debug("Quick-check ignoring missing file %s", file)
-                continue
-
-            logging.info("Quick-check could not find a match for file %s!", file)
-            result = False
-
-    # Save renames
-    if renames:
-        nzo.renamed_file(renames)
-
-    return result
-
-
 def unrar_check(rar: str) -> Tuple[int, bool]:
     """Return version number of unrar, where "5.01" returns 501
     Also return whether an original version is found

sabnzbd/utils/kronos.py

@@ -249,7 +249,11 @@ class Scheduler:
 
     def cancel(self, task):
         """Cancel given scheduled task."""
-        self.sched.cancel(task.event)
+        try:
+            self.sched.cancel(task.event)
+        except ValueError:
+            # Ignore if the task was already removed from the queue
+            pass
 
     def _getqueuetoptime(self):
         try:

sabnzbd/version.py

@@ -5,5 +5,5 @@
 
 # You MUST use double quotes (so " and not ')
 
-__version__ = "3.6.0RC2"
+__version__ = "3.6.1RC2"
 __baseline__ = "unknown"

tests/requirements.txt

@@ -1,5 +1,5 @@
 # Testing requirements
-selenium
+selenium<4.3.0 # needs code changes for 4.3.0
 requests
 pkginfo
 pytest

tests/test_filesystem.py

@@ -27,6 +27,7 @@ from pathlib import Path
 import tempfile
 
 import pyfakefs.fake_filesystem_unittest as ffs
+from pyfakefs.fake_filesystem import OSType
 
 import sabnzbd.cfg
 import sabnzbd.filesystem as filesystem
@@ -256,8 +257,7 @@ class TestFileFolderNameSanitizer:
 class TestSanitizeFiles(ffs.TestCase):
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.path_separator = "\\"
-        self.fs.is_windows_fs = True
+        self.fs.os = OSType.WINDOWS
         # Disable randomisation of directory listings
         self.fs.shuffle_listdir_results = False
 
@@ -445,9 +445,7 @@ class TestCheckMountMacOS(ffs.TestCase):
 
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.is_macos = True
-        self.fs.is_case_sensitive = False
-        self.fs.path_separator = "/"
+        self.fs.os = OSType.MACOS
         self.fs.create_dir(self.test_dir, perm_bits=755)
         # Verify the fake filesystem does its thing
         assert os.path.exists(self.test_dir) is True
@@ -489,9 +487,7 @@ class TestCheckMountWin(ffs.TestCase):
 
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.is_windows_fs = True
-        self.fs.is_case_sensitive = False
-        self.fs.path_separator = "\\"
+        self.fs.os = OSType.WINDOWS
         self.fs.create_dir(self.test_dir)
         # Sanity check the fake filesystem
         assert os.path.exists(self.test_dir) is True
@@ -616,9 +612,7 @@ class TestListdirFullWin(ffs.TestCase):
     # Basic fake filesystem setup stanza
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.is_windows_fs = True
-        self.fs.path_separator = "\\"
-        self.fs.is_case_sensitive = False
+        self.fs.os = OSType.WINDOWS
 
     def test_nonexistent_dir(self):
         assert filesystem.listdir_full(r"F:\foo\bar") == []
@@ -757,9 +751,7 @@ class TestGetUniqueDirFilenameWin(ffs.TestCase):
     # Basic fake filesystem setup stanza
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.is_windows_fs = True
-        self.fs.path_separator = "\\"
-        self.fs.is_case_sensitive = False
+        self.fs.os = OSType.WINDOWS
 
     # Reduce the waiting time when the function calls check_mount()
     @set_config({"wait_ext_drive": 1})
@@ -811,9 +803,7 @@ class TestCreateAllDirsWin(ffs.TestCase):
     # Basic fake filesystem setup stanza
     def setUp(self):
         self.setUpPyfakefs()
-        self.fs.is_windows_fs = True
-        self.fs.path_separator = "\\"
-        self.fs.is_case_sensitive = False
+        self.fs.os = OSType.WINDOWS
 
     @set_platform("win32")
     def test_create_all_dirs(self):