Update text files for 4.0.3Beta1

* run lowercasing on season pack setname

* also subject %fn to lowercasing

* add tests

* woops

README.mkd

@@ -1,21 +1,44 @@
-Release Notes - SABnzbd 4.0.0 Release Candidate 2
+Release Notes - SABnzbd 4.0.3 Beta 1
 =========================================================
 
+## Bugfixes and changes since 4.0.2
+- Direct Unpack could get stuck.
+- Sorters could not be modified.
+- Season Sorting did not respect desired capitalization.
+- Windows: Windows Service would fail to start on legacy release.
+- macOS: Failed to launch on macOS Sonoma Beta.
+
+## Breaking change
+- The `Parameters` setting of a `Notification Script` is now passed as
+  environment variable `SAB_NOTIFICATION_PARAMETERS` instead of as a
+  command-line parameter. This prevents the possibility of remote code
+  execution on systems exposed to the internet without a username/password.
+  If you use `nzb-notify` you need to update it to the latest version.
+
+## Bugfixes and changes since 4.0.1
+- Disabling a server during download did not stop it from downloading.
+- Show last line of post-processing script output even if it failed.
+- Prevent crash during Deobfuscate on non-unique paths.
+- Files that could not be parsed were removed from the `Watched Folder`.
+- Warn if the file system does not support unicode or long filenames.
+- Warn if `Scripts Folder` is inside the application directory.
+- Prevent output buffering of Python post-processing scripts.
+- The `PKG-INFO` file was removed from the `src` release.
+- Correctly decode partially malformed UUencoded posts.
+- macOS: Tray icon could not be disabled.
+
 ## Changes since 3.7.2
-- In this major update we replaced a core part of Python's SSL handling
-  with our own improved version. This results in large performance increases
-  when downloading from news servers with SSL enabled.
-  In addition, the general connection handling was overhauled, resulting in
-  performance improvements for all news servers.
+- In this major update we optimized a core part of the SSL handling.
+  This results in large performance increase when downloading from news
+  servers with SSL enabled. In addition, the general connection handling
+  was improved, resulting in performance improvements for all news servers.
   Special thanks to: mnightingale, puzzledsab and animetosho!
 - There are multiple settings that can tweak performance, see:
   https://github.com/sabnzbd/sabnzbd/discussions/2474
-  We are trying to find the most optimal default settings, so you
-  can help us by letting us know the results on your system!
 - When adding a new news server, SSL is enabled by default.
 - File assembly performance significantly improved by relying on the
   CRC32 instead of the MD5 to perform QuickCheck of files.
-- Slowdown more gracefully when the cache fills up.
+- Slow down more gracefully when the cache fills up.
 - Replaced separate Series/Movie/Date Sorting with general Sorter.
 - HTTPS files are included in the `Backup`.
 - Improved `Watched Folder` scanning and processing.

builder/osx/requirements.txt

@@ -1,3 +1,3 @@
 # Special requirements for macOS universal2 binary release
 # This way dependabot can auto-update them
-cryptography==40.0.2
+cryptography==41.0.1

builder/release.py

@@ -205,7 +205,7 @@ if RELEASE_THIS and gh_token:
 
         # Merge pull-request
         print("Merging pull request in sabnzbd/sabnzbd.github.io for the update")
-        update_pr.merge(method="squash")
+        update_pr.merge(merge_method="squash")
 
     # Only with GitHub success we proceed to Reddit
     if reddit_token := os.environ.get("REDDIT_TOKEN", ""):
@@ -227,18 +227,30 @@ if RELEASE_THIS and gh_token:
         with open(RELEASE_README, "r") as readme_file:
             readme_lines = readme_file.readlines()
 
+        # Put the download link after the title
+        readme_lines[2] = "## https://sabnzbd.org/downloads"
+
         # Use the header in the readme as title
         title = readme_lines[0]
         release_notes_text = "".join(readme_lines[3:])
 
-        # Post always to r/SABnzbd
-        print("Posting release notes to Reddit: r/sabnzbd")
-        submission = subreddit_sabnzbd.submit(title, selftext=release_notes_text)
+        # Only stable releases to r/usenet
+        if not prerelease:
+            print("Posting release notes to Reddit: r/usenet")
+            submission = subreddit_usenet.submit(title, selftext=release_notes_text)
+
+            # Cross-post to r/SABnzbd
+            print("Cross-posting release notes to Reddit: r/sabnzbd")
+            submission.crosspost(subreddit_sabnzbd)
+        else:
+            # Post always to r/SABnzbd
+            print("Posting release notes to Reddit: r/sabnzbd")
+            subreddit_sabnzbd.submit(title, selftext=release_notes_text)
 
         # Only stable releases to r/usenet
         if not prerelease:
-            print("Cross-posting release notes to Reddit: r/usenet")
-            submission.crosspost(subreddit_usenet)
+            print("Posting release notes to Reddit: r/usenet")
+            subreddit_usenet.submit(title, selftext=release_notes_text)
     else:
         print("Missing REDDIT_TOKEN")
 

builder/requirements.txt

@@ -1,6 +1,6 @@
 # Basic build requirements
 # Note that not all sub-dependencies are listed, but only ones we know could cause trouble
-pyinstaller==5.11.0
+pyinstaller==5.12.0
 pyinstaller-hooks-contrib==2023.3
 altgraph==0.17.3
 wrapt==1.15.0
@@ -9,7 +9,7 @@ certifi
 
 # orjson does not support 32bit Windows, exclude it based on Python-version
 # This way we also test ujson on Python 3.8 in the CI-tests
-orjson==3.8.14; python_version > '3.8'
+orjson==3.9.1; python_version > '3.8'
 
 # For the Windows build
 pefile==2023.2.7; sys_platform == 'win32'

interfaces/Config/templates/config_sorting.tmpl

@@ -472,7 +472,7 @@
                 ui.placeholder.height(hPlaceholder + hExtra);
                 \$('<div class="sorter-placeholder-anim" data-height="' + hPlaceholder + '"></div>').insertAfter(ui.placeholder);
             },
-            cancel: ".pattern-table",
+            cancel: "input,textarea,button,select,option,.pattern-table",
             change: function(event, ui) {
                 ui.placeholder.stop().height(0).animate({
                     height: ui.item.outerHeight() + hExtra

interfaces/Glitter/templates/static/javascripts/glitter.history.js

@@ -110,6 +110,8 @@ function HistoryListModel(parent) {
                 value: newValue
             })
         }
+        // Update pagination and counters
+        self.parent.refresh(true)
     });
 
     // Retry a job

interfaces/Glitter/templates/static/javascripts/glitter.queue.js

@@ -159,6 +159,8 @@ function QueueListModel(parent) {
                 value: newValue
             })
         }
+        // Update pagination and counters
+        self.parent.refresh(true)
     });
 
     // Do we show search box. So it doesn't dissapear when nothing is found

requirements.txt

@@ -9,15 +9,15 @@ configobj==5.0.8
 cheroot==10.0.0
 six==1.16.0
 cherrypy==18.8.0
-jaraco.functools==3.6.0
-jaraco.collections==4.1.0
+jaraco.functools==3.7.0
+jaraco.collections==4.2.0
 jaraco.text==3.8.1 # Newer version introduces irrelevant extra dependencies
 jaraco.classes==3.2.3
 jaraco.context==4.3.0
 more-itertools==9.1.0
 zc.lockfile==3.0.post1
 python-dateutil==2.8.2
-tempora==5.2.2
+tempora==5.3.0
 pytz==2023.3
 sgmllib3k==1.0.0
 portend==3.1.0
@@ -30,17 +30,17 @@ rebulk==3.2.0
 
 # Recent cryptography versions require Rust. If you run into issues compiling this
 # SABnzbd will also work with older pre-Rust versions such as cryptography==3.3.2
-cryptography==40.0.2
+cryptography==41.0.1
 
 # We recommend using "orjson" as it is 2x as fast as "ujson". However, it requires
 # Rust so SABnzbd works just as well with "ujson" or the Python built in "json" module
-ujson==5.7.0
+ujson==5.8.0
 
 # Windows system integration
 pywin32==306; sys_platform == 'win32'
 
 # macOS system calls
-pyobjc==9.1.1; sys_platform == 'darwin'
+pyobjc==9.2; sys_platform == 'darwin'
 
 # Linux notifications
 notify2==0.3.1; sys_platform != 'win32' and sys_platform != 'darwin'

sabnzbd/directunpacker.py

@@ -251,7 +251,7 @@ class DirectUnpacker(threading.Thread):
                     extracted = []
 
                     # Are there more files left?
-                    while self.nzo.files and not self.next_sets:
+                    while not self.removed_from_queue and not self.next_sets:
                         with self.next_file_lock:
                             self.next_file_lock.wait()
 

sabnzbd/dirscanner.py

@@ -66,7 +66,7 @@ class DirScanner(threading.Thread):
 
         self.loop: Optional[asyncio.AbstractEventLoop] = None
         self.scanner_task: Optional[asyncio.Task] = None
-        self.lock = asyncio.Lock()  # Prevents concurrent scans
+        self.lock: Optional[asyncio.Lock] = None  # Prevents concurrent scans
         self.error_reported = False  # Prevents multiple reporting of missing watched folder
         self.dirscan_dir = cfg.dirscan_dir.get_path()
         self.dirscan_speed = cfg.dirscan_speed()
@@ -109,7 +109,6 @@ class DirScanner(threading.Thread):
     def run(self):
         """Start the scanner"""
         logging.info("Dirscanner starting up")
-
         self.loop = asyncio.new_event_loop()
 
         try:
@@ -184,7 +183,6 @@ class DirScanner(threading.Thread):
 
     async def when_stable_add_nzbfile(self, path: str, catdir: Optional[str], stat_tuple: os.stat_result):
         """Try and import the NZB but wait until the attributes are stable for 1 second, but give up after 3 sec"""
-
         logging.info("Trying to import %s", path)
 
         # Wait until the attributes are stable for 1 second, but give up after 3 sec
@@ -225,6 +223,11 @@ class DirScanner(threading.Thread):
 
     async def scan_async(self, dirscan_dir: str):
         """Do one scan of the watched folder"""
+        # On Python 3.8 we first need an event loop before we can create a asyncio.Lock
+        if not self.lock:
+            with DIR_SCANNER_LOCK:
+                self.lock = asyncio.Lock()
+
         async with self.lock:
             if sabnzbd.PAUSED_ALL:
                 return

sabnzbd/osxmenu.py

@@ -19,14 +19,40 @@
 sabnzbd.osxmenu - macOS Top Menu
 """
 
-from Foundation import *
-from AppKit import *
-from objc import YES, NO
-
 import os
 import sys
 import time
 import logging
+from objc import YES, NO
+from Foundation import (
+    NSObject,
+    NSDate,
+    NSTimer,
+    NSRunLoop,
+    NSDefaultRunLoopMode,
+    NSColor,
+    NSFont,
+    NSImage,
+    NSAttributedString,
+)
+from AppKit import (
+    NSStatusBar,
+    NSMenu,
+    NSMenuItem,
+    NSAlternateKeyMask,
+    NSTerminateNow,
+    NSEventTrackingRunLoopMode,
+    NSVariableStatusItemLength,
+    NSForegroundColorAttributeName,
+    NSFontAttributeName,
+    NSOnState,
+    NSOffState,
+    NSBaselineOffsetAttributeName,
+    NSParagraphStyleAttributeName,
+    NSMutableParagraphStyle,
+    NSParagraphStyle,
+    NSCenterTextAlignment,
+)
 
 import sabnzbd
 import sabnzbd.cfg
@@ -315,7 +341,7 @@ class SABnzbdDelegate(NSObject):
                             },
                         )
                         menu_history_item.setAttributedTitle_(jobfailed)
-                    menu_history_item.setRepresentedObject_("%s" % history["storage"])
+                    menu_history_item.setRepresentedObject_(history["storage"])
                     self.menu_history.addItem_(menu_history_item)
             else:
                 menu_history_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Empty"), "", "")

sabnzbd/sorting.py

@@ -432,7 +432,7 @@ class Sorter:
                         ("%ext", f_ext.lstrip(".")),
                     ],
                 )
-                f_new = f_name_new + f_ext
+                f_new = to_lowercase(f_name_new + f_ext)
 
                 try:
                     logging.debug("Renaming season pack file %s to %s", f, f_new)
@@ -477,11 +477,13 @@ class Sorter:
             f_name, f_ext = os.path.splitext(os.path.split(f)[1])
             new_filepath = os.path.join(
                 base_path,
-                path_subst(
-                    self.filename_set + self.multipart_label,
-                    [("%1", str(index)), ("%fn", f_name), ("%ext", f_ext.lstrip("."))],
-                )
-                + f_ext,
+                to_lowercase(
+                    path_subst(
+                        self.filename_set + self.multipart_label,
+                        [("%1", str(index)), ("%fn", f_name), ("%ext", f_ext.lstrip("."))],
+                    )
+                    + f_ext,
+                ),
             )
             try:
                 logging.debug("Renaming %s to %s", filepath, new_filepath)
@@ -557,7 +559,8 @@ class Sorter:
         f_name, f_ext = os.path.splitext(largest_file.get("name"))
         filepath = self._to_filepath(largest_file.get("name"), base_path)
         new_filepath = os.path.join(
-            base_path, path_subst(self.filename_set, [("%fn", f_name), ("%ext", f_ext.lstrip("."))]) + f_ext
+            base_path,
+            to_lowercase(path_subst(self.filename_set, [("%fn", f_name), ("%ext", f_ext.lstrip("."))]) + f_ext),
         )
         if not os.path.exists(new_filepath):
             renamed_files = []

sabnzbd/version.py

@@ -5,5 +5,5 @@
 
 # You MUST use double quotes (so " and not ')
 
-__version__ = "4.1.0-develop"
+__version__ = "4.0.3Beta1"
 __baseline__ = "unknown"

tests/test_dirscanner.py

@@ -18,6 +18,7 @@
 """
 tests.test_dirscanner - Testing functions in dirscanner.py
 """
+import asyncio
 
 import pyfakefs.fake_filesystem_unittest as ffs
 

tests/test_sorting.py

@@ -617,10 +617,12 @@ class TestSortingSorter:
         [
             ("%sn s%0se%0e.%ext", True),  # %0e marker
             ("%sn s%se%e.%ext", True),  # %e marker
+            ("{%sn }s%se%e.%ext", True),  # Same with lowercasing; test for issue #2578
             ("%sn.%ext", False),  # No episode marker
             ("%sn_%0se%0e", False),  # No extension marker
             ("%r/%sn s%0se%0e.%ext", True),  # %0e marker, with dir in sort string
             ("%r/%sn s%se%e.%ext", True),  # %e marker, with dir in sort string
+            ("%r/{%sn} s%se%e.%ext", True),  # Same with lowercasing; test for issue #2578
             ("%r/%sn.%ext", False),  # No episode marker, with dir in sort string
             ("%r/%sn_%0se%0e", False),  # No extension marker, with dir in sort string
         ],
@@ -717,7 +719,11 @@ class TestSortingSorter:
                             # Also do a basic filename check
                             for root, dirs, files in os.walk(base_dir):
                                 for filename in files:
-                                    assert re.fullmatch(r"Pack Renamer s23e0?\d.*" + extension, filename)
+                                    if "{" in sort_string:
+                                        # Lowercasing marker in sort string, expect lowercase results
+                                        assert re.fullmatch(r"pack renamer s23e0?\d.*" + extension, filename)
+                                    else:
+                                        assert re.fullmatch(r"Pack Renamer s23e0?\d.*" + extension, filename)
                         else:
                             # No season pack renaming should happen, verify original files are still in place
                             assert os.path.exists(os.path.join(base_dir, f))