Update text files for 3.4.2RC3

Closes #1962, #1957

.github/workflows/integration_testing.yml

@@ -8,15 +8,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-20.04]
         include:
+          # TODO: Update to 3.10 when all packages are available, currently lxml is missing
           - name: macOS
             os: macos-latest
-            python-version: 3.9
+            python-version: "3.9"
           - name: Windows
             os: windows-latest
-            python-version: 3.9
+            python-version: "3.9"
 
     steps:
     - uses: actions/checkout@v2
@@ -30,7 +31,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python --version
-        pip install --upgrade pip
+        pip install --upgrade pip wheel
         pip install --upgrade -r requirements.txt
         pip install --upgrade -r tests/requirements.txt
     - name: Test SABnzbd

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.4.2RC2
-Summary: SABnzbd-3.4.2RC2
+Version: 3.4.2RC3
+Summary: SABnzbd-3.4.2RC3
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -2,15 +2,16 @@ Release Notes - SABnzbd 3.4.2 Release Candidate 2
 =========================================================
 
 ## Bugfixes since 3.4.1
-- Sorting requires at least 1 category te be selected since 3.4.0.
+- Sorting requires at least 1 category to be selected since 3.4.0.
   Warning will be shown if no category is selected.
+- Sorting would fail if `%ext` or `%fn` was not used.
 - Job failure due to Sorting-problems was not shown in the History.
 - Crash when `.par2` files were missing during download.
 - Prevent scanning the whole file to identify the correct extension.
 - `.rXX`, `.cbz` and `.cbr` extensions were wrongly renamed.
 - Processing unpacked `.par2` files would also process source
   `.par2` files and could result in duplicate (`.1`) filenames.
-- Always show number of MB missing during download.
+- Always show the number of MB missing during download.
 
 ## Bugfixes since 3.4.0
 - macOS: Failed to run on M1 systems or older macOS versions.
@@ -37,7 +38,7 @@ Release Notes - SABnzbd 3.4.2 Release Candidate 2
 
 ## Upgrade notices
 - The download statistics file `totals10.sab` is updated in 3.2.x 
-  version. If you downgrade to 3.1.x or lower, detailed download 
+  version. If you downgrade to 3.1.x or lower, all detailed download 
   statistics will be lost.
 
 ## Known problems and solutions

sabnzbd/constants.py

@@ -122,6 +122,7 @@ VALID_NZB_FILES = (".nzb", ".gz", ".bz2")
 CHEETAH_DIRECTIVES = {"directiveStartToken": "<!--#", "directiveEndToken": "#-->", "prioritizeSearchListOverSelf": True}
 
 IGNORED_FOLDERS = ("@eaDir", ".appleDouble")
+IGNORED_MOVIE_FOLDERS = ("video_ts", "audio_ts", "bdmv")
 
 EXCLUDED_GUESSIT_PROPERTIES = [
     "part",

sabnzbd/filesystem.py

@@ -480,6 +480,61 @@ def check_mount(path: str) -> bool:
     return not m
 
 
+RAR_RE = re.compile(r"\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$", re.I)
+SPLITFILE_RE = re.compile(r"\.(\d\d\d?\d$)", re.I)
+ZIP_RE = re.compile(r"\.(zip$)", re.I)
+SEVENZIP_RE = re.compile(r"\.7z$", re.I)
+SEVENMULTI_RE = re.compile(r"\.7z\.\d+$", re.I)
+TS_RE = re.compile(r"\.(\d+)\.(ts$)", re.I)
+
+
+def build_filelists(
+    workdir: Optional[str], workdir_complete: Optional[str] = None, check_both: bool = False, check_rar: bool = True
+) -> Tuple[List[str], List[str], List[str], List[str], List[str]]:
+    """Build filelists, if workdir_complete has files, ignore workdir.
+    Optionally scan both directories.
+    Optionally test content to establish RAR-ness
+    """
+    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])
+
+    if workdir_complete:
+        filelist.extend(listdir_full(workdir_complete))
+
+    if workdir and (not filelist or check_both):
+        filelist.extend(listdir_full(workdir, recursive=False))
+
+    for file in filelist:
+        # Extra check for rar (takes CPU/disk)
+        file_is_rar = False
+        if check_rar:
+            file_is_rar = rarfile.is_rarfile(file)
+
+        # Run through all the checks
+        if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file):
+            # 7zip
+            sevens.append(file)
+        elif SPLITFILE_RE.search(file) and not file_is_rar:
+            # Joinables, optional with RAR check
+            joinables.append(file)
+        elif ZIP_RE.search(file):
+            # ZIP files
+            zips.append(file)
+        elif RAR_RE.search(file):
+            # RAR files
+            rars.append(file)
+        elif TS_RE.search(file):
+            # TS split files
+            ts.append(file)
+
+    logging.debug("build_filelists(): joinables: %s", joinables)
+    logging.debug("build_filelists(): zips: %s", zips)
+    logging.debug("build_filelists(): rars: %s", rars)
+    logging.debug("build_filelists(): 7zips: %s", sevens)
+    logging.debug("build_filelists(): ts: %s", ts)
+
+    return joinables, zips, rars, sevens, ts
+
+
 def safe_fnmatch(f: str, pattern: str) -> bool:
     """fnmatch will fail if the pattern contains any of it's
     key characters, like [, ] or !.

sabnzbd/newsunpack.py

@@ -56,6 +56,8 @@ from sabnzbd.filesystem import (
     setname_from_path,
     get_ext,
     get_filename,
+    TS_RE,
+    build_filelists,
 )
 from sabnzbd.nzbstuff import NzbObject, NzbFile
 from sabnzbd.sorting import SeriesSorter
@@ -63,18 +65,12 @@ import sabnzbd.cfg as cfg
 from sabnzbd.constants import Status
 
 # Regex globals
-RAR_RE = re.compile(r"\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$", re.I)
 RAR_RE_V3 = re.compile(r"\.(?P<ext>part\d*)$", re.I)
-
 TARGET_RE = re.compile(r'^(?:File|Target): "(.+)" -')
 EXTRACTFROM_RE = re.compile(r"^Extracting\sfrom\s(.+)")
 EXTRACTED_RE = re.compile(r"^(Extracting|Creating|...)\s+(.*?)\s+OK\s*$")
-SPLITFILE_RE = re.compile(r"\.(\d\d\d?\d$)", re.I)
-ZIP_RE = re.compile(r"\.(zip$)", re.I)
-SEVENZIP_RE = re.compile(r"\.7z$", re.I)
-SEVENMULTI_RE = re.compile(r"\.7z\.\d+$", re.I)
-TS_RE = re.compile(r"\.(\d+)\.(ts$)", re.I)
 
+# Constants
 PAR2_COMMAND = None
 MULTIPAR_COMMAND = None
 RAR_COMMAND = None
@@ -1994,51 +1990,6 @@ def rar_sort(a, b):
         return cmp(a, b)
 
 
-def build_filelists(workdir, workdir_complete=None, check_both=False, check_rar=True):
-    """Build filelists, if workdir_complete has files, ignore workdir.
-    Optionally scan both directories.
-    Optionally test content to establish RAR-ness
-    """
-    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])
-
-    if workdir_complete:
-        filelist.extend(listdir_full(workdir_complete))
-
-    if workdir and (not filelist or check_both):
-        filelist.extend(listdir_full(workdir, recursive=False))
-
-    for file in filelist:
-        # Extra check for rar (takes CPU/disk)
-        file_is_rar = False
-        if check_rar:
-            file_is_rar = rarfile.is_rarfile(file)
-
-        # Run through all the checks
-        if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file):
-            # 7zip
-            sevens.append(file)
-        elif SPLITFILE_RE.search(file) and not file_is_rar:
-            # Joinables, optional with RAR check
-            joinables.append(file)
-        elif ZIP_RE.search(file):
-            # ZIP files
-            zips.append(file)
-        elif RAR_RE.search(file):
-            # RAR files
-            rars.append(file)
-        elif TS_RE.search(file):
-            # TS split files
-            ts.append(file)
-
-    logging.debug("build_filelists(): joinables: %s", joinables)
-    logging.debug("build_filelists(): zips: %s", zips)
-    logging.debug("build_filelists(): rars: %s", rars)
-    logging.debug("build_filelists(): 7zips: %s", sevens)
-    logging.debug("build_filelists(): ts: %s", ts)
-
-    return joinables, zips, rars, sevens, ts
-
-
 def quick_check_set(set, nzo):
     """Check all on-the-fly md5sums of a set"""
     md5pack = nzo.md5packs.get(set)

sabnzbd/postproc.py

@@ -74,6 +74,7 @@ from sabnzbd.constants import (
     JOB_ADMIN,
     Status,
     VERIFIED_FILE,
+    IGNORED_MOVIE_FOLDERS,
 )
 from sabnzbd.nzbparser import process_single_nzb
 import sabnzbd.emailer as emailer
@@ -499,7 +500,7 @@ def process_job(nzo: NzbObject):
                     )
                     logging.info("Traceback: ", exc_info=True)
                     # Better disable sorting because filenames are all off now
-                    file_sorter.sort_file = None
+                    file_sorter.sorter_active = None
 
             if empty:
                 job_result = -1
@@ -510,7 +511,7 @@ def process_job(nzo: NzbObject):
                 remove_samples(workdir_complete)
 
             # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
-            if all_ok and file_sorter.sort_file:
+            if all_ok and file_sorter.sorter_active:
                 if newfiles:
                     workdir_complete, ok = file_sorter.sorter.rename(newfiles, workdir_complete)
                     if not ok:
@@ -693,7 +694,7 @@ def prepare_extraction_path(nzo: NzbObject) -> Tuple[str, str, Sorter, bool, Opt
     else:
         file_sorter = Sorter(None, nzo.cat)
     complete_dir = file_sorter.detect(nzo.final_name, complete_dir)
-    if file_sorter.sort_file:
+    if file_sorter.sorter_active:
         one_folder = False
 
     complete_dir = sanitize_and_trim_path(complete_dir)
@@ -1177,7 +1178,7 @@ def rename_and_collapse_folder(oldpath, newpath, files):
     if len(items) == 1:
         folder = items[0]
         folder_path = os.path.join(oldpath, folder)
-        if os.path.isdir(folder_path) and folder not in ("VIDEO_TS", "AUDIO_TS"):
+        if os.path.isdir(folder_path) and folder.lower() not in IGNORED_MOVIE_FOLDERS:
             logging.info("Collapsing %s", os.path.join(newpath, folder))
             oldpath = folder_path
 

sabnzbd/sorting.py

@@ -38,7 +38,7 @@ from sabnzbd.filesystem import (
     clip_path,
 )
 import sabnzbd.cfg as cfg
-from sabnzbd.constants import EXCLUDED_GUESSIT_PROPERTIES
+from sabnzbd.constants import EXCLUDED_GUESSIT_PROPERTIES, IGNORED_MOVIE_FOLDERS
 from sabnzbd.nzbstuff import NzbObject, scan_password
 
 # Do not rename .vob files as they are usually DVD's
@@ -76,7 +76,7 @@ class BaseSorter:
         self.cat = cat
         self.filename_set = ""
         self.fname = ""  # Value for %fn substitution in folders
-        self.do_rename = False
+        self.rename_files = False
         self.info = {}
         self.type = None
         self.guess = guess
@@ -259,7 +259,7 @@ class BaseSorter:
         # Split the last part of the path up for the renamer
         if extension:
             path, self.filename_set = os.path.split(path)
-            self.do_rename = True
+            self.rename_files = True
 
         # The normpath function translates "" to "." which results in an incorrect path
         return os.path.normpath(path) if path else path
@@ -317,7 +317,7 @@ class Sorter:
 
     def __init__(self, nzo: Optional[NzbObject], cat: str):
         self.sorter: Optional[BaseSorter] = None
-        self.sort_file = False
+        self.sorter_active = False
         self.nzo = nzo
         self.cat = cat
 
@@ -334,9 +334,9 @@ class Sorter:
             self.sorter = MovieSorter(self.nzo, job_name, complete_dir, self.cat, guess)
 
         if self.sorter and self.sorter.matched:
-            self.sort_file = True
+            self.sorter_active = True
 
-        return self.sorter.get_final_path() if self.sort_file else complete_dir
+        return self.sorter.get_final_path() if self.sorter_active else complete_dir
 
 
 class SeriesSorter(BaseSorter):
@@ -398,8 +398,8 @@ class SeriesSorter(BaseSorter):
         """Rename for Series"""
         if min_size < 0:
             min_size = cfg.episode_rename_limit.get_int()
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
         else:
             logging.debug("Renaming series file(s)")
             return super().rename(files, current_path, min_size)
@@ -445,8 +445,9 @@ class MovieSorter(BaseSorter):
         if min_size < 0:
             min_size = cfg.movie_rename_limit.get_int()
 
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
+
         logging.debug("Renaming movie file(s)")
 
         def filter_files(f, current_path):
@@ -538,8 +539,8 @@ class DateSorter(BaseSorter):
         """Renaming Date file"""
         if min_size < 0:
             min_size = cfg.episode_rename_limit.get_int()
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
         else:
             logging.debug("Renaming date file(s)")
             return super().rename(files, current_path, min_size)
@@ -558,9 +559,11 @@ def move_to_parent_directory(workdir: str) -> Tuple[str, bool]:
     workdir = os.path.abspath(os.path.normpath(workdir))
     dest = os.path.abspath(os.path.normpath(os.path.join(workdir, "..")))
 
+    logging.debug("Moving all files from %s to %s", workdir, dest)
+
     # Check for DVD folders and bail out if found
     for item in os.listdir(workdir):
-        if item.lower() in ("video_ts", "audio_ts", "bdmv"):
+        if item.lower() in IGNORED_MOVIE_FOLDERS:
             return workdir, True
 
     for root, dirs, files in os.walk(workdir):
@@ -873,7 +876,7 @@ def eval_sort(sort_type: str, expression: str, name: str = None, multipart: str
     if "%fn" in path:
         path = path.replace("%fn", fname + ".ext")
     else:
-        if sorter.do_rename:
+        if sorter.rename_files:
             path = fpath + ".ext"
         else:
             path += "\\" if sabnzbd.WIN32 else "/"

sabnzbd/utils/file_extension.py

@@ -8,10 +8,8 @@ Note: extension always contains a leading dot
 import puremagic
 import os
 import sys
-import re
 from typing import List
-from sabnzbd.filesystem import get_ext
-
+from sabnzbd.filesystem import get_ext, RAR_RE
 
 # common extension from https://www.computerhope.com/issues/ch001789.htm
 POPULAR_EXT = (
@@ -242,14 +240,11 @@ ALL_EXT = tuple(set(POPULAR_EXT + DOWNLOAD_EXT))
 # Prepend a dot to each extension, because we work with a leading dot in extensions
 ALL_EXT = tuple(["." + i for i in ALL_EXT])
 
-# Match old-style multi-rar extensions
-SIMPLE_RAR_RE = re.compile(r"\.r\d\d\d?$", re.I)
-
 
 def has_popular_extension(file_path: str) -> bool:
     """returns boolean if the extension of file_path is a popular, well-known extension"""
     file_extension = get_ext(file_path)
-    return file_extension in ALL_EXT or SIMPLE_RAR_RE.match(file_extension)
+    return file_extension in ALL_EXT or RAR_RE.match(file_extension)
 
 
 def all_possible_extensions(file_path: str) -> List[str]:

tests/requirements.txt

@@ -7,6 +7,7 @@ pytest-httpbin
 pytest-httpserver
 flaky
 xmltodict
-tavern
+tavern<1.16.2; python_version == '3.6'
+tavern; python_version > '3.6'
 tavalidate
 lxml>=4.5.0 # needed by tavalidate

tests/test_file_extension.py

@@ -30,7 +30,7 @@ class Test_File_Extension:
         assert file_extension.has_popular_extension("blabla/blabla.srt")
         assert file_extension.has_popular_extension("djjddj/aaaaa.epub")
         assert file_extension.has_popular_extension("test/testing.r01")
-        assert file_extension.has_popular_extension("test/testing.r901")
+        assert file_extension.has_popular_extension("test/testing.s91")
         assert not file_extension.has_popular_extension("test/testing")
         assert not file_extension.has_popular_extension("test/testing.rar01")
         assert not file_extension.has_popular_extension("98ads098f098fa.a0ds98f098asdf")

tests/test_sorting.py

@@ -25,6 +25,7 @@ import sys
 from random import choice
 
 from sabnzbd import sorting
+from sabnzbd.constants import IGNORED_MOVIE_FOLDERS
 from tests.testhelper import *
 
 
@@ -315,7 +316,7 @@ class TestSortingFunctions:
             pyfakefs.fake_filesystem_unittest.set_uid(0)
             # Create a fake filesystem in a random base directory, and included a typical DVD directory
             base_dir = "/" + os.urandom(4).hex() + "/" + os.urandom(2).hex()
-            dvd = choice(("video_ts", "audio_ts", "bdmv"))
+            dvd = choice(IGNORED_MOVIE_FOLDERS)
             for test_dir in ["dir/2", "TEST/DIR2"]:
                 ffs.fs.create_dir(base_dir + "/" + test_dir, perm_bits=755)
                 assert os.path.exists(base_dir + "/" + test_dir) is True
@@ -373,7 +374,7 @@ class TestSortingFunctions:
             pyfakefs.fake_filesystem_unittest.set_uid(0)
             # Create a fake filesystem in a random base directory, and included a typical DVD directory
             base_dir = "D:\\" + os.urandom(4).hex() + "\\" + os.urandom(2).hex()
-            dvd = choice(("video_ts", "audio_ts", "bdmv"))
+            dvd = choice(IGNORED_MOVIE_FOLDERS)
             for test_dir in ["dir\\2", "TEST\\DIR2"]:
                 ffs.fs.create_dir(base_dir + "\\" + test_dir, perm_bits=755)
                 assert os.path.exists(base_dir + "\\" + test_dir) is True
@@ -553,11 +554,14 @@ class TestSortingSorters:
         _func()
 
     @pytest.mark.parametrize(
-        "s_class, job_tag, sort_string, sort_result",  # sort_result without extension
+        "s_class, job_tag, sort_string, sort_filename_result",  # Supply sort_filename_result without extension
         [
             (sorting.SeriesSorter, "S01E02", "%r/%sn s%0se%0e.%ext", "Simulated Job s01e02"),
+            (sorting.SeriesSorter, "S01E02", "%r/%sn s%0se%0e", ""),
             (sorting.MovieSorter, "2021", "%y_%.title.%r.%ext", "2021_Simulated.Job.2160p"),
-            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>", "Simulated.Job-SAB"),
+            (sorting.MovieSorter, "2021", "%y_%.title.%r", ""),
+            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>.%ext", "Simulated.Job-SAB"),
+            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>", ""),
         ],
     )
     @pytest.mark.parametrize("size_limit, file_size", [(512, 1024), (1024, 512)])
@@ -569,7 +573,7 @@ class TestSortingSorters:
         s_class,
         job_tag,
         sort_string,
-        sort_result,
+        sort_filename_result,
         size_limit,
         file_size,
         extension,
@@ -631,8 +635,10 @@ class TestSortingSorters:
 
             # Check the result
             try:
+                # If there's no "%ext" in the sort_string, no filenames should be changed
                 if (
                     is_ok
+                    and sort_filename_result
                     and file_size > size_limit
                     and extension not in sorting.EXCLUDED_FILE_EXTS
                     and not (sorter.type == "movie" and number_of_files > 1 and not generate_sequential_filenames)
@@ -642,10 +648,10 @@ class TestSortingSorters:
                     if number_of_files > 1 and generate_sequential_filenames and sorter.type == "movie":
                         # Movie sequential file handling
                         for n in range(1, number_of_files + 1):
-                            expected = os.path.join(sort_dest, sort_result + " CD" + str(n) + extension)
+                            expected = os.path.join(sort_dest, sort_filename_result + " CD" + str(n) + extension)
                             assert os.path.exists(expected)
                     else:
-                        expected = os.path.join(sort_dest, sort_result + extension)
+                        expected = os.path.join(sort_dest, sort_filename_result + extension)
                         assert os.path.exists(expected)
                 else:
                     # No renaming should happen
@@ -699,7 +705,7 @@ class TestSortingSorters:
             generic = sorting.Sorter(None, "test_cat")
             generic.detect(job_name, SAB_CACHE_DIR)
 
-            assert generic.sort_file is result_sort_file
+            assert generic.sorter_active is result_sort_file
             if result_sort_file:
                 assert generic.sorter
                 assert generic.sorter.__class__ is result_class