Update text files for 3.4.2RC2

Before 3.4.0, only for TV sorting we allowed to set 0 categories. But for Movies and Date Sorting we did require at least 1 category to be set. This was harmonized in 3.4.0, breaking existing setups. Added warning for those users.
The Sorting behavior is different from Notifications: in Notifications selecting Default only(!) means to apply it to all categories.
However, that has never been the case for Sorting. So for now added a bit more help texts to the Affected categories box on both pages.

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.4.1
+Version: 3.4.2RC2
-Summary: SABnzbd-3.4.1
+Summary: SABnzbd-3.4.2RC2
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,17 @@
-Release Notes - SABnzbd 3.4.1
+Release Notes - SABnzbd 3.4.2 Release Candidate 2
 =========================================================
 
+## Bugfixes since 3.4.1
+- Sorting requires at least 1 category te be selected since 3.4.0.
+  Warning will be shown if no category is selected.
+- Job failure due to Sorting-problems was not shown in the History.
+- Crash when `.par2` files were missing during download.
+- Prevent scanning the whole file to identify the correct extension.
+- `.rXX`, `.cbz` and `.cbr` extensions were wrongly renamed.
+- Processing unpacked `.par2` files would also process source
+  `.par2` files and could result in duplicate (`.1`) filenames.
+- Always show number of MB missing during download.
+
 ## Bugfixes since 3.4.0
 - macOS: Failed to run on M1 systems or older macOS versions.
 

SABnzbd.py

@@ -46,6 +46,8 @@ try:
     import portend
     import cryptography
     import chardet
+    import guessit
+    import puremagic
 except ImportError as e:
     print("Not all required Python modules are available, please check requirements.txt")
     print("Missing module:", e.name)
@@ -1439,12 +1441,11 @@ def main():
     try:
         cherrypy.engine.start()
     except:
+        # Since the webserver is started by cherrypy in a separate thread, we can't really catch any
+        # start-up errors. This try/except only catches very few errors, the rest is only shown in the console.
         logging.error(T("Failed to start web-interface: "), exc_info=True)
         abort_and_show_error(browserhost, cherryport)
 
-    # Wait for server to become ready
-    cherrypy.engine.wait(cherrypy.process.wspbus.states.STARTED)
-
     if sabnzbd.WIN32:
         if enable_https:
             mode = "s"

interfaces/Config/templates/config_notify.tmpl

@@ -22,6 +22,7 @@
                 <option value="$ct" <!--#if $ct in $getVar($section_label + '_cats') then 'selected="selected"' else ""#-->>$Tspec($ct)</option>
             <!--#end for#-->
         </select>
+        <p>$T('defaultNotifiesAll')</p>
     </div>
 <!--#end def#-->
 
@@ -40,6 +41,7 @@
                         <option value="$ct" <!--#if $ct in $email_cats then 'selected="selected"' else ""#-->>$Tspec($ct)</option>
                     <!--#end for#-->
                 </select>
+                <p>$T('defaultNotifiesAll')</p>
             </div>
         </div>
         <div class="col1">

interfaces/Config/templates/config_sorting.tmpl

@@ -11,12 +11,13 @@
                 <h3>$T('seriesSorting') <a href="$helpuri$help_uri#toc0" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="tv_categories" multiple="multiple" class="multiple_cats">
+                    <select name="tv_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $tv_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">
@@ -223,12 +224,13 @@
                 <h3>$T('movieSort') <a href="$helpuri$help_uri#toc6" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="movie_categories" multiple="multiple" class="multiple_cats">
+                    <select name="movie_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $movie_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">
@@ -419,12 +421,13 @@
                 <h3>$T('dateSorting') <a href="$helpuri$help_uri#toc9" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="date_categories" multiple="multiple" class="multiple_cats">
+                    <select name="date_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $date_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">

interfaces/Glitter/templates/static/javascripts/glitter.queue.js

@@ -531,11 +531,10 @@ function QueueModel(parent, data) {
         return self.name()
     })
     self.missingText = ko.pureComputed(function() {
-        // Check for missing data, the value is arbitrary! (1%)
+        // Check for missing data, can show 0 if article-size is smaller than 500K, but we accept that
-        if(self.missingMB()/self.totalMB() > 0.01) {
+        if(self.missingMB()) {
             return self.missingMB().toFixed(0) + ' MB ' + glitterTranslate.misingArt
         }
-        return;
     })
     self.statusText = ko.computed(function() {
         // Checking

sabnzbd/cfg.py

@@ -233,7 +233,7 @@ rating_filter_pause_keywords = OptionStr("misc", "rating_filter_pause_keywords")
 ##############################################################################
 enable_tv_sorting = OptionBool("misc", "enable_tv_sorting", False)
 tv_sort_string = OptionStr("misc", "tv_sort_string")
-tv_categories = OptionList("misc", "tv_categories", "")
+tv_categories = OptionList("misc", "tv_categories", ["tv"])
 
 enable_movie_sorting = OptionBool("misc", "enable_movie_sorting", False)
 movie_sort_string = OptionStr("misc", "movie_sort_string")

sabnzbd/deobfuscate_filenames.py

@@ -64,9 +64,9 @@ def decode_par2(parfile: str) -> List[str]:
             with open(filepath, "rb") as fileToMatch:
                 first16k_data = fileToMatch.read(16384)
 
-            # Check if we have this hash
+            # Check if we have this hash and the filename is different
             file_md5of16k = hashlib.md5(first16k_data).digest()
-            if file_md5of16k in md5of16k:
+            if file_md5of16k in md5of16k and fn != md5of16k[file_md5of16k]:
                 new_path = os.path.join(dirname, md5of16k[file_md5of16k])
                 # Make sure it's a unique name
                 unique_filename = get_unique_filename(new_path)
@@ -166,7 +166,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
     # 2. if no meaningful extension, add it
     # 3. based on detecting obfuscated filenames
 
-    # to be sure, only keep really exsiting files:
+    # to be sure, only keep really existing files:
     filelist = [f for f in filelist if os.path.isfile(f)]
 
     # let's see if there are files with uncommon/unpopular (so: obfuscated) extensions
@@ -176,7 +176,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
     for file in filelist:
         if file_extension.has_popular_extension(file):
             # common extension, like .doc or .iso, so assume OK and change nothing
-            logging.debug("extension of %s looks common", file)
+            logging.debug("Extension of %s looks common", file)
             newlist.append(file)
         else:
             # uncommon (so: obfuscated) extension
@@ -220,6 +220,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
         # check that file is still there (and not renamed by the secondary renaming process below)
         if not os.path.isfile(filename):
             continue
+
         logging.debug("Deobfuscate inspecting %s", filename)
         # Do we need to rename this file?
         # Criteria: big, not-excluded extension, obfuscated (in that order)

sabnzbd/newsunpack.py

@@ -1118,8 +1118,7 @@ def par2_repair(parfile_nzf: NzbFile, nzo: NzbObject, workdir, setname, single):
         readd = False
         for extrapar in nzo.extrapars[setname][:]:
             # Make sure we only get new par2 files
-            if extrapar not in nzo.finished_files and extrapar not in nzo.files:
+            if nzo.add_parfile(extrapar):
-                nzo.add_parfile(extrapar)
                 readd = True
         if readd:
             return readd, result

sabnzbd/nzbstuff.py

@@ -1105,8 +1105,7 @@ class NzbObject(TryList):
                 self.postpone_pars(nzf, setname)
             # Get the next one
             for new_nzf in self.extrapars[setname]:
-                if not new_nzf.completed:
+                if self.add_parfile(new_nzf):
-                    self.add_parfile(new_nzf)
                     # Add it to the top
                     self.files.remove(new_nzf)
                     self.files.insert(0, new_nzf)
@@ -1143,8 +1142,8 @@ class NzbObject(TryList):
             added_blocks = 0
             while added_blocks < needed_blocks:
                 new_nzf = block_list.pop()
-                self.add_parfile(new_nzf)
+                if self.add_parfile(new_nzf):
-                added_blocks += new_nzf.blocks
+                    added_blocks += new_nzf.blocks
 
             logging.info("Added %s blocks to %s", added_blocks, self.final_name)
             return added_blocks
@@ -1433,15 +1432,18 @@ class NzbObject(TryList):
             self.unwanted_ext = 2
 
     @synchronized(NZO_LOCK)
-    def add_parfile(self, parfile: NzbFile):
+    def add_parfile(self, parfile: NzbFile) -> bool:
         """Add parfile to the files to be downloaded
         Resets trylist just to be sure
         Adjust download-size accordingly
+        Returns False when the file couldn't be added
         """
         if not parfile.completed and parfile not in self.files and parfile not in self.finished_files:
             parfile.reset_try_list()
             self.files.append(parfile)
             self.bytes_tried -= parfile.bytes_left
+            return True
+        return False
 
     @synchronized(NZO_LOCK)
     def remove_parset(self, setname: str):
@@ -1468,12 +1470,12 @@ class NzbObject(TryList):
                 # from all the sets. This probably means we get too much par2, but it's worth it.
                 blocks_new = 0
                 for new_nzf in self.extrapars[parset]:
-                    self.add_parfile(new_nzf)
+                    if self.add_parfile(new_nzf):
-                    blocks_new += new_nzf.blocks
+                        blocks_new += new_nzf.blocks
-                    # Enough now?
+                        # Enough now?
-                    if blocks_new >= self.bad_articles:
+                        if blocks_new >= self.bad_articles:
-                        logging.info("Prospectively added %s repair blocks to %s", blocks_new, self.final_name)
+                            logging.info("Prospectively added %s repair blocks to %s", blocks_new, self.final_name)
-                        break
+                            break
             # Reset NZO TryList
             self.reset_try_list()
 

sabnzbd/postproc.py

@@ -515,12 +515,14 @@ def process_job(nzo: NzbObject):
                     workdir_complete, ok = file_sorter.sorter.rename(newfiles, workdir_complete)
                     if not ok:
                         nzo.set_unpack_info("Unpack", T("Failed to move files"))
+                        nzo.fail_msg = T("Failed to move files")
                         all_ok = False
 
             # Run further post-processing
             if (all_ok or not cfg.safe_postproc()) and not nzb_list:
                 # Use par2 files to deobfuscate unpacked file names
-                if cfg.process_unpacked_par2():
+                # Only if we also run cleanup, so not to process the "regular" par2 files
+                if flag_delete and cfg.process_unpacked_par2():
                     newfiles = deobfuscate.recover_par2_names(newfiles)
 
                 if cfg.deobfuscate_final_filenames():

sabnzbd/skintext.py

@@ -701,6 +701,9 @@ SKIN_TEXT = {
     "link-download": TT("Download"),  #: Config->RSS button "download item"
     "button-rssNow": TT("Read All Feeds Now"),  #: Config->RSS button
     # Config->Notifications
+    "defaultNotifiesAll": TT(
+        "If only the <em>Default</em> category is selected, notifications are enabled for jobs in all categories."
+    ),
     "opt-email_endjob": TT("Email Notification On Job Completion"),
     "email-never": TT("Never"),  #: When to send email
     "email-always": TT("Always"),  #: When to send email
@@ -773,6 +776,7 @@ SKIN_TEXT = {
     "catTags": TT("Indexer Categories / Groups"),
     "button-delCat": TT("X"),  #: Small delete button
     # Config->Sorting
+    "selectOneCat": TT("Select at least 1 category."),
     "seriesSorting": TT("Series Sorting"),
     "opt-tvsort": TT("Enable TV Sorting"),
     "sort-legenda": TT("Pattern Key"),

sabnzbd/sorting.py

@@ -305,7 +305,7 @@ class BaseSorter:
                 except:
                     logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath))
                     logging.info("Traceback: ", exc_info=True)
-                rename_similar(current_path, ext, self.filename_set, ())
+                rename_similar(current_path, ext, self.filename_set)
         else:
             logging.debug("Nothing to rename, %s", files)
 
@@ -363,6 +363,10 @@ class SeriesSorter(BaseSorter):
                 self.matched = True
                 self.type = "tv"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_tv_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Series Sorting"), T("Select at least 1 category."))
+
     def get_values(self):
         """Collect all values needed for path replacement"""
         self.get_year()
@@ -426,6 +430,10 @@ class MovieSorter(BaseSorter):
                 self.matched = True
                 self.type = "movie"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_movie_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Movie Sorting"), T("Select at least 1 category."))
+
     def get_values(self):
         """Collect all values needed for path replacement"""
         self.get_year()
@@ -506,6 +514,10 @@ class DateSorter(BaseSorter):
                 self.matched = True
                 self.type = "date"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_date_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Date Sorting"), T("Select at least 1 category."))
+
     def get_date(self):
         """Get month and day"""
         self.info["month"] = str(self.guess.get("date").month)
@@ -794,7 +806,7 @@ def strip_path_elements(path: str) -> str:
     return "\\\\" + path if is_unc else path
 
 
-def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: List[str]):
+def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: Optional[List[str]] = None):
     """Rename all other files in the 'folder' hierarchy after 'name'
     and move them to the root of 'folder'.
     Files having extension 'skip_ext' will be moved, but not renamed.
@@ -807,7 +819,7 @@ def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: List[st
     for root, dirs, files in os.walk(folder):
         for f in files:
             path = os.path.join(root, f)
-            if path in skipped_files:
+            if skipped_files and path in skipped_files:
                 continue
             org, ext = os.path.splitext(f)
             if ext.lower() == skip_ext:

sabnzbd/utils/file_extension.py

@@ -8,10 +8,11 @@ Note: extension always contains a leading dot
 import puremagic
 import os
 import sys
+import re
 from typing import List
-from pathlib import Path
 from sabnzbd.filesystem import get_ext
 
+
 # common extension from https://www.computerhope.com/issues/ch001789.htm
 POPULAR_EXT = (
     "3g2",
@@ -168,6 +169,8 @@ DOWNLOAD_EXT = (
     "bdmv",
     "bin",
     "bup",
+    "cbr",
+    "cbz",
     "clpi",
     "crx",
     "db",
@@ -234,16 +237,19 @@ DOWNLOAD_EXT = (
     "xpi",
 )
 
-# combine to one tuple, with unique entries:
+# Combine to one tuple, with unique entries:
 ALL_EXT = tuple(set(POPULAR_EXT + DOWNLOAD_EXT))
-# prepend a dot to each extension, because we work with a leading dot in extensions
+# Prepend a dot to each extension, because we work with a leading dot in extensions
 ALL_EXT = tuple(["." + i for i in ALL_EXT])
 
+# Match old-style multi-rar extensions
+SIMPLE_RAR_RE = re.compile(r"\.r\d\d\d?$", re.I)
+
 
 def has_popular_extension(file_path: str) -> bool:
     """returns boolean if the extension of file_path is a popular, well-known extension"""
     file_extension = get_ext(file_path)
-    return file_extension in ALL_EXT
+    return file_extension in ALL_EXT or SIMPLE_RAR_RE.match(file_extension)
 
 
 def all_possible_extensions(file_path: str) -> List[str]:
@@ -264,9 +270,12 @@ def what_is_most_likely_extension(file_path: str) -> str:
 
     # Check if text or NZB, as puremagic is not good at that.
     try:
-        txt = Path(file_path).read_text()
+        # Only read the start, don't need the whole file
+        with open(file_path, "r") as inp_file:
+            txt = inp_file.read(200).lower()
+
         # Yes, a text file ... so let's check if it's even an NZB:
-        if txt.lower().find("<nzb xmlns=") >= 0 or txt.lower().find("!doctype nzb public") >= 0:
+        if "!doctype nzb public" in txt or "<nzb xmlns=" in txt:
             # yes, contains NZB signals:
             return ".nzb"
         else:

sabnzbd/version.py

@@ -5,5 +5,5 @@
 
 # You MUST use double quotes (so " and not ')
 
-__version__ = "develop"
+__version__ = "3.4.1"
-__baseline__ = "unknown"
+__baseline__ = "447a7b684c32ca28dc8dfff285330c2c7de98377"

tests/test_file_extension.py

@@ -29,6 +29,10 @@ class Test_File_Extension:
         assert file_extension.has_popular_extension("blabla/blabla.mkv")
         assert file_extension.has_popular_extension("blabla/blabla.srt")
         assert file_extension.has_popular_extension("djjddj/aaaaa.epub")
+        assert file_extension.has_popular_extension("test/testing.r01")
+        assert file_extension.has_popular_extension("test/testing.r901")
+        assert not file_extension.has_popular_extension("test/testing")
+        assert not file_extension.has_popular_extension("test/testing.rar01")
         assert not file_extension.has_popular_extension("98ads098f098fa.a0ds98f098asdf")
 
     def test_what_is_most_likely_extension(self):