Update text files for 4.2.3RC3

Closes #2817

.github/workflows/build_release.yml

@@ -81,7 +81,7 @@ jobs:
       # We need the official Python, because the GA ones only support newer macOS versions
       # The deployment target is picked up by the Python build tools automatically
       # If updated, make sure to also set LSMinimumSystemVersion in SABnzbd.spec
-      PYTHON_VERSION: "3.12.1"
+      PYTHON_VERSION: "3.12.2"
       MACOSX_DEPLOYMENT_TARGET: "10.9"
       # We need to force compile for universal2 support
       CFLAGS: -arch x86_64 -arch arm64

README.mkd

@@ -1,4 +1,4 @@
-Release Notes - SABnzbd 4.2.3 Release Candidate 1
+Release Notes - SABnzbd 4.2.3 Release Candidate 3
 =========================================================
 
 This is the third bug-fix release of SABnzbd 4.2.0.
@@ -6,8 +6,10 @@ This is the third bug-fix release of SABnzbd 4.2.0.
 ## Bug-fixes and changes since 4.2.2:
 
 * **Bug-fixes:**
-    * Handle new status code for missing articles, which could result in timeouts.
+    * Handle new status code for missing articles, which would result in timeouts.
+      This specifically affects Giganews and its resellers.
     * Retry of failed job would not use the password provided.
+    * Optimize database handling in order to prevent locking errors.
     * macOS: System standby after finishing the queue would not always work.
 
 * **Changes:**

linux/org.sabnzbd.sabnzbd.appdata.xml

@@ -30,6 +30,7 @@
   <url type="faq">https://sabnzbd.org/wiki/faq</url>
   <url type="contact">https://sabnzbd.org/live-chat.html</url>
   <releases>
+    <release version="4.2.3" date="2024-03-10" type="stable"/>
     <release version="4.2.2" date="2024-01-31" type="stable"/>
     <release version="4.2.1" date="2024-01-05" type="stable"/>
     <release version="4.2.0" date="2024-01-03" type="stable"/>

sabnzbd/constants.py

@@ -75,6 +75,7 @@ DEF_LOG_CHERRY = "cherrypy.log"
 DEF_ARTICLE_CACHE_DEFAULT = "500M"
 DEF_ARTICLE_CACHE_MAX = "1G"
 DEF_TIMEOUT = 60
+DEF_TEST_TIMEOUT = 10
 DEF_SCANRATE = 5
 DEF_HTTPS_CERT_FILE = "server.cert"
 DEF_HTTPS_KEY_FILE = "server.key"

sabnzbd/database.py

@@ -38,7 +38,7 @@ from sabnzbd.encoding import ubtou, utob
 from sabnzbd.misc import int_conv, caller_name, opts_to_pp, to_units
 from sabnzbd.filesystem import remove_file, clip_path
 
-DB_LOCK = threading.RLock()
+DB_LOCK = threading.Lock()
 
 
 class HistoryDB:
@@ -50,66 +50,69 @@ class HistoryDB:
 
     # These class attributes will be accessed directly because
     # they need to be shared by all instances
-    db_path = None  # Will contain full path to history database
-    done_cleaning = False  # Ensure we only do one Vacuum per session
+    db_path = None  # Full path to history database
+    startup_done = False
 
     @synchronized(DB_LOCK)
     def __init__(self):
         """Determine database path and create connection"""
         self.connection: Optional[Connection] = None
         self.cursor: Optional[Cursor] = None
-        if not HistoryDB.db_path:
-            HistoryDB.db_path = os.path.join(sabnzbd.cfg.admin_dir.get_path(), DB_HISTORY_NAME)
         self.connect()
 
     def connect(self):
         """Create a connection to the database"""
-        create_table = not os.path.exists(HistoryDB.db_path)
+        if not HistoryDB.db_path:
+            HistoryDB.db_path = os.path.join(sabnzbd.cfg.admin_dir.get_path(), DB_HISTORY_NAME)
+        create_table = not HistoryDB.startup_done and not os.path.exists(HistoryDB.db_path)
+
         self.connection = sqlite3.connect(HistoryDB.db_path)
+        self.connection.isolation_level = None  # autocommit attribute only introduced in Python 3.12
         self.connection.row_factory = sqlite3.Row
         self.cursor = self.connection.cursor()
-        if create_table:
-            self.create_history_db()
-        elif not HistoryDB.done_cleaning:
-            # Run VACUUM on sqlite
+
+        # Perform initialization only once
+        if not HistoryDB.startup_done:
+            if create_table:
+                self.create_history_db()
+
             # When an object (table, index, or trigger) is dropped from the database, it leaves behind empty space
             # http://www.sqlite.org/lang_vacuum.html
-            HistoryDB.done_cleaning = True
             self.execute("VACUUM")
 
-        self.execute("PRAGMA user_version;")
-        try:
-            version = self.cursor.fetchone()["user_version"]
-        except (IndexError, TypeError):
-            version = 0
+            # See if we need to perform any updates
+            self.execute("PRAGMA user_version;")
+            try:
+                version = self.cursor.fetchone()["user_version"]
+            except (IndexError, TypeError):
+                version = 0
 
-        # Add any new columns added since last DB version
-        # Use "and" to stop when database has been reset due to corruption
-        if version < 1:
-            _ = (
-                self.execute("PRAGMA user_version = 1;", save=True)
-                and self.execute("ALTER TABLE history ADD COLUMN series TEXT;", save=True)
-                and self.execute("ALTER TABLE history ADD COLUMN md5sum TEXT;", save=True)
-            )
-        if version < 2:
-            _ = self.execute("PRAGMA user_version = 2;", save=True) and self.execute(
-                "ALTER TABLE history ADD COLUMN password TEXT;", save=True
-            )
-        if version < 3:
-            # Transfer data to new column (requires WHERE-hack), original column should be removed later
-            _ = (
-                self.execute("PRAGMA user_version = 3;", save=True)
-                and self.execute("ALTER TABLE history ADD COLUMN duplicate_key TEXT;", save=True)
-                and self.execute("UPDATE history SET duplicate_key = series WHERE 1 = 1;", save=True)
-            )
+            # Add any new columns added since last DB version
+            # Use "and" to stop when database has been reset due to corruption
+            if version < 1:
+                _ = (
+                    self.execute("PRAGMA user_version = 1;")
+                    and self.execute("ALTER TABLE history ADD COLUMN series TEXT;")
+                    and self.execute("ALTER TABLE history ADD COLUMN md5sum TEXT;")
+                )
+            if version < 2:
+                _ = self.execute("PRAGMA user_version = 2;") and self.execute(
+                    "ALTER TABLE history ADD COLUMN password TEXT;"
+                )
+            if version < 3:
+                # Transfer data to new column (requires WHERE-hack), original column should be removed later
+                _ = (
+                    self.execute("PRAGMA user_version = 3;")
+                    and self.execute("ALTER TABLE history ADD COLUMN duplicate_key TEXT;")
+                    and self.execute("UPDATE history SET duplicate_key = series WHERE 1 = 1;")
+                )
+            HistoryDB.startup_done = True
 
-    def execute(self, command: str, args: Sequence = (), save: bool = False) -> bool:
+    def execute(self, command: str, args: Sequence = ()) -> bool:
         """Wrapper for executing SQL commands"""
         for tries in (4, 3, 2, 1, 0):
             try:
                 self.cursor.execute(command, args)
-                if save:
-                    self.connection.commit()
                 return True
             except:
                 error = str(sys.exc_info()[1])
@@ -129,6 +132,7 @@ class HistoryDB:
                         remove_file(HistoryDB.db_path)
                     except:
                         pass
+                    HistoryDB.startup_done = False
                     self.connect()
                     # Return False in case of "duplicate column" error
                     # because the column addition in connect() must be terminated
@@ -141,6 +145,7 @@ class HistoryDB:
                     try:
                         self.connection.rollback()
                     except:
+                        # Can fail in case of automatic rollback
                         logging.debug("Rollback Failed:", exc_info=True)
             return False
 
@@ -178,10 +183,9 @@ class HistoryDB:
             "password" TEXT,
             "duplicate_key" TEXT
         )
-        """,
-            save=True,
+        """
         )
-        self.execute("PRAGMA user_version = 3;", save=True)
+        self.execute("PRAGMA user_version = 3;")
 
     def close(self):
         """Close database connection"""
@@ -196,9 +200,7 @@ class HistoryDB:
         """Remove all completed jobs from the database, optional with `search` pattern"""
         search = convert_search(search)
         logging.info("Removing all completed jobs from history")
-        return self.execute(
-            """DELETE FROM history WHERE name LIKE ? AND status = ?""", (search, Status.COMPLETED), save=True
-        )
+        return self.execute("""DELETE FROM history WHERE name LIKE ? AND status = ?""", (search, Status.COMPLETED))
 
     def get_failed_paths(self, search=None):
         """Return list of all storage paths of failed jobs (may contain non-existing or empty paths)"""
@@ -215,9 +217,7 @@ class HistoryDB:
         """Remove all failed jobs from the database, optional with `search` pattern"""
         search = convert_search(search)
         logging.info("Removing all failed jobs from history")
-        return self.execute(
-            """DELETE FROM history WHERE name LIKE ? AND status = ?""", (search, Status.FAILED), save=True
-        )
+        return self.execute("""DELETE FROM history WHERE name LIKE ? AND status = ?""", (search, Status.FAILED))
 
     def remove_history(self, jobs=None):
         """Remove all jobs in the list `jobs`, empty list will remove all completed jobs"""
@@ -228,7 +228,7 @@ class HistoryDB:
                 jobs = [jobs]
 
             for job in jobs:
-                self.execute("""DELETE FROM history WHERE nzo_id = ?""", (job,), save=True)
+                self.execute("""DELETE FROM history WHERE nzo_id = ?""", (job,))
                 logging.info("[%s] Removing job %s from history", caller_name(), job)
 
     def auto_history_purge(self):
@@ -247,9 +247,7 @@ class HistoryDB:
             if days_to_keep > 0:
                 logging.info("Removing completed jobs older than %s days from history", days_to_keep)
                 return self.execute(
-                    """DELETE FROM history WHERE status = ? AND completed < ?""",
-                    (Status.COMPLETED, seconds_to_keep),
-                    save=True,
+                    """DELETE FROM history WHERE status = ? AND completed < ?""", (Status.COMPLETED, seconds_to_keep)
                 )
         else:
             # How many to keep?
@@ -261,7 +259,6 @@ class HistoryDB:
                         SELECT id FROM history WHERE status = ? ORDER BY completed DESC LIMIT ?
                     )""",
                     (Status.COMPLETED, Status.COMPLETED, to_keep),
-                    save=True,
                 )
 
     def add_history_db(self, nzo, storage: str, postproc_time: int, script_output: str, script_line: str):
@@ -274,7 +271,6 @@ class HistoryDB:
             downloaded, fail_message, url_info, bytes, duplicate_key, md5sum, password)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
             t,
-            save=True,
         )
         logging.info("Added job %s to history", nzo.final_name)
 

sabnzbd/decoder.py

@@ -209,7 +209,7 @@ def decode_uu(article: Article, raw_data: bytes) -> bytes:
     """Try to uu-decode an article. The raw_data may or may not contain headers.
     If there are headers, they will be separated from the body by at least one
     empty line. In case of no headers, the first line seems to always be the nntp
-    response code (222) directly followed by the msg body."""
+    response code (220/222) directly followed by the msg body."""
     if not raw_data:
         logging.debug("No data to decode")
         raise BadUu
@@ -232,7 +232,7 @@ def decode_uu(article: Article, raw_data: bytes) -> bytes:
         uu_start = raw_data[:limit].index(b"") + 1
     except ValueError:
         # No empty line, look for a response code instead
-        if raw_data[0].startswith(b"222 "):
+        if raw_data[0].startswith(b"220 ") or raw_data[0].startswith(b"222 "):
             uu_start = 1
         else:
             # Invalid data?

sabnzbd/downloader.py

@@ -729,7 +729,9 @@ class Downloader(Thread):
                     time.sleep(0.01)
                     sabnzbd.BPSMeter.update()
 
-        if nw.status_code != 222 and not done:
+        # Response code depends on request command:
+        # # 220 = ARTICLE, 222 = BODY
+        if nw.status_code not in (220, 222) and not done:
             if not nw.connected or nw.status_code == 480:
                 if not self.__finish_connect_nw(nw):
                     return
@@ -766,12 +768,11 @@ class Downloader(Thread):
 
             else:
                 logging.warning(
-                    T("%s@%s recieved unknown status code %s for article %s: %s"),
+                    T("%s@%s: Received unknown status code %s for article %s"),
                     nw.thrdnum,
                     nw.server.host,
                     nw.status_code,
                     article.article,
-                    nw.nntp_msg.splitlines()[0],
                 )
                 done = True
                 nw.reset_data_buffer()

sabnzbd/interface.py

@@ -50,7 +50,6 @@ from sabnzbd.misc import (
     is_lan_addr,
     is_local_addr,
     is_loopback_addr,
-    helpful_warning,
     recursive_html_escape,
     is_none,
     get_cpu_name,
@@ -81,7 +80,7 @@ from sabnzbd.constants import (
     GUESSIT_SORT_TYPES,
     VALID_NZB_FILES,
     VALID_ARCHIVES,
-    DEF_TIMEOUT,
+    DEF_TEST_TIMEOUT,
 )
 from sabnzbd.lang import list_languages
 from sabnzbd.api import (
@@ -1144,7 +1143,7 @@ def handle_server(kwargs, root=None, new_svr=False):
         kwargs["connections"] = "1"
 
     if kwargs.get("enable") == "1":
-        if not happyeyeballs(host, int_conv(port), int_conv(kwargs.get("timeout"), default=DEF_TIMEOUT)):
+        if not happyeyeballs(host, int_conv(port), int_conv(kwargs.get("timeout"), default=DEF_TEST_TIMEOUT)):
             return badParameterResponse(T('Server address "%s:%s" is not valid.') % (host, port), ajax)
 
     # Default server name is just the host name

sabnzbd/newswrapper.py

@@ -383,7 +383,7 @@ class NNTP:
 
         # Ignore if the socket was already closed, resulting in errors
         if not self.closed:
-            msg = "Failed to connect: %s %s@%s:%s (%s)" % (
+            msg = T("Failed to connect: %s %s@%s:%s (%s)") % (
                 str(error),
                 self.nw.thrdnum,
                 self.nw.server.host,

sabnzbd/utils/servertests.py

@@ -20,9 +20,8 @@ sabnzbd.utils.servertests - Debugging server connections. Currently only NNTP se
 """
 
 import socket
-import sys
 
-from sabnzbd.constants import DEF_TIMEOUT
+from sabnzbd.constants import DEF_TEST_TIMEOUT
 from sabnzbd.newswrapper import NewsWrapper, NNTPPermanentError
 from sabnzbd.downloader import Server, clues_login, clues_too_many
 from sabnzbd.config import get_servers
@@ -37,7 +36,7 @@ def test_nntp_server_dict(kwargs):
     password = kwargs.get("password", "").strip()
     server = kwargs.get("server", "").strip()
     connections = int_conv(kwargs.get("connections", 0))
-    timeout = int_conv(kwargs.get("timeout", DEF_TIMEOUT))
+    timeout = int_conv(kwargs.get("timeout", DEF_TEST_TIMEOUT))
     ssl = int_conv(kwargs.get("ssl", 0))
     ssl_verify = int_conv(kwargs.get("ssl_verify", 1))
     ssl_ciphers = kwargs.get("ssl_ciphers", "").strip()
@@ -56,7 +55,7 @@ def test_nntp_server_dict(kwargs):
 
     if not timeout:
         # Lower value during new server testing
-        timeout = 10
+        timeout = DEF_TEST_TIMEOUT
 
     if "*" in password and not password.strip("*"):
         # If the password is masked, try retrieving it from the config

sabnzbd/version.py

@@ -6,5 +6,5 @@
 # You MUST use double quotes (so " and not ')
 # Do not forget to update the appdata file for every major release!
 
-__version__ = "4.2.3RC1"
+__version__ = "4.2.3RC3"
 __baseline__ = "unknown"