Update text files for 4.0.0 Alpha 2

* Update last_max_chunk_size for each call to recv

* Reduce _DEFAULT_CHUNK_SIZE

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 4.0.0Alpha1
-Summary: SABnzbd-4.0.0Alpha1
+Version: 4.0.0Alpha2
+Summary: SABnzbd-4.0.0Alpha2
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,12 @@
-Release Notes - SABnzbd 4.0.0 Alpha 1
+Release Notes - SABnzbd 4.0.0 Alpha 2
 =========================================================
 
+## Changes since 4.0.0 Alpha 1
+- There are now multiple settings that can tweak performance, see:
+  https://github.com/sabnzbd/sabnzbd/discussions/2474
+  We are trying to find the most optimal default settings, so you
+  can help us by letting us know the results on your system!
+
 ## Changes since 3.7.2
 - In this major update we replaced a core part of Python's SSL handling
   with our own improved version. This results in large performance increases

po/email/SABemail.pot

@@ -5,7 +5,7 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: SABnzbd-4.0.0-develop\n"
+"Project-Id-Version: SABnzbd-4.0.0Alpha1\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: team@sabnzbd.org\n"
 "Language-Team: SABnzbd <team@sabnzbd.org>\n"

po/main/SABnzbd.pot

@@ -5,7 +5,7 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: SABnzbd-4.0.0-develop\n"
+"Project-Id-Version: SABnzbd-4.0.0Alpha1\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: team@sabnzbd.org\n"
 "Language-Team: SABnzbd <team@sabnzbd.org>\n"

po/nsis/SABnsis.pot

@@ -5,7 +5,7 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: SABnzbd-4.0.0-develop\n"
+"Project-Id-Version: SABnzbd-4.0.0Alpha1\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: team@sabnzbd.org\n"
 "Language-Team: SABnzbd <team@sabnzbd.org>\n"

sabnzbd/__init__.py

@@ -232,6 +232,8 @@ def initialize(pause_downloader=False, clean_up=False, repair=0):
 
     sabnzbd.__SHUTTING_DOWN__ = False
 
+    sys.setswitchinterval(cfg.switchinterval())
+
     # Set global database connection for Web-UI threads
     cherrypy.engine.subscribe("start_thread", get_db_connection)
 

sabnzbd/cfg.py

@@ -260,7 +260,6 @@ configlock = OptionBool("misc", "config_lock", False)
 # One time trackers
 ##############################################################################
 fixed_ports = OptionBool("misc", "fixed_ports", False)
-sched_converted = OptionBool("misc", "sched_converted", False)
 notified_new_skin = OptionNumber("misc", "notified_new_skin", 0)
 direct_unpack_tested = OptionBool("misc", "direct_unpack_tested", False)
 
@@ -438,6 +437,7 @@ max_url_retries = OptionNumber("misc", "max_url_retries", 10, minval=1)
 downloader_sleep_time = OptionNumber("misc", "downloader_sleep_time", 10, minval=0)
 receive_threads = OptionNumber("misc", "receive_threads", 2, minval=1)
 num_simd_decoders = OptionNumber("misc", "num_simd_decoders", 2, minval=1)
+switchinterval = OptionNumber("misc", "switchinterval", 0.005)
 ssdp_broadcast_interval = OptionNumber("misc", "ssdp_broadcast_interval", 15, minval=1, maxval=600)
 ext_rename_ignore = OptionList("misc", "ext_rename_ignore", validation=lower_case_ext)
 

sabnzbd/decoder.py

@@ -149,9 +149,6 @@ class DecoderWorker(Thread):
                     logging.debug("Decoding %s", art_id)
 
                 if article.nzf.type == "uu":
-                    # TODO: UU needs to be fixed
-                    n = 250000
-                    raw_data = [bytes(raw_data[i : min(raw_data_size, i + n)]) for i in range(0, len(raw_data), n)]
                     decoded_data = decode_uu(article, raw_data)
                 else:
                     decoded_data = decode_yenc(article, raw_data, raw_data_size)
@@ -274,7 +271,7 @@ def decode_yenc(article: Article, data: bytearray, raw_data_size: int) -> bytear
     return data
 
 
-def decode_uu(article: Article, raw_data: List[bytes]) -> bytes:
+def decode_uu(article: Article, raw_data: bytearray) -> bytes:
     """Try to uu-decode an article. The raw_data may or may not contain headers.
     If there are headers, they will be separated from the body by at least one
     empty line. In case of no headers, the first line seems to always be the nntp
@@ -284,10 +281,7 @@ def decode_uu(article: Article, raw_data: List[bytes]) -> bytes:
         raise BadUu
 
     # Line up the raw_data
-    with BytesIO() as encoded_data:
-        for data in raw_data:
-            encoded_data.write(data)
-        raw_data = encoded_data.getvalue().split(b"\r\n")
+    raw_data = raw_data.split(b"\r\n")
 
     # Index of the uu payload start in raw_data
     uu_start = 0
@@ -301,7 +295,7 @@ def decode_uu(article: Article, raw_data: List[bytes]) -> bytes:
     # Try to find an empty line separating the body from headers or response
     # code and set the expected payload start to the next line.
     try:
-        uu_start = raw_data[:limit].index(b"") + 1
+        uu_start = raw_data[:limit].index(bytearray(b"")) + 1
     except ValueError:
         # No empty line, look for a response code instead
         if raw_data[0].startswith(b"222 "):

sabnzbd/downloader.py

@@ -58,7 +58,7 @@ _BPSMETER_UPDATE_DELAY = 0.05
 # How many articles should be prefetched when checking the next articles?
 _ARTICLE_PREFETCH = 20
 # Minimum expected size of TCP receive buffer
-_DEFAULT_CHUNK_SIZE = 65536
+_DEFAULT_CHUNK_SIZE = 32768
 
 TIMER_LOCK = RLock()
 
@@ -586,6 +586,8 @@ class Downloader(Thread):
         sabnzbd.EXTERNAL_IPV6 = sabnzbd.misc.test_ipv6()
         logging.debug("External IPv6 test result: %s", sabnzbd.EXTERNAL_IPV6)
 
+        logging.debug("switchinterval = %s", sys.getswitchinterval())
+
         # Then we check SSL certificate checking
         sabnzbd.CERTIFICATE_VALIDATION = sabnzbd.misc.test_cert_checking()
         logging.debug("SSL verification test: %s", sabnzbd.CERTIFICATE_VALIDATION)
@@ -770,12 +772,16 @@ class Downloader(Thread):
 
             if self.recv_threads > 1:
                 for nw, bytes_received, done in self.recv_pool.map(self.__recv, read):
+                    if bytes_received > last_max_chunk_size:
+                        last_max_chunk_size = bytes_received
                     self.__handle_recv_result(nw, bytes_received, done)
                 if self.bandwidth_limit:
                     self.__check_speed()
             else:
                 for selected in read:
                     nw, bytes_received, done = self.__recv(selected)
+                    if bytes_received > last_max_chunk_size:
+                        last_max_chunk_size = bytes_received
                     self.__handle_recv_result(nw, bytes_received, done)
                     if self.bandwidth_limit and bytes_received:
                         self.__check_speed()

sabnzbd/interface.py

@@ -888,6 +888,7 @@ SPECIAL_VALUE_LIST = (
     "url_base",
     "receive_threads",
     "num_simd_decoders",
+    "switchinterval",
     "direct_unpack_threads",
     "ipv6_servers",
     "selftest_host",

sabnzbd/version.py

@@ -5,5 +5,5 @@
 
 # You MUST use double quotes (so " and not ')
 
-__version__ = "4.0.0Alpha1"
+__version__ = "4.0.0Alpha2"
 __baseline__ = "unknown"

tests/data/tavern/api_get_files_format.yaml

@@ -56,7 +56,7 @@ stages:
         pragma: "no-cache"
         access-control-allow-origin: "*"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           # Verify response parameters with a regex using positive
           # looksaheads, so parameters match regardless of their order

tests/data/tavern/api_history_empty.yaml

@@ -54,7 +54,7 @@ stages:
         access-control-allow-origin: "*"
         content-length: !re_match "[0-9]+"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           # Verify response with a regex using positive looksaheads so
           # parameters match regardless of their order of appearance

tests/data/tavern/api_history_format.yaml

@@ -87,7 +87,7 @@ stages:
         access-control-allow-origin: "*"
         content-length: !re_match "[0-9]+"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           # Verify response parameters with a regex using positive looksaheads,
           # so parameters match regardless of their order of appearance. Note

tests/data/tavern/api_queue_empty.yaml

@@ -78,7 +78,7 @@ stages:
         access-control-allow-origin: "*"
         content-length: !re_match "[0-9]+"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           # Verify only a bunch of key response parameters with a regex using
           # positive looksaheads, so parameters match regardless of their order

tests/data/tavern/api_queue_format.yaml

@@ -98,7 +98,7 @@ stages:
         access-control-allow-origin: "*"
         content-length: !re_match "[0-9]+"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           # Verify only a bunch of key response parameters with a regex using
           # positive looksaheads, so parameters match regardless of their order

tests/data/tavern/api_server_stats.yaml

@@ -47,7 +47,7 @@ stages:
         content-length: !re_match "[0-9]+"
       status_code: 200
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           expression: "{{(?=.*'total': [0-9]+)(?=.*'month': [0-9]+)(?=.*'week': [0-9]+)(?=.*'day': [0-9]+)(?=.*'servers': .*).*}}"
 

tests/data/tavern/api_version.yaml

@@ -35,7 +35,7 @@ stages:
         content-type: !re_match "text/plain"
         content-type: !re_search "charset=(UTF|utf)-8"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           expression: "{SAB_VERSION}"
 
@@ -53,6 +53,6 @@ stages:
         content-type: !re_match "text/xml"
         content-type: !re_search "charset=(UTF|utf)-8"
       verify_response_with:
-        function: tavern.testutils.helpers:validate_regex
+        function: tavern.helpers:validate_regex
         extra_kwargs:
           expression: '<\?xml version="1.0" encoding="UTF-8" \?>\r?\n?<version>{SAB_VERSION}</version>'

tests/requirements.txt

@@ -9,7 +9,7 @@ pytest-httpbin
 pytest-httpserver
 flaky
 xmltodict
-tavern<2.0.0 # tavalidate does not support tavern 2.0.0+
+tavern>=2.0.0
 tavalidate
 importlib_metadata
 lxml

tests/test_decoder.py

@@ -102,7 +102,7 @@ class TestUuDecoder:
         # Concatenate expected result
         result = b"".join(result)
 
-        return article, data, result
+        return article, bytearray(data), result
 
     def test_no_data(self):
         with pytest.raises(decoder.BadUu):
@@ -111,28 +111,28 @@ class TestUuDecoder:
     @pytest.mark.parametrize(
         "raw_data",
         [
-            [b""],
-            [b"\r\n\r\n"],
-            [b"f", b"o", b"o", b"b", b"a", b"r", b"\r\n"],  # Plenty of list items, but (too) few actual lines
-            [b"222 0 <artid@woteva>\r\nX-Too-Short: yup\r\n"],
+            b"",
+            b"\r\n\r\n",
+            b"foobar\r\n",  # Plenty of list items, but (too) few actual lines
+            b"222 0 <artid@woteva>\r\nX-Too-Short: yup\r\n",
         ],
     )
     def test_short_data(self, raw_data):
         with pytest.raises(decoder.BadUu):
-            assert decoder.decode_uu(None, raw_data)
+            assert decoder.decode_uu(None, bytearray(raw_data))
 
     @pytest.mark.parametrize(
         "raw_data",
         [
-            [b"222 0 <foo@bar>\r\n\r\n"],  # Missing altogether
-            [b"222 0 <foo@bar>\r\n\r\nbeing\r\n"],  # Typo in 'begin'
-            [b"222 0 <foo@bar>\r\n\r\nx-header: begin 644 foobar\r\n"],  # Not at start of the line
-            [b"666 0 <foo@bar>\r\nbegin\r\n"],  # No empty line + wrong response code
-            [b"OMG 0 <foo@bar>\r\nbegin\r\n"],  # No empty line + invalid response code
-            [b"222 0 <foo@bar>\r\nbegin\r\n"],  # No perms
-            [b"222 0 <foo@bar>\r\nbegin ABC DEF\r\n"],  # Permissions not octal
-            [b"222 0 <foo@bar>\r\nbegin 755\r\n"],  # No filename
-            [b"222 0 <foo@bar>\r\nbegin 644 \t \t\r\n"],  # Filename empty after stripping
+            b"222 0 <foo@bar>\r\n\r\n",  # Missing altogether
+            b"222 0 <foo@bar>\r\n\r\nbeing\r\n",  # Typo in 'begin'
+            b"222 0 <foo@bar>\r\n\r\nx-header: begin 644 foobar\r\n",  # Not at start of the line
+            b"666 0 <foo@bar>\r\nbegin\r\n",  # No empty line + wrong response code
+            b"OMG 0 <foo@bar>\r\nbegin\r\n",  # No empty line + invalid response code
+            b"222 0 <foo@bar>\r\nbegin\r\n",  # No perms
+            b"222 0 <foo@bar>\r\nbegin ABC DEF\r\n",  # Permissions not octal
+            b"222 0 <foo@bar>\r\nbegin 755\r\n",  # No filename
+            b"222 0 <foo@bar>\r\nbegin 644 \t \t\r\n",  # Filename empty after stripping
         ],
     )
     def test_missing_uu_begin(self, raw_data):
@@ -140,7 +140,9 @@ class TestUuDecoder:
         article.lowest_partnum = True
         filler = b"\r\n" * 4
         with pytest.raises(decoder.BadUu):
-            assert decoder.decode_uu(article, raw_data.append(filler))
+            raw_data = bytearray(raw_data)
+            raw_data.extend(filler)
+            assert decoder.decode_uu(article, raw_data)
 
     @pytest.mark.parametrize("insert_empty_line", [True, False])
     @pytest.mark.parametrize("insert_excess_empty_lines", [True, False])
@@ -161,7 +163,7 @@ class TestUuDecoder:
         article, raw_data, expected_result = self._generate_msg_part(
             "single", insert_empty_line, insert_excess_empty_lines, insert_headers, insert_end, begin_line
         )
-        assert decoder.decode_uu(article, [raw_data]) == expected_result
+        assert decoder.decode_uu(article, raw_data) == expected_result
         assert article.nzf.filename_checked
 
     @pytest.mark.parametrize("insert_empty_line", [True, False])
@@ -172,7 +174,7 @@ class TestUuDecoder:
         decoded_data = expected_data = b""
         for part in ("begin", "middle", "middle", "end"):
             article, data, result = self._generate_msg_part(part, insert_empty_line, False, False, True)
-            decoded_data += decoder.decode_uu(article, [data])
+            decoded_data += decoder.decode_uu(article, data)
             expected_data += result
 
         # Verify results
@@ -191,4 +193,4 @@ class TestUuDecoder:
         article.lowest_partnum = False
         filler = b"\r\n".join(VALID_UU_LINES[:4]) + b"\r\n"
         with pytest.raises(decoder.BadData):
-            assert decoder.decode_uu(article, [b"222 0 <foo@bar>\r\n" + filler + bad_data + b"\r\n"])
+            assert decoder.decode_uu(article, bytearray(b"222 0 <foo@bar>\r\n" + filler + bad_data + b"\r\n"))