sabnzbd/tests/test_encoding.py

#!/usr/bin/python3 -OO
# Copyright 2007-2026 by The SABnzbd-Team (sabnzbd.org)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

"""
tests.test_misc - Testing functions in encoding.py
"""

import unicodedata


import sabnzbd.encoding as enc


class TestEncoding:
    def test_correct_unknown_encoding(self):
        # Windows encoding in bytes
        assert "frènch_german_demö" == enc.correct_unknown_encoding(b"fr\xe8nch_german_dem\xf6")
        # Windows encoding in string that's already UTF8
        assert "demotöwers" == enc.correct_unknown_encoding("demot\udcf6wers")

    def test_correct_unknown_encoding_nfc_normalization(self):
        """Verify that correct_unknown_encoding always returns NFC-normalized strings.

        This is the fix for GitHub issues #1633 and #2858: par2 file metadata may
        carry NFC filenames while macOS / some archivers produce NFD.
        Both must end up identical after passing through this function so that
        filename comparisons (e.g. quick_check_set) do not cause double-unpacking.
        """
        # NFD form: 'e' + combining-grave (U+0300), 'o' + combining-diaeresis (U+0308)
        nfd_bytes = "fre\u0300nch_german_demo\u0308".encode("utf-8")
        # NFC form: precomposed è (U+00E8) and ö (U+00F6)
        nfc_string = "frènch_german_demö"  # U+00E8 / U+00F6

        result_from_nfd = enc.correct_unknown_encoding(nfd_bytes)
        result_from_nfc = enc.correct_unknown_encoding(nfc_string.encode("utf-8"))

        # Both should produce the same NFC result
        assert result_from_nfd == nfc_string
        assert result_from_nfc == nfc_string
        # Confirm the output is actually NFC, not NFD
        assert unicodedata.is_normalized("NFC", result_from_nfd)
        assert unicodedata.is_normalized("NFC", result_from_nfc)
        # Confirm NFD input was different at the byte level before normalization
        assert nfd_bytes != nfc_string.encode("utf-8")

    def test_limit_encoded_length(self):
        # Test with empty string
        assert enc.limit_encoded_length("", 10) == "", "Empty string should return empty string"

        # Test with string shorter than the limit
        assert enc.limit_encoded_length("hello", 10) == "hello"

        # Test with string equal to the limit
        assert enc.limit_encoded_length("hello", 5) == "hello", "String equal to limit should return the same string"

        # Test with string longer than the limit
        assert enc.limit_encoded_length("hello world", 5) == "hello", "String longer than limit should be truncated"

        # Test with UTF-8 characters
        assert enc.limit_encoded_length("héllö wørld", 10) == "héllö w", "UTF-8 characters should be handled correctly"

        # Test with emojis (multibyte characters)
        assert enc.limit_encoded_length("😀😂🤣😃😄😅😆😉😊😋😎😍", 30) == "😀😂🤣😃😄😅😆"

        # Test with invalid UTF-8 (single surrogate)
        invalid_utf8 = b"\xed\xa0\x80".decode("latin-1")
        limited_string = enc.limit_encoded_length(invalid_utf8, 3)
        assert len(limited_string) == 1, "Invalid UTF-8 should be handled without raising an exception"

        # Test with mixed valid and invalid UTF-8
        mixed_string = "hello" + b"\xed\xa0\x80".decode("latin-1") + "world"
        limited_string = enc.limit_encoded_length(mixed_string, 8)
        assert "hello" in limited_string, "Valid part of mixed string should be present"
        assert len(limited_string) <= 8, "Length of mixed string should be limited"

        # Parametrized tests for various string and length combinations
        test_cases = [
            ("", 5, ""),
            ("short", 10, "short"),
            ("longstring", 4, "long"),
            ("üöä", 4, "üö"),  # Test for umlauts
            ("你好世界", 4, "你"),  # Test for CJK characters
        ]
        for input_string, max_len, expected in test_cases:
            assert enc.limit_encoded_length(input_string, max_len) == expected