sabnzbd/builder/package.py

#!/usr/bin/python3 -OO
# Copyright 2008-2025 by The SABnzbd-Team (sabnzbd.org)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import glob
import re
import sys
import os
import tempfile
import time
import shutil
import subprocess
import tarfile
import urllib.request
import urllib.error
import configobj
import packaging.version

from constants import (
    RELEASE_VERSION,
    RELEASE_VERSION_TUPLE,
    VERSION_FILE,
    RELEASE_README,
    RELEASE_NAME,
    RELEASE_WIN_BIN,
    RELEASE_WIN_INSTALLER,
    ON_GITHUB_ACTIONS,
    RELEASE_THIS,
    RELEASE_SRC,
    EXTRA_FILES,
    EXTRA_FOLDERS,
)


# Support functions
def safe_remove(path):
    """Remove file without errors if the file doesn't exist
    Can also handle folders
    """
    if os.path.exists(path):
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)


def delete_files_glob(glob_pattern: str, allow_no_matches: bool = False):
    """Delete one file or set of files from wild-card spec.
    We expect to match at least 1 file, to force expected behavior"""
    if files_to_remove := glob.glob(glob_pattern):
        for path in files_to_remove:
            if os.path.exists(path):
                os.remove(path)
    else:
        if not allow_no_matches:
            raise FileNotFoundError(f"No files found that match '{glob_pattern}'")


def run_external_command(command: list[str], print_output: bool = True, **kwargs):
    """Wrapper to ease the use of calling external programs"""
    process = subprocess.Popen(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
    output, _ = process.communicate()
    ret = process.wait()
    if (output and print_output) or ret != 0:
        print(output)
    if ret != 0:
        raise RuntimeError("Command returned non-zero exit code %s!" % ret)
    return output


def run_git_command(parms):
    """Run git command, raise error if it failed"""
    return run_external_command(["git"] + parms)


def patch_version_file(release_name):
    """Patch in the Git commit hash, but only when this is
    an unmodified checkout
    """
    git_output = run_git_command(["log", "-1"])
    for line in git_output.split("\n"):
        if "commit " in line:
            commit = line.split(" ")[1].strip()
            break
    else:
        raise TypeError("Commit hash not found")

    with open(VERSION_FILE, "r") as ver:
        version_file = ver.read()

    version_file = re.sub(r'__baseline__\s*=\s*"[^"]*"', '__baseline__ = "%s"' % commit, version_file)
    version_file = re.sub(r'__version__\s*=\s*"[^"]*"', '__version__ = "%s"' % release_name, version_file)

    with open(VERSION_FILE, "w") as ver:
        ver.write(version_file)


def test_macos_min_version(binary_path: str):
    # Skip check if nothing was set
    if macos_min_version := os.environ.get("MACOSX_DEPLOYMENT_TARGET"):
        # Skip any arm64 specific files
        if "arm64" in binary_path:
            print(f"Skipping arm64 binary {binary_path}")
            return

        # Check minimum macOS version is at least mac OS10.13
        # We only check the x86_64 since for arm64 it's always macOS 11+
        print(f"Checking if binary supports macOS {macos_min_version} and above: {binary_path}")
        otool_output = run_external_command(
            [
                "otool",
                "-arch",
                "x86_64",
                "-l",
                binary_path,
            ],
            print_output=False,
        )

        # Parse the output for LC_BUILD_VERSION minos
        # The output is very large, so that's why we enumerate over it
        req_version = packaging.version.parse(macos_min_version)
        bin_version = None
        lines = otool_output.split("\n")
        for line_nr, line in enumerate(lines):
            if "LC_VERSION_MIN_MACOSX" in line:
                # Display the version in the next lines
                bin_version = packaging.version.parse(lines[line_nr + 2].split()[1])
            elif "minos" in line:
                bin_version = packaging.version.parse(line.split()[1])

            if bin_version and bin_version > req_version:
                raise ValueError(f"{binary_path} requires {bin_version}, we want {req_version}")
            else:
                # We got the information we need
                break
        else:
            print(lines)
            raise RuntimeError(f"Could not determine minimum macOS version for {binary_path}")
    else:
        print(f"Skipping macOS version check, MACOSX_DEPLOYMENT_TARGET not set")


def test_sab_binary(binary_path: str):
    """Wrapper to have a simple start-up test for the binary"""
    with tempfile.TemporaryDirectory() as config_dir:
        sabnzbd_process = subprocess.Popen(
            [binary_path, "--browser", "0", "--logging", "2", "--config", config_dir],
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )

        # Wait for SAB to respond
        base_url = "http://127.0.0.1:8080/"
        for _ in range(30):
            try:
                urllib.request.urlopen(base_url, timeout=1).read()
                break
            except Exception:
                time.sleep(1)
        else:
            # Print console output and give some time to print
            print(sabnzbd_process.stdout.read())
            time.sleep(1)
            raise urllib.error.URLError("Could not connect to SABnzbd")

        # Open a number of API calls and pages, to see if we are really up
        pages_to_test = [
            "",
            "wizard",
            "config",
            "config/server",
            "config/categories",
            "config/scheduling",
            "config/rss",
            "config/general",
            "config/folders",
            "config/switches",
            "config/sorting",
            "config/notify",
            "config/special",
            "api?mode=version",
        ]
        for url in pages_to_test:
            print("Testing: %s%s" % (base_url, url))
            if b"500 Internal Server Error" in urllib.request.urlopen(base_url + url, timeout=1).read():
                raise RuntimeError("Crash in %s" % url)

        # Parse API-key so we can do a graceful shutdown
        sab_config = configobj.ConfigObj(os.path.join(config_dir, "sabnzbd.ini"))
        urllib.request.urlopen(base_url + "shutdown/?apikey=" + sab_config["misc"]["api_key"], timeout=10)
        sabnzbd_process.wait()

        # Print logs for verification
        with open(os.path.join(config_dir, "logs", "sabnzbd.log"), "r") as log_file:
            # Wait after printing so the output is nicely displayed in case of problems
            print(log_text := log_file.read())
            time.sleep(5)

            # Make sure no extra errors/warnings were reported
            if "ERROR" in log_text or "WARNING" in log_text:
                raise RuntimeError("Warning or error reported during execution")


if __name__ == "__main__":
    # Was any option supplied?
    if len(sys.argv) < 2:
        raise TypeError("Please specify what to do")

    # Make sure we are in the src folder
    if not os.path.exists("builder"):
        raise FileNotFoundError("Run from the main SABnzbd source folder: python builder/package.py")

    # Check if we have the needed certificates
    try:
        import certifi
    except ImportError:
        raise FileNotFoundError("Need certifi module")

    # Patch release file
    patch_version_file(RELEASE_VERSION)

    # Rename release notes file
    safe_remove("README.txt")
    shutil.copyfile(RELEASE_README, "README.txt")

    # Compile translations
    if not os.path.exists("locale"):
        run_external_command([sys.executable, "tools/make_mo.py"])

        # Check again if translations exist, fail otherwise
        if not os.path.exists("locale"):
            raise FileNotFoundError("Failed to compile language files")

    if "binary" in sys.argv:
        # Must be run on Windows
        if sys.platform != "win32":
            raise RuntimeError("Binary should be created on Windows")

        # Make sure we remove any existing build-folders
        safe_remove("build")
        safe_remove("dist")

        # Remove any leftovers
        safe_remove(RELEASE_NAME)
        safe_remove(RELEASE_WIN_BIN)

        # Run PyInstaller and check output
        shutil.copyfile("builder/SABnzbd.spec", "SABnzbd.spec")
        run_external_command([sys.executable, "-O", "-m", "PyInstaller", "SABnzbd.spec"])

        shutil.copytree("dist/SABnzbd-console", "dist/SABnzbd", dirs_exist_ok=True)
        safe_remove("dist/SABnzbd-console")

        # Remove unwanted DLL's
        shutil.rmtree("dist/SABnzbd/Pythonwin")
        delete_files_glob("dist/SABnzbd/api-ms-win*.dll", allow_no_matches=True)
        delete_files_glob("dist/SABnzbd/ucrtbase.dll", allow_no_matches=True)

        # Test the release
        test_sab_binary("dist/SABnzbd/SABnzbd.exe")

        # Create the archive
        run_external_command(["win/7zip/7za.exe", "a", RELEASE_WIN_BIN, "SABnzbd"], cwd="dist")
        shutil.move(f"dist/{RELEASE_WIN_BIN}", RELEASE_WIN_BIN)

    if "installer" in sys.argv:
        # Check if we have the dist folder
        if not os.path.exists("dist/SABnzbd/SABnzbd.exe"):
            raise FileNotFoundError("SABnzbd executable not found, run binary creation first")

        # Check if we have a signed version
        if os.path.exists(f"signed/{RELEASE_WIN_BIN}"):
            print("Using signed version of SABnzbd binaries")
            safe_remove("dist/SABnzbd")
            run_external_command(["win/7zip/7za.exe", "x", "-odist", f"signed/{RELEASE_WIN_BIN}"])

            # Make sure it exists
            if not os.path.exists("dist/SABnzbd/SABnzbd.exe"):
                raise FileNotFoundError("SABnzbd executable not found, signed zip extraction failed")
        elif RELEASE_THIS:
            raise FileNotFoundError("Signed SABnzbd executable not found, required for release!")
        else:
            print("Using unsigned version of SABnzbd binaries")

        # Compile NSIS translations
        safe_remove("NSIS_Installer.nsi")
        safe_remove("NSIS_Installer.nsi.tmp")
        shutil.copyfile("builder/win/NSIS_Installer.nsi", "NSIS_Installer.nsi")
        run_external_command([sys.executable, "tools/make_mo.py", "nsis"])

        # Run NSIS to build installer
        run_external_command(
            [
                "makensis.exe",
                "/V3",
                "/DSAB_VERSION=%s" % RELEASE_VERSION,
                "/DSAB_VERSIONKEY=%s" % ".".join(map(str, RELEASE_VERSION_TUPLE)),
                "/DSAB_FILE=%s" % RELEASE_WIN_INSTALLER,
                "NSIS_Installer.nsi.tmp",
            ]
        )

    if "app" in sys.argv:
        # Must be run on macOS
        if sys.platform != "darwin":
            raise RuntimeError("App should be created on macOS")

        # Who will sign and notarize this?
        authority = os.environ.get("SIGNING_AUTH")
        notarization_user = os.environ.get("NOTARIZATION_USER")
        notarization_pass = os.environ.get("NOTARIZATION_PASS")

        # We need to sign all the included binaries before packaging them
        # Otherwise the signature of the main application becomes invalid
        if authority:
            files_to_sign = [
                "macos/par2/par2",
                "macos/unrar/unrar",
                "macos/unrar/arm64/unrar",
                "macos/7zip/7zz",
            ]
            for file_to_sign in files_to_sign:
                # Make sure it supports the macOS versions we want first
                test_macos_min_version(file_to_sign)

                # Then sign in
                print("Signing %s with hardened runtime" % file_to_sign)
                run_external_command(
                    [
                        "codesign",
                        "--deep",
                        "--force",
                        "--timestamp",
                        "--options",
                        "runtime",
                        "--entitlements",
                        "builder/macos/entitlements.plist",
                        "-s",
                        authority,
                        file_to_sign,
                    ],
                    print_output=False,
                )
                print("Signed %s!" % file_to_sign)

        # Run PyInstaller and check output
        shutil.copyfile("builder/SABnzbd.spec", "SABnzbd.spec")
        run_external_command([sys.executable, "-O", "-m", "PyInstaller", "SABnzbd.spec"])

        # Make sure we created a fully universal2 release when releasing or during CI
        if RELEASE_THIS or ON_GITHUB_ACTIONS:
            for bin_to_check in glob.glob("dist/SABnzbd.app/**/*.so", recursive=True):
                print("Checking if binary is universal2: %s" % bin_to_check)
                file_output = run_external_command(["file", bin_to_check], print_output=False)
                # Make sure we have both arm64 and x86
                if not ("x86_64" in file_output and "arm64" in file_output):
                    raise RuntimeError("Non-universal2 binary found!")

                # Make sure it supports the macOS versions we want
                test_macos_min_version(bin_to_check)

        # Only continue if we can sign
        if authority:
            # We use PyInstaller to sign the main SABnzbd executable and the SABnzbd.app
            files_already_signed = [
                "dist/SABnzbd.app/Contents/MacOS/SABnzbd",
                "dist/SABnzbd.app",
            ]
            for file_to_check in files_already_signed:
                print("Checking signature of %s" % file_to_check)
                sign_result = run_external_command(
                    [
                        "codesign",
                        "-dv",
                        "-r-",
                        file_to_check,
                    ],
                    print_output=False,
                ) + run_external_command(
                    [
                        "codesign",
                        "--verify",
                        "--deep",
                        file_to_check,
                    ],
                    print_output=False,
                )
                if authority not in sign_result or "adhoc" in sign_result or "invalid" in sign_result:
                    raise RuntimeError("Signature of %s seems invalid!" % file_to_check)

            # Always notarize, as newer macOS versions don't allow any code without it
            if notarization_user and notarization_pass:
                # Prepare zip to upload to notarization service
                print("Creating zip to send to Apple notarization service")
                # We need to use ditto, otherwise the signature gets lost!
                notarization_zip = RELEASE_NAME + ".zip"
                run_external_command(
                    ["ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", "dist/SABnzbd.app", notarization_zip]
                )

                # Upload to Apple
                print("Sending zip to Apple notarization service")
                upload_result = run_external_command(
                    [
                        "xcrun",
                        "notarytool",
                        "submit",
                        notarization_zip,
                        "--apple-id",
                        notarization_user,
                        "--team-id",
                        authority,
                        "--password",
                        notarization_pass,
                        "--wait",
                    ],
                )

                # Check if success
                if "status: accepted" not in upload_result.lower():
                    raise RuntimeError("Failed to notarize..")

                # Staple the notarization!
                print("Approved! Stapling the result to the app")
                run_external_command(["xcrun", "stapler", "staple", "dist/SABnzbd.app"])
            else:
                print("Notarization skipped, NOTARIZATION_USER or NOTARIZATION_PASS missing.")
        else:
            print("Signing skipped, missing SIGNING_AUTH.")

        # Test the release, as the very last step to not mess with any release code
        test_sab_binary("dist/SABnzbd.app/Contents/MacOS/SABnzbd")

    if "source" in sys.argv:
        # Prepare Source distribution package.
        # We assume the sources are freshly cloned from the repo
        # Make sure all source files are Unix format
        src_folder = "srcdist"
        safe_remove(src_folder)
        os.mkdir(src_folder)

        # Remove any leftovers
        safe_remove(RELEASE_SRC)

        # Add extra files and folders need for source dist
        EXTRA_FOLDERS.extend(["sabnzbd/", "po/", "linux/", "tools/", "tests/"])
        EXTRA_FILES.extend(["SABnzbd.py", "requirements.txt"])

        # Copy all folders and files to the new folder
        for source_folder in EXTRA_FOLDERS:
            shutil.copytree(source_folder, os.path.join(src_folder, source_folder), dirs_exist_ok=True)

        # Copy all files
        for source_file in EXTRA_FILES:
            shutil.copyfile(source_file, os.path.join(src_folder, source_file))

        # Make sure all line-endings are correct
        for input_filename in glob.glob("%s/**/*.*" % src_folder, recursive=True):
            base, ext = os.path.splitext(input_filename)
            if ext.lower() not in (".py", ".txt", ".css", ".js", ".tmpl", ".sh", ".cmd"):
                continue
            print(input_filename)

            with open(input_filename, "rb") as input_data:
                data = input_data.read()
            data = data.replace(b"\r", b"")
            with open(input_filename, "wb") as output_data:
                output_data.write(data)

        # Create tar.gz file for source distro
        with tarfile.open(RELEASE_SRC, "w:gz") as tar_output:
            for root, dirs, files in os.walk(src_folder):
                for _file in files:
                    input_path = os.path.join(root, _file)
                    if sys.platform == "win32":
                        tar_path = input_path.replace("srcdist\\", RELEASE_NAME + "/").replace("\\", "/")
                    else:
                        tar_path = input_path.replace("srcdist/", RELEASE_NAME + "/")
                    tarinfo = tar_output.gettarinfo(input_path, tar_path)
                    tarinfo.uid = 0
                    tarinfo.gid = 0
                    if _file in ("SABnzbd.py", "Sample-PostProc.sh", "make_mo.py", "msgfmt.py"):
                        # Force Linux/macOS scripts as executable
                        tarinfo.mode = 0o755
                    else:
                        tarinfo.mode = 0o644

                    with open(input_path, "rb") as f:
                        tar_output.addfile(tarinfo, f)

        # Remove source folder
        safe_remove(src_folder)

    # Reset!
    run_git_command(["reset", "--hard"])
    run_git_command(["clean", "-f"])