From c582bdd67337652b16de12390bcdc99679cde13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Heath=20Dutton=F0=9F=95=B4=EF=B8=8F?= Date: Sat, 20 Dec 2025 20:28:59 -0500 Subject: [PATCH] bugfix: Handle MacMon errors gracefully --- src/exo/worker/utils/macmon.py | 18 ++++-- src/exo/worker/utils/profile.py | 1 + src/exo/worker/utils/tests/__init__.py | 0 src/exo/worker/utils/tests/test_macmon.py | 77 +++++++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/exo/worker/utils/tests/__init__.py create mode 100644 src/exo/worker/utils/tests/test_macmon.py diff --git a/src/exo/worker/utils/macmon.py b/src/exo/worker/utils/macmon.py index 3e4e29e1..1e892d77 100644 --- a/src/exo/worker/utils/macmon.py +++ b/src/exo/worker/utils/macmon.py @@ -1,6 +1,7 @@ import platform import shutil from subprocess import CalledProcessError +from typing import cast from anyio import run_process from pydantic import BaseModel, ConfigDict, ValidationError @@ -80,7 +81,6 @@ async def get_metrics_async() -> Metrics: """ path = _get_binary_path() - result = None try: # TODO: Keep Macmon running in the background? result = await run_process([path, "pipe", "-s", "1"]) @@ -90,8 +90,14 @@ async def get_metrics_async() -> Metrics: except ValidationError as e: raise MacMonError(f"Error parsing JSON output: {e}") from e except CalledProcessError as e: - if result: - raise MacMonError( - f"MacMon failed with return code {result.returncode}" - ) from e - raise e + stderr_msg = "no stderr" + stderr_output = cast(bytes | str | None, e.stderr) + if stderr_output is not None: + stderr_msg = ( + stderr_output.decode() + if isinstance(stderr_output, bytes) + else str(stderr_output) + ) + raise MacMonError( + f"MacMon failed with return code {e.returncode}: {stderr_msg}" + ) from e diff --git a/src/exo/worker/utils/profile.py b/src/exo/worker/utils/profile.py index 30aca08c..316973ad 100644 --- a/src/exo/worker/utils/profile.py +++ b/src/exo/worker/utils/profile.py @@ -109,5 +109,6 @@ async def start_polling_node_metrics( ) except MacMonError as e: logger.opt(exception=e).error("Resource Monitor encountered error") + return finally: await anyio.sleep(poll_interval_s) diff --git a/src/exo/worker/utils/tests/__init__.py b/src/exo/worker/utils/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/exo/worker/utils/tests/test_macmon.py b/src/exo/worker/utils/tests/test_macmon.py new file mode 100644 index 00000000..afb19aa4 --- /dev/null +++ b/src/exo/worker/utils/tests/test_macmon.py @@ -0,0 +1,77 @@ +"""Tests for macmon error handling. + +These tests verify that MacMon errors are handled gracefully without +crashing the application or spamming logs. +""" + +import platform +from subprocess import CalledProcessError +from unittest.mock import AsyncMock, patch + +import pytest + +from exo.worker.utils.macmon import MacMonError, get_metrics_async + + +@pytest.mark.skipif( + platform.system().lower() != "darwin" or "arm" not in platform.machine().lower(), + reason="MacMon only supports macOS with Apple Silicon", +) +class TestMacMonErrorHandling: + """Test MacMon error handling.""" + + async def test_called_process_error_wrapped_as_macmon_error(self) -> None: + """CalledProcessError should be wrapped as MacMonError.""" + mock_error = CalledProcessError( + returncode=1, + cmd=["macmon", "pipe", "-s", "1"], + stderr=b"some error message", + ) + + with ( + patch( + "exo.worker.utils.macmon.shutil.which", return_value="/usr/bin/macmon" + ), + patch( + "exo.worker.utils.macmon.run_process", new_callable=AsyncMock + ) as mock_run, + ): + mock_run.side_effect = mock_error + + with pytest.raises(MacMonError) as exc_info: + await get_metrics_async() + + assert "MacMon failed with return code 1" in str(exc_info.value) + assert "some error message" in str(exc_info.value) + + async def test_called_process_error_with_no_stderr(self) -> None: + """CalledProcessError with no stderr should be handled gracefully.""" + mock_error = CalledProcessError( + returncode=1, + cmd=["macmon", "pipe", "-s", "1"], + stderr=None, + ) + + with ( + patch( + "exo.worker.utils.macmon.shutil.which", return_value="/usr/bin/macmon" + ), + patch( + "exo.worker.utils.macmon.run_process", new_callable=AsyncMock + ) as mock_run, + ): + mock_run.side_effect = mock_error + + with pytest.raises(MacMonError) as exc_info: + await get_metrics_async() + + assert "MacMon failed with return code 1" in str(exc_info.value) + assert "no stderr" in str(exc_info.value) + + async def test_macmon_not_found_raises_macmon_error(self) -> None: + """When macmon is not found in PATH, MacMonError should be raised.""" + with patch("exo.worker.utils.macmon.shutil.which", return_value=None): + with pytest.raises(MacMonError) as exc_info: + await get_metrics_async() + + assert "MacMon not found in PATH" in str(exc_info.value)