mirror of
https://github.com/exo-explore/exo.git
synced 2026-06-02 11:21:47 -04:00
fa57131374
## Motivation No automated integration tests exist for exo. Manual testing against real hardware clusters is slow and error-prone. We need a pytest framework that deploys clusters via `eco`, runs inference scenarios, and tears down cleanly. ## Changes - **`tools/src/exo_tools/`** — New workspace member shared by bench, eval, and tests: - `client.py` — `ExoClient` HTTP client (extracted from `bench/harness.py`) - `harness.py` — instance lifecycle helpers (placement, wait-for-ready, etc.) - `cluster.py` — `EcoSession` for eco cluster lifecycle (deploy/stop/start/release/logs/exec) with unique `USER=<prefix>-<uuid>` per session and atexit/signal cleanup - **`tests/integration/`** — 17 pytest tests across 5 files: - `test_1node.py` — place, chat, multi-turn, delete, state/models endpoints, cluster snapshot, download-from-scratch - `test_2node.py` — parametrized tensor/jaccl + pipeline/ring inference and multi-turn - `test_4node.py` — parametrized 4-node pipeline/ring inference, cluster state - `test_resilience.py` — full disconnect/reconnect cycle (2-node → disconnect → 1-node → reconnect → 2-node) - `test_dashboard.py` — Playwright: dashboard loads, shows node info, chat flow - `helpers.py` — placement/inference helpers, re-exports from `exo_tools` - `conftest.py` — session-scoped cluster fixtures with constraint-based eco reservations; `--hosts` override; `EXO_REF` env var for CI deployments from a GitHub branch - **`bench/`** — Updated imports from `exo_tools.client` / `exo_tools.harness` - **`pyproject.toml`** — Added `tools` workspace member, `playwright` dev dep, `--ignore=tests/integration` ## Why It Works Tests use `eco` for cluster lifecycle and `ExoClient` for API interactions — same tools humans use. Session-scoped fixtures deploy once per file. Unique eco users prevent test runs from interfering with each other or manual usage. ## Test Plan ### Automated Testing - `uv run pytest tests/integration/ -v -s` — full suite (~4-5 min, 17/17 passing) - `uv run pytest tests/integration/ -v -s --hosts s4,s9,s10,s22` — pin specific hosts - `EXO_REF=main uv run pytest tests/integration/ -v` — deploy from a GitHub branch (CI) - `uv run pytest` — confirms integration tests are excluded from default runs
57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
# type: ignore
|
|
"""Resilience tests: disconnect/reconnect nodes and verify cluster recovery.
|
|
|
|
Run with:
|
|
uv run pytest tests/test_resilience.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from exo_tools.cluster import Thunderbolt
|
|
from exo_tools.harness import Comm, Sharding, cleanup_all_instances, place_instance
|
|
|
|
from .framework import DEFAULT_MODEL, InstanceSpec
|
|
|
|
|
|
@pytest.mark.cluster(count=2, thunderbolt=Thunderbolt.A2A)
|
|
@pytest.mark.instance(
|
|
DEFAULT_MODEL, sharding=Sharding.PIPELINE, comm=Comm.RING, min_nodes=2
|
|
)
|
|
def test_node_recovery(session):
|
|
"""Full disconnect/reconnect cycle.
|
|
|
|
1. Place a 2-node instance, verify inference
|
|
2. Disconnect one node
|
|
3. Place a 1-node instance on remaining node, verify inference
|
|
4. Reconnect the stopped node, wait for the cluster to reform
|
|
5. Place a 2-node instance again, verify inference
|
|
"""
|
|
# --- Phase 1: 2-node inference ---
|
|
resp = session.chat("Hello")
|
|
assert len(resp) > 0
|
|
|
|
# --- Phase 2: disconnect one node ---
|
|
session.disconnect_node(1)
|
|
session.wait_ready(60)
|
|
|
|
# Clean up the now-broken 2-node instance
|
|
cleanup_all_instances(session.client)
|
|
|
|
# --- Phase 3: 1-node inference on the remaining node ---
|
|
place_instance(session.client, DEFAULT_MODEL, min_nodes=1)
|
|
session.instance_spec = InstanceSpec(model_id=DEFAULT_MODEL, min_nodes=1)
|
|
resp = session.chat("Hello")
|
|
assert len(resp) > 0
|
|
|
|
# --- Phase 4: reconnect and restore 2-node cluster ---
|
|
cleanup_all_instances(session.client)
|
|
session.reconnect_node(1)
|
|
session.wait_ready(60)
|
|
|
|
# --- Phase 5: 2-node inference again ---
|
|
place_instance(session.client, DEFAULT_MODEL, min_nodes=2)
|
|
session.instance_spec = InstanceSpec(model_id=DEFAULT_MODEL, min_nodes=2)
|
|
resp = session.chat("Hello again")
|
|
assert len(resp) > 0
|