mirror of
https://github.com/exo-explore/exo.git
synced 2026-05-24 06:35:32 -04:00
fa57131374
## Motivation No automated integration tests exist for exo. Manual testing against real hardware clusters is slow and error-prone. We need a pytest framework that deploys clusters via `eco`, runs inference scenarios, and tears down cleanly. ## Changes - **`tools/src/exo_tools/`** — New workspace member shared by bench, eval, and tests: - `client.py` — `ExoClient` HTTP client (extracted from `bench/harness.py`) - `harness.py` — instance lifecycle helpers (placement, wait-for-ready, etc.) - `cluster.py` — `EcoSession` for eco cluster lifecycle (deploy/stop/start/release/logs/exec) with unique `USER=<prefix>-<uuid>` per session and atexit/signal cleanup - **`tests/integration/`** — 17 pytest tests across 5 files: - `test_1node.py` — place, chat, multi-turn, delete, state/models endpoints, cluster snapshot, download-from-scratch - `test_2node.py` — parametrized tensor/jaccl + pipeline/ring inference and multi-turn - `test_4node.py` — parametrized 4-node pipeline/ring inference, cluster state - `test_resilience.py` — full disconnect/reconnect cycle (2-node → disconnect → 1-node → reconnect → 2-node) - `test_dashboard.py` — Playwright: dashboard loads, shows node info, chat flow - `helpers.py` — placement/inference helpers, re-exports from `exo_tools` - `conftest.py` — session-scoped cluster fixtures with constraint-based eco reservations; `--hosts` override; `EXO_REF` env var for CI deployments from a GitHub branch - **`bench/`** — Updated imports from `exo_tools.client` / `exo_tools.harness` - **`pyproject.toml`** — Added `tools` workspace member, `playwright` dev dep, `--ignore=tests/integration` ## Why It Works Tests use `eco` for cluster lifecycle and `ExoClient` for API interactions — same tools humans use. Session-scoped fixtures deploy once per file. Unique eco users prevent test runs from interfering with each other or manual usage. ## Test Plan ### Automated Testing - `uv run pytest tests/integration/ -v -s` — full suite (~4-5 min, 17/17 passing) - `uv run pytest tests/integration/ -v -s --hosts s4,s9,s10,s22` — pin specific hosts - `EXO_REF=main uv run pytest tests/integration/ -v` — deploy from a GitHub branch (CI) - `uv run pytest` — confirms integration tests are excluded from default runs
50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
# type: ignore
|
|
"""Two-node integration tests (ring + jaccl parallelism).
|
|
|
|
Run with:
|
|
uv run pytest tests/test_2node.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from exo_tools.cluster import Thunderbolt
|
|
from exo_tools.harness import Comm, Sharding
|
|
|
|
from .framework import DEFAULT_MODEL
|
|
|
|
|
|
@pytest.mark.cluster(count=2, thunderbolt=Thunderbolt.A2A)
|
|
@pytest.mark.instance(
|
|
DEFAULT_MODEL, sharding=Sharding.TENSOR, comm=Comm.JACCL, min_nodes=2
|
|
)
|
|
def test_2node_jaccl(session):
|
|
resp = session.chat("Say hello in one sentence.")
|
|
assert len(resp) > 0
|
|
|
|
|
|
@pytest.mark.cluster(count=2, thunderbolt=Thunderbolt.A2A)
|
|
@pytest.mark.instance(
|
|
DEFAULT_MODEL, sharding=Sharding.PIPELINE, comm=Comm.RING, min_nodes=2
|
|
)
|
|
def test_2node_ring(session):
|
|
resp = session.chat("Say hello in one sentence.")
|
|
assert len(resp) > 0
|
|
|
|
|
|
@pytest.mark.cluster(count=2, thunderbolt=Thunderbolt.A2A)
|
|
@pytest.mark.instance(
|
|
DEFAULT_MODEL, sharding=Sharding.TENSOR, comm=Comm.JACCL, min_nodes=2
|
|
)
|
|
def test_2node_jaccl_multi_turn(session):
|
|
first = session.chat("What is the capital of France?")
|
|
assert len(first) > 0
|
|
second = session.multi_turn(
|
|
[
|
|
{"role": "user", "content": "What is the capital of France?"},
|
|
{"role": "assistant", "content": first},
|
|
{"role": "user", "content": "What country is it in?"},
|
|
]
|
|
)
|
|
assert len(second) > 0
|