mirror of
https://github.com/exo-explore/exo.git
synced 2026-02-20 15:57:29 -05:00
facf2d4d03
## Motivation There is an issue on Macs that means that an explicit synchronization is necessary for memory to be updated from L1 cache. This means that GPU locks can occur when a spin wait does not see the updated timestamp. ## Changes Updated in my own personal fork. ## Why It Works https://github.com/ARM-software/acle/releases ## Test Plan ### Manual Testing Tested manually that no GPU locks occur (even with multiple simultaneous instances running) and that the performance differential is negligible (267 vs 269 tps on Llama 3.2 1B at an approx 10k context.) ------------------------------------------------------ I have seen a GPU lock, specifically when sending a particularly large chat completion while the model was loading. However, I have since been unable to reproduce and this may be something I did wrong. Please do create an issue and tag me if any GPU locks do occur. --------- Co-authored-by: Jake Hillion <jake@hillion.co.uk> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
140 lines
3.3 KiB
TOML
140 lines
3.3 KiB
TOML
[project]
|
|
name = "exo"
|
|
version = "0.3.0"
|
|
description = "Exo"
|
|
readme = "README.md"
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"aiofiles>=24.1.0",
|
|
"aiohttp>=3.12.14",
|
|
"types-aiofiles>=24.1.0.20250708",
|
|
"pydantic>=2.11.7",
|
|
"fastapi>=0.116.1",
|
|
"filelock>=3.18.0",
|
|
"rustworkx>=0.17.1",
|
|
"huggingface-hub>=0.33.4",
|
|
"psutil>=7.0.0",
|
|
"loguru>=0.7.3",
|
|
"exo_pyo3_bindings", # rust bindings
|
|
"anyio==4.11.0",
|
|
"mlx; sys_platform == 'darwin'",
|
|
"mlx[cpu]==0.30.6; sys_platform == 'linux'",
|
|
"mlx-lm==0.30.6",
|
|
"tiktoken>=0.12.0", # required for kimi k2 tokenizer
|
|
"hypercorn>=0.18.0",
|
|
"openai-harmony>=0.0.8",
|
|
"httpx>=0.28.1",
|
|
"tomlkit>=0.14.0",
|
|
"pillow>=11.0,<12.0", # compatibility with mflux
|
|
"mflux==0.15.5",
|
|
"python-multipart>=0.0.21",
|
|
"msgspec>=0.19.0",
|
|
"zstandard>=0.23.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
exo = "exo.main:main"
|
|
|
|
# dependencies only required for development
|
|
[dependency-groups]
|
|
dev = [
|
|
"basedpyright>=1.29.0",
|
|
"pyinstaller>=6.17.0",
|
|
"pytest>=8.4.0",
|
|
"pytest-asyncio>=1.0.0",
|
|
"pytest-env",
|
|
"ruff>=0.11.13",
|
|
]
|
|
|
|
# mlx[cuda] requires a newer version of mlx. the ideal on linux is: default to mlx[cpu] unless[cuda] specified.
|
|
[project.optional-dependencies]
|
|
# cuda = [
|
|
# "mlx[cuda]==0.26.3",
|
|
# ]
|
|
|
|
###
|
|
# workspace configuration
|
|
###
|
|
|
|
[tool.uv.workspace]
|
|
members = [
|
|
"rust/exo_pyo3_bindings",
|
|
"bench",
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
exo_pyo3_bindings = { workspace = true }
|
|
mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
|
|
#mlx-lm = { git = "https://github.com/davidmcc73/mlx-lm", branch = "stable" }
|
|
# Uncomment to use local mlx/mlx-lm development versions:
|
|
# mlx = { path = "/Users/Shared/mlx", editable=true }
|
|
# mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }
|
|
|
|
[build-system]
|
|
requires = ["uv_build>=0.8.9,<0.9.0"]
|
|
build-backend = "uv_build"
|
|
|
|
###
|
|
# type-checker configuration
|
|
###
|
|
|
|
[tool.basedpyright]
|
|
include = [".venv/lib/mlx", ".venv/lib/mlx_lm", "src", "bench"]
|
|
typeCheckingMode = "strict"
|
|
failOnWarnings = true
|
|
|
|
reportAny = "error"
|
|
reportUnknownVariableType = "error"
|
|
reportUnknownParameterType = "error"
|
|
reportMissingParameterType = "error"
|
|
reportMissingTypeStubs = "error"
|
|
reportInvalidCast = "error"
|
|
reportUnnecessaryCast = "error"
|
|
reportUnnecessaryTypeIgnoreComment = "error"
|
|
|
|
pythonVersion = "3.13"
|
|
pythonPlatform = "Darwin"
|
|
|
|
exclude = ["**/.venv", "**/venv", "**/__pycache__", "**/exo_scripts", "**/.direnv", "**/rust", "**/.github"]
|
|
stubPath = ".mlx_typings"
|
|
|
|
[[tool.basedpyright.executionEnvironments]]
|
|
root = "src"
|
|
|
|
###
|
|
# uv configuration
|
|
###
|
|
|
|
# supported platforms for this project
|
|
[tool.uv]
|
|
required-version = ">=0.8.6"
|
|
prerelease = "allow"
|
|
environments = [
|
|
"sys_platform == 'darwin'",
|
|
"sys_platform == 'linux'",
|
|
]
|
|
|
|
###
|
|
# ruff configuration
|
|
###
|
|
|
|
[tool.ruff]
|
|
extend-exclude = ["shared/protobufs/**", "*mlx_typings/**", "rust/exo_pyo3_bindings/**"]
|
|
|
|
[tool.ruff.lint]
|
|
extend-select = ["I", "N", "B", "A", "PIE", "SIM"]
|
|
|
|
[tool.pytest.ini_options]
|
|
pythonpath = "."
|
|
asyncio_mode = "auto"
|
|
markers = [
|
|
"slow: marks tests as slow (deselected by default)"
|
|
]
|
|
env = [
|
|
"EXO_TESTS=1"
|
|
]
|
|
addopts = "-m 'not slow' --ignore=tests/start_distributed_test.py"
|
|
filterwarnings = [
|
|
"ignore:builtin type Swig:DeprecationWarning",
|
|
]
|