From c9e2062f6e9182bca64d962bc051866ea1917f7d Mon Sep 17 00:00:00 2001 From: Evan Quiney Date: Fri, 5 Dec 2025 17:29:06 +0000 Subject: [PATCH] switch from uvicorn to hypercorn --- TODO.md | 2 + pyproject.toml | 3 +- src/exo/main.py | 3 +- src/exo/master/api.py | 113 +++++++++--------- src/exo/shared/constants.py | 17 --- src/exo/shared/logging.py | 53 ++++++++ src/exo/shared/tests/conftest.py | 15 +++ .../shared/tests/test_node_id_persistence.py | 33 ++--- src/exo/utils/banner.py | 10 +- src/exo/utils/channels.py | 11 +- .../worker/download/impl_shard_downloader.py | 10 +- src/exo/worker/engines/mlx/constants.py | 2 - uv.lock | 85 +++++++++---- 13 files changed, 213 insertions(+), 144 deletions(-) diff --git a/TODO.md b/TODO.md index fb5ef0d9..89d7a525 100644 --- a/TODO.md +++ b/TODO.md @@ -17,6 +17,8 @@ 23. Do we need cache_limit? We went back and forth on that a lot because we thought it might be causing issues. One problem is it sets it relative to model size. So if you have multiple models loaded in it will take the most recent model size for the cache_limit. This is problematic if you launch DeepSeek -> Llama for example. 24. further openai/lmstudio api compatibility 25. Rethink retry logic +26. Task cancellation. When API http request gets cancelled, it should cancel corresponding task. +27. Log cleanup - per-module log filters and default to DEBUG log levels Potential refactors: diff --git a/pyproject.toml b/pyproject.toml index 83cafc67..b0c3e18b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ "base58>=2.1.1", "cryptography>=45.0.5", "fastapi>=0.116.1", - "uvicorn>=0.35.0", "filelock>=3.18.0", "aiosqlite>=0.21.0", "networkx>=3.5", @@ -26,7 +25,6 @@ dependencies = [ "greenlet>=3.2.4", "huggingface-hub>=0.33.4", "psutil>=7.0.0", - "cobs>=1.2.2", "loguru>=0.7.3", "textual>=5.3.0", "exo_pyo3_bindings", # rust bindings @@ -35,6 +33,7 @@ dependencies = [ "mlx>=0.29.3", "mlx-lm>=0.28.3", "tiktoken>=0.12.0", # required for kimi k2 tokenizer + "hypercorn>=0.18.0", ] [project.scripts] diff --git a/src/exo/main.py b/src/exo/main.py index 382b957a..0f16d6c2 100644 --- a/src/exo/main.py +++ b/src/exo/main.py @@ -6,6 +6,7 @@ from typing import Self import anyio from anyio.abc import TaskGroup +from loguru import logger from pydantic import PositiveInt import exo.routing.topics as topics @@ -14,7 +15,7 @@ from exo.master.main import Master from exo.routing.router import Router, get_node_id_keypair from exo.shared.constants import EXO_LOG from exo.shared.election import Election, ElectionResult -from exo.shared.logging import logger, logger_cleanup, logger_setup +from exo.shared.logging import logger_cleanup, logger_setup from exo.shared.types.commands import KillCommand from exo.shared.types.common import NodeId, SessionId from exo.utils.channels import Receiver, channel diff --git a/src/exo/master/api.py b/src/exo/master/api.py index f0ed302b..9d65c7c1 100644 --- a/src/exo/master/api.py +++ b/src/exo/master/api.py @@ -1,21 +1,23 @@ -import asyncio import os import time from collections.abc import AsyncGenerator -from typing import final +from typing import cast -import uvicorn -from anyio import Event as AsyncTaskEvent +import anyio from anyio import create_task_group from anyio.abc import TaskGroup from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse from fastapi.staticfiles import StaticFiles +from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType] +from hypercorn.config import Config +from hypercorn.typing import ASGIFramework from loguru import logger from exo.shared.apply import apply from exo.shared.election import ElectionMessage +from exo.shared.logging import InterceptLogger from exo.shared.models.model_cards import MODEL_CARDS from exo.shared.models.model_meta import get_model_meta from exo.shared.types.api import ( @@ -46,9 +48,10 @@ from exo.shared.types.state import State from exo.shared.types.tasks import ChatCompletionTaskParams from exo.shared.types.worker.instances import Instance, InstanceId from exo.utils.banner import print_startup_banner -from exo.utils.channels import Receiver, Sender +from exo.utils.channels import Receiver, Sender, channel from exo.utils.event_buffer import OrderedBuffer -from exo.worker.engines.mlx.constants import HIDE_THINKING + +HIDE_THINKING = False def chunk_to_response( @@ -76,7 +79,6 @@ async def resolve_model_meta(model_id: str) -> ModelMetadata: return await get_model_meta(model_id) -@final class API: def __init__( self, @@ -101,7 +103,7 @@ class API: self.port = port self.paused: bool = False - self.paused_ev: AsyncTaskEvent = AsyncTaskEvent() + self.paused_ev: anyio.Event = anyio.Event() self.app = FastAPI() self._setup_cors() @@ -121,7 +123,7 @@ class API: name="dashboard", ) - self._chat_completion_queues: dict[CommandId, asyncio.Queue[TokenChunk]] = {} + self._chat_completion_queues: dict[CommandId, Sender[TokenChunk]] = {} self._tg: TaskGroup | None = None def reset(self, new_session_id: SessionId, result_clock: int): @@ -135,7 +137,7 @@ class API: self.last_completed_election = result_clock self.paused = False self.paused_ev.set() - self.paused_ev = AsyncTaskEvent() + self.paused_ev = anyio.Event() def _setup_cors(self) -> None: self.app.add_middleware( @@ -210,37 +212,40 @@ class API: ) -> AsyncGenerator[str, None]: """Generate chat completion stream as JSON strings.""" - self._chat_completion_queues[command_id] = asyncio.Queue() + try: + self._chat_completion_queues[command_id], recv = channel[TokenChunk]() - finished = False - is_thinking = False - while not finished: - # TODO: how long should this timeout be? - chunk = await asyncio.wait_for( - self._chat_completion_queues[command_id].get(), timeout=600 + is_thinking = False + with recv as token_chunks: + async for chunk in token_chunks: + if HIDE_THINKING: + if chunk.text == "": + is_thinking = True + if chunk.text == "": + is_thinking = False + chunk_response: ChatCompletionResponse = chunk_to_response( + chunk, command_id + ) + if not (is_thinking and HIDE_THINKING): + logger.debug(f"chunk_response: {chunk_response}") + yield f"data: {chunk_response.model_dump_json()}\n\n" + + if chunk.finish_reason is not None: + yield "data: [DONE]\n\n" + break + + except anyio.get_cancelled_exc_class(): + # TODO: TaskCancelled + """ + self.command_sender.send_nowait( + ForwarderCommand(origin=self.node_id, command=command) ) - assert isinstance(chunk, TokenChunk) - # TODO: Do we want this? - if HIDE_THINKING: - if chunk.text == "": - chunk.text = "\n" - if chunk.text == "": - chunk.text = "\n" - chunk_response: ChatCompletionResponse = chunk_to_response( - chunk, command_id - ) - logger.debug(f"chunk_response: {chunk_response}") - - if not HIDE_THINKING or not is_thinking: - yield f"data: {chunk_response.model_dump_json()}\n\n" - - if chunk.finish_reason is not None: - yield "data: [DONE]\n\n" - finished = True - - command = TaskFinished(finished_command_id=command_id) - await self._send(command) - del self._chat_completion_queues[command_id] + """ + raise + finally: + command = TaskFinished(finished_command_id=command_id) + await self._send(command) + del self._chat_completion_queues[command_id] async def _trigger_notify_user_to_download_model(self, model_id: str) -> None: logger.warning( @@ -298,30 +303,28 @@ class API: ) async def run(self): - uvicorn_config = uvicorn.Config( - self.app, host="0.0.0.0", port=self.port, access_log=False - ) - uvicorn_server = uvicorn.Server(uvicorn_config) + cfg = Config() + cfg.bind = f"0.0.0.0:{self.port}" + # nb: shared.logging needs updating if any of this changes + cfg.accesslog = None + cfg.errorlog = "-" + cfg.logger_class = InterceptLogger async with create_task_group() as tg: self._tg = tg logger.info("Starting API") - tg.start_soon(uvicorn_server.serve) tg.start_soon(self._apply_state) tg.start_soon(self._pause_on_new_election) - tg.start_soon(self._print_banner_when_ready, uvicorn_server) + print_startup_banner(self.port) + await serve( + cast(ASGIFramework, self.app), + cfg, + shutdown_trigger=lambda: anyio.sleep_forever(), + ) + self.command_sender.close() self.global_event_receiver.close() - async def _print_banner_when_ready(self, uvicorn_server: uvicorn.Server): - """Wait for the uvicorn server to be ready, then print the startup banner.""" - # TODO: Is this the best condition to check for? - # The point is this should log when exo is ready. - while not uvicorn_server.started: - await asyncio.sleep(0.1) - - print_startup_banner(self.port) - async def _apply_state(self): with self.global_event_receiver as events: async for f_event in events: @@ -333,7 +336,7 @@ class API: and event.command_id in self._chat_completion_queues ): assert isinstance(event.chunk, TokenChunk) - self._chat_completion_queues[event.command_id].put_nowait( + await self._chat_completion_queues[event.command_id].send( event.chunk ) diff --git a/src/exo/shared/constants.py b/src/exo/shared/constants.py index 489b871a..63ff8526 100644 --- a/src/exo/shared/constants.py +++ b/src/exo/shared/constants.py @@ -1,4 +1,3 @@ -import inspect import os from pathlib import Path @@ -34,19 +33,3 @@ LIBP2P_COMMANDS_TOPIC = "commands" LB_TFLOPS = 2.3 LB_MEMBW_GBPS = 68 LB_DISK_GBPS = 1.5 - - -# little helper function to get the name of the module that raised the error -def get_caller_module_name() -> str: - frm = inspect.stack()[1] - mod = inspect.getmodule(frm[0]) - if mod is None: - return "UNKNOWN MODULE" - return mod.__name__ - - -def get_error_reporting_message() -> str: - return ( - f"THIS IS A BUG IN THE EXO SOFTWARE, PLEASE REPORT IT AT https://github.com/exo-explore/exo/\n" - f"The module that raised the error was: {get_caller_module_name()}" - ) diff --git a/src/exo/shared/logging.py b/src/exo/shared/logging.py index 66ba1700..75040cfd 100644 --- a/src/exo/shared/logging.py +++ b/src/exo/shared/logging.py @@ -1,12 +1,39 @@ +import logging import sys from pathlib import Path +from hypercorn import Config +from hypercorn.logging import Logger as HypercornLogger from loguru import logger +class InterceptLogger(HypercornLogger): + def __init__(self, config: Config): + super().__init__(config) + assert self.error_logger + # TODO: Decide if we want to provide access logs + # assert self.access_logger + # self.access_logger.handlers = [_InterceptHandler()] + self.error_logger.handlers = [_InterceptHandler()] + + +class _InterceptHandler(logging.Handler): + def emit(self, record: logging.LogRecord): + try: + level = logger.level(record.levelname).name + except ValueError: + level = record.levelno + + logger.opt(depth=3, exception=record.exc_info).log(level, record.getMessage()) + + def logger_setup(log_file: Path | None, verbosity: int = 0): """Set up logging for this process - formatting, file handles, verbosity and output""" logger.remove() + + # replace all stdlib loggers with _InterceptHandlers that log to loguru + logging.basicConfig(handlers=[_InterceptHandler()], level=0) + if verbosity == 0: logger.add( sys.__stderr__, # type: ignore @@ -37,3 +64,29 @@ def logger_setup(log_file: Path | None, verbosity: int = 0): def logger_cleanup(): """Flush all queues before shutting down so any in-flight logs are written to disk""" logger.complete() + + +""" --- TODO: Capture MLX Log output: +import contextlib +import sys +from loguru import logger + +class StreamToLogger: + + def __init__(self, level="INFO"): + self._level = level + + def write(self, buffer): + for line in buffer.rstrip().splitlines(): + logger.opt(depth=1).log(self._level, line.rstrip()) + + def flush(self): + pass + +logger.remove() +logger.add(sys.__stdout__) + +stream = StreamToLogger() +with contextlib.redirect_stdout(stream): + print("Standard output is sent to added handlers.") +""" diff --git a/src/exo/shared/tests/conftest.py b/src/exo/shared/tests/conftest.py index 4b982c42..1a6092f1 100644 --- a/src/exo/shared/tests/conftest.py +++ b/src/exo/shared/tests/conftest.py @@ -4,6 +4,8 @@ import asyncio from typing import Generator import pytest +from _pytest.logging import LogCaptureFixture +from loguru import logger from exo.shared.types.memory import Memory from exo.shared.types.models import ModelId, ModelMetadata @@ -41,3 +43,16 @@ def get_pipeline_shard_metadata( end_layer=32, n_layers=32, ) + + +@pytest.fixture +def caplog(caplog: LogCaptureFixture): + handler_id = logger.add( + caplog.handler, + format="{message}", + level=0, + filter=lambda record: record["level"].no >= caplog.handler.level, + enqueue=True, # Set to 'True' if your test is spawning child processes. + ) + yield caplog + logger.remove(handler_id) diff --git a/src/exo/shared/tests/test_node_id_persistence.py b/src/exo/shared/tests/test_node_id_persistence.py index cdcf19ca..8b241aa5 100644 --- a/src/exo/shared/tests/test_node_id_persistence.py +++ b/src/exo/shared/tests/test_node_id_persistence.py @@ -1,5 +1,4 @@ import contextlib -import logging import multiprocessing import os from multiprocessing import Event, Queue, Semaphore @@ -8,6 +7,7 @@ from multiprocessing.queues import Queue as QueueT from multiprocessing.synchronize import Event as EventT from multiprocessing.synchronize import Semaphore as SemaphoreT +from loguru import logger from pytest import LogCaptureFixture from exo.routing.router import get_node_id_keypair @@ -17,20 +17,13 @@ NUM_CONCURRENT_PROCS = 10 def _get_keypair_concurrent_subprocess_task( - pid: int, sem: SemaphoreT, ev: EventT, queue: QueueT[bytes] + sem: SemaphoreT, ev: EventT, queue: QueueT[bytes] ) -> None: - try: - # synchronise with parent process - logging.info(msg=f"SUBPROCESS {pid}: Started") - sem.release() - - # wait to be told to begin simultaneous read - ev.wait() - logging.info(msg=f"SUBPROCESS {pid}: Reading start") - queue.put(get_node_id_keypair().to_protobuf_encoding()) - logging.info(msg=f"SUBPROCESS {pid}: Reading end") - except Exception as e: - logging.error(msg=f"SUBPROCESS {pid}: Error encountered: {e}") + # synchronise with parent process + sem.release() + # wait to be told to begin simultaneous read + ev.wait() + queue.put(get_node_id_keypair().to_protobuf_encoding()) def _get_keypair_concurrent(num_procs: int) -> bytes: @@ -41,11 +34,11 @@ def _get_keypair_concurrent(num_procs: int) -> bytes: queue: QueueT[bytes] = Queue(maxsize=num_procs) # make parent process wait for all subprocesses to start - logging.info(msg=f"PARENT: Starting {num_procs} subprocesses") + logger.info(f"PARENT: Starting {num_procs} subprocesses") ps: list[BaseProcess] = [] - for i in range(num_procs): + for _ in range(num_procs): p = multiprocessing.get_context("fork").Process( - target=_get_keypair_concurrent_subprocess_task, args=(i + 1, sem, ev, queue) + target=_get_keypair_concurrent_subprocess_task, args=(sem, ev, queue) ) ps.append(p) p.start() @@ -53,7 +46,7 @@ def _get_keypair_concurrent(num_procs: int) -> bytes: sem.acquire() # start all the sub processes simultaneously - logging.info(msg="PARENT: Beginning read") + logger.info("PARENT: Beginning read") ev.set() # wait until all subprocesses are done & read results @@ -62,7 +55,7 @@ def _get_keypair_concurrent(num_procs: int) -> bytes: # check that the input/output order match, and that # all subprocesses end up reading the same file - logging.info(msg="PARENT: Checking consistency") + logger.info("PARENT: Checking consistency") keypair: bytes | None = None qsize = 0 # cannot use Queue.qsize due to MacOS incompatibility :( while not queue.empty(): @@ -88,7 +81,7 @@ def test_node_id_fetching(caplog: LogCaptureFixture): _delete_if_exists(EXO_NODE_ID_KEYPAIR) kp = _get_keypair_concurrent(NUM_CONCURRENT_PROCS) - with caplog.at_level(logging.CRITICAL): # supress logs + with caplog.at_level(101): # supress logs # make sure that continuous fetches return the same value for _ in range(reps): assert kp == _get_keypair_concurrent(NUM_CONCURRENT_PROCS) diff --git a/src/exo/utils/banner.py b/src/exo/utils/banner.py index cae6eac3..eb6d7b08 100644 --- a/src/exo/utils/banner.py +++ b/src/exo/utils/banner.py @@ -1,6 +1,7 @@ def print_startup_banner(port: int) -> None: """Print a prominent startup banner with API endpoint information.""" - banner = """ + dashboard_url = f"http://localhost:{port}" + banner = f""" ╔═══════════════════════════════════════════════════════════════════════╗ ║ ║ ║ ███████╗██╗ ██╗ ██████╗ ║ @@ -13,11 +14,7 @@ def print_startup_banner(port: int) -> None: ║ Distributed AI Inference Cluster ║ ║ ║ ╚═══════════════════════════════════════════════════════════════════════╝ -""" - dashboard_url = f"http://localhost:{port}" - - api_info = f""" ╔═══════════════════════════════════════════════════════════════════════╗ ║ ║ ║ 🌐 Dashboard & API Ready ║ @@ -27,8 +24,7 @@ def print_startup_banner(port: int) -> None: ║ Click the URL above to open the dashboard in your browser ║ ║ ║ ╚═══════════════════════════════════════════════════════════════════════╝ + """ print(banner) - print(api_info) - print() diff --git a/src/exo/utils/channels.py b/src/exo/utils/channels.py index 72caa7ea..3db08d6b 100644 --- a/src/exo/utils/channels.py +++ b/src/exo/utils/channels.py @@ -77,9 +77,6 @@ class _MpEndOfStream: pass -MP_END_OF_STREAM = _MpEndOfStream() - - class MpState[T]: def __init__(self, max_buffer_size: float): if max_buffer_size == inf: @@ -133,7 +130,7 @@ class MpSender[T]: def close(self) -> None: if not self._state.closed.is_set(): self._state.closed.set() - self._state.buffer.put(MP_END_OF_STREAM) + self._state.buffer.put(_MpEndOfStream()) self._state.buffer.close() # == unique to Mp channels == @@ -177,10 +174,9 @@ class MpReceiver[T]: try: item = self._state.buffer.get(block=False) - if item == MP_END_OF_STREAM: + if isinstance(item, _MpEndOfStream): self.close() raise EndOfStream - assert not isinstance(item, _MpEndOfStream) return item except Empty: raise WouldBlock from None @@ -193,10 +189,9 @@ class MpReceiver[T]: return self.receive_nowait() except WouldBlock: item = self._state.buffer.get() - if item == MP_END_OF_STREAM: + if isinstance(item, _MpEndOfStream): self.close() raise EndOfStream from None - assert not isinstance(item, _MpEndOfStream) return item # nb: this function will not cancel particularly well diff --git a/src/exo/worker/download/impl_shard_downloader.py b/src/exo/worker/download/impl_shard_downloader.py index d6c59a80..46f55ff9 100644 --- a/src/exo/worker/download/impl_shard_downloader.py +++ b/src/exo/worker/download/impl_shard_downloader.py @@ -20,7 +20,6 @@ def exo_shard_downloader(max_parallel_downloads: int = 8) -> ShardDownloader: async def build_base_shard(model_id: str) -> ShardMetadata: model_meta = await get_model_meta(model_id) - # print(f"build_base_shard {model_id=} {model_meta=}") return PipelineShardMetadata( model_meta=model_meta, device_rank=0, @@ -92,10 +91,8 @@ class CachedShardDownloader(ShardDownloader): self, shard: ShardMetadata, config_only: bool = False ) -> Path: if (shard.model_meta.model_id, shard) in self.cache: - # print(f"ensure_shard cache hit {shard=}") return self.cache[(shard.model_meta.model_id, shard)] - # print(f"ensure_shard cache miss {shard=}") target_dir = await self.shard_downloader.ensure_shard(shard, config_only) self.cache[(shard.model_meta.model_id, shard)] = target_dir return target_dir @@ -135,7 +132,6 @@ class ResumableShardDownloader(ShardDownloader): ) -> Path: allow_patterns = ["config.json"] if config_only else None - # print(f"ensure_shard {shard=} {config_only=} {allow_patterns=}") target_dir, _ = await download_shard( shard, self.on_progress_wrapper, @@ -147,7 +143,6 @@ class ResumableShardDownloader(ShardDownloader): async def get_shard_download_status( self, ) -> AsyncIterator[tuple[Path, RepoDownloadProgress]]: - # print("get_shard_download_status") async def _status_for_model( model_id: str, ) -> tuple[Path, RepoDownloadProgress]: @@ -165,9 +160,8 @@ class ResumableShardDownloader(ShardDownloader): for task in asyncio.as_completed(tasks): try: - result = await task - path, progress = result - yield (path, progress) + yield await task + # TODO: except Exception except Exception as e: print("Error downloading shard:", e) diff --git a/src/exo/worker/engines/mlx/constants.py b/src/exo/worker/engines/mlx/constants.py index 91c20de4..9b5db542 100644 --- a/src/exo/worker/engines/mlx/constants.py +++ b/src/exo/worker/engines/mlx/constants.py @@ -14,5 +14,3 @@ TEMPERATURE: float = 1.0 # TODO: We should really make this opt-in, but Kimi requires trust_remote_code=True TRUST_REMOTE_CODE: bool = True -# TODO: Do we really want this? -HIDE_THINKING: bool = False diff --git a/uv.lock b/uv.lock index 1b4e594a..d162d6b5 100644 --- a/uv.lock +++ b/uv.lock @@ -258,21 +258,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] -[[package]] -name = "click" -version = "8.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, -] - -[[package]] -name = "cobs" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/ef/ea149311227a4fc3160cc885fce06da7c7d76782a308ef070b8065c69953/cobs-1.2.2.tar.gz", hash = "sha256:dbdd5e32111d72786f83d0c269215dcd6ac629b1ac1962c6878221f3b2ca98da", size = 14582, upload-time = "2025-07-20T01:08:35.434Z" } - [[package]] name = "cryptography" version = "46.0.3" @@ -331,13 +316,13 @@ dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "base58", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "bidict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "cobs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "exo-pyo3-bindings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "greenlet", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "hypercorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "loguru", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "mlx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "mlx-lm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -354,7 +339,6 @@ dependencies = [ { name = "tiktoken", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typeguard", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "types-aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "uvicorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.dev-dependencies] @@ -373,13 +357,13 @@ requires-dist = [ { name = "anyio", specifier = "==4.11.0" }, { name = "base58", specifier = ">=2.1.1" }, { name = "bidict", specifier = ">=0.23.1" }, - { name = "cobs", specifier = ">=1.2.2" }, { name = "cryptography", specifier = ">=45.0.5" }, { name = "exo-pyo3-bindings", editable = "rust/exo_pyo3_bindings" }, { name = "fastapi", specifier = ">=0.116.1" }, { name = "filelock", specifier = ">=3.18.0" }, { name = "greenlet", specifier = ">=3.2.4" }, { name = "huggingface-hub", specifier = ">=0.33.4" }, + { name = "hypercorn", specifier = ">=0.18.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "mlx", specifier = ">=0.29.3" }, { name = "mlx-lm", specifier = ">=0.28.3" }, @@ -396,7 +380,6 @@ requires-dist = [ { name = "tiktoken", specifier = ">=0.12.0" }, { name = "typeguard", specifier = ">=4.4.4" }, { name = "types-aiofiles", specifier = ">=24.1.0.20250708" }, - { name = "uvicorn", specifier = ">=0.35.0" }, ] [package.metadata.requires-dev] @@ -557,6 +540,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "hyperframe", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, +] + [[package]] name = "hf-xet" version = "1.2.0" @@ -583,6 +579,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/68/89ac4e5b12a9ff6286a12174c8538a5930e2ed662091dd2572bbe0a18c8a/hf_xet-1.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a55558084c16b09b5ed32ab9ed38421e2d87cf3f1f89815764d1177081b99865", size = 3508920, upload-time = "2025-10-24T19:04:26.927Z" }, ] +[[package]] +name = "hpack" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, +] + [[package]] name = "huggingface-hub" version = "0.36.0" @@ -602,6 +607,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/bd/1a875e0d592d447cbc02805fd3fe0f497714d6a2583f59d14fa9ebad96eb/huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d", size = 566094, upload-time = "2025-10-23T12:11:59.557Z" }, ] +[[package]] +name = "hypercorn" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "h2", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "priority", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "wsproto", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/01/39f41a014b83dd5c795217362f2ca9071cf243e6a75bdcd6cd5b944658cc/hypercorn-0.18.0.tar.gz", hash = "sha256:d63267548939c46b0247dc8e5b45a9947590e35e64ee73a23c074aa3cf88e9da", size = 68420, upload-time = "2025-11-08T13:54:04.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/35/850277d1b17b206bd10874c8a9a3f52e059452fb49bb0d22cbb908f6038b/hypercorn-0.18.0-py3-none-any.whl", hash = "sha256:225e268f2c1c2f28f6d8f6db8f40cb8c992963610c5725e13ccfcddccb24b1cd", size = 61640, upload-time = "2025-11-08T13:54:03.202Z" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -926,6 +955,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "priority" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/3c/eb7c35f4dcede96fca1842dac5f4f5d15511aa4b52f3a961219e68ae9204/priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0", size = 24792, upload-time = "2021-06-27T10:15:05.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/5f/82c8074f7e84978129347c2c6ec8b6c59f3584ff1a20bc3c940a3e061790/priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", size = 8946, upload-time = "2021-06-27T10:15:03.856Z" }, +] + [[package]] name = "propcache" version = "0.4.1" @@ -1524,16 +1562,15 @@ wheels = [ ] [[package]] -name = "uvicorn" -version = "0.38.0" +name = "wsproto" +version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "h11", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/ce/f06b84e2697fef4688ca63bdb2fdf113ca0a3be33f94488f2cadb690b0cf/uvicorn-0.38.0.tar.gz", hash = "sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d", size = 80605, upload-time = "2025-10-18T13:46:44.63Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, ] [[package]]