mirror of
https://github.com/exo-explore/exo.git
synced 2026-09-08 19:41:32 -04:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex CheemaandClaude Opus 4.6
|
b0825335c7 |
No files matched your search
@@ -32,6 +32,7 @@ jobs:
|
||||
SPARKLE_ED25519_PRIVATE: ${{ secrets.SPARKLE_ED25519_PRIVATE }}
|
||||
SPARKLE_S3_BUCKET: ${{ secrets.SPARKLE_S3_BUCKET }}
|
||||
SPARKLE_S3_PREFIX: ${{ secrets.SPARKLE_S3_PREFIX }}
|
||||
EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT: ${{ secrets.EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
EXO_BUILD_NUMBER: ${{ github.run_number }}
|
||||
EXO_LIBP2P_NAMESPACE: ${{ github.ref_name }}
|
||||
@@ -158,7 +159,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Install Homebrew packages
|
||||
run: brew install just awscli
|
||||
run: brew install just awscli macmon
|
||||
|
||||
- name: Install UV
|
||||
uses: astral-sh/setup-uv@v6
|
||||
@@ -238,92 +239,10 @@ jobs:
|
||||
# Export keychain path for other steps
|
||||
echo "BUILD_KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV
|
||||
|
||||
# ============================================================
|
||||
# Pre-flight credential / profile validation
|
||||
# Runs BEFORE the ~16 min build so auth/expiry failures surface in <1 min.
|
||||
# ============================================================
|
||||
|
||||
- name: Validate Apple notarization credentials
|
||||
env:
|
||||
APPLE_NOTARIZATION_USERNAME: ${{ secrets.APPLE_NOTARIZATION_USERNAME }}
|
||||
APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
|
||||
APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
|
||||
run: |
|
||||
# All-or-nothing: either all three creds are set, or none are.
|
||||
CRED_COUNT=0
|
||||
for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
|
||||
[[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
|
||||
done
|
||||
if [[ "$CRED_COUNT" -eq 0 ]]; then
|
||||
echo "No notarization credentials configured — skipping notarization for this build."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$CRED_COUNT" -ne 3 ]]; then
|
||||
echo "ERROR: partial notarization credentials set ($CRED_COUNT/3). Aborting before build."
|
||||
exit 1
|
||||
fi
|
||||
# Cheap, ~5s, auth-only call. Fails instantly with a clear message if
|
||||
# the app-specific password is stale, wrong team-id, etc.
|
||||
echo "Verifying Apple notarization credentials via notarytool history..."
|
||||
if ! xcrun notarytool history \
|
||||
--apple-id "$APPLE_NOTARIZATION_USERNAME" \
|
||||
--password "$APPLE_NOTARIZATION_PASSWORD" \
|
||||
--team-id "$APPLE_NOTARIZATION_TEAM" >/dev/null; then
|
||||
echo "ERROR: notarytool rejected the provided credentials. Fix before rerunning."
|
||||
echo "Common causes: app-specific password expired/revoked, wrong team-id,"
|
||||
echo "Apple ID not on the team, or 2FA not configured for this Apple ID."
|
||||
exit 1
|
||||
fi
|
||||
echo "Apple notarization credentials OK."
|
||||
|
||||
- name: Validate provisioning profile expiry
|
||||
run: |
|
||||
PROFILE="$HOME/Library/Developer/Xcode/UserData/Provisioning Profiles/EXO.provisionprofile"
|
||||
if [[ ! -f "$PROFILE" ]]; then
|
||||
echo "ERROR: provisioning profile not found at $PROFILE"
|
||||
exit 1
|
||||
fi
|
||||
EXPIRY=$(security cms -D -i "$PROFILE" | plutil -extract ExpirationDate raw -o - - 2>/dev/null || true)
|
||||
if [[ -z "$EXPIRY" ]]; then
|
||||
echo "WARNING: could not read ExpirationDate from provisioning profile; skipping expiry check."
|
||||
exit 0
|
||||
fi
|
||||
# Try a couple of known plutil date formats. If none parse, skip the check rather
|
||||
# than risk a false-positive "expired" block on a format we didn't anticipate.
|
||||
EXPIRY_EPOCH=""
|
||||
for fmt in "%Y-%m-%dT%H:%M:%SZ" "%Y-%m-%d %H:%M:%S %z" "%Y-%m-%d %H:%M:%S +0000"; do
|
||||
if parsed=$(date -j -f "$fmt" "$EXPIRY" +%s 2>/dev/null); then
|
||||
EXPIRY_EPOCH="$parsed"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -z "$EXPIRY_EPOCH" ]]; then
|
||||
echo "WARNING: could not parse ExpirationDate '$EXPIRY'; skipping expiry check."
|
||||
exit 0
|
||||
fi
|
||||
NOW_EPOCH=$(date +%s)
|
||||
if [[ "$EXPIRY_EPOCH" -le "$NOW_EPOCH" ]]; then
|
||||
echo "ERROR: provisioning profile expired on $EXPIRY. Regenerate it before rerunning."
|
||||
exit 1
|
||||
fi
|
||||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
|
||||
echo "Provisioning profile valid until $EXPIRY ($DAYS_LEFT days remaining)."
|
||||
if [[ "$DAYS_LEFT" -lt 14 ]]; then
|
||||
echo "WARNING: profile expires in under 14 days — regenerate soon."
|
||||
fi
|
||||
|
||||
# ============================================================
|
||||
# Build the bundle
|
||||
# ============================================================
|
||||
|
||||
- name: Add pinned macmon to PATH
|
||||
run: |
|
||||
MACMON_DIR=$(nix develop --command sh -c 'dirname $(which macmon)')
|
||||
echo "Using macmon from: $MACMON_DIR"
|
||||
echo "$MACMON_DIR" >> $GITHUB_PATH
|
||||
# Remove any Homebrew macmon so PyInstaller can't accidentally pick it up
|
||||
brew uninstall macmon 2>/dev/null || true
|
||||
|
||||
- name: Build PyInstaller bundle
|
||||
run: uv run pyinstaller packaging/pyinstaller/exo.spec
|
||||
|
||||
@@ -346,6 +265,7 @@ jobs:
|
||||
EXO_BUILD_COMMIT="$GITHUB_SHA" \
|
||||
SPARKLE_FEED_URL="$SPARKLE_FEED_URL" \
|
||||
SPARKLE_ED25519_PUBLIC="$SPARKLE_ED25519_PUBLIC" \
|
||||
EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT="$EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT" \
|
||||
CODE_SIGNING_IDENTITY="$SIGNING_IDENTITY" \
|
||||
CODE_SIGN_INJECT_BASE_ENTITLEMENTS=YES
|
||||
mkdir -p ../../output
|
||||
@@ -378,41 +298,11 @@ jobs:
|
||||
APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
|
||||
APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
cd output
|
||||
security unlock-keychain -p "$MACOS_CERTIFICATE_PASSWORD" "$BUILD_KEYCHAIN_PATH"
|
||||
SIGNING_IDENTITY=$(security find-identity -v -p codesigning "$BUILD_KEYCHAIN_PATH" | awk -F '"' '{print $2}')
|
||||
|
||||
# Fail fast if notarization creds are partial. All-or-nothing.
|
||||
CRED_COUNT=0
|
||||
for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
|
||||
[[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
|
||||
done
|
||||
if [[ "$CRED_COUNT" -ne 0 && "$CRED_COUNT" -ne 3 ]]; then
|
||||
echo "ERROR: partial Apple notarization credentials set ($CRED_COUNT/3). Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
/usr/bin/codesign --deep --force --timestamp --options runtime \
|
||||
--sign "$SIGNING_IDENTITY" EXO.app
|
||||
|
||||
# Pre-flight: verify the signed app BEFORE building DMG and submitting to Apple.
|
||||
# If this fails, notarization will fail too — cheap way to fail in seconds, not 15 minutes.
|
||||
echo "===== codesign --verify EXO.app ====="
|
||||
if ! /usr/bin/codesign --verify --deep --strict --verbose=2 EXO.app; then
|
||||
echo "ERROR: EXO.app failed codesign verification. Dumping signing status of every executable:"
|
||||
find EXO.app -type f \( -perm -111 -o -name "*.dylib" -o -name "*.so" -o -name "*.framework" \) -print0 |
|
||||
while IFS= read -r -d '' f; do
|
||||
printf -- '--- %s\n' "$f"
|
||||
/usr/bin/codesign -dv --verbose=2 "$f" 2>&1 | sed 's/^/ /' || true
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Gatekeeper assessment. A failure here strongly predicts notarization rejection.
|
||||
echo "===== spctl assessment (predicts notarization outcome) ====="
|
||||
/usr/bin/spctl -a -vvv -t install EXO.app || echo "WARNING: spctl assessment failed — notarization is likely to fail too."
|
||||
|
||||
mkdir -p dmg-root
|
||||
cp -R EXO.app dmg-root/
|
||||
ln -s /Applications dmg-root/Applications
|
||||
@@ -420,22 +310,12 @@ jobs:
|
||||
hdiutil create -volname "EXO" -srcfolder dmg-root -ov -format UDZO "$DMG_NAME"
|
||||
/usr/bin/codesign --force --timestamp --options runtime \
|
||||
--sign "$SIGNING_IDENTITY" "$DMG_NAME"
|
||||
|
||||
echo "===== codesign --verify DMG ====="
|
||||
if ! /usr/bin/codesign --verify --verbose=2 "$DMG_NAME"; then
|
||||
echo "ERROR: DMG failed codesign verification."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "$APPLE_NOTARIZATION_USERNAME" ]]; then
|
||||
echo "===== notarytool submit ====="
|
||||
# `|| true` so set -e doesn't abort before we can echo output / fetch the log.
|
||||
# We rely on the parsed STATUS below to decide pass/fail.
|
||||
SUBMISSION_OUTPUT=$(xcrun notarytool submit "$DMG_NAME" \
|
||||
--apple-id "$APPLE_NOTARIZATION_USERNAME" \
|
||||
--password "$APPLE_NOTARIZATION_PASSWORD" \
|
||||
--team-id "$APPLE_NOTARIZATION_TEAM" \
|
||||
--wait --timeout 15m 2>&1) || true
|
||||
--wait --timeout 15m 2>&1)
|
||||
echo "$SUBMISSION_OUTPUT"
|
||||
|
||||
SUBMISSION_ID=$(echo "$SUBMISSION_OUTPUT" | awk 'tolower($1)=="id:" && $2 ~ /^[0-9a-fA-F-]+$/ {print $2; exit}')
|
||||
|
||||
@@ -91,6 +91,9 @@ jobs:
|
||||
nix build .#metal-toolchain
|
||||
fi
|
||||
|
||||
# Build mlx (depends on metal-toolchain)
|
||||
nix build .#mlx
|
||||
|
||||
- name: Build all Nix outputs
|
||||
run: |
|
||||
nix flake show --json | jq -r '
|
||||
|
||||
@@ -38,5 +38,3 @@ bench/**/*.json
|
||||
|
||||
# tmp
|
||||
tmp/models
|
||||
/build/exo
|
||||
/.claude/skills
|
||||
Generated
+31
@@ -0,0 +1,31 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="EMPTY_MODULE" version="4">
|
||||
<component name="FacetManager">
|
||||
<facet type="Python" name="Python facet">
|
||||
<configuration sdkName="Python 3.13 virtualenv at ~/Desktop/exo/.venv" />
|
||||
</facet>
|
||||
</component>
|
||||
<component name="Go" enabled="true" />
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/scripts/src" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/exo_pyo3_bindings/src" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/exo_pyo3_bindings/tests" isTestSource="true" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/util/src" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/networking/examples" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/networking/src" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/networking/tests" isTestSource="true" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/rust/system_custodian/src" isTestSource="false" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.venv" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.direnv" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/build" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/dist" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.go_cache" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/rust/target" />
|
||||
</content>
|
||||
<orderEntry type="jdk" jdkName="Python 3.13 (exo)" jdkType="Python SDK" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="library" name="Python 3.13 virtualenv at ~/Desktop/exo/.venv interpreter library" level="application" />
|
||||
</component>
|
||||
</module>
|
||||
Generated
+1
-1
@@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalDependencies">
|
||||
<plugin id="al.aoli.intellijdirenv" />
|
||||
<plugin id="systems.fehn.intellijdirenv" />
|
||||
</component>
|
||||
</project>
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<option name="ourVersions">
|
||||
<value>
|
||||
<list size="1">
|
||||
<item index="0" class="java.lang.String" itemvalue="3.14" />
|
||||
</list>
|
||||
</value>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
Generated
+3
@@ -4,4 +4,7 @@
|
||||
<option name="sdkName" value="Python 3.13 (exo)" />
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13 (exo)" project-jdk-type="Python SDK" />
|
||||
<component name="PythonCompatibilityInspectionAdvertiser">
|
||||
<option name="version" value="3" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -215,22 +215,6 @@ class StreamContext:
|
||||
traceback: object | None = ...,
|
||||
) -> None: ...
|
||||
|
||||
def device_info() -> dict[str, str | int]:
|
||||
"""
|
||||
Get information about the GPU device and system settings.
|
||||
|
||||
Currently returns:
|
||||
|
||||
* ``architecture``
|
||||
* ``max_buffer_size``
|
||||
* ``max_recommended_working_set_size``
|
||||
* ``memory_size``
|
||||
* ``resource_limit``
|
||||
|
||||
Returns:
|
||||
dict: A dictionary with string keys and string or integer values.
|
||||
"""
|
||||
|
||||
def abs(a: array, /, *, stream: Stream | Device | None = ...) -> array:
|
||||
"""
|
||||
Element-wise absolute value.
|
||||
@@ -1767,12 +1751,12 @@ def clip(
|
||||
array: The clipped array.
|
||||
"""
|
||||
|
||||
def compile[F: Callable[..., object]](
|
||||
fun: F,
|
||||
def compile(
|
||||
fun: Callable,
|
||||
inputs: object | None = ...,
|
||||
outputs: object | None = ...,
|
||||
shapeless: bool = ...,
|
||||
) -> F:
|
||||
) -> Callable:
|
||||
"""
|
||||
Returns a compiled function which produces the same output as ``fun``.
|
||||
|
||||
@@ -2396,7 +2380,7 @@ def degrees(a: array, /, *, stream: Stream | Device | None = ...) -> array:
|
||||
array: The angles in degrees.
|
||||
"""
|
||||
|
||||
def depends[T](inputs: T, dependencies: array | Sequence[array]) -> T:
|
||||
def depends(inputs: array | Sequence[array], dependencies: array | Sequence[array]):
|
||||
"""
|
||||
Insert dependencies between arrays in the graph. The outputs are
|
||||
identical to ``inputs`` but with dependencies on ``dependencies``.
|
||||
@@ -2915,8 +2899,8 @@ def gather_mm(
|
||||
a: array,
|
||||
b: array,
|
||||
/,
|
||||
lhs_indices: array | None = ...,
|
||||
rhs_indices: array | None = ...,
|
||||
lhs_indices: array,
|
||||
rhs_indices: array,
|
||||
*,
|
||||
sorted_indices: bool = ...,
|
||||
stream: Stream | Device | None = ...,
|
||||
@@ -4707,7 +4691,6 @@ def softmax(
|
||||
/,
|
||||
axis: int | Sequence[int] | None = ...,
|
||||
*,
|
||||
precise: bool = ...,
|
||||
stream: Stream | Device | None = ...,
|
||||
) -> array:
|
||||
"""
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from .layers import *
|
||||
from .utils import *
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from layers import *
|
||||
from utils import *
|
||||
|
||||
from . import init as init
|
||||
from . import losses as losses
|
||||
@@ -1,16 +1,20 @@
|
||||
from .activations import *
|
||||
from .base import *
|
||||
from .containers import *
|
||||
from .convolution import *
|
||||
from .convolution_transpose import *
|
||||
from .distributed import *
|
||||
from .dropout import *
|
||||
from .embedding import *
|
||||
from .linear import *
|
||||
from .normalization import *
|
||||
from .pooling import *
|
||||
from .positional_encoding import *
|
||||
from .quantized import *
|
||||
from .recurrent import *
|
||||
from .transformer import *
|
||||
from .upsample import *
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from activations import *
|
||||
from base import *
|
||||
from containers import *
|
||||
from convolution import *
|
||||
from convolution_transpose import *
|
||||
from distributed import *
|
||||
from dropout import *
|
||||
from embedding import *
|
||||
from linear import *
|
||||
from normalization import *
|
||||
from pooling import *
|
||||
from positional_encoding import *
|
||||
from quantized import *
|
||||
from recurrent import *
|
||||
from transformer import *
|
||||
from upsample import *
|
||||
@@ -53,14 +53,10 @@ class Module(dict):
|
||||
mx.eval(model.parameters())
|
||||
"""
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> mx.array: ...
|
||||
__call__: Callable
|
||||
def __init__(self) -> None:
|
||||
"""Should be called by the subclasses of ``Module``."""
|
||||
|
||||
def __getitem__(self, key: str) -> mx.array | Module: ...
|
||||
def get(
|
||||
self, key: str, default: mx.array | Module | None = ...
|
||||
) -> mx.array | Module | None: ...
|
||||
@property
|
||||
def training(self): # -> bool:
|
||||
"""Boolean indicating if the model is in training mode."""
|
||||
|
||||
@@ -32,7 +32,6 @@ class Conv1d(Module):
|
||||
"""
|
||||
|
||||
weight: mx.array
|
||||
bias: mx.array | None
|
||||
groups: int
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -40,10 +40,6 @@ class Linear(Module):
|
||||
bias (bool, optional): If set to ``False`` then the layer will
|
||||
not use a bias. Default is ``True``.
|
||||
"""
|
||||
|
||||
weight: mx.array
|
||||
bias: mx.array | None
|
||||
|
||||
def __init__(self, input_dims: int, output_dims: int, bias: bool = ...) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
def to_quantized(
|
||||
|
||||
@@ -88,9 +88,6 @@ class RMSNorm(Module):
|
||||
dims (int): The feature dimension of the input to normalize over
|
||||
eps (float): A small additive constant for numerical stability
|
||||
"""
|
||||
|
||||
weight: mx.array
|
||||
|
||||
def __init__(self, dims: int, eps: float = ...) -> None: ...
|
||||
def __call__(self, x) -> mx.array: ...
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
import mlx.core as mx
|
||||
from base import Module
|
||||
@@ -13,10 +13,8 @@ def quantize(
|
||||
bits: int = ...,
|
||||
*,
|
||||
mode: str = ...,
|
||||
class_predicate: Optional[
|
||||
Callable[[str, Module], Union[bool, dict[str, Any]]]
|
||||
] = ...,
|
||||
) -> None:
|
||||
class_predicate: Optional[Callable[[str, Module], Union[bool, dict]]] = ...,
|
||||
): # -> None:
|
||||
"""Quantize the sub-modules of a module according to a predicate.
|
||||
|
||||
By default all layers that define a ``to_quantized(group_size, bits)``
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
__version__ = ...
|
||||
@@ -3,12 +3,13 @@ This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Generator, List, Optional, Tuple, Union
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
from dataclasses import dataclass
|
||||
from collections import deque
|
||||
from typing import Any, Callable, Generator, List, Optional, Sequence, Tuple, Union
|
||||
from transformers import PreTrainedTokenizer
|
||||
|
||||
from .tokenizer_utils import TokenizerWrapper
|
||||
|
||||
DEFAULT_PROMPT = ...
|
||||
@@ -28,9 +29,8 @@ def str2bool(string): # -> bool:
|
||||
...
|
||||
def setup_arg_parser(): # -> ArgumentParser:
|
||||
"""Set up and return the argument parser."""
|
||||
...
|
||||
|
||||
generation_stream: mx.Stream
|
||||
generation_stream = ...
|
||||
|
||||
@contextlib.contextmanager
|
||||
def wired_limit(
|
||||
@@ -43,7 +43,6 @@ def wired_limit(
|
||||
async eval could be running pass in the streams to synchronize with prior
|
||||
to exiting the context manager.
|
||||
"""
|
||||
...
|
||||
@dataclass
|
||||
class GenerationResponse:
|
||||
"""
|
||||
@@ -74,11 +73,9 @@ class GenerationResponse:
|
||||
finish_reason: Optional[str] = ...
|
||||
|
||||
def maybe_quantize_kv_cache(
|
||||
prompt_cache: Any,
|
||||
quantized_kv_start: int | None,
|
||||
kv_group_size: int | None,
|
||||
kv_bits: int | None,
|
||||
) -> None: ...
|
||||
prompt_cache, quantized_kv_start, kv_group_size, kv_bits
|
||||
): # -> None:
|
||||
...
|
||||
def generate_step(
|
||||
prompt: mx.array,
|
||||
model: nn.Module,
|
||||
@@ -92,7 +89,7 @@ def generate_step(
|
||||
kv_bits: Optional[int] = ...,
|
||||
kv_group_size: int = ...,
|
||||
quantized_kv_start: int = ...,
|
||||
prompt_progress_callback: Optional[Callable[[int, int], None]] = ...,
|
||||
prompt_progress_callback: Optional[Callable[[int], int]] = ...,
|
||||
input_embeddings: Optional[mx.array] = ...,
|
||||
) -> Generator[Tuple[mx.array, mx.array], None, None]:
|
||||
"""
|
||||
@@ -118,7 +115,7 @@ def generate_step(
|
||||
kv_group_size (int): Group size for KV cache quantization. Default: ``64``.
|
||||
quantized_kv_start (int): Step to begin using a quantized KV cache.
|
||||
when ``kv_bits`` is non-None. Default: ``0``.
|
||||
prompt_progress_callback (Callable[[int, int], None]): A call-back which takes the
|
||||
prompt_progress_callback (Callable[[int], int]): A call-back which takes the
|
||||
prompt tokens processed so far and the total number of prompt tokens.
|
||||
input_embeddings (mx.array, optional): Input embeddings to use instead of or in
|
||||
conjunction with prompt tokens. Default: ``None``.
|
||||
@@ -126,7 +123,6 @@ def generate_step(
|
||||
Yields:
|
||||
Tuple[mx.array, mx.array]: One token and a vector of log probabilities.
|
||||
"""
|
||||
...
|
||||
|
||||
def speculative_generate_step(
|
||||
prompt: mx.array,
|
||||
@@ -172,7 +168,6 @@ def speculative_generate_step(
|
||||
Tuple[mx.array, mx.array, bool]: One token, a vector of log probabilities,
|
||||
and a bool indicating if the token was generated by the draft model
|
||||
"""
|
||||
...
|
||||
|
||||
def stream_generate(
|
||||
model: nn.Module,
|
||||
@@ -180,7 +175,7 @@ def stream_generate(
|
||||
prompt: Union[str, mx.array, List[int]],
|
||||
max_tokens: int = ...,
|
||||
draft_model: Optional[nn.Module] = ...,
|
||||
**kwargs: Any,
|
||||
**kwargs: object,
|
||||
) -> Generator[GenerationResponse, None, None]:
|
||||
"""
|
||||
A generator producing text based on the given prompt from the model.
|
||||
@@ -202,7 +197,6 @@ def stream_generate(
|
||||
GenerationResponse: An instance containing the generated text segment and
|
||||
associated metadata. See :class:`GenerationResponse` for details.
|
||||
"""
|
||||
...
|
||||
|
||||
def generate(
|
||||
model: nn.Module,
|
||||
@@ -223,9 +217,6 @@ def generate(
|
||||
kwargs: The remaining options get passed to :func:`stream_generate`.
|
||||
See :func:`stream_generate` for more details.
|
||||
"""
|
||||
...
|
||||
|
||||
def _merge_caches(caches: List[List[Any]]) -> List[Any]: ...
|
||||
@dataclass
|
||||
class BatchStats:
|
||||
"""
|
||||
@@ -249,263 +240,10 @@ class BatchStats:
|
||||
generation_time: float = ...
|
||||
peak_memory: float = ...
|
||||
|
||||
class SequenceStateMachine:
|
||||
"""A state machine that uses one Aho-Corasick trie per state to efficiently
|
||||
track state across a generated sequence.
|
||||
|
||||
The transitions are provided as state -> [(sequence, new_state)].
|
||||
|
||||
Example:
|
||||
|
||||
sm = SequenceStateMachine(
|
||||
transitions={
|
||||
"normal": [
|
||||
(think_start_tokens, "reasoning"),
|
||||
(tool_start_tokens, "tool"),
|
||||
(eos, None),
|
||||
],
|
||||
"reasoning": [
|
||||
(think_end_tokens, "normal"),
|
||||
(eos, None),
|
||||
],
|
||||
"tool": [
|
||||
(tool_end_tokens, None),
|
||||
(eos, None)
|
||||
],
|
||||
},
|
||||
initial="normal"
|
||||
)
|
||||
"""
|
||||
def __init__(self, transitions=..., initial=...) -> None: ...
|
||||
def __deepcopy__(self, memo): # -> SequenceStateMachine:
|
||||
...
|
||||
def make_state(self): # -> tuple[str, Any, dict[Any, Any]]:
|
||||
...
|
||||
@staticmethod
|
||||
def match(state, x): # -> tuple[tuple[Any, Any | None, Any], Any | None, Any]:
|
||||
...
|
||||
|
||||
class PromptProcessingBatch:
|
||||
"""
|
||||
A batch processor for prompt tokens with support for incremental processing.
|
||||
|
||||
This class handles batched prompt processing, managing KV caches and preparing
|
||||
tokens for generation. It supports extending, filtering, and splitting batches.
|
||||
"""
|
||||
@dataclass
|
||||
class Response:
|
||||
uid: int
|
||||
progress: tuple
|
||||
end_of_segment: bool
|
||||
end_of_prompt: bool
|
||||
...
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
uids: List[int],
|
||||
caches: List[List[Any]],
|
||||
tokens: Optional[List[List[int]]] = ...,
|
||||
prefill_step_size: int = ...,
|
||||
samplers: Optional[List[Callable[[mx.array], mx.array]]] = ...,
|
||||
fallback_sampler: Optional[Callable[[mx.array], mx.array]] = ...,
|
||||
logits_processors: Optional[
|
||||
List[List[Callable[[mx.array, mx.array], mx.array]]]
|
||||
] = ...,
|
||||
state_machines: Optional[List[SequenceStateMachine]] = ...,
|
||||
max_tokens: Optional[List[int]] = ...,
|
||||
) -> None: ...
|
||||
def __len__(self): # -> int:
|
||||
...
|
||||
def extract_cache(self, idx: int) -> List[Any]: ...
|
||||
def extend(self, batch): # -> None:
|
||||
...
|
||||
def split(self, indices: List[int]): # -> Self:
|
||||
...
|
||||
def filter(self, keep: List[int]): # -> None:
|
||||
...
|
||||
def prompt(self, tokens: List[List[int]]): # -> None:
|
||||
"""
|
||||
Process prompt tokens through the model.
|
||||
|
||||
Args:
|
||||
tokens: List of token sequences to process.
|
||||
"""
|
||||
...
|
||||
|
||||
def generate(self, tokens: List[List[int]]): # -> GenerationBatch:
|
||||
"""
|
||||
Transition from prompt processing to generation.
|
||||
|
||||
Args:
|
||||
tokens: Final tokens for each sequence to start generation.
|
||||
|
||||
Returns:
|
||||
A GenerationBatch ready for token generation.
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def empty(
|
||||
cls,
|
||||
model: nn.Module,
|
||||
fallback_sampler: Callable[[mx.array], mx.array],
|
||||
prefill_step_size: int = ...,
|
||||
): # -> Self:
|
||||
...
|
||||
|
||||
class GenerationBatch:
|
||||
"""
|
||||
A batched token generator that manages multiple sequences in parallel.
|
||||
|
||||
This class handles the generation phase after prompt processing, managing
|
||||
KV caches, sampling, and stop sequence detection for multiple sequences.
|
||||
"""
|
||||
@dataclass
|
||||
class Response:
|
||||
uid: int
|
||||
token: int
|
||||
logprobs: mx.array
|
||||
finish_reason: Optional[str]
|
||||
current_state: Optional[str]
|
||||
match_sequence: Optional[List[int]]
|
||||
prompt_cache: Optional[List[Any]]
|
||||
all_tokens: Optional[List[int]]
|
||||
...
|
||||
|
||||
model: nn.Module
|
||||
uids: List[int]
|
||||
prompt_cache: List[Any]
|
||||
tokens: List[List[int]]
|
||||
samplers: Optional[List[Callable[[mx.array], mx.array]]]
|
||||
fallback_sampler: Callable[[mx.array], mx.array]
|
||||
logits_processors: Optional[List[List[Callable[[mx.array, mx.array], mx.array]]]]
|
||||
state_machines: List[SequenceStateMachine]
|
||||
max_tokens: List[int]
|
||||
_current_tokens: Optional[mx.array]
|
||||
_current_logprobs: mx.array | List[mx.array]
|
||||
_next_tokens: Optional[mx.array]
|
||||
_next_logprobs: mx.array | List[mx.array]
|
||||
_token_context: List[Any]
|
||||
_num_tokens: List[int]
|
||||
_matcher_states: List[Any]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
uids: List[int],
|
||||
inputs: mx.array,
|
||||
prompt_cache: List[Any],
|
||||
tokens: List[List[int]],
|
||||
samplers: Optional[List[Callable[[mx.array], mx.array]]],
|
||||
fallback_sampler: Callable[[mx.array], mx.array],
|
||||
logits_processors: Optional[
|
||||
List[List[Callable[[mx.array, mx.array], mx.array]]]
|
||||
],
|
||||
state_machines: List[SequenceStateMachine],
|
||||
max_tokens: List[int],
|
||||
) -> None: ...
|
||||
def __len__(self) -> int: ...
|
||||
def extend(self, batch: GenerationBatch) -> None: ...
|
||||
def extract_cache(self, idx: int) -> List[Any]: ...
|
||||
def filter(self, keep: List[int]) -> None: ...
|
||||
def _step(self) -> Tuple[List[int], List[mx.array]]: ...
|
||||
def next(self) -> List[Response]:
|
||||
"""
|
||||
Generate the next batch of tokens.
|
||||
|
||||
Returns:
|
||||
List of Response objects for each sequence in the batch.
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def empty(
|
||||
cls, model: nn.Module, fallback_sampler: Callable[[mx.array], mx.array]
|
||||
): # -> Self:
|
||||
...
|
||||
|
||||
class BatchGenerator:
|
||||
"""
|
||||
A batch generator implements continuous batching.
|
||||
|
||||
This class provides automatic management of prompt processing and generation
|
||||
batches, handling the transition between the two.
|
||||
|
||||
It also allows for segmented prompt processing which guarantees that the
|
||||
generator will stop at these boundaries when processing an input.
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
max_tokens: int = ...,
|
||||
stop_tokens: Optional[Sequence[Sequence[int]]] = ...,
|
||||
sampler: Optional[Callable[[mx.array], mx.array]] = ...,
|
||||
logits_processors: Optional[
|
||||
List[Callable[[mx.array, mx.array], mx.array]]
|
||||
] = ...,
|
||||
completion_batch_size: int = ...,
|
||||
prefill_batch_size: int = ...,
|
||||
prefill_step_size: int = ...,
|
||||
) -> None: ...
|
||||
def close(self) -> None: ...
|
||||
def __del__(self): # -> None:
|
||||
...
|
||||
@contextlib.contextmanager
|
||||
def stats(self, stats=...): # -> Generator[Any | BatchStats, Any, None]:
|
||||
...
|
||||
_unprocessed_sequences: deque[tuple[Any, ...]]
|
||||
_prompt_batch: PromptProcessingBatch
|
||||
_generation_batch: GenerationBatch
|
||||
_currently_processing: list[Any]
|
||||
_gen_tokens_counter: int
|
||||
_steps_counter: int
|
||||
def _next(
|
||||
self,
|
||||
) -> tuple[
|
||||
List[PromptProcessingBatch.Response], List[GenerationBatch.Response]
|
||||
]: ...
|
||||
def insert(
|
||||
self,
|
||||
prompts: List[List[int]],
|
||||
max_tokens: Optional[List[int]] = ...,
|
||||
caches: Optional[List[List[Any]]] = ...,
|
||||
all_tokens: Optional[List[List[int]]] = ...,
|
||||
samplers: Optional[List[Callable[[mx.array], mx.array]]] = ...,
|
||||
logits_processors: Optional[
|
||||
List[List[Callable[[mx.array, mx.array], mx.array]]]
|
||||
] = ...,
|
||||
state_machines: Optional[List[SequenceStateMachine]] = ...,
|
||||
) -> List[int]: ...
|
||||
def insert_segments(
|
||||
self,
|
||||
segments: List[List[List[int]]],
|
||||
max_tokens: Optional[List[int]] = ...,
|
||||
caches: Optional[List[List[Any]]] = ...,
|
||||
all_tokens: Optional[List[List[int]]] = ...,
|
||||
samplers: Optional[List[Callable[[mx.array], mx.array]]] = ...,
|
||||
logits_processors: Optional[
|
||||
List[List[Callable[[mx.array, mx.array], mx.array]]]
|
||||
] = ...,
|
||||
state_machines: Optional[List[SequenceStateMachine]] = ...,
|
||||
) -> List[int]: ...
|
||||
def extract_cache(self, uids: List[int]) -> dict[int, Any]: ...
|
||||
def remove(
|
||||
self, uids: List[int], return_prompt_caches: bool = ...
|
||||
) -> dict[int, Any]: ...
|
||||
@property
|
||||
def prompt_cache_nbytes(self) -> int: ...
|
||||
def next(
|
||||
self,
|
||||
) -> tuple[
|
||||
List[PromptProcessingBatch.Response], List[GenerationBatch.Response]
|
||||
]: ...
|
||||
def next_generated(self) -> List[GenerationBatch.Response]: ...
|
||||
|
||||
@dataclass
|
||||
class BatchResponse:
|
||||
"""
|
||||
A data object to hold a batch generation response.
|
||||
An data object to hold a batch generation response.
|
||||
|
||||
Args:
|
||||
texts: (List[str]): The generated text for each prompt.
|
||||
@@ -514,18 +252,55 @@ class BatchResponse:
|
||||
|
||||
texts: List[str]
|
||||
stats: BatchStats
|
||||
caches: Optional[List[List[Any]]]
|
||||
...
|
||||
|
||||
@dataclass
|
||||
class Batch:
|
||||
uids: List[int]
|
||||
y: mx.array
|
||||
logprobs: mx.array
|
||||
max_tokens: List[int]
|
||||
num_tokens: List[int]
|
||||
cache: List[Any]
|
||||
def __len__(self): # -> int:
|
||||
...
|
||||
def filter(self, keep_idx: List[int]): # -> None:
|
||||
...
|
||||
def extend(self, other): # -> None:
|
||||
...
|
||||
|
||||
class BatchGenerator:
|
||||
@dataclass
|
||||
class Response:
|
||||
uid: int
|
||||
token: int
|
||||
logprobs: mx.array
|
||||
finish_reason: Optional[str]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model,
|
||||
max_tokens: int = ...,
|
||||
stop_tokens: Optional[set] = ...,
|
||||
sampler: Optional[Callable[[mx.array], mx.array]] = ...,
|
||||
completion_batch_size: int = ...,
|
||||
prefill_batch_size: int = ...,
|
||||
prefill_step_size: int = ...,
|
||||
) -> None: ...
|
||||
def insert(
|
||||
self, prompts, max_tokens: Union[List[int], int, None] = ...
|
||||
): # -> list[Any]:
|
||||
...
|
||||
def stats(self): # -> BatchStats:
|
||||
...
|
||||
def next(self): # -> list[Any]:
|
||||
...
|
||||
|
||||
def batch_generate(
|
||||
model,
|
||||
tokenizer,
|
||||
prompts: List[List[int]],
|
||||
prompt_caches: Optional[List[List[Any]]] = ...,
|
||||
prompts: List[int],
|
||||
max_tokens: Union[int, List[int]] = ...,
|
||||
verbose: bool = ...,
|
||||
return_prompt_caches: bool = ...,
|
||||
logits_processors: Optional[List[Callable[[mx.array, mx.array], mx.array]]] = ...,
|
||||
**kwargs,
|
||||
) -> BatchResponse:
|
||||
"""
|
||||
@@ -534,22 +309,14 @@ def batch_generate(
|
||||
Args:
|
||||
model (nn.Module): The language model.
|
||||
tokenizer (PreTrainedTokenizer): The tokenizer.
|
||||
prompts (List[List[int]]): The input prompts.
|
||||
prompt_caches (List[List[Any]], optional): Pre-computed prompt-caches
|
||||
for each input prompt. Note, unlike ``generate_step``, the caches
|
||||
won't be updated in-place.
|
||||
prompt (List[List[int]]): The input prompts.
|
||||
verbose (bool): If ``True``, print tokens and timing information.
|
||||
Default: ``False``.
|
||||
max_tokens (Union[int, List[int]): Maximum number of output tokens. This
|
||||
can be per prompt if a list is provided.
|
||||
return_prompt_caches (bool): Return the prompt caches in the batch
|
||||
responses. Default: ``False``.
|
||||
logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional):
|
||||
A list of functions that take tokens and logits and return the processed logits. Default: ``None``.
|
||||
kwargs: The remaining options get passed to :obj:`BatchGenerator`.
|
||||
See :obj:`BatchGenerator` for more details.
|
||||
"""
|
||||
...
|
||||
|
||||
def main(): # -> None:
|
||||
...
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
from functools import partial
|
||||
|
||||
@partial(mx.compile, shapeless=True)
|
||||
def swiglu(gate, x): ...
|
||||
@partial(mx.compile, shapeless=True)
|
||||
def xielu(x, alpha_p, alpha_n, beta, eps): # -> array:
|
||||
...
|
||||
|
||||
class XieLU(nn.Module):
|
||||
def __init__(
|
||||
self, alpha_p_init=..., alpha_n_init=..., beta=..., eps=...
|
||||
) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
@@ -3,7 +3,7 @@ This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
from typing import Optional
|
||||
|
||||
import mlx.core as mx
|
||||
|
||||
@@ -37,10 +37,10 @@ def quantized_scaled_dot_product_attention(
|
||||
bits: int = ...,
|
||||
) -> mx.array: ...
|
||||
def scaled_dot_product_attention(
|
||||
queries: mx.array,
|
||||
keys: mx.array,
|
||||
values: mx.array,
|
||||
cache: Optional[Any],
|
||||
queries,
|
||||
keys,
|
||||
values,
|
||||
cache,
|
||||
scale: float,
|
||||
mask: Optional[mx.array],
|
||||
sinks: Optional[mx.array] = ...,
|
||||
|
||||
@@ -16,7 +16,7 @@ class Cache(Protocol):
|
||||
self, keys: mx.array, values: mx.array
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
@property
|
||||
def state(self) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
def state(self) -> tuple[mx.array, mx.array]: ...
|
||||
@state.setter
|
||||
def state(self, v) -> None: ...
|
||||
|
||||
@@ -88,18 +88,17 @@ def create_attention_mask(
|
||||
) -> array | Literal["causal"] | None: ...
|
||||
|
||||
class _BaseCache(Cache):
|
||||
keys: mx.array | None
|
||||
values: mx.array | None
|
||||
keys: mx.array
|
||||
values: mx.array
|
||||
offset: int
|
||||
@property
|
||||
def state(self) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
def state(self) -> tuple[mx.array, mx.array]: ...
|
||||
@state.setter
|
||||
def state(self, v) -> None: ...
|
||||
@property
|
||||
def meta_state(self) -> Literal[""]: ...
|
||||
@meta_state.setter
|
||||
def meta_state(self, v) -> None: ...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def is_trimmable(self) -> Literal[False]: ...
|
||||
@classmethod
|
||||
def from_state(cls, state, meta_state) -> Self: ...
|
||||
@@ -115,13 +114,15 @@ class ConcatenateKVCache(_BaseCache):
|
||||
def update_and_fetch(self, keys, values): # -> tuple[Any | array, Any | array]:
|
||||
...
|
||||
@property
|
||||
def state(self) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
def state(self): # -> tuple[Any | array | None, Any | array | None]:
|
||||
...
|
||||
@state.setter
|
||||
def state(self, v): # -> None:
|
||||
...
|
||||
def is_trimmable(self): # -> Literal[True]:
|
||||
...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def trim(self, n): # -> int:
|
||||
...
|
||||
def make_mask(self, *args, **kwargs): # -> array | Literal['causal'] | None:
|
||||
...
|
||||
|
||||
@@ -131,7 +132,10 @@ class QuantizedKVCache(_BaseCache):
|
||||
def update_and_fetch(self, keys, values): # -> Any:
|
||||
...
|
||||
@property
|
||||
def state(self) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
def state(
|
||||
self,
|
||||
): # -> tuple[Any | tuple[array, array, array] | None, Any | tuple[array, array, array] | None] | Any:
|
||||
...
|
||||
@state.setter
|
||||
def state(self, v): # -> None:
|
||||
...
|
||||
@@ -143,7 +147,8 @@ class QuantizedKVCache(_BaseCache):
|
||||
...
|
||||
def is_trimmable(self): # -> Literal[True]:
|
||||
...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def trim(self, n): # -> int:
|
||||
...
|
||||
def make_mask(self, *args, **kwargs): # -> array | Literal['causal'] | None:
|
||||
...
|
||||
|
||||
@@ -155,30 +160,22 @@ class KVCache(_BaseCache):
|
||||
@property
|
||||
def state(
|
||||
self,
|
||||
) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
) -> tuple[array, array]: ...
|
||||
@state.setter
|
||||
def state(self, v) -> None: ...
|
||||
def is_trimmable(self): # -> Literal[True]:
|
||||
...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def trim(self, n): # -> int:
|
||||
...
|
||||
def to_quantized(
|
||||
self, group_size: int = ..., bits: int = ...
|
||||
) -> QuantizedKVCache: ...
|
||||
def make_mask(
|
||||
self, *args: Any, **kwargs: Any
|
||||
) -> mx.array | Literal["causal"] | None: ...
|
||||
def make_mask(self, *args, **kwargs): # -> array | Literal['causal'] | None:
|
||||
...
|
||||
|
||||
class RotatingKVCache(_BaseCache):
|
||||
step = ...
|
||||
keys: mx.array | None
|
||||
values: mx.array | None
|
||||
keep: int
|
||||
max_size: int
|
||||
_idx: int
|
||||
def __init__(self, max_size, keep=...) -> None: ...
|
||||
def _trim(
|
||||
self, trim_size: int, v: mx.array, append: mx.array | None = ...
|
||||
) -> mx.array: ...
|
||||
def update_and_fetch(
|
||||
self, keys, values
|
||||
): # -> tuple[array | Any, array | Any] | tuple[array | Any, array | Any | None]:
|
||||
@@ -186,16 +183,21 @@ class RotatingKVCache(_BaseCache):
|
||||
@property
|
||||
def state(
|
||||
self,
|
||||
) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
): # -> tuple[Any | array, Any | array] | tuple[Any | array | None, Any | array | None]:
|
||||
...
|
||||
@state.setter
|
||||
def state(self, v): # -> None:
|
||||
...
|
||||
@property
|
||||
def meta_state(self) -> tuple[str, ...]: ...
|
||||
def meta_state(self): # -> tuple[str, ...]:
|
||||
...
|
||||
@meta_state.setter
|
||||
def meta_state(self, v: tuple[str, ...]) -> None: ...
|
||||
def is_trimmable(self) -> bool: ...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def meta_state(self, v): # -> None:
|
||||
...
|
||||
def is_trimmable(self): # -> bool:
|
||||
...
|
||||
def trim(self, n): # -> int:
|
||||
...
|
||||
def to_quantized(
|
||||
self, group_size: int = ..., bits: int = ...
|
||||
) -> QuantizedKVCache: ...
|
||||
@@ -210,7 +212,8 @@ class ArraysCache(_BaseCache):
|
||||
...
|
||||
def __getitem__(self, idx): ...
|
||||
@property
|
||||
def state(self) -> tuple[mx.array | None, mx.array | None]: ...
|
||||
def state(self): # -> list[Any | array] | list[array]:
|
||||
...
|
||||
@state.setter
|
||||
def state(self, v): # -> None:
|
||||
...
|
||||
@@ -224,7 +227,8 @@ class ArraysCache(_BaseCache):
|
||||
In-place extend this cache with the other cache.
|
||||
"""
|
||||
|
||||
def make_mask(self, N: int) -> mx.array | None: ...
|
||||
def make_mask(self, N: int): # -> array | None:
|
||||
...
|
||||
|
||||
class MambaCache(ArraysCache):
|
||||
def __init__(self, left_padding: Optional[List[int]] = ...) -> None: ...
|
||||
@@ -235,7 +239,8 @@ class ChunkedKVCache(KVCache):
|
||||
...
|
||||
def update_and_fetch(self, keys, values): # -> tuple[array, array]:
|
||||
...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def trim(self, n): # -> int:
|
||||
...
|
||||
@property
|
||||
def meta_state(self): # -> tuple[str, ...]:
|
||||
...
|
||||
@@ -248,9 +253,10 @@ class CacheList(_BaseCache):
|
||||
def __getitem__(self, idx): ...
|
||||
def is_trimmable(self): # -> bool:
|
||||
...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def trim(self, n): ...
|
||||
@property
|
||||
def state(self) -> list[tuple[mx.array | None, mx.array | None]]: ...
|
||||
def state(self): # -> list[Any]:
|
||||
...
|
||||
@state.setter
|
||||
def state(self, v): # -> None:
|
||||
...
|
||||
@@ -265,14 +271,29 @@ class CacheList(_BaseCache):
|
||||
"""
|
||||
|
||||
class BatchKVCache(_BaseCache):
|
||||
step: int
|
||||
keys: array | None
|
||||
values: array | None
|
||||
offset: array
|
||||
left_padding: array
|
||||
_idx: int
|
||||
def __init__(self, left_padding: List[int]) -> None: ...
|
||||
def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
|
||||
step = ...
|
||||
def __init__(self, left_padding: List[int]) -> None:
|
||||
"""
|
||||
The BatchKV cache expects inputs to be left-padded.
|
||||
|
||||
E.g. the following prompts:
|
||||
|
||||
[1, 3, 5]
|
||||
[7]
|
||||
[2, 6, 8, 9]
|
||||
|
||||
Should be padded like so:
|
||||
|
||||
[0, 1, 3, 5]
|
||||
[0, 0, 0, 7]
|
||||
[2, 6, 8, 9]
|
||||
|
||||
And ``left_padding`` specifies the amount of padding for each.
|
||||
In this case, ``left_padding = [1, 3, 0]``.
|
||||
"""
|
||||
|
||||
def update_and_fetch(self, keys, values): # -> tuple[array | Any, array | Any]:
|
||||
...
|
||||
@property
|
||||
def state(
|
||||
self,
|
||||
@@ -298,21 +319,12 @@ class BatchKVCache(_BaseCache):
|
||||
"""
|
||||
|
||||
class BatchRotatingKVCache(_BaseCache):
|
||||
step: int
|
||||
keys: array | None
|
||||
values: array | None
|
||||
offset: array
|
||||
left_padding: array
|
||||
max_size: int
|
||||
_idx: int
|
||||
_offset: int
|
||||
rotated: bool
|
||||
_lengths: array | None
|
||||
def __init__(self, max_size: int, left_padding: List[int]) -> None: ...
|
||||
def _trim(self, trim_size: int, v: array, append: array | None = ...) -> array: ...
|
||||
def _update_in_place(self, keys: array, values: array) -> tuple[array, array]: ...
|
||||
def _update_concat(self, keys: array, values: array) -> tuple[array, array]: ...
|
||||
def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
|
||||
step = ...
|
||||
def __init__(self, max_size, left_padding: List[int]) -> None: ...
|
||||
def update_and_fetch(
|
||||
self, keys, values
|
||||
): # -> tuple[array | Any, array | Any] | tuple[array | Any, array | Any | None]:
|
||||
...
|
||||
@property
|
||||
def state(
|
||||
self,
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
"""Type stubs for mlx_lm.models.deepseek_v4"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .base import BaseModelArgs
|
||||
from .cache import ArraysCache, RotatingKVCache
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
@dataclass
|
||||
class ModelArgs(BaseModelArgs):
|
||||
model_type: str
|
||||
vocab_size: int
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
moe_intermediate_size: int
|
||||
num_hidden_layers: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
n_shared_experts: Optional[int]
|
||||
n_routed_experts: int
|
||||
num_experts_per_tok: int
|
||||
head_dim: int
|
||||
qk_rope_head_dim: int
|
||||
q_lora_rank: int
|
||||
o_lora_rank: int
|
||||
o_groups: int
|
||||
sliding_window: int
|
||||
hc_mult: int
|
||||
hc_sinkhorn_iters: int
|
||||
hc_eps: float
|
||||
compress_ratios: Optional[List[int]]
|
||||
compress_rope_theta: float
|
||||
rope_theta: float
|
||||
rope_scaling: Optional[Dict[str, Any]]
|
||||
rms_norm_eps: float
|
||||
swiglu_limit: float
|
||||
attention_bias: bool
|
||||
max_position_embeddings: int
|
||||
|
||||
class DeepseekV4RoPE(nn.Module):
|
||||
dims: int
|
||||
freqs: mx.array
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dims: int,
|
||||
base: float,
|
||||
scaling_config: Optional[Dict[str, Any]] = None,
|
||||
) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
offset: int = 0,
|
||||
inverse: bool = False,
|
||||
) -> mx.array: ...
|
||||
|
||||
class HyperConnection(nn.Module):
|
||||
dim: int
|
||||
hc_mult: int
|
||||
norm_eps: float
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
hc_mult: int,
|
||||
norm_eps: float,
|
||||
sinkhorn_iters: int,
|
||||
hc_eps: float,
|
||||
) -> None: ...
|
||||
|
||||
class HyperHead(nn.Module):
|
||||
dim: int
|
||||
hc_mult: int
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
hc_mult: int,
|
||||
norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class Compressor(nn.Module):
|
||||
dim: int
|
||||
head_dim: int
|
||||
rope_head_dim: int
|
||||
compress_ratio: int
|
||||
overlap: bool
|
||||
wkv_gate: nn.Linear
|
||||
ape: mx.array
|
||||
norm: nn.RMSNorm
|
||||
rope: DeepseekV4RoPE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
compress_ratio: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
rms_norm_eps: float,
|
||||
rope: DeepseekV4RoPE,
|
||||
) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
cache: "DeepseekV4Cache",
|
||||
offset: Any,
|
||||
key: str = ...,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Indexer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
args: ModelArgs,
|
||||
compress_ratio: int,
|
||||
rope: DeepseekV4RoPE,
|
||||
) -> None: ...
|
||||
|
||||
class _CompressorBranch:
|
||||
buffer_kv: Optional[mx.array]
|
||||
buffer_gate: Optional[mx.array]
|
||||
prev_kv: Optional[mx.array]
|
||||
prev_gate: Optional[mx.array]
|
||||
pool: Optional[mx.array]
|
||||
buffer_lengths: Optional[List[int]]
|
||||
pool_lengths: Optional[List[int]]
|
||||
buffer_count: int
|
||||
_new_pool_lengths: Optional[List[int]]
|
||||
|
||||
def __init__(self) -> None: ...
|
||||
|
||||
class DeepseekV4Cache:
|
||||
local: RotatingKVCache
|
||||
offset: int
|
||||
keys: Optional[mx.array]
|
||||
values: Optional[mx.array]
|
||||
state: Any
|
||||
meta_state: Any
|
||||
nbytes: int
|
||||
_branches: Dict[str, _CompressorBranch]
|
||||
_pending_lengths: Optional[List[int]]
|
||||
|
||||
def __init__(self, sliding_window: int) -> None: ...
|
||||
def update_and_fetch(
|
||||
self, keys: mx.array, values: mx.array
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
def is_trimmable(self) -> bool: ...
|
||||
def trim(self, n: int) -> int: ...
|
||||
def empty(self) -> bool: ...
|
||||
def size(self) -> int: ...
|
||||
def prepare(
|
||||
self,
|
||||
*,
|
||||
left_padding: Optional[List[int]] = None,
|
||||
lengths: Optional[List[int]] = None,
|
||||
right_padding: Optional[List[int]] = None,
|
||||
) -> None: ...
|
||||
def finalize(self) -> None: ...
|
||||
def filter(self, batch_indices: mx.array) -> None: ...
|
||||
def extend(self, other: "DeepseekV4Cache") -> None: ...
|
||||
def extract(self, idx: int) -> "DeepseekV4Cache": ...
|
||||
@classmethod
|
||||
def merge(cls, caches: List["DeepseekV4Cache"]) -> "DeepseekV4Cache": ...
|
||||
|
||||
class V4Attention(nn.Module):
|
||||
args: ModelArgs
|
||||
layer_id: int
|
||||
dim: int
|
||||
n_heads: int
|
||||
head_dim: int
|
||||
rope_head_dim: int
|
||||
nope_head_dim: int
|
||||
n_groups: int
|
||||
q_lora_rank: int
|
||||
o_lora_rank: int
|
||||
window: int
|
||||
eps: float
|
||||
scale: float
|
||||
compress_ratio: int
|
||||
wqkv_a: nn.Linear
|
||||
q_norm: nn.RMSNorm
|
||||
wq_b: nn.Linear
|
||||
kv_norm: nn.RMSNorm
|
||||
attn_sink: mx.array
|
||||
wo_a: nn.Linear
|
||||
wo_b: nn.Linear
|
||||
rope: DeepseekV4RoPE
|
||||
compressor: Compressor
|
||||
indexer: Indexer
|
||||
|
||||
def __init__(self, args: ModelArgs, layer_id: int) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
gate_proj: nn.Linear
|
||||
up_proj: nn.Linear
|
||||
down_proj: nn.Linear
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class MoEGate(nn.Module):
|
||||
weight: mx.array
|
||||
|
||||
def __init__(self, args: ModelArgs, layer_id: int) -> None: ...
|
||||
def __call__(
|
||||
self, x: mx.array, input_ids: mx.array
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
num_experts_per_tok: int
|
||||
switch_mlp: SwitchGLU
|
||||
gate: MoEGate
|
||||
shared_experts: DeepseekV4MLP
|
||||
|
||||
def __init__(self, args: ModelArgs, layer_id: int) -> None: ...
|
||||
def __call__(self, x: mx.array, input_ids: mx.array) -> mx.array: ...
|
||||
|
||||
class DeepseekV4Block(nn.Module):
|
||||
attn_norm: nn.RMSNorm
|
||||
attn: V4Attention
|
||||
hc_attn: HyperConnection
|
||||
ffn_norm: nn.RMSNorm
|
||||
ffn: DeepseekV4MoE
|
||||
hc_ffn: HyperConnection
|
||||
|
||||
def __init__(self, args: ModelArgs, layer_id: int) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
h: mx.array,
|
||||
cache: Optional[Any],
|
||||
input_ids: mx.array,
|
||||
) -> mx.array: ...
|
||||
|
||||
class DeepseekV4Model(nn.Module):
|
||||
args: ModelArgs
|
||||
vocab_size: int
|
||||
embed_tokens: nn.Embedding
|
||||
layers: list[DeepseekV4Block]
|
||||
norm: nn.RMSNorm
|
||||
hc_head: HyperHead
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[List[Any]] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
args: ModelArgs
|
||||
model_type: str
|
||||
model: DeepseekV4Model
|
||||
lm_head: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[List[Any]] = None,
|
||||
) -> mx.array: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
def make_cache(self) -> list[RotatingKVCache | DeepseekV4Cache]: ...
|
||||
@property
|
||||
def layers(self) -> list[DeepseekV4Block]: ...
|
||||
@@ -1,35 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
import mlx.core as mx
|
||||
|
||||
def compute_g(A_log: mx.array, a: mx.array, dt_bias: mx.array) -> mx.array: ...
|
||||
def gated_delta_update(
|
||||
q: mx.array,
|
||||
k: mx.array,
|
||||
v: mx.array,
|
||||
a: mx.array,
|
||||
b: mx.array,
|
||||
A_log: mx.array,
|
||||
dt_bias: mx.array,
|
||||
state: Optional[mx.array] = ...,
|
||||
mask: Optional[mx.array] = ...,
|
||||
use_kernel: bool = ...,
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
def gated_delta_ops(
|
||||
q: mx.array,
|
||||
k: mx.array,
|
||||
v: mx.array,
|
||||
g: mx.array,
|
||||
beta: mx.array,
|
||||
state: Optional[mx.array] = ...,
|
||||
mask: Optional[mx.array] = ...,
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
def gated_delta_kernel(
|
||||
q: mx.array,
|
||||
k: mx.array,
|
||||
v: mx.array,
|
||||
g: mx.array,
|
||||
beta: mx.array,
|
||||
state: mx.array,
|
||||
mask: Optional[mx.array] = ...,
|
||||
) -> tuple[mx.array, mx.array]: ...
|
||||
@@ -1,31 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from . import gemma4_text
|
||||
from .base import BaseModelArgs
|
||||
from .cache import KVCache, RotatingKVCache
|
||||
|
||||
@dataclass
|
||||
class ModelArgs(BaseModelArgs):
|
||||
model_type: str
|
||||
text_config: Optional[dict[str, Any]]
|
||||
vocab_size: int
|
||||
|
||||
def __post_init__(self) -> None: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
args: ModelArgs
|
||||
model_type: str
|
||||
language_model: gemma4_text.Model
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> mx.array: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@property
|
||||
def layers(self) -> list[gemma4_text.DecoderLayer]: ...
|
||||
@property
|
||||
def quant_predicate(self) -> Any: ...
|
||||
def make_cache(self) -> list[KVCache | RotatingKVCache]: ...
|
||||
@@ -1,179 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .base import BaseModelArgs
|
||||
from .cache import KVCache, RotatingKVCache
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
@dataclass
|
||||
class ModelArgs(BaseModelArgs):
|
||||
model_type: str
|
||||
hidden_size: int
|
||||
num_hidden_layers: int
|
||||
intermediate_size: int
|
||||
num_attention_heads: int
|
||||
head_dim: int
|
||||
global_head_dim: int
|
||||
global_partial_rotary_factor: float
|
||||
rms_norm_eps: float
|
||||
vocab_size: int
|
||||
vocab_size_per_layer_input: int
|
||||
num_key_value_heads: int
|
||||
num_global_key_value_heads: Optional[int]
|
||||
num_kv_shared_layers: int
|
||||
pad_token_id: int
|
||||
hidden_size_per_layer_input: int
|
||||
rope_traditional: bool
|
||||
partial_rotary_factor: float
|
||||
rope_parameters: Optional[Dict[str, Any]]
|
||||
sliding_window: int
|
||||
sliding_window_pattern: int
|
||||
max_position_embeddings: int
|
||||
attention_k_eq_v: bool
|
||||
final_logit_softcapping: float
|
||||
use_double_wide_mlp: bool
|
||||
enable_moe_block: bool
|
||||
num_experts: Optional[int]
|
||||
top_k_experts: Optional[int]
|
||||
moe_intermediate_size: Optional[int]
|
||||
layer_types: Optional[List[str]]
|
||||
tie_word_embeddings: bool
|
||||
|
||||
def __post_init__(self) -> None: ...
|
||||
|
||||
class MLP(nn.Module):
|
||||
gate_proj: nn.Linear
|
||||
down_proj: nn.Linear
|
||||
up_proj: nn.Linear
|
||||
|
||||
def __init__(self, config: ModelArgs, layer_idx: int = 0) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class Router(nn.Module):
|
||||
proj: nn.Linear
|
||||
scale: mx.array
|
||||
per_expert_scale: mx.array
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(self, x: mx.array) -> tuple[mx.array, mx.array]: ...
|
||||
|
||||
class Experts(nn.Module):
|
||||
switch_glu: SwitchGLU
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self, x: mx.array, top_k_indices: mx.array, top_k_weights: mx.array
|
||||
) -> mx.array: ...
|
||||
|
||||
class Attention(nn.Module):
|
||||
layer_idx: int
|
||||
layer_type: str
|
||||
is_sliding: bool
|
||||
head_dim: int
|
||||
n_heads: int
|
||||
n_kv_heads: int
|
||||
use_k_eq_v: bool
|
||||
scale: float
|
||||
q_proj: nn.Linear
|
||||
k_proj: nn.Linear
|
||||
v_proj: nn.Linear
|
||||
o_proj: nn.Linear
|
||||
q_norm: nn.Module
|
||||
k_norm: nn.Module
|
||||
v_norm: nn.Module
|
||||
rope: nn.Module
|
||||
|
||||
def __init__(self, config: ModelArgs, layer_idx: int) -> None: ...
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
|
||||
|
||||
class DecoderLayer(nn.Module):
|
||||
layer_idx: int
|
||||
layer_type: str
|
||||
self_attn: Attention
|
||||
mlp: MLP
|
||||
enable_moe: bool
|
||||
router: Router
|
||||
experts: Experts
|
||||
input_layernorm: nn.Module
|
||||
post_attention_layernorm: nn.Module
|
||||
pre_feedforward_layernorm: nn.Module
|
||||
post_feedforward_layernorm: nn.Module
|
||||
post_feedforward_layernorm_1: nn.Module
|
||||
post_feedforward_layernorm_2: nn.Module
|
||||
pre_feedforward_layernorm_2: nn.Module
|
||||
hidden_size_per_layer_input: int
|
||||
per_layer_input_gate: Optional[nn.Linear]
|
||||
per_layer_projection: Optional[nn.Linear]
|
||||
post_per_layer_input_norm: Optional[nn.Module]
|
||||
layer_scalar: mx.array
|
||||
|
||||
def __init__(self, config: ModelArgs, layer_idx: int) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = ...,
|
||||
cache: Optional[Any] = ...,
|
||||
per_layer_input: Optional[mx.array] = ...,
|
||||
shared_kv: Optional[tuple[mx.array, mx.array]] = ...,
|
||||
offset: Optional[mx.array] = ...,
|
||||
) -> tuple[mx.array, tuple[mx.array, mx.array], mx.array]: ...
|
||||
|
||||
class Gemma4TextModel(nn.Module):
|
||||
config: ModelArgs
|
||||
vocab_size: int
|
||||
window_size: int
|
||||
sliding_window_pattern: int
|
||||
num_hidden_layers: int
|
||||
embed_tokens: nn.Embedding
|
||||
embed_scale: float
|
||||
layers: list[DecoderLayer]
|
||||
norm: nn.Module
|
||||
hidden_size_per_layer_input: int
|
||||
embed_tokens_per_layer: Optional[nn.Embedding]
|
||||
per_layer_model_projection: Optional[nn.Linear]
|
||||
per_layer_projection_norm: Optional[nn.Module]
|
||||
previous_kvs: list[int]
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: Optional[mx.array] = ...,
|
||||
cache: Optional[list[Any]] = ...,
|
||||
input_embeddings: Optional[mx.array] = ...,
|
||||
per_layer_inputs: Optional[mx.array] = ...,
|
||||
) -> mx.array: ...
|
||||
def _get_per_layer_inputs(
|
||||
self,
|
||||
input_ids: Optional[mx.array],
|
||||
input_embeddings: Optional[mx.array] = ...,
|
||||
) -> mx.array: ...
|
||||
def _project_per_layer_inputs(
|
||||
self,
|
||||
input_embeddings: mx.array,
|
||||
per_layer_inputs: Optional[mx.array] = ...,
|
||||
) -> mx.array: ...
|
||||
def _make_masks(self, h: mx.array, cache: list[Any]) -> list[Any]: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
args: ModelArgs
|
||||
model_type: str
|
||||
model: Gemma4TextModel
|
||||
final_logit_softcapping: float
|
||||
tie_word_embeddings: bool
|
||||
lm_head: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> mx.array: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@property
|
||||
def layers(self) -> list[DecoderLayer]: ...
|
||||
@property
|
||||
def head_dim(self) -> int: ...
|
||||
@property
|
||||
def n_kv_heads(self) -> int: ...
|
||||
@property
|
||||
def quant_predicate(self) -> Any: ...
|
||||
def make_cache(self) -> list[KVCache | RotatingKVCache]: ...
|
||||
@@ -1,103 +0,0 @@
|
||||
"""Type stubs for mlx_lm.models.gpt_oss"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .base import BaseModelArgs
|
||||
from .cache import KVCache
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
@dataclass
|
||||
class ModelArgs(BaseModelArgs):
|
||||
model_type: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_hidden_layers: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
num_local_experts: int
|
||||
num_experts_per_tok: int
|
||||
vocab_size: int
|
||||
rms_norm_eps: float
|
||||
sliding_window: int
|
||||
layer_types: Optional[List[str]]
|
||||
|
||||
def mlx_topk(a: mx.array, k: int, axis: int = -1) -> tuple[mx.array, mx.array]: ...
|
||||
|
||||
class AttentionBlock(nn.Module):
|
||||
head_dim: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
num_key_value_groups: int
|
||||
sinks: mx.array
|
||||
q_proj: nn.Linear
|
||||
k_proj: nn.Linear
|
||||
v_proj: nn.Linear
|
||||
o_proj: nn.Linear
|
||||
sm_scale: float
|
||||
rope: nn.Module
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class TransformerBlock(nn.Module):
|
||||
self_attn: AttentionBlock
|
||||
mlp: MLPBlock
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class MLPBlock(nn.Module):
|
||||
hidden_size: int
|
||||
num_local_experts: int
|
||||
num_experts_per_tok: int
|
||||
experts: SwitchGLU
|
||||
router: nn.Linear
|
||||
sharding_group: Optional[mx.distributed.Group]
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class GptOssMoeModel(nn.Module):
|
||||
embed_tokens: nn.Embedding
|
||||
norm: nn.RMSNorm
|
||||
layer_types: List[str]
|
||||
layers: list[TransformerBlock]
|
||||
window_size: int
|
||||
swa_idx: int
|
||||
ga_idx: int
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
model_type: str
|
||||
model: GptOssMoeModel
|
||||
lm_head: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
@property
|
||||
def layers(self) -> list[nn.Module]: ...
|
||||
def make_cache(self) -> list[KVCache]: ...
|
||||
@@ -1,94 +0,0 @@
|
||||
"""Type stubs for mlx_lm.models.minimax"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .base import BaseModelArgs
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
@dataclass
|
||||
class ModelArgs(BaseModelArgs):
|
||||
model_type: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_hidden_layers: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
num_local_experts: int
|
||||
num_experts_per_tok: int
|
||||
max_position_embeddings: int
|
||||
|
||||
class MiniMaxAttention(nn.Module):
|
||||
num_heads: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
head_dim: int
|
||||
scale: float
|
||||
q_proj: nn.Linear
|
||||
k_proj: nn.Linear
|
||||
v_proj: nn.Linear
|
||||
o_proj: nn.Linear
|
||||
q_norm: nn.Module
|
||||
k_norm: nn.Module
|
||||
rope: nn.Module
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class MiniMaxSparseMoeBlock(nn.Module):
|
||||
num_experts_per_tok: int
|
||||
gate: nn.Linear
|
||||
switch_mlp: SwitchGLU
|
||||
e_score_correction_bias: mx.array
|
||||
sharding_group: Optional[mx.distributed.Group]
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class MiniMaxDecoderLayer(nn.Module):
|
||||
self_attn: MiniMaxAttention
|
||||
block_sparse_moe: MiniMaxSparseMoeBlock
|
||||
input_layernorm: nn.RMSNorm
|
||||
post_attention_layernorm: nn.RMSNorm
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class MiniMaxModel(nn.Module):
|
||||
embed_tokens: nn.Embedding
|
||||
layers: list[MiniMaxDecoderLayer]
|
||||
norm: nn.RMSNorm
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
model_type: str
|
||||
model: MiniMaxModel
|
||||
lm_head: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
@property
|
||||
def layers(self) -> list[MiniMaxDecoderLayer]: ...
|
||||
@@ -1,168 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .cache import ArraysCache, KVCache
|
||||
from .switch_layers import SwitchMLP
|
||||
|
||||
@dataclass
|
||||
class ModelArgs:
|
||||
model_type: str
|
||||
vocab_size: int
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_hidden_layers: int
|
||||
max_position_embeddings: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
attention_bias: bool
|
||||
mamba_num_heads: int
|
||||
mamba_head_dim: int
|
||||
mamba_proj_bias: bool
|
||||
ssm_state_size: int
|
||||
conv_kernel: int
|
||||
n_groups: int
|
||||
mlp_bias: bool
|
||||
layer_norm_epsilon: float
|
||||
use_bias: bool
|
||||
use_conv_bias: bool
|
||||
hybrid_override_pattern: List[str]
|
||||
head_dim: Optional[int]
|
||||
moe_intermediate_size: Optional[int]
|
||||
moe_shared_expert_intermediate_size: Optional[int]
|
||||
n_group: Optional[int]
|
||||
n_routed_experts: Optional[int]
|
||||
n_shared_experts: Optional[int]
|
||||
topk_group: Optional[int]
|
||||
num_experts_per_tok: Optional[int]
|
||||
norm_topk_prob: Optional[bool]
|
||||
routed_scaling_factor: Optional[float]
|
||||
time_step_limit: Optional[Tuple[float, float]]
|
||||
time_step_min: Optional[float]
|
||||
time_step_max: Optional[float]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params: dict[str, Any]) -> ModelArgs: ...
|
||||
def __post_init__(self) -> None: ...
|
||||
|
||||
class NemotronHMamba2Mixer(nn.Module):
|
||||
num_heads: int
|
||||
hidden_size: int
|
||||
ssm_state_size: int
|
||||
conv_kernel_size: int
|
||||
intermediate_size: int
|
||||
n_groups: int
|
||||
head_dim: int
|
||||
conv_dim: int
|
||||
conv1d: nn.Conv1d
|
||||
in_proj: nn.Linear
|
||||
dt_bias: mx.array
|
||||
A_log: mx.array
|
||||
D: mx.array
|
||||
norm: nn.RMSNorm
|
||||
heads_per_group: int
|
||||
out_proj: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: mx.array,
|
||||
mask: Optional[mx.array],
|
||||
cache: Optional[ArraysCache] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class NemotronHAttention(nn.Module):
|
||||
hidden_size: int
|
||||
num_heads: int
|
||||
head_dim: int
|
||||
num_key_value_heads: int
|
||||
scale: float
|
||||
q_proj: nn.Linear
|
||||
k_proj: nn.Linear
|
||||
v_proj: nn.Linear
|
||||
o_proj: nn.Linear
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[KVCache] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class MoEGate(nn.Module):
|
||||
config: ModelArgs
|
||||
top_k: int
|
||||
norm_topk_prob: bool
|
||||
weight: mx.array
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(self, x: mx.array) -> tuple[mx.array, mx.array]: ...
|
||||
|
||||
class NemotronHMLP(nn.Module):
|
||||
up_proj: nn.Linear
|
||||
down_proj: nn.Linear
|
||||
|
||||
def __init__(
|
||||
self, args: ModelArgs, intermediate_size: Optional[int] = None
|
||||
) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class NemotronHMoE(nn.Module):
|
||||
config: ModelArgs
|
||||
num_experts_per_tok: int
|
||||
moe_latent_size: Optional[int]
|
||||
switch_mlp: SwitchMLP
|
||||
gate: MoEGate
|
||||
shared_experts: NemotronHMLP
|
||||
fc1_latent_proj: nn.Linear
|
||||
fc2_latent_proj: nn.Linear
|
||||
|
||||
def __init__(self, config: ModelArgs) -> None: ...
|
||||
def __call__(self, x: mx.array) -> mx.array: ...
|
||||
|
||||
class NemotronHBlock(nn.Module):
|
||||
block_type: str
|
||||
norm: nn.RMSNorm
|
||||
mixer: NemotronHMamba2Mixer | NemotronHAttention | NemotronHMLP | NemotronHMoE
|
||||
|
||||
def __init__(self, args: ModelArgs, block_type: str) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class NemotronHModel(nn.Module):
|
||||
embeddings: nn.Embedding
|
||||
layers: list[NemotronHBlock]
|
||||
norm_f: nn.RMSNorm
|
||||
fa_idx: int
|
||||
ssm_idx: int
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
args: ModelArgs
|
||||
backbone: NemotronHModel
|
||||
lm_head: nn.Linear
|
||||
model_type: str
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
@property
|
||||
def layers(self) -> list[NemotronHBlock]: ...
|
||||
def make_cache(self) -> list[ArraysCache | KVCache]: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@@ -1,153 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .cache import ArraysCache, KVCache
|
||||
from .qwen3_next import (
|
||||
Qwen3NextAttention as Attention,
|
||||
Qwen3NextMLP as MLP,
|
||||
Qwen3NextRMSNormGated as RMSNormGated,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
)
|
||||
|
||||
SparseMoeBlock = Qwen3NextSparseMoeBlock
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
@dataclass
|
||||
class TextModelArgs:
|
||||
model_type: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_hidden_layers: int
|
||||
num_attention_heads: int
|
||||
rms_norm_eps: float
|
||||
vocab_size: int
|
||||
num_key_value_heads: int
|
||||
max_position_embeddings: int
|
||||
linear_num_value_heads: int
|
||||
linear_num_key_heads: int
|
||||
linear_key_head_dim: int
|
||||
linear_value_head_dim: int
|
||||
linear_conv_kernel_dim: int
|
||||
tie_word_embeddings: bool
|
||||
attention_bias: bool
|
||||
head_dim: Optional[int]
|
||||
full_attention_interval: int
|
||||
num_experts: int
|
||||
num_experts_per_tok: int
|
||||
decoder_sparse_step: int
|
||||
shared_expert_intermediate_size: int
|
||||
moe_intermediate_size: int
|
||||
norm_topk_prob: bool
|
||||
rope_parameters: Optional[dict[str, Any]]
|
||||
partial_rotary_factor: float
|
||||
rope_theta: float
|
||||
rope_scaling: Optional[dict[str, Any]]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params: dict[str, Any]) -> TextModelArgs: ...
|
||||
def __post_init__(self) -> None: ...
|
||||
|
||||
class GatedDeltaNet(nn.Module):
|
||||
hidden_size: int
|
||||
num_v_heads: int
|
||||
num_k_heads: int
|
||||
head_k_dim: int
|
||||
head_v_dim: int
|
||||
key_dim: int
|
||||
value_dim: int
|
||||
conv_kernel_size: int
|
||||
conv_dim: int
|
||||
conv1d: nn.Conv1d
|
||||
in_proj_qkv: nn.Linear
|
||||
in_proj_z: nn.Linear
|
||||
in_proj_b: nn.Linear
|
||||
in_proj_a: nn.Linear
|
||||
dt_bias: mx.array
|
||||
A_log: mx.array
|
||||
norm: RMSNormGated
|
||||
out_proj: nn.Linear
|
||||
|
||||
def __init__(self, config: TextModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class DecoderLayer(nn.Module):
|
||||
is_linear: bool
|
||||
linear_attn: GatedDeltaNet
|
||||
self_attn: Attention
|
||||
input_layernorm: nn.RMSNorm
|
||||
post_attention_layernorm: nn.RMSNorm
|
||||
mlp: MLP | SparseMoeBlock
|
||||
|
||||
def __init__(self, args: TextModelArgs, layer_idx: int) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
x: mx.array,
|
||||
mask: Optional[mx.array] = None,
|
||||
cache: Optional[Any] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class Qwen3_5TextModel(nn.Module):
|
||||
embed_tokens: nn.Embedding
|
||||
layers: list[DecoderLayer]
|
||||
norm: nn.RMSNorm
|
||||
ssm_idx: int
|
||||
fa_idx: int
|
||||
|
||||
def __init__(self, args: TextModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
input_embeddings: Optional[mx.array] = None,
|
||||
) -> mx.array: ...
|
||||
|
||||
class TextModel(nn.Module):
|
||||
args: TextModelArgs
|
||||
model_type: str
|
||||
model: Qwen3_5TextModel
|
||||
lm_head: nn.Linear
|
||||
|
||||
def __init__(self, args: TextModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
input_embeddings: Optional[mx.array] = None,
|
||||
) -> mx.array: ...
|
||||
@property
|
||||
def layers(self) -> list[DecoderLayer]: ...
|
||||
def make_cache(self) -> list[ArraysCache | KVCache]: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
|
||||
@dataclass
|
||||
class ModelArgs:
|
||||
model_type: str
|
||||
text_config: dict[str, Any]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params: dict[str, Any]) -> ModelArgs: ...
|
||||
|
||||
class Model(nn.Module):
|
||||
args: ModelArgs
|
||||
model_type: str
|
||||
language_model: TextModel
|
||||
|
||||
def __init__(self, args: ModelArgs) -> None: ...
|
||||
def __call__(
|
||||
self,
|
||||
inputs: mx.array,
|
||||
cache: Optional[Any] = None,
|
||||
input_embeddings: Optional[mx.array] = None,
|
||||
) -> mx.array: ...
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@property
|
||||
def layers(self) -> list[DecoderLayer]: ...
|
||||
def make_cache(self) -> list[ArraysCache | KVCache]: ...
|
||||
@@ -1,19 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .cache import ArraysCache, KVCache
|
||||
from .qwen3_5 import DecoderLayer, Model as Qwen3_5Model, TextModel
|
||||
|
||||
@dataclass
|
||||
class ModelArgs:
|
||||
model_type: str
|
||||
text_config: dict[str, Any]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params: dict[str, Any]) -> ModelArgs: ...
|
||||
|
||||
class Model(Qwen3_5Model):
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@@ -5,18 +5,8 @@ from typing import Any, Optional
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
|
||||
from .cache import ArraysCache, KVCache
|
||||
from .switch_layers import SwitchGLU
|
||||
|
||||
class Qwen3NextRMSNormGated(nn.Module):
|
||||
eps: float
|
||||
weight: mx.array
|
||||
|
||||
def __init__(self, hidden_size: int, eps: float = ...) -> None: ...
|
||||
def __call__(
|
||||
self, hidden_states: mx.array, gate: mx.array | None = None
|
||||
) -> mx.array: ...
|
||||
|
||||
class Qwen3NextMLP(nn.Module):
|
||||
gate_proj: nn.Linear
|
||||
down_proj: nn.Linear
|
||||
@@ -71,7 +61,6 @@ class Qwen3NextAttention(nn.Module):
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
norm_topk_prob: bool
|
||||
num_experts: int
|
||||
num_experts_per_tok: int
|
||||
top_k: int
|
||||
gate: nn.Linear
|
||||
switch_mlp: SwitchGLU
|
||||
@@ -101,8 +90,6 @@ class Qwen3NextModel(nn.Module):
|
||||
embed_tokens: nn.Embedding
|
||||
layers: list[Qwen3NextDecoderLayer]
|
||||
norm: nn.RMSNorm
|
||||
ssm_idx: int
|
||||
fa_idx: int
|
||||
|
||||
def __init__(self, args: Any) -> None: ...
|
||||
def __call__(
|
||||
@@ -125,4 +112,3 @@ class Model(nn.Module):
|
||||
def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@property
|
||||
def layers(self) -> list[Qwen3NextDecoderLayer]: ...
|
||||
def make_cache(self) -> list[ArraysCache | KVCache]: ...
|
||||
@@ -1,51 +0,0 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
import mlx.nn as nn
|
||||
|
||||
class YarnRoPE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dims: int,
|
||||
traditional: bool = ...,
|
||||
max_position_embeddings: int = ...,
|
||||
base: float = ...,
|
||||
scaling_factor: float = ...,
|
||||
original_max_position_embeddings: int = ...,
|
||||
beta_fast: float = ...,
|
||||
beta_slow: float = ...,
|
||||
mscale: float = ...,
|
||||
mscale_all_dim: float = ...,
|
||||
) -> None: ...
|
||||
|
||||
class Llama3RoPE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dims: int,
|
||||
traditional: bool = ...,
|
||||
max_position_embeddings: int = ...,
|
||||
base: float = ...,
|
||||
scaling_factor: float = ...,
|
||||
original_max_position_embeddings: int = ...,
|
||||
low_freq_factor: float = ...,
|
||||
high_freq_factor: float = ...,
|
||||
) -> None: ...
|
||||
|
||||
class SuScaledRoPE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dims: int,
|
||||
traditional: bool = ...,
|
||||
max_position_embeddings: int = ...,
|
||||
base: float = ...,
|
||||
short_factor: Any = ...,
|
||||
long_factor: Any = ...,
|
||||
original_max_position_embeddings: int = ...,
|
||||
) -> None: ...
|
||||
|
||||
def initialize_rope(
|
||||
dims: int,
|
||||
base: float = ...,
|
||||
traditional: bool = ...,
|
||||
scaling_config: Optional[dict[str, Any]] = ...,
|
||||
max_position_embeddings: Optional[int] = ...,
|
||||
) -> nn.Module: ...
|
||||
@@ -73,9 +73,6 @@ class SwitchGLU(nn.Module):
|
||||
def __call__(self, x, indices) -> mx.array: ...
|
||||
|
||||
class SwitchMLP(nn.Module):
|
||||
fc1: SwitchLinear
|
||||
fc2: SwitchLinear
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_dims: int,
|
||||
|
||||
@@ -48,11 +48,7 @@ def make_logits_processors(
|
||||
logit_bias: Optional[Dict[int, float]] = ...,
|
||||
repetition_penalty: Optional[float] = ...,
|
||||
repetition_context_size: Optional[int] = ...,
|
||||
presence_penalty: Optional[float] = ...,
|
||||
presence_context_size: Optional[int] = ...,
|
||||
frequency_penalty: Optional[float] = ...,
|
||||
frequency_context_size: Optional[int] = ...,
|
||||
) -> list[Callable[[mx.array, mx.array], mx.array]]:
|
||||
): # -> list[Any]:
|
||||
"""
|
||||
Make logits processors for use with ``generate_step``.
|
||||
|
||||
|
||||
@@ -39,11 +39,11 @@ class StreamingDetokenizer:
|
||||
"""
|
||||
|
||||
__slots__ = ...
|
||||
def reset(self) -> None: ...
|
||||
def add_token(self, token: int) -> None: ...
|
||||
def finalize(self) -> None: ...
|
||||
def reset(self): ...
|
||||
def add_token(self, token): ...
|
||||
def finalize(self): ...
|
||||
@property
|
||||
def last_segment(self) -> str:
|
||||
def last_segment(self):
|
||||
"""Return the last segment of readable text since last time this property was accessed."""
|
||||
|
||||
class NaiveStreamingDetokenizer(StreamingDetokenizer):
|
||||
@@ -117,8 +117,6 @@ class TokenizerWrapper:
|
||||
think_end: str | None
|
||||
think_start_id: int | None
|
||||
think_end_id: int | None
|
||||
think_start_tokens: list[int] | None
|
||||
think_end_tokens: list[int] | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
import mlx.nn as nn
|
||||
|
||||
class DoRALinear(nn.Module):
|
||||
@staticmethod
|
||||
def from_base(
|
||||
linear: nn.Linear, r: int = ..., dropout: float = ..., scale: float = ...
|
||||
): # -> DoRALinear:
|
||||
...
|
||||
def fuse(self, dequantize: bool = ...): # -> QuantizedLinear | Linear:
|
||||
...
|
||||
def __init__(
|
||||
self,
|
||||
input_dims: int,
|
||||
output_dims: int,
|
||||
r: int = ...,
|
||||
dropout: float = ...,
|
||||
scale: float = ...,
|
||||
bias: bool = ...,
|
||||
) -> None: ...
|
||||
def set_linear(self, linear): # -> None:
|
||||
"""
|
||||
Set the self.linear layer and recompute self.m.
|
||||
"""
|
||||
...
|
||||
|
||||
def __call__(self, x): ...
|
||||
|
||||
class DoRAEmbedding(nn.Module):
|
||||
def from_base(
|
||||
embedding: nn.Embedding, r: int = ..., dropout: float = ..., scale: float = ...
|
||||
): # -> DoRAEmbedding:
|
||||
...
|
||||
def fuse(self, dequantize: bool = ...): # -> Embedding:
|
||||
...
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
dims: int,
|
||||
r: int = ...,
|
||||
dropout: float = ...,
|
||||
scale: float = ...,
|
||||
) -> None: ...
|
||||
def set_embedding(self, embedding: nn.Module): # -> None:
|
||||
...
|
||||
def __call__(self, x): ...
|
||||
def as_linear(self, x): ...
|
||||
@@ -1,66 +0,0 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
import mlx.nn as nn
|
||||
|
||||
class LoRALinear(nn.Module):
|
||||
@staticmethod
|
||||
def from_base(
|
||||
linear: nn.Linear, r: int = ..., dropout: float = ..., scale: float = ...
|
||||
): # -> LoRALinear:
|
||||
...
|
||||
def fuse(self, dequantize: bool = ...): # -> QuantizedLinear | Linear:
|
||||
...
|
||||
def __init__(
|
||||
self,
|
||||
input_dims: int,
|
||||
output_dims: int,
|
||||
r: int = ...,
|
||||
dropout: float = ...,
|
||||
scale: float = ...,
|
||||
bias: bool = ...,
|
||||
) -> None: ...
|
||||
def __call__(self, x): # -> array:
|
||||
...
|
||||
|
||||
class LoRASwitchLinear(nn.Module):
|
||||
@staticmethod
|
||||
def from_base(
|
||||
linear: nn.Module, r: int = ..., dropout: float = ..., scale: float = ...
|
||||
): # -> LoRASwitchLinear:
|
||||
...
|
||||
def fuse(self, dequantize: bool = ...): # -> QuantizedSwitchLinear | SwitchLinear:
|
||||
...
|
||||
def __init__(
|
||||
self,
|
||||
input_dims: int,
|
||||
output_dims: int,
|
||||
num_experts: int,
|
||||
r: int = ...,
|
||||
dropout: float = ...,
|
||||
scale: float = ...,
|
||||
bias: bool = ...,
|
||||
) -> None: ...
|
||||
def __call__(self, x, indices, sorted_indices=...): ...
|
||||
|
||||
class LoRAEmbedding(nn.Module):
|
||||
@staticmethod
|
||||
def from_base(
|
||||
embedding: nn.Embedding, r: int = ..., dropout: float = ..., scale: float = ...
|
||||
): # -> LoRAEmbedding:
|
||||
...
|
||||
def fuse(self, dequantize: bool = ...): # -> QuantizedEmbedding | Embedding:
|
||||
...
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
dims: int,
|
||||
r: int = ...,
|
||||
dropout: float = ...,
|
||||
scale: float = ...,
|
||||
) -> None: ...
|
||||
def __call__(self, x): # -> array:
|
||||
...
|
||||
def as_linear(self, x): # -> array:
|
||||
...
|
||||
@@ -1,57 +0,0 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
import mlx.nn as nn
|
||||
from typing import Dict
|
||||
|
||||
def build_schedule(schedule_config: Dict): # -> Any:
|
||||
"""
|
||||
Build a learning rate schedule from the given config.
|
||||
"""
|
||||
...
|
||||
|
||||
def linear_to_lora_layers(
|
||||
model: nn.Module, num_layers: int, config: Dict, use_dora: bool = ...
|
||||
): # -> None:
|
||||
"""
|
||||
Convert some of the models linear layers to lora layers.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The neural network model.
|
||||
num_layers (int): The number of blocks to convert to lora layers
|
||||
starting from the last layer.
|
||||
config (dict): More configuration parameters for LoRA, including the
|
||||
rank, scale, and optional layer keys.
|
||||
use_dora (bool): If True, uses DoRA instead of LoRA.
|
||||
Default: ``False``
|
||||
"""
|
||||
...
|
||||
|
||||
def load_adapters(model: nn.Module, adapter_path: str) -> nn.Module:
|
||||
"""
|
||||
Load any fine-tuned adapters / layers.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The neural network model.
|
||||
adapter_path (str): Path to the adapter configuration file.
|
||||
|
||||
Returns:
|
||||
nn.Module: The updated model with LoRA layers applied.
|
||||
"""
|
||||
...
|
||||
|
||||
def remove_lora_layers(model: nn.Module) -> nn.Module:
|
||||
"""
|
||||
Remove the LoRA layers from the model.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The model with LoRA layers.
|
||||
|
||||
Returns:
|
||||
nn.Module: The model without LoRA layers.
|
||||
"""
|
||||
...
|
||||
|
||||
def print_trainable_parameters(model): # -> None:
|
||||
...
|
||||
Whitespace-only changes.
@@ -1,12 +0,0 @@
|
||||
from typing import Any
|
||||
|
||||
def get_message_json(
|
||||
model_name: str,
|
||||
prompt: str,
|
||||
role: str = "user",
|
||||
skip_image_token: bool = False,
|
||||
skip_audio_token: bool = False,
|
||||
num_images: int = 0,
|
||||
num_audios: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> dict[str, Any]: ...
|
||||
@@ -1,15 +0,0 @@
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
class ImageProcessor:
|
||||
def preprocess(
|
||||
self, images: list[dict[str, Any]], **kwargs: Any
|
||||
) -> dict[str, Any]: ...
|
||||
def __call__(self, **kwargs: Any) -> dict[str, Any]: ...
|
||||
|
||||
def load_image_processor(
|
||||
model_path: str | Path, **kwargs: Any
|
||||
) -> ImageProcessor | None: ...
|
||||
def load_processor(
|
||||
model_path: str | Path, add_detokenizer: bool = ..., **kwargs: Any
|
||||
) -> ImageProcessor: ...
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,8 +0,0 @@
|
||||
from typing import Any, Self
|
||||
|
||||
class safe_open:
|
||||
def __init__(self, filename: str, framework: str = "pt") -> None: ...
|
||||
def __enter__(self) -> Self: ...
|
||||
def __exit__(self, *args: Any) -> None: ...
|
||||
def keys(self) -> list[str]: ...
|
||||
def get_tensor(self, name: str) -> Any: ...
|
||||
Vendored
+1
-2
@@ -1,8 +1,7 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"detachhead.basedpyright",
|
||||
"ms-python.python",
|
||||
"jnoortheen.nix-ide"
|
||||
"ms-python.python"
|
||||
],
|
||||
"unwantedRecommendations": [
|
||||
"ms-python.vscode-pylance",
|
||||
|
||||
Vendored
+1
-30
@@ -1,32 +1,3 @@
|
||||
{
|
||||
"files.associations": {
|
||||
"*.nix": "nix",
|
||||
},
|
||||
"nix.enableLanguageServer": true,
|
||||
"nix.serverPath": "nixd",
|
||||
"nix.serverSettings": {
|
||||
"nixd": {
|
||||
"formatting": {
|
||||
"command": ["nixpkgs-fmt"]
|
||||
},
|
||||
"nixpkgs": {
|
||||
"expr": "(builtins.getFlake \"path:${workspaceFolder}\").currentSystem.config._module.args.pkgs"
|
||||
},
|
||||
"options": {
|
||||
"flake-parts": {
|
||||
"expr": "(builtins.getFlake \"path:${workspaceFolder}\").debug.options"
|
||||
},
|
||||
"flake-parts-perSystem": {
|
||||
"expr": "(builtins.getFlake \"path:${workspaceFolder}\").currentSystem.options"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"[nix]": {
|
||||
"editor.defaultFormatter": "jnoortheen.nix-ide"
|
||||
},
|
||||
|
||||
"python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
|
||||
"basedpyright.analysis.configFilePath": "${workspaceFolder}/pyproject.toml",
|
||||
"basedpyright.importStrategy": "fromEnvironment",
|
||||
"basedpyright.importStrategy": "fromEnvironment"
|
||||
}
|
||||
+1
-123
@@ -11,18 +11,9 @@ To run EXO from source:
|
||||
```bash
|
||||
brew install uv
|
||||
```
|
||||
- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
|
||||
```bash
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
rustup toolchain install nightly
|
||||
```
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
|
||||
```bash
|
||||
cargo install --git https://github.com/vladkens/macmon \
|
||||
--rev a1cd06b6cc0d5e61db24fd8832e74cd992097a7d \
|
||||
macmon \
|
||||
--force
|
||||
brew install macmon
|
||||
```
|
||||
|
||||
```bash
|
||||
@@ -48,119 +39,6 @@ Write pure functions where possible. When adding new code, prefer Rust unless th
|
||||
|
||||
Run `nix fmt` to auto-format your code before submitting.
|
||||
|
||||
## Model Cards
|
||||
|
||||
EXO uses TOML-based model cards to define model metadata and capabilities. Model cards are stored in:
|
||||
- `resources/inference_model_cards/` for text generation models
|
||||
- `resources/image_model_cards/` for image generation models
|
||||
- `~/.exo/custom_model_cards/` for user-added custom models
|
||||
|
||||
### Adding a Model Card
|
||||
|
||||
To add a new model, create a TOML file with the following structure:
|
||||
|
||||
```toml
|
||||
model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
|
||||
n_layers = 16
|
||||
hidden_size = 2048
|
||||
supports_tensor = true
|
||||
tasks = ["TextGeneration"]
|
||||
family = "llama"
|
||||
quantization = "4bit"
|
||||
base_model = "Llama 3.2 1B"
|
||||
capabilities = ["text"]
|
||||
|
||||
[storage_size]
|
||||
in_bytes = 729808896
|
||||
```
|
||||
|
||||
### Required Fields
|
||||
|
||||
- `model_id`: Hugging Face model identifier
|
||||
- `n_layers`: Number of transformer layers
|
||||
- `hidden_size`: Hidden dimension size
|
||||
- `supports_tensor`: Whether the model supports tensor parallelism
|
||||
- `tasks`: List of supported tasks (`TextGeneration`, `TextToImage`, `ImageToImage`)
|
||||
- `family`: Model family (e.g., "llama", "deepseek", "qwen")
|
||||
- `quantization`: Quantization level (e.g., "4bit", "8bit", "bf16")
|
||||
- `base_model`: Human-readable base model name
|
||||
- `capabilities`: List of capabilities (e.g., `["text"]`, `["text", "thinking"]`)
|
||||
|
||||
### Optional Fields
|
||||
|
||||
- `components`: For multi-component models (like image models with separate text encoders and transformers)
|
||||
- `uses_cfg`: Whether the model uses classifier-free guidance (for image models)
|
||||
- `trust_remote_code`: Whether to allow remote code execution (defaults to `false` for security)
|
||||
|
||||
### Capabilities
|
||||
|
||||
The `capabilities` field defines what the model can do:
|
||||
- `text`: Standard text generation
|
||||
- `thinking`: Model supports chain-of-thought reasoning
|
||||
- `thinking_toggle`: Thinking can be enabled/disabled via `enable_thinking` parameter
|
||||
- `image_edit`: Model supports image-to-image editing (FLUX.1-Kontext)
|
||||
|
||||
### Security Note
|
||||
|
||||
By default, `trust_remote_code` is set to `false` for security. Only enable it if the model explicitly requires remote code execution from the Hugging Face hub.
|
||||
|
||||
## API Adapters
|
||||
|
||||
EXO supports multiple API formats through an adapter pattern. Adapters convert API-specific request formats to the internal `TextGenerationTaskParams` format and convert internal token chunks back to API-specific responses.
|
||||
|
||||
### Adapter Architecture
|
||||
|
||||
All adapters live in `src/exo/master/adapters/` and follow the same pattern:
|
||||
|
||||
1. Convert API-specific requests to `TextGenerationTaskParams`
|
||||
2. Handle both streaming and non-streaming response generation
|
||||
3. Convert internal `TokenChunk` objects to API-specific formats
|
||||
4. Manage error handling and edge cases
|
||||
|
||||
### Existing Adapters
|
||||
|
||||
- `chat_completions.py`: OpenAI Chat Completions API
|
||||
- `claude.py`: Anthropic Claude Messages API
|
||||
- `responses.py`: OpenAI Responses API
|
||||
- `ollama.py`: Ollama API (for OpenWebUI compatibility)
|
||||
|
||||
### Adding a New API Adapter
|
||||
|
||||
To add support for a new API format:
|
||||
|
||||
1. Create a new adapter file in `src/exo/master/adapters/`
|
||||
2. Implement a request conversion function:
|
||||
```python
|
||||
def your_api_request_to_text_generation(
|
||||
request: YourAPIRequest,
|
||||
) -> TextGenerationTaskParams:
|
||||
# Convert API request to internal format
|
||||
pass
|
||||
```
|
||||
3. Implement streaming response generation:
|
||||
```python
|
||||
async def generate_your_api_stream(
|
||||
command_id: CommandId,
|
||||
chunk_stream: AsyncGenerator[TokenChunk | ErrorChunk | ToolCallChunk, None],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
# Convert internal chunks to API-specific streaming format
|
||||
pass
|
||||
```
|
||||
4. Implement non-streaming response collection:
|
||||
```python
|
||||
async def collect_your_api_response(
|
||||
command_id: CommandId,
|
||||
chunk_stream: AsyncGenerator[TokenChunk | ErrorChunk | ToolCallChunk, None],
|
||||
) -> AsyncGenerator[str]:
|
||||
# Collect all chunks and return single response
|
||||
pass
|
||||
```
|
||||
5. Register the adapter endpoints in `src/exo/master/api.py`
|
||||
|
||||
The adapter pattern keeps API-specific logic isolated from core inference systems. Internal systems (worker, runner, event sourcing) only see `TextGenerationTaskParams` and `TokenChunk` objects - no API-specific types cross the adapter boundary.
|
||||
|
||||
For detailed API documentation, see [docs/api.md](docs/api.md).
|
||||
|
||||
## Testing
|
||||
|
||||
EXO relies heavily on manual testing at this point in the project, but this is evolving. Before submitting a change, test both before and after to demonstrate how your change improves behavior. Do the best you can with the hardware you have available - if you need help testing, ask and we'll do our best to assist. Add automated tests where possible - we're actively working to substantially improve our automated testing story.
|
||||
|
||||
Generated
-24
@@ -216,28 +216,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
|
||||
dependencies = [
|
||||
"async-stream-impl",
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream-impl"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.111",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.89"
|
||||
@@ -2781,7 +2759,6 @@ dependencies = [
|
||||
name = "networking"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"delegate",
|
||||
"either",
|
||||
"extend",
|
||||
@@ -2790,7 +2767,6 @@ dependencies = [
|
||||
"keccak-const",
|
||||
"libp2p",
|
||||
"log",
|
||||
"pin-project",
|
||||
"tokio",
|
||||
"tracing-subscriber",
|
||||
"util",
|
||||
|
||||
+5
-2
@@ -1,6 +1,10 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["rust/networking", "rust/exo_pyo3_bindings", "rust/util"]
|
||||
members = [
|
||||
"rust/networking",
|
||||
"rust/exo_pyo3_bindings",
|
||||
"rust/util",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.0.1"
|
||||
@@ -30,7 +34,6 @@ delegate = "0.13"
|
||||
keccak-const = "0.2"
|
||||
|
||||
# Async dependencies
|
||||
async-stream = "0.3"
|
||||
tokio = "1.46"
|
||||
futures-lite = "2.6.1"
|
||||
futures-timer = "3.0"
|
||||
|
||||
@@ -26,8 +26,6 @@ exo connects all your devices into an AI cluster. Not only does exo enable runni
|
||||
- **Topology-Aware Auto Parallel**: exo figures out the best way to split your model across all available devices based on a realtime view of your device topology. It takes into account device resources and network latency/bandwidth between each link.
|
||||
- **Tensor Parallelism**: exo supports sharding models, for up to 1.8x speedup on 2 devices and 3.2x speedup on 4 devices.
|
||||
- **MLX Support**: exo uses [MLX](https://github.com/ml-explore/mlx) as an inference backend and [MLX distributed](https://ml-explore.github.io/mlx/build/html/usage/distributed.html) for distributed communication.
|
||||
- **Multiple API Compatibility**: Compatible with OpenAI Chat Completions API, Claude Messages API, OpenAI Responses API, and Ollama API - use your existing tools and clients.
|
||||
- **Custom Model Support**: Load custom models from HuggingFace hub to expand the range of available models.
|
||||
|
||||
## Dashboard
|
||||
|
||||
@@ -95,10 +93,11 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
||||
```
|
||||
- [uv](https://github.com/astral-sh/uv) (for Python dependency management)
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
- [node](https://github.com/nodejs/node) (for building the dashboard)
|
||||
|
||||
```bash
|
||||
brew install uv node
|
||||
brew install uv macmon node
|
||||
```
|
||||
- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
|
||||
|
||||
@@ -106,17 +105,6 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
rustup toolchain install nightly
|
||||
```
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
|
||||
Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
|
||||
Homebrew `macmon 0.6.1` still crashes on Apple M5.
|
||||
|
||||
```bash
|
||||
cargo install --git https://github.com/vladkens/macmon \
|
||||
--rev a1cd06b6cc0d5e61db24fd8832e74cd992097a7d \
|
||||
macmon \
|
||||
--force
|
||||
```
|
||||
|
||||
Clone the repo, build the dashboard, and run exo:
|
||||
|
||||
@@ -208,8 +196,6 @@ exo follows the [XDG Base Directory Specification](https://specifications.freede
|
||||
- **Configuration files**: `~/.config/exo/` (or `$XDG_CONFIG_HOME/exo/`)
|
||||
- **Data files**: `~/.local/share/exo/` (or `$XDG_DATA_HOME/exo/`)
|
||||
- **Cache files**: `~/.cache/exo/` (or `$XDG_CACHE_HOME/exo/`)
|
||||
- **Log files**: `~/.cache/exo/exo_log/` (with automatic log rotation)
|
||||
- **Custom model cards**: `~/.local/share/exo/custom_model_cards/`
|
||||
|
||||
You can override these locations by setting the corresponding XDG environment variables.
|
||||
|
||||
@@ -289,51 +275,8 @@ After that, RDMA will be enabled in macOS and exo will take care of the rest.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
exo supports several environment variables for configuration:
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `EXO_DEFAULT_MODELS_DIR` | Default directory for model downloads and caches. Always first in the writable dirs list. | `~/.local/share/exo/models` (Linux) or `~/.exo/models` (macOS) |
|
||||
| `EXO_MODELS_DIRS` | Colon-separated additional writable directories for model downloads. Checked in order after the default; first with enough free space is used. | None |
|
||||
| `EXO_MODELS_READ_ONLY_DIRS` | Colon-separated read-only directories to search for pre-downloaded models (e.g., NFS mounts, shared storage). Models here cannot be deleted. | None |
|
||||
| `EXO_OFFLINE` | Run without internet connection (uses only local models) | `false` |
|
||||
| `EXO_ENABLE_IMAGE_MODELS` | Enable image model support | `false` |
|
||||
| `EXO_LIBP2P_NAMESPACE` | Custom namespace for cluster isolation | None |
|
||||
| `EXO_FAST_SYNCH` | Control MLX_METAL_FAST_SYNCH behavior (for JACCL backend) | Auto |
|
||||
| `EXO_TRACING_ENABLED` | Enable distributed tracing for performance analysis | `false` |
|
||||
|
||||
**Example usage:**
|
||||
|
||||
```bash
|
||||
# Use pre-downloaded models from NFS mount (read-only)
|
||||
EXO_MODELS_READ_ONLY_DIRS=/mnt/nfs/models:/opt/ai-models uv run exo
|
||||
|
||||
# Download models to an external SSD (falls back to default dir if full)
|
||||
EXO_MODELS_DIRS=/Volumes/ExternalSSD/exo-models uv run exo
|
||||
|
||||
# Run in offline mode
|
||||
EXO_OFFLINE=true uv run exo
|
||||
|
||||
# Enable image models
|
||||
EXO_ENABLE_IMAGE_MODELS=true uv run exo
|
||||
|
||||
# Use custom namespace for cluster isolation
|
||||
EXO_LIBP2P_NAMESPACE=my-dev-cluster uv run exo
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Using the API
|
||||
|
||||
exo provides multiple API-compatible interfaces for maximum compatibility with existing tools:
|
||||
|
||||
- **OpenAI Chat Completions API** - Compatible with OpenAI clients
|
||||
- **Claude Messages API** - Compatible with Anthropic's Claude format
|
||||
- **OpenAI Responses API** - Compatible with OpenAI's Responses format
|
||||
- **Ollama API** - Compatible with Ollama and tools like OpenWebUI
|
||||
|
||||
If you prefer to interact with exo via the API, here is an example creating an instance of a small model (`mlx-community/Llama-3.2-1B-Instruct-4bit`), sending a chat completions request and deleting the instance.
|
||||
|
||||
---
|
||||
@@ -423,85 +366,14 @@ When you're done, delete the instance by its ID (find it via `/state` or `/insta
|
||||
curl -X DELETE http://localhost:52415/instance/YOUR_INSTANCE_ID
|
||||
```
|
||||
|
||||
### Claude Messages API Compatibility
|
||||
|
||||
Use the Claude Messages API format with the `/v1/messages` endpoint:
|
||||
|
||||
```bash
|
||||
curl -N -X POST http://localhost:52415/v1/messages \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "mlx-community/Llama-3.2-1B-Instruct-4bit",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
"max_tokens": 1024,
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
### OpenAI Responses API Compatibility
|
||||
|
||||
Use the OpenAI Responses API format with the `/v1/responses` endpoint:
|
||||
|
||||
```bash
|
||||
curl -N -X POST http://localhost:52415/v1/responses \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "mlx-community/Llama-3.2-1B-Instruct-4bit",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
### Ollama API Compatibility
|
||||
|
||||
exo supports Ollama API endpoints for compatibility with tools like OpenWebUI:
|
||||
|
||||
```bash
|
||||
# Ollama chat
|
||||
curl -X POST http://localhost:52415/ollama/api/chat \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "mlx-community/Llama-3.2-1B-Instruct-4bit",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
"stream": false
|
||||
}'
|
||||
|
||||
# List models (Ollama format)
|
||||
curl http://localhost:52415/ollama/api/tags
|
||||
```
|
||||
|
||||
### Custom Model Loading from HuggingFace
|
||||
|
||||
You can add custom models from the HuggingFace hub:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:52415/models/add \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model_id": "mlx-community/my-custom-model"
|
||||
}'
|
||||
```
|
||||
|
||||
**Security Note:**
|
||||
|
||||
Custom models requiring `trust_remote_code` in their configuration must be explicitly enabled (default is false) for security. Only enable this if you trust the model's remote code execution. Models are fetched from HuggingFace and stored locally as custom model cards.
|
||||
|
||||
**Other useful API endpoints*:**
|
||||
|
||||
- List all models: `curl http://localhost:52415/models`
|
||||
- List downloaded models only: `curl http://localhost:52415/models?status=downloaded`
|
||||
- Search HuggingFace: `curl "http://localhost:52415/models/search?query=llama&limit=10"`
|
||||
- Inspect instance IDs and deployment state: `curl http://localhost:52415/state`
|
||||
|
||||
For further details, see:
|
||||
|
||||
- API documentation in [docs/api.md](docs/api.md).
|
||||
- API basic documentation in [docs/api.md](docs/api.md).
|
||||
- API types and endpoints in [src/exo/master/api.py](src/exo/master/api.py).
|
||||
|
||||
---
|
||||
@@ -560,4 +432,4 @@ On macOS, exo uses the GPU. On Linux, exo currently runs on CPU. We are working
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to exo.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to exo.
|
||||
+145
-221
@@ -15,23 +15,14 @@ struct ContentView: View {
|
||||
@EnvironmentObject private var localNetworkChecker: LocalNetworkChecker
|
||||
@EnvironmentObject private var updater: SparkleUpdater
|
||||
@EnvironmentObject private var thunderboltBridgeService: ThunderboltBridgeService
|
||||
@EnvironmentObject private var settingsWindowController: SettingsWindowController
|
||||
@State private var focusedNode: NodeViewModel?
|
||||
@State private var deletingInstanceIDs: Set<String> = []
|
||||
@State private var showAllNodes = false
|
||||
@State private var showAllInstances = false
|
||||
@State private var baseURLCopied = false
|
||||
@State private var showAdvanced = false
|
||||
@State private var showDebugInfo = false
|
||||
private enum BugReportPhase: Equatable {
|
||||
case idle
|
||||
case prompting
|
||||
case sending(String)
|
||||
case success(String)
|
||||
case failure(String)
|
||||
}
|
||||
@State private var bugReportPhase: BugReportPhase = .idle
|
||||
@State private var bugReportUserDescription: String = ""
|
||||
@State private var bugReportInFlight = false
|
||||
@State private var bugReportMessage: String?
|
||||
@State private var uninstallInProgress = false
|
||||
@State private var pendingNamespace: String = ""
|
||||
@State private var pendingHFToken: String = ""
|
||||
@@ -267,79 +258,139 @@ struct ContentView: View {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
if controller.status != .stopped {
|
||||
dashboardButton
|
||||
baseURLRow
|
||||
Divider()
|
||||
.padding(.vertical, 8)
|
||||
} else {
|
||||
Divider()
|
||||
.padding(.vertical, 4)
|
||||
}
|
||||
HoverButton(
|
||||
title: "Settings",
|
||||
tint: .primary,
|
||||
trailingSystemImage: "gear"
|
||||
) {
|
||||
settingsWindowController.open(
|
||||
controller: controller,
|
||||
updater: updater,
|
||||
networkStatusService: networkStatusService,
|
||||
thunderboltBridgeService: thunderboltBridgeService,
|
||||
stateService: stateService
|
||||
)
|
||||
}
|
||||
HoverButton(
|
||||
title: "Check for Updates",
|
||||
tint: .primary,
|
||||
trailingSystemImage: "arrow.triangle.2.circlepath"
|
||||
) {
|
||||
updater.checkForUpdates()
|
||||
}
|
||||
.padding(.bottom, 8)
|
||||
HoverButton(title: "Quit", tint: .secondary) {
|
||||
advancedSection
|
||||
.padding(.bottom, 8)
|
||||
controlButton(title: "Quit", tint: .secondary) {
|
||||
controller.stop()
|
||||
NSApplication.shared.terminate(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var dashboardButton: some View {
|
||||
HoverButton(
|
||||
title: "Web Dashboard",
|
||||
tint: .primary,
|
||||
trailingSystemImage: "arrow.up.right"
|
||||
) {
|
||||
guard let url = URL(string: "http://localhost:52415/") else { return }
|
||||
NSWorkspace.shared.open(url)
|
||||
private var advancedSection: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
HStack {
|
||||
Text("Advanced")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
Spacer()
|
||||
collapseButton(isExpanded: $showAdvanced)
|
||||
}
|
||||
.animation(nil, value: showAdvanced)
|
||||
if showAdvanced {
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Text("Cluster Namespace")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
HStack {
|
||||
TextField("optional", text: $pendingNamespace)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.caption2)
|
||||
.onAppear {
|
||||
pendingNamespace = controller.customNamespace
|
||||
}
|
||||
Button("Save & Restart") {
|
||||
controller.customNamespace = pendingNamespace
|
||||
if controller.status == .running || controller.status == .starting {
|
||||
controller.restart()
|
||||
}
|
||||
}
|
||||
.font(.caption2)
|
||||
.disabled(pendingNamespace == controller.customNamespace)
|
||||
}
|
||||
}
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Text("HuggingFace Token")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
HStack {
|
||||
SecureField("optional", text: $pendingHFToken)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.caption2)
|
||||
.onAppear {
|
||||
pendingHFToken = controller.hfToken
|
||||
}
|
||||
Button("Save & Restart") {
|
||||
controller.hfToken = pendingHFToken
|
||||
if controller.status == .running || controller.status == .starting {
|
||||
controller.restart()
|
||||
}
|
||||
}
|
||||
.font(.caption2)
|
||||
.disabled(pendingHFToken == controller.hfToken)
|
||||
}
|
||||
}
|
||||
Divider()
|
||||
HStack {
|
||||
Toggle(
|
||||
"Enable Image Models (experimental)", isOn: $pendingEnableImageModels
|
||||
)
|
||||
.toggleStyle(.switch)
|
||||
.font(.caption2)
|
||||
.onAppear {
|
||||
pendingEnableImageModels = controller.enableImageModels
|
||||
}
|
||||
|
||||
Spacer()
|
||||
|
||||
Button("Save & Restart") {
|
||||
controller.enableImageModels = pendingEnableImageModels
|
||||
if controller.status == .running || controller.status == .starting {
|
||||
controller.restart()
|
||||
}
|
||||
}
|
||||
.font(.caption2)
|
||||
.disabled(pendingEnableImageModels == controller.enableImageModels)
|
||||
}
|
||||
HoverButton(title: "Check for Updates", small: true) {
|
||||
updater.checkForUpdates()
|
||||
}
|
||||
debugSection
|
||||
HoverButton(title: "Uninstall", tint: .red, small: true) {
|
||||
showUninstallConfirmationAlert()
|
||||
}
|
||||
.disabled(uninstallInProgress)
|
||||
}
|
||||
.transition(.opacity)
|
||||
}
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.25), value: showAdvanced)
|
||||
}
|
||||
|
||||
private var baseURLRow: some View {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "link")
|
||||
.imageScale(.small)
|
||||
.foregroundColor(.secondary)
|
||||
Text("localhost:52415/v1")
|
||||
.font(.system(.caption, design: .monospaced))
|
||||
.foregroundColor(.primary)
|
||||
Spacer()
|
||||
Button {
|
||||
NSPasteboard.general.clearContents()
|
||||
NSPasteboard.general.setString("http://localhost:52415/v1", forType: .string)
|
||||
baseURLCopied = true
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
|
||||
baseURLCopied = false
|
||||
}
|
||||
} label: {
|
||||
Image(systemName: baseURLCopied ? "checkmark" : "doc.on.doc")
|
||||
private func controlButton(title: String, tint: Color = .primary, action: @escaping () -> Void)
|
||||
-> some View
|
||||
{
|
||||
HoverButton(title: title, tint: tint, trailingSystemImage: nil, action: action)
|
||||
}
|
||||
|
||||
private var dashboardButton: some View {
|
||||
Button {
|
||||
guard let url = URL(string: "http://localhost:52415/") else { return }
|
||||
NSWorkspace.shared.open(url)
|
||||
} label: {
|
||||
HStack {
|
||||
Image(systemName: "arrow.up.right.square")
|
||||
.imageScale(.small)
|
||||
.foregroundColor(baseURLCopied ? .green : .secondary)
|
||||
.contentTransition(.symbolEffect(.replace))
|
||||
Text("Dashboard")
|
||||
.fontWeight(.medium)
|
||||
Spacer()
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.help("Copy API base URL")
|
||||
.padding(.vertical, 8)
|
||||
.padding(.horizontal, 10)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: 8, style: .continuous)
|
||||
.fill(Color(red: 1.0, green: 0.87, blue: 0.0).opacity(0.2))
|
||||
)
|
||||
}
|
||||
.padding(.vertical, 4)
|
||||
.padding(.horizontal, 8)
|
||||
.buttonStyle(.plain)
|
||||
.padding(.bottom, 4)
|
||||
}
|
||||
|
||||
private func collapseButton(isExpanded: Binding<Bool>) -> some View {
|
||||
@@ -560,124 +611,39 @@ struct ContentView: View {
|
||||
}
|
||||
|
||||
private var sendBugReportButton: some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
switch bugReportPhase {
|
||||
case .idle:
|
||||
Button {
|
||||
bugReportPhase = .prompting
|
||||
bugReportUserDescription = ""
|
||||
} label: {
|
||||
HStack {
|
||||
Text("Send Bug Report")
|
||||
.font(.caption)
|
||||
.fontWeight(.semibold)
|
||||
Spacer()
|
||||
}
|
||||
.padding(.vertical, 6)
|
||||
.padding(.horizontal, 8)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: 6)
|
||||
.fill(Color.accentColor.opacity(0.12))
|
||||
)
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Button {
|
||||
Task {
|
||||
await sendBugReport()
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
|
||||
case .prompting:
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text("Tell us what went wrong (optional)")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
Text(
|
||||
"A quick description of what you were doing and what happened helps us track down the bug for you."
|
||||
)
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
.opacity(0.8)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
TextEditor(text: $bugReportUserDescription)
|
||||
.font(.caption2)
|
||||
.frame(height: 60)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: 4)
|
||||
.stroke(Color.secondary.opacity(0.3), lineWidth: 1)
|
||||
)
|
||||
HStack(spacing: 8) {
|
||||
Button("Send") {
|
||||
Task {
|
||||
await sendBugReport()
|
||||
}
|
||||
}
|
||||
.font(.caption2)
|
||||
.buttonStyle(.borderedProminent)
|
||||
.controlSize(.small)
|
||||
Button("Cancel") {
|
||||
bugReportPhase = .idle
|
||||
}
|
||||
.font(.caption2)
|
||||
.buttonStyle(.bordered)
|
||||
.controlSize(.small)
|
||||
} label: {
|
||||
HStack {
|
||||
if bugReportInFlight {
|
||||
ProgressView()
|
||||
.scaleEffect(0.6)
|
||||
}
|
||||
Text("Send Bug Report")
|
||||
.font(.caption)
|
||||
.fontWeight(.semibold)
|
||||
Spacer()
|
||||
}
|
||||
.padding(8)
|
||||
.padding(.vertical, 6)
|
||||
.padding(.horizontal, 8)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: 6)
|
||||
.fill(Color.accentColor.opacity(0.06))
|
||||
.fill(Color.accentColor.opacity(0.12))
|
||||
)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.disabled(bugReportInFlight)
|
||||
|
||||
case .sending(let message):
|
||||
HStack(spacing: 6) {
|
||||
ProgressView()
|
||||
.scaleEffect(0.6)
|
||||
Text(message)
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
case .success(let message):
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
Text(message)
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
Button {
|
||||
openGitHubIssue()
|
||||
} label: {
|
||||
HStack(spacing: 4) {
|
||||
Image(systemName: "arrow.up.right.square")
|
||||
.imageScale(.small)
|
||||
Text("Create GitHub Issue")
|
||||
.font(.caption2)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.bordered)
|
||||
.controlSize(.small)
|
||||
Button("Done") {
|
||||
bugReportPhase = .idle
|
||||
bugReportUserDescription = ""
|
||||
}
|
||||
if let message = bugReportMessage {
|
||||
Text(message)
|
||||
.font(.caption2)
|
||||
.buttonStyle(.plain)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
case .failure(let message):
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Text(message)
|
||||
.font(.caption2)
|
||||
.foregroundColor(.red)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
Button("Dismiss") {
|
||||
bugReportPhase = .idle
|
||||
}
|
||||
.font(.caption2)
|
||||
.buttonStyle(.plain)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.2), value: bugReportPhase)
|
||||
}
|
||||
|
||||
private var processToggleBinding: Binding<Bool> {
|
||||
@@ -721,58 +687,16 @@ struct ContentView: View {
|
||||
}
|
||||
|
||||
private func sendBugReport() async {
|
||||
bugReportPhase = .sending("Collecting logs...")
|
||||
bugReportInFlight = true
|
||||
bugReportMessage = "Collecting logs..."
|
||||
let service = BugReportService()
|
||||
let description = bugReportUserDescription.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
do {
|
||||
let outcome = try await service.sendReport(
|
||||
isManual: true,
|
||||
userDescription: description.isEmpty ? nil : description
|
||||
)
|
||||
if outcome.success {
|
||||
bugReportPhase = .success(outcome.message)
|
||||
} else {
|
||||
bugReportPhase = .failure(outcome.message)
|
||||
}
|
||||
let outcome = try await service.sendReport(isManual: true)
|
||||
bugReportMessage = outcome.message
|
||||
} catch {
|
||||
bugReportPhase = .failure(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func openGitHubIssue() {
|
||||
let description = bugReportUserDescription.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
var bodyParts: [String] = []
|
||||
bodyParts.append("## Describe the bug")
|
||||
bodyParts.append("")
|
||||
if !description.isEmpty {
|
||||
bodyParts.append(description)
|
||||
} else {
|
||||
bodyParts.append("A clear and concise description of what the bug is.")
|
||||
}
|
||||
bodyParts.append("")
|
||||
bodyParts.append("## Environment")
|
||||
bodyParts.append("")
|
||||
bodyParts.append("- macOS Version: \(ProcessInfo.processInfo.operatingSystemVersionString)")
|
||||
bodyParts.append("- EXO Version: \(buildTag) (\(buildCommit))")
|
||||
bodyParts.append("")
|
||||
bodyParts.append("## Additional context")
|
||||
bodyParts.append("")
|
||||
bodyParts.append("A bug report with diagnostic logs was submitted via the app.")
|
||||
|
||||
let body = bodyParts.joined(separator: "\n")
|
||||
|
||||
var components = URLComponents(string: "https://github.com/exo-explore/exo/issues/new")!
|
||||
components.queryItems = [
|
||||
URLQueryItem(name: "template", value: "bug_report.md"),
|
||||
URLQueryItem(name: "title", value: "[BUG] "),
|
||||
URLQueryItem(name: "body", value: body),
|
||||
URLQueryItem(name: "labels", value: "bug"),
|
||||
]
|
||||
|
||||
if let url = components.url {
|
||||
NSWorkspace.shared.open(url)
|
||||
bugReportMessage = error.localizedDescription
|
||||
}
|
||||
bugReportInFlight = false
|
||||
}
|
||||
|
||||
private func showUninstallConfirmationAlert() {
|
||||
|
||||
@@ -21,9 +21,7 @@ struct EXOApp: App {
|
||||
@StateObject private var localNetworkChecker: LocalNetworkChecker
|
||||
@StateObject private var updater: SparkleUpdater
|
||||
@StateObject private var thunderboltBridgeService: ThunderboltBridgeService
|
||||
@StateObject private var settingsWindowController: SettingsWindowController
|
||||
private let terminationObserver: TerminationObserver
|
||||
private let firstLaunchPopout = FirstLaunchPopout()
|
||||
private let ciContext = CIContext(options: nil)
|
||||
|
||||
init() {
|
||||
@@ -45,13 +43,12 @@ struct EXOApp: App {
|
||||
_updater = StateObject(wrappedValue: updater)
|
||||
let thunderboltBridge = ThunderboltBridgeService(clusterStateService: service)
|
||||
_thunderboltBridgeService = StateObject(wrappedValue: thunderboltBridge)
|
||||
_settingsWindowController = StateObject(wrappedValue: SettingsWindowController())
|
||||
enableLaunchAtLoginIfNeeded()
|
||||
// Install LaunchDaemon to disable Thunderbolt Bridge on startup (prevents network loops)
|
||||
NetworkSetupHelper.promptAndInstallIfNeeded()
|
||||
// Check local network access periodically (warning disappears when user grants permission)
|
||||
localNetwork.startPeriodicChecking(interval: 10)
|
||||
controller.scheduleLaunch(after: 5)
|
||||
controller.scheduleLaunch(after: 15)
|
||||
service.startPolling()
|
||||
networkStatus.startPolling()
|
||||
}
|
||||
@@ -65,19 +62,8 @@ struct EXOApp: App {
|
||||
.environmentObject(localNetworkChecker)
|
||||
.environmentObject(updater)
|
||||
.environmentObject(thunderboltBridgeService)
|
||||
.environmentObject(settingsWindowController)
|
||||
} label: {
|
||||
menuBarIcon
|
||||
.onReceive(controller.$isFirstLaunchReady) { ready in
|
||||
if ready {
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) {
|
||||
self.firstLaunchPopout.onComplete = { [weak controller] in
|
||||
controller?.markOnboardingCompleted()
|
||||
}
|
||||
self.firstLaunchPopout.show()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.menuBarExtraStyle(.window)
|
||||
}
|
||||
|
||||
@@ -4,30 +4,7 @@ import Foundation
|
||||
|
||||
private let customNamespaceKey = "EXOCustomNamespace"
|
||||
private let hfTokenKey = "EXOHFToken"
|
||||
private let hfEndpointKey = "EXOHFEndpoint"
|
||||
private let enableImageModelsKey = "EXOEnableImageModels"
|
||||
private let offlineModeKey = "EXOOfflineMode"
|
||||
private let fastSynchEnabledKey = "EXOFastSynchEnabled"
|
||||
private let onboardingCompletedKey = "EXOOnboardingCompleted"
|
||||
private let defaultModelsDirKey = "EXODefaultModelsDir"
|
||||
private let additionalModelsDirsKey = "EXOAdditionalModelsDirs"
|
||||
private let readOnlyModelsDirsKey = "EXOReadOnlyModelsDirs"
|
||||
private let customEnvironmentVariablesKey = "EXOCustomEnvironmentVariables"
|
||||
|
||||
/// A user-defined environment variable that is injected into the exo child
|
||||
/// process at launch. Used as an escape hatch for env vars that don't have
|
||||
/// first-class typed UI in Settings.
|
||||
struct CustomEnvironmentVariable: Codable, Identifiable, Equatable {
|
||||
var id: UUID
|
||||
var key: String
|
||||
var value: String
|
||||
|
||||
init(id: UUID = UUID(), key: String = "", value: String = "") {
|
||||
self.id = id
|
||||
self.key = key
|
||||
self.value = value
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class ExoProcessController: ObservableObject {
|
||||
@@ -74,14 +51,6 @@ final class ExoProcessController: ObservableObject {
|
||||
UserDefaults.standard.set(hfToken, forKey: hfTokenKey)
|
||||
}
|
||||
}
|
||||
@Published var hfEndpoint: String = {
|
||||
return UserDefaults.standard.string(forKey: hfEndpointKey) ?? ""
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(hfEndpoint, forKey: hfEndpointKey)
|
||||
}
|
||||
}
|
||||
@Published var enableImageModels: Bool = {
|
||||
return UserDefaults.standard.bool(forKey: enableImageModelsKey)
|
||||
}()
|
||||
@@ -90,71 +59,6 @@ final class ExoProcessController: ObservableObject {
|
||||
UserDefaults.standard.set(enableImageModels, forKey: enableImageModelsKey)
|
||||
}
|
||||
}
|
||||
@Published var offlineMode: Bool = {
|
||||
return UserDefaults.standard.bool(forKey: offlineModeKey)
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(offlineMode, forKey: offlineModeKey)
|
||||
}
|
||||
}
|
||||
@Published var fastSynchEnabled: Bool = {
|
||||
if UserDefaults.standard.object(forKey: fastSynchEnabledKey) == nil {
|
||||
return true
|
||||
}
|
||||
return UserDefaults.standard.bool(forKey: fastSynchEnabledKey)
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(fastSynchEnabled, forKey: fastSynchEnabledKey)
|
||||
}
|
||||
}
|
||||
@Published var defaultModelsDir: String = {
|
||||
return UserDefaults.standard.string(forKey: defaultModelsDirKey) ?? ""
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(defaultModelsDir, forKey: defaultModelsDirKey)
|
||||
}
|
||||
}
|
||||
@Published var additionalModelsDirs: String = {
|
||||
return UserDefaults.standard.string(forKey: additionalModelsDirsKey) ?? ""
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(additionalModelsDirs, forKey: additionalModelsDirsKey)
|
||||
}
|
||||
}
|
||||
@Published var readOnlyModelsDirs: String = {
|
||||
return UserDefaults.standard.string(forKey: readOnlyModelsDirsKey) ?? ""
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
UserDefaults.standard.set(readOnlyModelsDirs, forKey: readOnlyModelsDirsKey)
|
||||
}
|
||||
}
|
||||
@Published var customEnvironmentVariables: [CustomEnvironmentVariable] = {
|
||||
guard
|
||||
let data = UserDefaults.standard.data(forKey: customEnvironmentVariablesKey),
|
||||
let decoded = try? JSONDecoder().decode(
|
||||
[CustomEnvironmentVariable].self, from: data
|
||||
)
|
||||
else {
|
||||
return []
|
||||
}
|
||||
return decoded
|
||||
}()
|
||||
{
|
||||
didSet {
|
||||
guard let data = try? JSONEncoder().encode(customEnvironmentVariables) else {
|
||||
return
|
||||
}
|
||||
UserDefaults.standard.set(data, forKey: customEnvironmentVariablesKey)
|
||||
}
|
||||
}
|
||||
|
||||
/// Fires once when EXO transitions to `.running` for the very first time (fresh install).
|
||||
@Published private(set) var isFirstLaunchReady = false
|
||||
|
||||
private var process: Process?
|
||||
private var runtimeDirectoryURL: URL?
|
||||
@@ -209,9 +113,6 @@ final class ExoProcessController: ObservableObject {
|
||||
try child.run()
|
||||
process = child
|
||||
status = .running
|
||||
|
||||
// Show welcome popout on every launch
|
||||
isFirstLaunchReady = true
|
||||
} catch {
|
||||
process = nil
|
||||
status = .failed(message: "Launch error")
|
||||
@@ -263,17 +164,6 @@ final class ExoProcessController: ObservableObject {
|
||||
launch()
|
||||
}
|
||||
|
||||
/// Mark onboarding as completed (user interacted with the welcome popout).
|
||||
func markOnboardingCompleted() {
|
||||
UserDefaults.standard.set(true, forKey: onboardingCompletedKey)
|
||||
}
|
||||
|
||||
/// Reset onboarding so the welcome popout appears on next launch.
|
||||
func resetOnboarding() {
|
||||
UserDefaults.standard.removeObject(forKey: onboardingCompletedKey)
|
||||
isFirstLaunchReady = false
|
||||
}
|
||||
|
||||
func scheduleLaunch(after seconds: TimeInterval) {
|
||||
cancelPendingLaunch()
|
||||
let start = max(1, Int(ceil(seconds)))
|
||||
@@ -356,16 +246,9 @@ final class ExoProcessController: ObservableObject {
|
||||
if !hfToken.isEmpty {
|
||||
environment["HF_TOKEN"] = hfToken
|
||||
}
|
||||
if !hfEndpoint.isEmpty {
|
||||
environment["HF_ENDPOINT"] = hfEndpoint
|
||||
}
|
||||
if enableImageModels {
|
||||
environment["EXO_ENABLE_IMAGE_MODELS"] = "true"
|
||||
}
|
||||
if offlineMode {
|
||||
environment["EXO_OFFLINE"] = "true"
|
||||
}
|
||||
environment["EXO_FAST_SYNCH"] = fastSynchEnabled ? "true" : "false"
|
||||
|
||||
var paths: [String] = []
|
||||
if let existing = environment["PATH"], !existing.isEmpty {
|
||||
@@ -390,29 +273,6 @@ final class ExoProcessController: ObservableObject {
|
||||
}
|
||||
|
||||
environment["PATH"] = paths.joined(separator: ":")
|
||||
|
||||
let trimmedDefaultModelsDir = defaultModelsDir.trimmingCharacters(in: .whitespaces)
|
||||
if !trimmedDefaultModelsDir.isEmpty {
|
||||
environment["EXO_DEFAULT_MODELS_DIR"] = trimmedDefaultModelsDir
|
||||
}
|
||||
let trimmedAdditionalModelsDirs = additionalModelsDirs.trimmingCharacters(in: .whitespaces)
|
||||
if !trimmedAdditionalModelsDirs.isEmpty {
|
||||
environment["EXO_MODELS_DIRS"] = trimmedAdditionalModelsDirs
|
||||
}
|
||||
let trimmedReadOnlyModelsDirs = readOnlyModelsDirs.trimmingCharacters(in: .whitespaces)
|
||||
if !trimmedReadOnlyModelsDirs.isEmpty {
|
||||
environment["EXO_MODELS_READ_ONLY_DIRS"] = trimmedReadOnlyModelsDirs
|
||||
}
|
||||
|
||||
// Apply user-defined arbitrary environment variables last so that
|
||||
// power users can override any of the typed fields above when
|
||||
// necessary. Empty keys are ignored.
|
||||
for variable in customEnvironmentVariables {
|
||||
let trimmedKey = variable.key.trimmingCharacters(in: .whitespaces)
|
||||
guard !trimmedKey.isEmpty else { continue }
|
||||
environment[trimmedKey] = variable.value
|
||||
}
|
||||
|
||||
return environment
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
<key>EXOBuildCommit</key>
|
||||
<string>$(EXO_BUILD_COMMIT)</string>
|
||||
<key>EXOBugReportPresignedUrlEndpoint</key>
|
||||
<string>https://reports.exolabs.net/presigned-urls</string>
|
||||
<string>$(EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT)</string>
|
||||
<key>NSLocalNetworkUsageDescription</key>
|
||||
<string>EXO needs local network access to discover and connect to other devices in your cluster for distributed AI inference.</string>
|
||||
<key>NSBonjourServices</key>
|
||||
|
||||
@@ -38,8 +38,7 @@ struct BugReportService {
|
||||
func sendReport(
|
||||
baseURL: URL = URL(string: "http://127.0.0.1:52415")!,
|
||||
now: Date = Date(),
|
||||
isManual: Bool = false,
|
||||
userDescription: String? = nil
|
||||
isManual: Bool = false
|
||||
) async throws -> BugReportOutcome {
|
||||
let timestamp = Self.runTimestampString(now)
|
||||
let dayPrefix = Self.dayPrefixString(now)
|
||||
@@ -61,8 +60,7 @@ struct BugReportService {
|
||||
ifconfig: ifconfigText,
|
||||
debugInfo: debugInfo,
|
||||
isManual: isManual,
|
||||
clusterTbBridgeStatus: clusterTbBridgeStatus,
|
||||
userDescription: userDescription
|
||||
clusterTbBridgeStatus: clusterTbBridgeStatus
|
||||
)
|
||||
|
||||
let eventLogFiles = readAllEventLogs()
|
||||
@@ -308,8 +306,7 @@ struct BugReportService {
|
||||
ifconfig: String,
|
||||
debugInfo: DebugInfo,
|
||||
isManual: Bool,
|
||||
clusterTbBridgeStatus: [[String: Any]]?,
|
||||
userDescription: String? = nil
|
||||
clusterTbBridgeStatus: [[String: Any]]?
|
||||
) -> Data? {
|
||||
let system = readSystemMetadata()
|
||||
let exo = readExoMetadata()
|
||||
@@ -326,9 +323,6 @@ struct BugReportService {
|
||||
if let tbStatus = clusterTbBridgeStatus {
|
||||
payload["cluster_thunderbolt_bridge"] = tbStatus
|
||||
}
|
||||
if let desc = userDescription, !desc.isEmpty {
|
||||
payload["user_description"] = desc
|
||||
}
|
||||
return try? JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted])
|
||||
}
|
||||
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
/// A popover callout anchored to the menu bar icon on every launch,
|
||||
/// pointing the user to the web dashboard with an arrow connecting to the icon.
|
||||
@MainActor
|
||||
final class FirstLaunchPopout {
|
||||
private var popover: NSPopover?
|
||||
private var countdownTask: Task<Void, Never>?
|
||||
private static let dashboardURL = "http://localhost:52415/"
|
||||
|
||||
/// Called when the user completes onboarding (clicks Open Dashboard or dismisses).
|
||||
var onComplete: (() -> Void)?
|
||||
|
||||
func show() {
|
||||
guard popover == nil else { return }
|
||||
|
||||
// The status bar button may not exist yet on first launch; retry generously.
|
||||
showWithRetry(attemptsRemaining: 15)
|
||||
}
|
||||
|
||||
private func showWithRetry(attemptsRemaining: Int) {
|
||||
guard attemptsRemaining > 0 else {
|
||||
// Exhausted retries — fall back to just opening the dashboard directly.
|
||||
openDashboard()
|
||||
onComplete?()
|
||||
return
|
||||
}
|
||||
|
||||
guard let button = Self.findStatusItemButton() else {
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { [weak self] in
|
||||
self?.showWithRetry(attemptsRemaining: attemptsRemaining - 1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
let pop = NSPopover()
|
||||
pop.behavior = .applicationDefined
|
||||
pop.animates = true
|
||||
pop.contentSize = NSSize(width: 280, height: 120)
|
||||
pop.contentViewController = NSHostingController(
|
||||
rootView: WelcomeCalloutView(
|
||||
countdownDuration: 10,
|
||||
onDismiss: { [weak self] in
|
||||
self?.onComplete?()
|
||||
self?.dismiss()
|
||||
},
|
||||
onOpen: { [weak self] in
|
||||
self?.openDashboard()
|
||||
self?.onComplete?()
|
||||
self?.dismiss()
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
self.popover = pop
|
||||
pop.show(relativeTo: button.bounds, of: button, preferredEdge: .minY)
|
||||
|
||||
// Auto-open dashboard after 10s then dismiss
|
||||
countdownTask = Task {
|
||||
try? await Task.sleep(nanoseconds: 10_000_000_000)
|
||||
if !Task.isCancelled {
|
||||
openDashboard()
|
||||
onComplete?()
|
||||
dismiss()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func dismiss() {
|
||||
countdownTask?.cancel()
|
||||
countdownTask = nil
|
||||
guard let pop = popover else { return }
|
||||
popover = nil
|
||||
pop.performClose(nil)
|
||||
}
|
||||
|
||||
private func openDashboard() {
|
||||
guard let url = URL(string: Self.dashboardURL) else { return }
|
||||
NSWorkspace.shared.open(url)
|
||||
}
|
||||
|
||||
/// Finds the NSStatusBarButton created by SwiftUI's MenuBarExtra.
|
||||
/// Walks the view hierarchy to find the actual button rather than the content view.
|
||||
private static func findStatusItemButton() -> NSView? {
|
||||
for window in NSApp.windows {
|
||||
let className = NSStringFromClass(type(of: window))
|
||||
// Match NSStatusBarWindow or any internal SwiftUI status bar window
|
||||
guard className.contains("StatusBar") || className.contains("MenuBarExtra") else {
|
||||
continue
|
||||
}
|
||||
if let content = window.contentView {
|
||||
if let button = findButton(in: content) {
|
||||
return button
|
||||
}
|
||||
// Fall back to the content view itself if it has a non-zero frame
|
||||
if content.frame.width > 0 {
|
||||
return content
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// Recursively searches the view hierarchy for an NSStatusBarButton.
|
||||
private static func findButton(in view: NSView) -> NSView? {
|
||||
let className = NSStringFromClass(type(of: view))
|
||||
if className.contains("StatusBarButton") || className.contains("StatusItem") {
|
||||
return view
|
||||
}
|
||||
for subview in view.subviews {
|
||||
if let found = findButton(in: subview) {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal welcome callout — friendly pointer, not a wall of text.
|
||||
/// Rendered inside the NSPopover which provides its own chrome and arrow.
|
||||
private struct WelcomeCalloutView: View {
|
||||
let countdownDuration: Int
|
||||
let onDismiss: () -> Void
|
||||
let onOpen: () -> Void
|
||||
@State private var countdown: Int
|
||||
@State private var timerTask: Task<Void, Never>?
|
||||
|
||||
init(countdownDuration: Int, onDismiss: @escaping () -> Void, onOpen: @escaping () -> Void) {
|
||||
self.countdownDuration = countdownDuration
|
||||
self.onDismiss = onDismiss
|
||||
self.onOpen = onOpen
|
||||
self._countdown = State(initialValue: countdownDuration)
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 10) {
|
||||
HStack(alignment: .top) {
|
||||
Text("EXO is running")
|
||||
.font(.system(.headline, design: .rounded))
|
||||
.fontWeight(.semibold)
|
||||
.foregroundColor(.primary)
|
||||
Spacer()
|
||||
Button {
|
||||
onDismiss()
|
||||
} label: {
|
||||
Image(systemName: "xmark.circle.fill")
|
||||
.font(.system(size: 14))
|
||||
.foregroundStyle(.tertiary)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
Text("Run your first model here:")
|
||||
.font(.system(.subheadline, design: .default))
|
||||
.foregroundColor(.secondary)
|
||||
|
||||
HStack {
|
||||
Button {
|
||||
onOpen()
|
||||
} label: {
|
||||
Label("Open Dashboard", systemImage: "arrow.up.right.square")
|
||||
.font(.system(.caption, design: .default))
|
||||
.fontWeight(.medium)
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.tint(.accentColor)
|
||||
.controlSize(.small)
|
||||
|
||||
Spacer()
|
||||
|
||||
if countdown > 0 {
|
||||
Text("Launching in \(countdown) secs...")
|
||||
.font(.system(.caption2, design: .default))
|
||||
.foregroundColor(.secondary.opacity(0.6))
|
||||
.monospacedDigit()
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(14)
|
||||
.onAppear {
|
||||
startCountdown()
|
||||
}
|
||||
.onDisappear {
|
||||
timerTask?.cancel()
|
||||
timerTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private func startCountdown() {
|
||||
timerTask = Task {
|
||||
while countdown > 0 {
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
if !Task.isCancelled {
|
||||
countdown -= 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,764 +0,0 @@
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
/// Native macOS Settings window following Apple HIG.
|
||||
/// Organized into General, Model, Advanced, and About sections.
|
||||
struct SettingsView: View {
|
||||
@EnvironmentObject private var controller: ExoProcessController
|
||||
@EnvironmentObject private var updater: SparkleUpdater
|
||||
@EnvironmentObject private var networkStatusService: NetworkStatusService
|
||||
@EnvironmentObject private var thunderboltBridgeService: ThunderboltBridgeService
|
||||
@EnvironmentObject private var stateService: ClusterStateService
|
||||
|
||||
@State private var pendingNamespace: String = ""
|
||||
@State private var pendingHFToken: String = ""
|
||||
@State private var pendingHFEndpoint: String = ""
|
||||
@State private var pendingEnableImageModels = false
|
||||
@State private var pendingOfflineMode = false
|
||||
@State private var pendingFastSynchEnabled = false
|
||||
@State private var pendingDefaultModelsDir: String = ""
|
||||
@State private var pendingAdditionalModelsDirs: String = ""
|
||||
@State private var pendingReadOnlyModelsDirs: String = ""
|
||||
@State private var pendingCustomEnvironmentVariables: [CustomEnvironmentVariable] = []
|
||||
@State private var needsRestart = false
|
||||
@State private var bugReportInFlight = false
|
||||
@State private var bugReportMessage: String?
|
||||
@State private var uninstallInProgress = false
|
||||
|
||||
var body: some View {
|
||||
TabView {
|
||||
generalTab
|
||||
.tabItem {
|
||||
Label("General", systemImage: "gear")
|
||||
}
|
||||
modelTab
|
||||
.tabItem {
|
||||
Label("Model", systemImage: "cube")
|
||||
}
|
||||
advancedTab
|
||||
.tabItem {
|
||||
Label("Advanced", systemImage: "wrench.and.screwdriver")
|
||||
}
|
||||
environmentTab
|
||||
.tabItem {
|
||||
Label("Environment", systemImage: "terminal")
|
||||
}
|
||||
aboutTab
|
||||
.tabItem {
|
||||
Label("About", systemImage: "info.circle")
|
||||
}
|
||||
}
|
||||
.frame(width: 640, height: 560)
|
||||
.onAppear {
|
||||
pendingNamespace = controller.customNamespace
|
||||
pendingHFToken = controller.hfToken
|
||||
pendingHFEndpoint = controller.hfEndpoint
|
||||
pendingEnableImageModels = controller.enableImageModels
|
||||
pendingOfflineMode = controller.offlineMode
|
||||
pendingFastSynchEnabled = controller.fastSynchEnabled
|
||||
pendingDefaultModelsDir = controller.defaultModelsDir
|
||||
pendingAdditionalModelsDirs = controller.additionalModelsDirs
|
||||
pendingReadOnlyModelsDirs = controller.readOnlyModelsDirs
|
||||
pendingCustomEnvironmentVariables = controller.customEnvironmentVariables
|
||||
needsRestart = false
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - General Tab
|
||||
|
||||
private var generalTab: some View {
|
||||
Form {
|
||||
Section {
|
||||
LabeledContent("Cluster Namespace") {
|
||||
TextField("", text: $pendingNamespace, prompt: Text("default"))
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Nodes with the same namespace form a cluster. Leave empty for default.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section {
|
||||
LabeledContent("HuggingFace Token") {
|
||||
SecureField("", text: $pendingHFToken, prompt: Text("optional"))
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Required for gated models. Get yours at huggingface.co/settings/tokens")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section {
|
||||
LabeledContent("HuggingFace Endpoint") {
|
||||
TextField("", text: $pendingHFEndpoint, prompt: Text("default"))
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Defaults to huggingface.co. Use a mirror (e.g. hf-mirror.com) for China.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section {
|
||||
Toggle("Offline Mode", isOn: $pendingOfflineMode)
|
||||
Text("Skip internet checks and use only locally available models.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section {
|
||||
HStack {
|
||||
Spacer()
|
||||
Button("Save & Restart") {
|
||||
applyGeneralSettings()
|
||||
}
|
||||
.disabled(!hasGeneralChanges)
|
||||
}
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// MARK: - Model Tab
|
||||
|
||||
private var modelTab: some View {
|
||||
Form {
|
||||
Section {
|
||||
Toggle("Enable Image Models (experimental)", isOn: $pendingEnableImageModels)
|
||||
Text("Allow text-to-image and image-to-image models in the model picker.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section {
|
||||
HStack {
|
||||
Spacer()
|
||||
Button("Save & Restart") {
|
||||
applyModelSettings()
|
||||
}
|
||||
.disabled(!hasModelChanges)
|
||||
}
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// MARK: - Advanced Tab
|
||||
|
||||
private var advancedTab: some View {
|
||||
Form {
|
||||
Section("Performance") {
|
||||
Toggle("Fast Synch Enabled", isOn: $pendingFastSynchEnabled)
|
||||
Text(
|
||||
"Experimental: enables fast CPU to GPU synchronization. Can sometimes cause a \"GPU lock\" where inference hangs for ~10 seconds before starting. Necessary for low latency with RDMA and Tensor Parallelism."
|
||||
)
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
|
||||
HStack {
|
||||
Spacer()
|
||||
Button("Save & Restart") {
|
||||
applyAdvancedSettings()
|
||||
}
|
||||
.disabled(!hasAdvancedChanges)
|
||||
}
|
||||
}
|
||||
|
||||
Section("Onboarding") {
|
||||
HStack {
|
||||
VStack(alignment: .leading) {
|
||||
Text("Reset Onboarding")
|
||||
Text("Opens the dashboard and resets the onboarding wizard.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
Spacer()
|
||||
Button("Reset") {
|
||||
guard let url = URL(string: "http://localhost:52415/?reset-onboarding")
|
||||
else { return }
|
||||
NSWorkspace.shared.open(url)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Section("Debug Info") {
|
||||
LabeledContent("Thunderbolt Bridge") {
|
||||
Text(thunderboltStatusText)
|
||||
.foregroundColor(thunderboltStatusColor)
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
clusterThunderboltBridgeView
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
interfaceIpList
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
rdmaStatusView
|
||||
}
|
||||
|
||||
sendBugReportButton
|
||||
}
|
||||
|
||||
Section("Danger Zone") {
|
||||
Button(role: .destructive) {
|
||||
showUninstallConfirmationAlert()
|
||||
} label: {
|
||||
HStack {
|
||||
Text("Uninstall EXO")
|
||||
Spacer()
|
||||
Image(systemName: "trash")
|
||||
.imageScale(.small)
|
||||
}
|
||||
}
|
||||
.disabled(uninstallInProgress)
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// MARK: - Environment Tab
|
||||
|
||||
private var environmentTab: some View {
|
||||
Form {
|
||||
Section("Models Directories") {
|
||||
LabeledContent("Default Models Directory") {
|
||||
TextField(
|
||||
"",
|
||||
text: $pendingDefaultModelsDir,
|
||||
prompt: Text("~/.exo/models")
|
||||
)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Sets EXO_DEFAULT_MODELS_DIR. Where models are downloaded.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
|
||||
LabeledContent("Additional Directories") {
|
||||
TextField(
|
||||
"",
|
||||
text: $pendingAdditionalModelsDirs,
|
||||
prompt: Text("optional, colon-separated")
|
||||
)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Sets EXO_MODELS_DIRS. Extra writable model directories.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
|
||||
LabeledContent("Read-Only Directories") {
|
||||
TextField(
|
||||
"",
|
||||
text: $pendingReadOnlyModelsDirs,
|
||||
prompt: Text("optional, colon-separated")
|
||||
)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
.frame(width: 260)
|
||||
}
|
||||
Text("Sets EXO_MODELS_READ_ONLY_DIRS. Never written to.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
|
||||
Section("Custom Environment Variables") {
|
||||
Text(
|
||||
"Escape hatch for env vars that don't have typed fields above. "
|
||||
+ "Values here override the typed fields on conflict."
|
||||
)
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
|
||||
if pendingCustomEnvironmentVariables.isEmpty {
|
||||
Text("No custom variables.")
|
||||
.font(.caption)
|
||||
.foregroundColor(.secondary)
|
||||
} else {
|
||||
ForEach($pendingCustomEnvironmentVariables) { $variable in
|
||||
HStack(alignment: .center, spacing: 8) {
|
||||
VStack(spacing: 4) {
|
||||
TextField("key", text: $variable.key)
|
||||
.labelsHidden()
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
TextField("value", text: $variable.value)
|
||||
.labelsHidden()
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
}
|
||||
VStack(spacing: 4) {
|
||||
Button {
|
||||
pendingCustomEnvironmentVariables.removeAll {
|
||||
$0.id == variable.id
|
||||
}
|
||||
} label: {
|
||||
Image(systemName: "minus.circle")
|
||||
}
|
||||
.buttonStyle(.borderless)
|
||||
.help("Remove variable")
|
||||
if !isValidEnvironmentVariableName(variable.key) {
|
||||
Image(systemName: "exclamationmark.triangle.fill")
|
||||
.foregroundColor(.orange)
|
||||
.help(
|
||||
"Invalid environment variable name. "
|
||||
+ "Must match [A-Za-z_][A-Za-z0-9_]*."
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
HStack {
|
||||
Button {
|
||||
pendingCustomEnvironmentVariables.append(
|
||||
CustomEnvironmentVariable()
|
||||
)
|
||||
} label: {
|
||||
Label("Add Variable", systemImage: "plus")
|
||||
}
|
||||
Spacer()
|
||||
}
|
||||
}
|
||||
|
||||
Section {
|
||||
HStack {
|
||||
Spacer()
|
||||
Button("Save & Restart") {
|
||||
applyEnvironmentSettings()
|
||||
}
|
||||
.disabled(!hasEnvironmentChanges)
|
||||
}
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// MARK: - About Tab
|
||||
|
||||
private var aboutTab: some View {
|
||||
Form {
|
||||
Section {
|
||||
LabeledContent("Version") {
|
||||
Text(buildTag)
|
||||
.textSelection(.enabled)
|
||||
}
|
||||
LabeledContent("Commit") {
|
||||
Text(buildCommit)
|
||||
.font(.system(.body, design: .monospaced))
|
||||
.textSelection(.enabled)
|
||||
}
|
||||
}
|
||||
|
||||
Section {
|
||||
Button("Check for Updates") {
|
||||
updater.checkForUpdates()
|
||||
}
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// MARK: - Debug Info Views (moved from ContentView)
|
||||
|
||||
private var thunderboltStatusText: String {
|
||||
switch networkStatusService.status.thunderboltBridgeState {
|
||||
case .some(.disabled):
|
||||
return "Disabled"
|
||||
case .some(.deleted):
|
||||
return "Deleted"
|
||||
case .some(.enabled):
|
||||
return "Enabled"
|
||||
case nil:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
private var thunderboltStatusColor: Color {
|
||||
switch networkStatusService.status.thunderboltBridgeState {
|
||||
case .some(.disabled), .some(.deleted):
|
||||
return .green
|
||||
case .some(.enabled):
|
||||
return .red
|
||||
case nil:
|
||||
return .secondary
|
||||
}
|
||||
}
|
||||
|
||||
private var clusterThunderboltBridgeView: some View {
|
||||
let bridgeStatuses = stateService.latestSnapshot?.nodeThunderboltBridge ?? [:]
|
||||
let localNodeId = stateService.localNodeId
|
||||
let nodeProfiles = stateService.latestSnapshot?.nodeProfiles ?? [:]
|
||||
|
||||
return VStack(alignment: .leading, spacing: 1) {
|
||||
if bridgeStatuses.isEmpty {
|
||||
Text("Cluster TB Bridge: No data")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
} else {
|
||||
Text("Cluster TB Bridge Status:")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
ForEach(Array(bridgeStatuses.keys.sorted()), id: \.self) { nodeId in
|
||||
if let status = bridgeStatuses[nodeId] {
|
||||
let nodeName =
|
||||
nodeProfiles[nodeId]?.friendlyName ?? String(nodeId.prefix(8))
|
||||
let isLocal = nodeId == localNodeId
|
||||
let prefix = isLocal ? " \(nodeName) (local):" : " \(nodeName):"
|
||||
let statusText =
|
||||
!status.exists
|
||||
? "N/A"
|
||||
: (status.enabled ? "Enabled" : "Disabled")
|
||||
let color: Color =
|
||||
!status.exists
|
||||
? .secondary
|
||||
: (status.enabled ? .red : .green)
|
||||
Text("\(prefix) \(statusText)")
|
||||
.font(.caption2)
|
||||
.foregroundColor(color)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var interfaceIpList: some View {
|
||||
let statuses = networkStatusService.status.interfaceStatuses
|
||||
return VStack(alignment: .leading, spacing: 1) {
|
||||
Text("Interfaces (en0–en7):")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
if statuses.isEmpty {
|
||||
Text(" Unknown")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
} else {
|
||||
ForEach(statuses, id: \.interfaceName) { status in
|
||||
let ipText = status.ipAddress ?? "No IP"
|
||||
Text(" \(status.interfaceName): \(ipText)")
|
||||
.font(.caption2)
|
||||
.foregroundColor(status.ipAddress == nil ? .red : .green)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var rdmaStatusView: some View {
|
||||
let rdmaStatuses = stateService.latestSnapshot?.nodeRdmaCtl ?? [:]
|
||||
let localNodeId = stateService.localNodeId
|
||||
let nodeProfiles = stateService.latestSnapshot?.nodeProfiles ?? [:]
|
||||
let localDevices = networkStatusService.status.localRdmaDevices
|
||||
let localPorts = networkStatusService.status.localRdmaActivePorts
|
||||
|
||||
return VStack(alignment: .leading, spacing: 1) {
|
||||
if rdmaStatuses.isEmpty {
|
||||
Text("Cluster RDMA: No data")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
} else {
|
||||
Text("Cluster RDMA Status:")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
ForEach(Array(rdmaStatuses.keys.sorted()), id: \.self) { nodeId in
|
||||
if let status = rdmaStatuses[nodeId] {
|
||||
let nodeName =
|
||||
nodeProfiles[nodeId]?.friendlyName ?? String(nodeId.prefix(8))
|
||||
let isLocal = nodeId == localNodeId
|
||||
let prefix = isLocal ? " \(nodeName) (local):" : " \(nodeName):"
|
||||
let statusText = status.enabled ? "Enabled" : "Disabled"
|
||||
let color: Color = status.enabled ? .green : .orange
|
||||
Text("\(prefix) \(statusText)")
|
||||
.font(.caption2)
|
||||
.foregroundColor(color)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !localDevices.isEmpty {
|
||||
Text(" Local Devices: \(localDevices.joined(separator: ", "))")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
}
|
||||
if !localPorts.isEmpty {
|
||||
Text(" Local Active Ports:")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
ForEach(localPorts, id: \.device) { port in
|
||||
Text(" \(port.device) port \(port.port): \(port.state)")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.green)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var sendBugReportButton: some View {
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Button {
|
||||
Task {
|
||||
await sendBugReport()
|
||||
}
|
||||
} label: {
|
||||
HStack {
|
||||
if bugReportInFlight {
|
||||
ProgressView()
|
||||
.scaleEffect(0.6)
|
||||
}
|
||||
Text("Send Bug Report")
|
||||
.font(.caption)
|
||||
.fontWeight(.semibold)
|
||||
Spacer()
|
||||
}
|
||||
}
|
||||
.disabled(bugReportInFlight)
|
||||
|
||||
if let message = bugReportMessage {
|
||||
Text(message)
|
||||
.font(.caption2)
|
||||
.foregroundColor(.secondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Actions
|
||||
|
||||
private func sendBugReport() async {
|
||||
bugReportInFlight = true
|
||||
bugReportMessage = "Collecting logs..."
|
||||
let service = BugReportService()
|
||||
do {
|
||||
let outcome = try await service.sendReport(isManual: true)
|
||||
bugReportMessage = outcome.message
|
||||
} catch {
|
||||
bugReportMessage = error.localizedDescription
|
||||
}
|
||||
bugReportInFlight = false
|
||||
}
|
||||
|
||||
private func showUninstallConfirmationAlert() {
|
||||
let alert = NSAlert()
|
||||
alert.messageText = "Uninstall EXO"
|
||||
alert.informativeText = """
|
||||
This will remove EXO and all its components:
|
||||
|
||||
• Network configuration daemon
|
||||
• Launch at login registration
|
||||
• EXO network location
|
||||
• EXO data directory (~/.exo)
|
||||
|
||||
The app will be moved to Trash.
|
||||
"""
|
||||
alert.alertStyle = .warning
|
||||
|
||||
let checkbox = NSButton(
|
||||
checkboxWithTitle: "Keep downloaded models (~/.exo/models)",
|
||||
target: nil, action: nil)
|
||||
checkbox.state = .off
|
||||
checkbox.sizeToFit()
|
||||
alert.accessoryView = checkbox
|
||||
|
||||
alert.addButton(withTitle: "Uninstall")
|
||||
alert.addButton(withTitle: "Cancel")
|
||||
|
||||
if let uninstallButton = alert.buttons.first {
|
||||
uninstallButton.hasDestructiveAction = true
|
||||
}
|
||||
|
||||
let response = alert.runModal()
|
||||
if response == .alertFirstButtonReturn {
|
||||
performUninstall(keepModels: checkbox.state == .on)
|
||||
}
|
||||
}
|
||||
|
||||
private func performUninstall(keepModels: Bool) {
|
||||
uninstallInProgress = true
|
||||
|
||||
controller.cancelPendingLaunch()
|
||||
controller.stop()
|
||||
stateService.stopPolling()
|
||||
|
||||
DispatchQueue.global(qos: .utility).async {
|
||||
do {
|
||||
try NetworkSetupHelper.uninstall()
|
||||
try Self.removeExoDirectory(keepModels: keepModels)
|
||||
|
||||
DispatchQueue.main.async {
|
||||
LaunchAtLoginHelper.disable()
|
||||
self.moveAppToTrash()
|
||||
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
|
||||
NSApplication.shared.terminate(nil)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
DispatchQueue.main.async {
|
||||
let errorAlert = NSAlert()
|
||||
errorAlert.messageText = "Uninstall Failed"
|
||||
errorAlert.informativeText = error.localizedDescription
|
||||
errorAlert.alertStyle = .critical
|
||||
errorAlert.addButton(withTitle: "OK")
|
||||
errorAlert.runModal()
|
||||
self.uninstallInProgress = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func removeExoDirectory(keepModels: Bool) throws {
|
||||
let fm = FileManager.default
|
||||
let exoDir = ExoProcessController.exoDirectoryURL
|
||||
guard fm.fileExists(atPath: exoDir.path) else { return }
|
||||
|
||||
if !keepModels {
|
||||
try fm.removeItem(at: exoDir)
|
||||
return
|
||||
}
|
||||
|
||||
let contents = try fm.contentsOfDirectory(
|
||||
at: exoDir, includingPropertiesForKeys: nil, options: [])
|
||||
for entry in contents where entry.lastPathComponent != "models" {
|
||||
try? fm.removeItem(at: entry)
|
||||
}
|
||||
}
|
||||
|
||||
private func moveAppToTrash() {
|
||||
guard let appURL = Bundle.main.bundleURL as URL? else { return }
|
||||
do {
|
||||
try FileManager.default.trashItem(at: appURL, resultingItemURL: nil)
|
||||
} catch {
|
||||
// If we can't trash the app, that's OK - user can do it manually
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
private var hasGeneralChanges: Bool {
|
||||
pendingNamespace != controller.customNamespace || pendingHFToken != controller.hfToken
|
||||
|| pendingHFEndpoint != controller.hfEndpoint
|
||||
|| pendingOfflineMode != controller.offlineMode
|
||||
}
|
||||
|
||||
private var hasModelChanges: Bool {
|
||||
pendingEnableImageModels != controller.enableImageModels
|
||||
}
|
||||
|
||||
private var hasAdvancedChanges: Bool {
|
||||
pendingFastSynchEnabled != controller.fastSynchEnabled
|
||||
}
|
||||
|
||||
private var hasEnvironmentChanges: Bool {
|
||||
pendingDefaultModelsDir != controller.defaultModelsDir
|
||||
|| pendingAdditionalModelsDirs != controller.additionalModelsDirs
|
||||
|| pendingReadOnlyModelsDirs != controller.readOnlyModelsDirs
|
||||
|| pendingCustomEnvironmentVariables != controller.customEnvironmentVariables
|
||||
}
|
||||
|
||||
private func applyGeneralSettings() {
|
||||
controller.customNamespace = pendingNamespace
|
||||
controller.hfToken = pendingHFToken
|
||||
controller.hfEndpoint = pendingHFEndpoint
|
||||
controller.offlineMode = pendingOfflineMode
|
||||
restartIfRunning()
|
||||
}
|
||||
|
||||
private func applyModelSettings() {
|
||||
controller.enableImageModels = pendingEnableImageModels
|
||||
restartIfRunning()
|
||||
}
|
||||
|
||||
private func applyAdvancedSettings() {
|
||||
controller.fastSynchEnabled = pendingFastSynchEnabled
|
||||
restartIfRunning()
|
||||
}
|
||||
|
||||
private func applyEnvironmentSettings() {
|
||||
controller.defaultModelsDir = pendingDefaultModelsDir.trimmingCharacters(
|
||||
in: .whitespaces)
|
||||
controller.additionalModelsDirs = pendingAdditionalModelsDirs.trimmingCharacters(
|
||||
in: .whitespaces)
|
||||
controller.readOnlyModelsDirs = pendingReadOnlyModelsDirs.trimmingCharacters(
|
||||
in: .whitespaces)
|
||||
|
||||
pendingDefaultModelsDir = controller.defaultModelsDir
|
||||
pendingAdditionalModelsDirs = controller.additionalModelsDirs
|
||||
pendingReadOnlyModelsDirs = controller.readOnlyModelsDirs
|
||||
|
||||
// Trim whitespace from keys and drop empty ones so that the stored
|
||||
// form matches what is actually injected into the child process and
|
||||
// hasEnvironmentChanges doesn't show a stale diff after save.
|
||||
let trimmed: [CustomEnvironmentVariable] =
|
||||
pendingCustomEnvironmentVariables.compactMap { variable in
|
||||
let key = variable.key.trimmingCharacters(in: .whitespaces)
|
||||
guard !key.isEmpty else { return nil }
|
||||
return CustomEnvironmentVariable(
|
||||
id: variable.id, key: key, value: variable.value
|
||||
)
|
||||
}
|
||||
|
||||
// De-duplicate keys, keeping the last occurrence. This matches the
|
||||
// effective semantics of the dictionary assignment in
|
||||
// ExoProcessController.makeEnvironment and avoids silently losing
|
||||
// visible rows after save.
|
||||
var seenKeys = Set<String>()
|
||||
var deduplicatedReversed: [CustomEnvironmentVariable] = []
|
||||
for variable in trimmed.reversed() {
|
||||
if seenKeys.insert(variable.key).inserted {
|
||||
deduplicatedReversed.append(variable)
|
||||
}
|
||||
}
|
||||
let sanitized = Array(deduplicatedReversed.reversed())
|
||||
|
||||
pendingCustomEnvironmentVariables = sanitized
|
||||
controller.customEnvironmentVariables = sanitized
|
||||
|
||||
restartIfRunning()
|
||||
}
|
||||
|
||||
/// Validates a POSIX-style environment variable name:
|
||||
/// `[A-Za-z_][A-Za-z0-9_]*`. Uses an ASCII-only charset so that
|
||||
/// Unicode letters (e.g. `ñ`, Cyrillic) are rejected in line with what
|
||||
/// the help tooltip advertises. Empty strings are treated as valid
|
||||
/// here so that a freshly added blank row does not immediately look
|
||||
/// broken; the save step filters empty keys out instead.
|
||||
private func isValidEnvironmentVariableName(_ key: String) -> Bool {
|
||||
if key.isEmpty { return true }
|
||||
let headAllowed = CharacterSet(
|
||||
charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
|
||||
)
|
||||
let tailAllowed = headAllowed.union(CharacterSet(charactersIn: "0123456789"))
|
||||
guard let first = key.unicodeScalars.first, headAllowed.contains(first) else {
|
||||
return false
|
||||
}
|
||||
for scalar in key.unicodeScalars.dropFirst() {
|
||||
if !tailAllowed.contains(scalar) { return false }
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private func restartIfRunning() {
|
||||
if controller.status == .running || controller.status == .starting {
|
||||
controller.restart()
|
||||
}
|
||||
}
|
||||
|
||||
private var buildTag: String {
|
||||
Bundle.main.infoDictionary?["EXOBuildTag"] as? String ?? "unknown"
|
||||
}
|
||||
|
||||
private var buildCommit: String {
|
||||
Bundle.main.infoDictionary?["EXOBuildCommit"] as? String ?? "unknown"
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
/// Manages a standalone native macOS Settings window.
|
||||
/// Ensures only one instance exists and brings it to front on repeated opens.
|
||||
@MainActor
|
||||
final class SettingsWindowController: ObservableObject {
|
||||
private var window: NSWindow?
|
||||
|
||||
func open(
|
||||
controller: ExoProcessController,
|
||||
updater: SparkleUpdater,
|
||||
networkStatusService: NetworkStatusService,
|
||||
thunderboltBridgeService: ThunderboltBridgeService,
|
||||
stateService: ClusterStateService
|
||||
) {
|
||||
if let existing = window, existing.isVisible {
|
||||
existing.makeKeyAndOrderFront(nil)
|
||||
NSApp.activate(ignoringOtherApps: true)
|
||||
return
|
||||
}
|
||||
|
||||
let settingsView = SettingsView()
|
||||
.environmentObject(controller)
|
||||
.environmentObject(updater)
|
||||
.environmentObject(networkStatusService)
|
||||
.environmentObject(thunderboltBridgeService)
|
||||
.environmentObject(stateService)
|
||||
|
||||
let hostingView = NSHostingView(rootView: settingsView)
|
||||
|
||||
let newWindow = NSWindow(
|
||||
contentRect: NSRect(x: 0, y: 0, width: 640, height: 560),
|
||||
styleMask: [.titled, .closable],
|
||||
backing: .buffered,
|
||||
defer: false
|
||||
)
|
||||
newWindow.title = "EXO Settings"
|
||||
newWindow.contentView = hostingView
|
||||
newWindow.center()
|
||||
newWindow.isReleasedWhenClosed = false
|
||||
newWindow.makeKeyAndOrderFront(nil)
|
||||
NSApp.activate(ignoringOtherApps: true)
|
||||
|
||||
window = newWindow
|
||||
}
|
||||
}
|
||||
@@ -3,55 +3,25 @@
|
||||
# EXO Uninstaller Script
|
||||
#
|
||||
# This script removes all EXO system components that persist after deleting the app.
|
||||
# Run with: sudo ./uninstall-exo.sh [--keep-models]
|
||||
#
|
||||
# Options:
|
||||
# --keep-models Preserve ~/.exo/models when removing the EXO data directory.
|
||||
# Run with: sudo ./uninstall-exo.sh
|
||||
#
|
||||
# Components removed:
|
||||
# - LaunchDaemon: /Library/LaunchDaemons/io.exo.networksetup.plist
|
||||
# - Network script: /Library/Application Support/EXO/
|
||||
# - Log files: /var/log/io.exo.networksetup.*
|
||||
# - Network location: "exo"
|
||||
# - EXO data directory: ~/.exo (or all of ~/.exo except models/ when --keep-models is set)
|
||||
# - Launch at login registration
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
KEEP_MODELS=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--keep-models)
|
||||
KEEP_MODELS=1
|
||||
;;
|
||||
-h | --help)
|
||||
echo "Usage: sudo ./uninstall-exo.sh [--keep-models]"
|
||||
echo " --keep-models Preserve ~/.exo/models when removing the EXO data directory."
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $arg" >&2
|
||||
echo "Usage: sudo ./uninstall-exo.sh [--keep-models]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
LABEL="io.exo.networksetup"
|
||||
# Current script path. Older installs used a different filename; keep the
|
||||
# legacy path here so a fresh uninstall still cleans up upgraded machines.
|
||||
CURRENT_SCRIPT_DEST="/Library/Application Support/EXO/disable_bridge.sh"
|
||||
LEGACY_SCRIPT_DEST="/Library/Application Support/EXO/disable_bridge_enable_dhcp.sh"
|
||||
SCRIPT_DEST="/Library/Application Support/EXO/disable_bridge_enable_dhcp.sh"
|
||||
PLIST_DEST="/Library/LaunchDaemons/io.exo.networksetup.plist"
|
||||
LOG_OUT="/var/log/${LABEL}.log"
|
||||
LOG_ERR="/var/log/${LABEL}.err.log"
|
||||
APP_BUNDLE_ID="io.exo.EXO"
|
||||
|
||||
# Resolve the invoking user's home, even when run via sudo.
|
||||
USER_HOME="$(eval echo "~${SUDO_USER:-$USER}")"
|
||||
EXO_DIR="$USER_HOME/.exo"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@@ -99,17 +69,11 @@ else
|
||||
echo_warn "LaunchDaemon plist not found (already removed?)"
|
||||
fi
|
||||
|
||||
# Remove the script (current and legacy filenames) — backwards-compatible:
|
||||
# tolerate either, both, or neither being present.
|
||||
removed_any_script=0
|
||||
for script in "$CURRENT_SCRIPT_DEST" "$LEGACY_SCRIPT_DEST"; do
|
||||
if [[ -f $script ]]; then
|
||||
rm -f "$script"
|
||||
echo_info "Removed network setup script: $script"
|
||||
removed_any_script=1
|
||||
fi
|
||||
done
|
||||
if [[ $removed_any_script -eq 0 ]]; then
|
||||
# Remove the script and parent directory
|
||||
if [[ -f $SCRIPT_DEST ]]; then
|
||||
rm -f "$SCRIPT_DEST"
|
||||
echo_info "Removed network setup script"
|
||||
else
|
||||
echo_warn "Network setup script not found (already removed?)"
|
||||
fi
|
||||
|
||||
@@ -151,22 +115,6 @@ if networksetup -listnetworkservices 2>/dev/null | grep -q "Thunderbolt Bridge";
|
||||
echo_info "Re-enabled Thunderbolt Bridge"
|
||||
fi
|
||||
|
||||
# Remove EXO data directory (~/.exo)
|
||||
EXO_DIR_REMOVED=""
|
||||
if [[ -d $EXO_DIR ]]; then
|
||||
if [[ $KEEP_MODELS == "1" && -d "$EXO_DIR/models" ]]; then
|
||||
find "$EXO_DIR" -mindepth 1 -maxdepth 1 ! -name models -exec rm -rf {} +
|
||||
EXO_DIR_REMOVED="kept_models"
|
||||
echo_info "Removed ~/.exo (preserved models/)"
|
||||
else
|
||||
rm -rf "$EXO_DIR"
|
||||
EXO_DIR_REMOVED="full"
|
||||
echo_info "Removed ~/.exo"
|
||||
fi
|
||||
else
|
||||
echo_warn "~/.exo not found (already removed?)"
|
||||
fi
|
||||
|
||||
# Note about launch at login registration
|
||||
# SMAppService-based login items cannot be removed from a shell script.
|
||||
# They can only be unregistered from within the app itself or manually via System Settings.
|
||||
@@ -196,10 +144,6 @@ echo " • Network setup LaunchDaemon"
|
||||
echo " • Network configuration script"
|
||||
echo " • Log files"
|
||||
echo " • 'exo' network location"
|
||||
case "$EXO_DIR_REMOVED" in
|
||||
full) echo " • EXO data directory (~/.exo)" ;;
|
||||
kept_models) echo " • EXO data directory (~/.exo, models preserved)" ;;
|
||||
esac
|
||||
echo ""
|
||||
echo "Your network has been restored to use the 'Automatic' location."
|
||||
echo "Thunderbolt Bridge has been re-enabled (if present)."
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
# Exo-Bench — Methodology
|
||||
|
||||
exo bench measures inference throughput and resource consumption of an exo cluster under controlled conditions. It sends prompts to the `/bench/chat/completions` endpoint, collects server-reported timing statistics, and records system-level metrics (power, GPU utilisation, temperature) throughout each run.
|
||||
|
||||
The goal is to have accurate, transparent and reproducible numbers to compare speed and scaling across different models and different setups, and to be able to track these results as optimizations and features are added to EXO.
|
||||
|
||||
Below is a technical summary of how Exo-Bench works. While the methodology and benchmark may change over time, this document will be kept up to date whenever this happens. If you find an issue with the methodology, or would like a feature to be added, please open a GitHub issue!
|
||||
|
||||
---
|
||||
|
||||
## Prompt Construction
|
||||
|
||||
Benchmarks need prompts of an exact token length. Unfortunately, we do not have direct access to the model but just the chat completion endpoint. To get around this fact, we create a request that will tokenise to a certain prompt length.
|
||||
|
||||
This is achieved by:
|
||||
|
||||
1. Tokenising a sample message through the model's `apply_chat_template()` to measure overhead (system tokens, special tokens, chat formatting).
|
||||
2. Binary-searching over a repeated atom string (default `"a "`) to find the content length that produces exactly the target number of tokens after template expansion.
|
||||
3. Returning both the content string and the verified token count.
|
||||
|
||||
The actual token count is recorded in every result row as `pp_tokens`, so downstream analysis can confirm the prompt hit its target.
|
||||
|
||||
Chat template formatting means that it may be impossible to attain very small pp benchmarks. e.g. pp=32 may not work. This tradeoff was made because the result of such a small prompt does not seem very interesting or useful for any real-world use cases.
|
||||
|
||||
---
|
||||
|
||||
## Bench Endpoint
|
||||
|
||||
When a request reaches the server via the `/bench/chat/completions` endpoint, three things change compared to a normal chat completion:
|
||||
|
||||
- **KV prefix cache is disabled by default**. Every request starts from a cold cache, ensuring prefill timing is not affected by prior requests. See [Prefix Cache Mode](#prefix-cache-mode) for the `--use-prefix-cache` option.
|
||||
- **EOS tokens are banned**. A logits processor suppresses all end-of-sequence tokens, forcing the model to generate exactly `max_tokens` tokens. This guarantees consistent generation length for fair TPS comparison — the model cannot short-circuit a run by stopping early.
|
||||
- **No model output parsing**. The bench collection path concatenates raw token text without any model-specific post-processing (thinking tag extraction, structured output handling, etc.). This is to avoid model outputs such as tool parsing or any structural mistakes from breaking the benchmark - we are testing for speed; see Exo-Eval for performance metrics.
|
||||
|
||||
---
|
||||
|
||||
## Timing
|
||||
|
||||
### Prefill TPS
|
||||
|
||||
Measured server-side per task.
|
||||
|
||||
```
|
||||
prefill_tps = num_prompt_tokens / prefill_wall_seconds
|
||||
```
|
||||
|
||||
### Generation TPS
|
||||
|
||||
Measured server-side per task. Each task records wall-clock timestamps as tokens arrive:
|
||||
|
||||
- First generated token: timestamp recorded
|
||||
- Every subsequent token: timestamp updated
|
||||
|
||||
When generation completes:
|
||||
|
||||
```
|
||||
gen_span = last_token_time - first_token_time
|
||||
generation_tps = (completion_tokens - 1) / gen_span
|
||||
```
|
||||
|
||||
The first token is excluded from the numerator because the rate measures inter-token throughput — the time between the first and last token divided by the number of intervals.
|
||||
|
||||
This does mean that tg=1 will not work.
|
||||
|
||||
---
|
||||
|
||||
## Concurrency
|
||||
|
||||
### Single Request
|
||||
|
||||
The client records wall-clock `elapsed_s` around the HTTP round-trip (network latency + server prefill + generation + response serialisation). This is a convenience metric for end-to-end latency. The authoritative TPS numbers come from the server-side per-task timing in the `generation_stats` response.
|
||||
|
||||
### Concurrent Requests
|
||||
|
||||
When `--concurrency N` is set with N > 1, all N requests must hit the server at the same instant. The mechanism:
|
||||
|
||||
1. The prompt is built once and shared across all threads.
|
||||
2. Each thread gets its own HTTP connection.
|
||||
3. A thread barrier blocks all threads until every thread is ready.
|
||||
4. The first thread past the barrier records the batch start time and signals the others.
|
||||
5. All threads use the same start time as their reference, then fire their HTTP request.
|
||||
6. Each thread's `elapsed_s` is measured from the shared start time to its own response completion.
|
||||
|
||||
**Batch wall time** is the maximum `elapsed_s` across all N requests — the time until the last request finishes.
|
||||
|
||||
### Aggregate TPS
|
||||
|
||||
```
|
||||
per_req_tps = max(generation_tps across N concurrent requests)
|
||||
agg_gen_tps = per_req_tps * concurrency
|
||||
```
|
||||
|
||||
`max` is used instead of `mean` because all requests run in parallel against the same model. The fastest request's generation rate represents the system's per-stream throughput capacity; multiplying by concurrency gives aggregate throughput.
|
||||
|
||||
---
|
||||
|
||||
## Prefix Cache Mode
|
||||
|
||||
When `--use-prefix-cache` is passed, the KV prefix cache remains active during benchmarking. This speeds up repeated runs by skipping redundant prefill work, which is useful when prompt processing is not the focus of the benchmark (e.g. when measuring generation throughput or power consumption across many configurations).
|
||||
|
||||
Each response includes a `prefix_cache_hit` field (`"none"`, `"partial"`, or `"exact"`):
|
||||
|
||||
- **none**: Cold prefill — no cached KV state was available. The reported `prompt_tps` is the real prefill throughput.
|
||||
- **partial**: A prefix of the prompt was found in cache. Only the remaining tokens were prefilled. The reported `prompt_tps` reflects the real throughput on the uncached portion. This occurs when multiple ascending `--pp` values share a common prefix (e.g. `--pp 1000,5000` — the 5000-token prompt reuses the 1000-token cache entry and prefills the remaining 4000 tokens).
|
||||
- **exact**: The entire prompt was found in cache (e.g. same `--pp` value on a `--repeat`). No prefill work was done. The reported `prompt_tps` is the TPS from when the cache entry was originally created, not a new measurement.
|
||||
|
||||
**Prompt TPS is approximate in this mode.** Exact-hit runs report the stored TPS from the original cold/partial prefill rather than a freshly measured value. For accurate cold prefill numbers, run without `--use-prefix-cache`.
|
||||
|
||||
Ascending `--pp` order (e.g. `--pp 1000,5000,10000`) gives the most useful data: each size gets a meaningful partial hit except the first which is cold. Descending order produces exact hits with approximate TPS from a longer prompt's original run.
|
||||
|
||||
---
|
||||
|
||||
## Warmup
|
||||
|
||||
Before measurement begins, `--warmup N` (default: 0) discarded requests are sent using the first pp/tg pair. Warmup results are not included in the output.
|
||||
|
||||
---
|
||||
|
||||
## System Metrics
|
||||
|
||||
A background thread polls each node at 1 Hz, collecting:
|
||||
|
||||
- GPU utilisation (%)
|
||||
- Temperature (C)
|
||||
- System power draw (W)
|
||||
- CPU cluster usage (performance and efficiency cores)
|
||||
|
||||
**Energy** is computed via trapezoidal integration of the power samples over each inference window (the wall-clock span of each benchmark request or concurrent batch). Average power is `total_joules / total_inference_seconds`.
|
||||
|
||||
---
|
||||
|
||||
## Output Format
|
||||
|
||||
Results are written as JSON with three top-level keys:
|
||||
|
||||
- **`runs`**: Array of per-request result objects, each containing:
|
||||
- `elapsed_s`, `output_text_preview` (first 200 chars)
|
||||
- `stats`: `{ prompt_tps, generation_tps, prompt_tokens, generation_tokens, peak_memory_usage }`
|
||||
- Placement metadata: `model_id`, `placement_sharding`, `placement_instance_meta`, `placement_nodes`
|
||||
- Run metadata: `pp_tokens`, `tg`, `repeat_index`, `concurrency`, `concurrent_index`
|
||||
- `download_duration_s` (if model was freshly downloaded)
|
||||
- **`cluster`**: Cluster state snapshot at time of benchmarking.
|
||||
- **`system_metrics`**: Per-node time-series samples (GPU, power, temperature).
|
||||
|
||||
---
|
||||
|
||||
## Reproducing Results
|
||||
|
||||
```bash
|
||||
cd bench && uv run python exo_bench.py \
|
||||
--model "mlx-community/Qwen3.5-27B-4bit" \
|
||||
--instance-meta jaccl \
|
||||
--sharding tensor \
|
||||
--min-nodes 2 --max-nodes 2 \
|
||||
--pp 512 4096 --tg 128 \
|
||||
--repeat 3 \
|
||||
--warmup 1
|
||||
```
|
||||
|
||||
Run --help for all the available flags.
|
||||
+3
-1
@@ -2,4 +2,6 @@
|
||||
#
|
||||
# Lists the suite files to include. Each file defines benchmarks
|
||||
# with shared constraints, topology, and default args.
|
||||
include = ["single-m3-ultra.toml"]
|
||||
include = [
|
||||
"single-m3-ultra.toml",
|
||||
]
|
||||
@@ -1,347 +0,0 @@
|
||||
# Model evaluation configurations for exo_eval.
|
||||
#
|
||||
# Each [[model]] entry uses `patterns` — a list of substrings matched
|
||||
# against the model_id. First matching entry wins.
|
||||
#
|
||||
# Required fields:
|
||||
# name, patterns, reasoning
|
||||
#
|
||||
# Optional per-model overrides (CLI flags take priority over these):
|
||||
# temperature, top_p, max_tokens, reasoning_effort, enable_thinking
|
||||
#
|
||||
# Fallback defaults (when no per-model config):
|
||||
# reasoning: temperature=1.0, max_tokens=131072, reasoning_effort="high"
|
||||
# non-reasoning: temperature=0.0, max_tokens=16384
|
||||
#
|
||||
# All per-model values below are sourced from official model cards,
|
||||
# generation_config.json files, and vendor documentation.
|
||||
|
||||
# ─── Qwen3.5 (Feb 2026) ─────────────────────────────────────────────
|
||||
# Source: HuggingFace model cards (Qwen/Qwen3.5-*)
|
||||
# Model card recommends: temp=0.6, top_p=0.95, top_k=20
|
||||
# We omit top_k to match vllm eval (which doesn't set it).
|
||||
# max_tokens=121072 to match vllm eval (131072 context - 10000 safety margin).
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 2B"
|
||||
patterns = ["Qwen3.5-2B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 9B"
|
||||
patterns = ["Qwen3.5-9B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 27B"
|
||||
patterns = ["Qwen3.5-27B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 35B A3B"
|
||||
patterns = ["Qwen3.5-35B-A3B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 122B A10B"
|
||||
patterns = ["Qwen3.5-122B-A10B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3.5 397B A17B"
|
||||
patterns = ["Qwen3.5-397B-A17B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
# ─── Qwen3 (Apr 2025) ───────────────────────────────────────────────
|
||||
# Source: HuggingFace model cards (Qwen/Qwen3-*)
|
||||
# Model card recommends: temp=0.6, top_p=0.95, top_k=20
|
||||
# We omit top_k to match vllm eval (which doesn't set it).
|
||||
# Non-thinking: temp=0.7, top_p=0.8
|
||||
# max_tokens: 32768 general, 38912 for complex math/code
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 0.6B"
|
||||
patterns = ["Qwen3-0.6B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 38912
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 30B A3B"
|
||||
patterns = ["Qwen3-30B-A3B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 38912
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 235B A22B"
|
||||
patterns = ["Qwen3-235B-A22B"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 38912
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 Next 80B Thinking"
|
||||
patterns = ["Qwen3-Next-80B-A3B-Thinking"]
|
||||
reasoning = true
|
||||
temperature = 0.6
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 38912
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 Next 80B Instruct"
|
||||
patterns = ["Qwen3-Next-80B-A3B-Instruct"]
|
||||
reasoning = false
|
||||
temperature = 0.7
|
||||
top_p = 0.8
|
||||
max_tokens = 16384
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 Coder 480B"
|
||||
patterns = ["Qwen3-Coder-480B"]
|
||||
reasoning = false
|
||||
temperature = 0.7
|
||||
top_p = 0.8
|
||||
max_tokens = 16384
|
||||
|
||||
[[model]]
|
||||
name = "Qwen3 Coder Next"
|
||||
patterns = ["Qwen3-Coder-Next"]
|
||||
reasoning = false
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
max_tokens = 121072
|
||||
|
||||
# ─── GPT-OSS (OpenAI) ───────────────────────────────────────────────
|
||||
# Source: OpenAI GitHub README + HuggingFace discussion #21
|
||||
# temp=1.0, top_p=1.0, NO top_k, NO repetition_penalty
|
||||
# reasoning_effort supported: low/medium/high
|
||||
|
||||
[[model]]
|
||||
name = "GPT-OSS 20B"
|
||||
patterns = ["gpt-oss-20b"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 1.0
|
||||
|
||||
[[model]]
|
||||
name = "GPT-OSS 120B"
|
||||
patterns = ["gpt-oss-120b"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 1.0
|
||||
|
||||
# ─── DeepSeek ────────────────────────────────────────────────────────
|
||||
# Source: https://api-docs.deepseek.com/quick_start/parameter_settings
|
||||
# Coding/Math: temp=0.0, General: temp=1.3, Creative: temp=1.5
|
||||
# NOTE: DeepSeek API applies nonlinear temp mapping. These are API values.
|
||||
# When running model directly: API temp 1.0 = model temp 0.3
|
||||
# We use temp=0.0 for eval (coding/math focus).
|
||||
|
||||
[[model]]
|
||||
name = "DeepSeek V3.1"
|
||||
patterns = ["DeepSeek-V3.1"]
|
||||
reasoning = true
|
||||
temperature = 0.0
|
||||
|
||||
[[model]]
|
||||
name = "DeepSeek V3.2"
|
||||
patterns = ["DeepSeek-V3.2"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
|
||||
# ─── NVIDIA Nemotron ───────────────────────────────────────────────────
|
||||
# Source: HuggingFace model cards
|
||||
# All variants: temp=1.0, top_p=0.95, enable_thinking=true
|
||||
|
||||
[[model]]
|
||||
name = "Nemotron Cascade 2 30B A3B"
|
||||
patterns = ["Nemotron-Cascade-2-30B-A3B"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
|
||||
[[model]]
|
||||
name = "Nemotron 3 Super 120B A12B"
|
||||
patterns = ["Nemotron-3-Super-120B-A12B", "NVIDIA-Nemotron-3-Super-120B-A12B"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
|
||||
# ─── GLM (ZhipuAI / THUDM) ──────────────────────────────────────────
|
||||
# Source: HuggingFace model cards + generation_config.json + docs.z.ai
|
||||
# GLM 4.5+: temp=1.0, top_p=0.95
|
||||
# max_tokens=121072 to match vllm eval (131072 context - 10000 safety margin)
|
||||
|
||||
[[model]]
|
||||
name = "GLM-5"
|
||||
patterns = ["GLM-5"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "GLM 4.5 Air"
|
||||
patterns = ["GLM-4.5-Air"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
|
||||
[[model]]
|
||||
name = "GLM 4.7"
|
||||
patterns = ["GLM-4.7-"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
# Note: matches both GLM-4.7 and GLM-4.7-Flash
|
||||
|
||||
# ─── Kimi (Moonshot AI) ─────────────────────────────────────────────
|
||||
# Source: HuggingFace model cards (moonshotai/Kimi-K2-*)
|
||||
# K2-Instruct: temp=0.6
|
||||
# K2-Thinking: temp=1.0, max_length=262144
|
||||
# K2.5: thinking temp=1.0, top_p=0.95; instant temp=0.6, top_p=0.95
|
||||
|
||||
[[model]]
|
||||
name = "Kimi K2 Thinking"
|
||||
patterns = ["Kimi-K2-Thinking"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
max_tokens = 131072
|
||||
|
||||
[[model]]
|
||||
name = "Kimi K2.5"
|
||||
patterns = ["Kimi-K2.5"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
[[model]]
|
||||
name = "Kimi K2 Instruct"
|
||||
patterns = ["Kimi-K2-Instruct"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
|
||||
# ─── MiniMax ─────────────────────────────────────────────────────────
|
||||
# Source: HuggingFace model cards + generation_config.json
|
||||
# All models: temp=1.0, top_p=0.95
|
||||
# max_tokens=90000 to match vllm eval (100000 context - 10000 safety margin)
|
||||
|
||||
[[model]]
|
||||
name = "MiniMax M2.7"
|
||||
patterns = ["MiniMax-M2.7"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 90000
|
||||
|
||||
[[model]]
|
||||
name = "MiniMax M2.5"
|
||||
patterns = ["MiniMax-M2.5"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 90000
|
||||
|
||||
[[model]]
|
||||
name = "MiniMax M2.1"
|
||||
patterns = ["MiniMax-M2.1"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
|
||||
# ─── Step (StepFun) ─────────────────────────────────────────────────
|
||||
# Source: HuggingFace model card (stepfun-ai/Step-3.5-Flash)
|
||||
# Reasoning: temp=1.0, top_p=0.95
|
||||
# General chat: temp=0.6, top_p=0.95
|
||||
# We use reasoning settings for eval.
|
||||
|
||||
[[model]]
|
||||
name = "Step 3.5 Flash"
|
||||
patterns = ["Step-3.5-Flash"]
|
||||
reasoning = true
|
||||
temperature = 1.0
|
||||
top_p = 0.95
|
||||
enable_thinking = true
|
||||
max_tokens = 121072
|
||||
|
||||
# ─── Llama (Meta) ───────────────────────────────────────────────────
|
||||
# Source: generation_config.json + meta-llama/llama-models generation.py
|
||||
# All variants: temp=0.6, top_p=0.9
|
||||
|
||||
[[model]]
|
||||
name = "Llama 3.2 1B"
|
||||
patterns = ["Llama-3.2-1B"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
top_p = 0.9
|
||||
|
||||
[[model]]
|
||||
name = "Llama 3.2 3B"
|
||||
patterns = ["Llama-3.2-3B"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
top_p = 0.9
|
||||
|
||||
[[model]]
|
||||
name = "Llama 3.1 8B"
|
||||
patterns = ["Llama-3.1-8B", "Meta-Llama-3.1-8B"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
top_p = 0.9
|
||||
|
||||
[[model]]
|
||||
name = "Llama 3.1 70B"
|
||||
patterns = ["Llama-3.1-70B", "Meta-Llama-3.1-70B"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
top_p = 0.9
|
||||
|
||||
[[model]]
|
||||
name = "Llama 3.3 70B"
|
||||
patterns = ["Llama-3.3-70B", "llama-3.3-70b"]
|
||||
reasoning = false
|
||||
temperature = 0.6
|
||||
top_p = 0.9
|
||||
+39
-89
@@ -3,13 +3,11 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
@@ -19,7 +17,6 @@ from harness import (
|
||||
ExoClient,
|
||||
ExoHttpError,
|
||||
add_common_instance_args,
|
||||
capture_cluster_snapshot,
|
||||
instance_id_from_instance,
|
||||
nodes_used_in_instance,
|
||||
resolve_model_short_id,
|
||||
@@ -211,7 +208,7 @@ def _openai_build_request(
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
return "/v1/chat/completions", body
|
||||
@@ -278,7 +275,7 @@ def _openai_build_followup(
|
||||
"model": model,
|
||||
"messages": followup_messages,
|
||||
"tools": tools,
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
return "/v1/chat/completions", body
|
||||
@@ -381,7 +378,7 @@ def _claude_build_request(
|
||||
"model": model,
|
||||
"messages": claude_messages,
|
||||
"tools": claude_tools,
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
if system_content is not None:
|
||||
@@ -491,7 +488,7 @@ def _claude_build_followup(
|
||||
"model": model,
|
||||
"messages": claude_messages,
|
||||
"tools": claude_tools,
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
if system_content is not None:
|
||||
@@ -915,12 +912,6 @@ Examples:
|
||||
default=1,
|
||||
help="Repeat each scenario N times (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Run up to N scenarios in parallel against the same instance (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scenarios",
|
||||
nargs="*",
|
||||
@@ -943,13 +934,6 @@ Examples:
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.concurrency < 1:
|
||||
print(
|
||||
f"--concurrency must be >= 1 (got {args.concurrency})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
all_scenarios = load_scenarios(SCENARIOS_PATH)
|
||||
if args.scenarios:
|
||||
scenarios = [s for s in all_scenarios if s.name in args.scenarios]
|
||||
@@ -1022,75 +1006,44 @@ Examples:
|
||||
sys.exit(1)
|
||||
|
||||
time.sleep(1)
|
||||
cluster_snapshot = capture_cluster_snapshot(exo)
|
||||
all_results: list[ScenarioResult] = []
|
||||
|
||||
tasks: list[tuple[int, Scenario, ApiName]] = [
|
||||
(run_idx, scenario, api_name)
|
||||
for run_idx in range(args.repeat)
|
||||
for scenario in scenarios
|
||||
for api_name in api_names
|
||||
]
|
||||
|
||||
def _run_one(
|
||||
http_client: httpx.Client,
|
||||
task: tuple[int, Scenario, ApiName],
|
||||
) -> tuple[tuple[int, Scenario, ApiName], list[ScenarioResult], str]:
|
||||
run_idx, scenario, api_name = task
|
||||
buf = io.StringIO()
|
||||
run_tag = f"[run {run_idx + 1}/{args.repeat}]" if args.repeat > 1 else ""
|
||||
print(
|
||||
f"\n {run_tag}[{api_name:>9}] {scenario.name}: {scenario.description}",
|
||||
file=buf,
|
||||
)
|
||||
scenario_results = run_scenario(
|
||||
http_client,
|
||||
args.host,
|
||||
args.port,
|
||||
full_model_id,
|
||||
scenario,
|
||||
api_name,
|
||||
args.timeout,
|
||||
args.verbose,
|
||||
)
|
||||
for r in scenario_results:
|
||||
status = "PASS" if r.passed else "FAIL"
|
||||
print(
|
||||
f" [{r.phase:>10}] {status} ({r.latency_ms:.0f}ms)",
|
||||
file=buf,
|
||||
)
|
||||
for check_name, check_ok in r.checks.items():
|
||||
mark = "+" if check_ok else "-"
|
||||
print(f" {mark} {check_name}", file=buf)
|
||||
if r.error:
|
||||
print(f" ! {r.error}", file=buf)
|
||||
return task, scenario_results, buf.getvalue()
|
||||
|
||||
try:
|
||||
with httpx.Client() as http_client:
|
||||
if args.concurrency == 1:
|
||||
current_run = -1
|
||||
for task in tasks:
|
||||
run_idx = task[0]
|
||||
if args.repeat > 1 and run_idx != current_run:
|
||||
print(f"\n--- Run {run_idx + 1}/{args.repeat} ---", file=log)
|
||||
current_run = run_idx
|
||||
_, scenario_results, buffered = _run_one(http_client, task)
|
||||
all_results.extend(scenario_results)
|
||||
log.write(buffered)
|
||||
log.flush()
|
||||
else:
|
||||
print(
|
||||
f"Running {len(tasks)} tasks with concurrency={args.concurrency}",
|
||||
file=log,
|
||||
)
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = [pool.submit(_run_one, http_client, t) for t in tasks]
|
||||
for fut in as_completed(futures):
|
||||
_, scenario_results, buffered = fut.result()
|
||||
for run_idx in range(args.repeat):
|
||||
if args.repeat > 1:
|
||||
print(f"\n--- Run {run_idx + 1}/{args.repeat} ---", file=log)
|
||||
|
||||
for scenario in scenarios:
|
||||
for api_name in api_names:
|
||||
print(
|
||||
f"\n [{api_name:>9}] {scenario.name}: {scenario.description}",
|
||||
file=log,
|
||||
)
|
||||
|
||||
scenario_results = run_scenario(
|
||||
http_client,
|
||||
args.host,
|
||||
args.port,
|
||||
full_model_id,
|
||||
scenario,
|
||||
api_name,
|
||||
args.timeout,
|
||||
args.verbose,
|
||||
)
|
||||
all_results.extend(scenario_results)
|
||||
log.write(buffered)
|
||||
log.flush()
|
||||
|
||||
for r in scenario_results:
|
||||
status = "PASS" if r.passed else "FAIL"
|
||||
print(
|
||||
f" [{r.phase:>10}] {status} ({r.latency_ms:.0f}ms)",
|
||||
file=log,
|
||||
)
|
||||
for check_name, check_ok in r.checks.items():
|
||||
mark = "+" if check_ok else "-"
|
||||
print(f" {mark} {check_name}", file=log)
|
||||
if r.error:
|
||||
print(f" ! {r.error}", file=log)
|
||||
finally:
|
||||
try:
|
||||
exo.request_json("DELETE", f"/instance/{instance_id}")
|
||||
@@ -1131,19 +1084,16 @@ Examples:
|
||||
print(f" - {r.name} [{r.api}/{r.phase}]: {r.error}", file=log)
|
||||
|
||||
json_results = [result_to_dict(r) for r in all_results]
|
||||
output: dict[str, Any] = {"results": json_results}
|
||||
if cluster_snapshot:
|
||||
output["cluster"] = cluster_snapshot
|
||||
|
||||
if args.stdout:
|
||||
print(json.dumps(output, indent=2))
|
||||
print(json.dumps(json_results, indent=2))
|
||||
else:
|
||||
json_path = args.json_out
|
||||
parent = os.path.dirname(json_path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(output, f, indent=2)
|
||||
json.dump(json_results, f, indent=2)
|
||||
f.write("\n")
|
||||
print(f"\nJSON results written to {json_path}", file=log)
|
||||
|
||||
|
||||
+124
-581
@@ -22,10 +22,8 @@ import contextlib
|
||||
import itertools
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Any
|
||||
@@ -34,10 +32,7 @@ from harness import (
|
||||
ExoClient,
|
||||
ExoHttpError,
|
||||
add_common_instance_args,
|
||||
capture_cluster_snapshot,
|
||||
find_existing_instance,
|
||||
instance_id_from_instance,
|
||||
node_ids_from_instance,
|
||||
nodes_used_in_instance,
|
||||
resolve_model_short_id,
|
||||
run_planning_phase,
|
||||
@@ -80,7 +75,7 @@ def load_tokenizer_for_bench(model_id: str) -> Any:
|
||||
model_path = Path(
|
||||
snapshot_download(
|
||||
model_id,
|
||||
allow_patterns=["*.json", "*.py", "*.tiktoken", "*.model", "*.jinja"],
|
||||
allow_patterns=["*.json", "*.py", "*.tiktoken"],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -123,48 +118,8 @@ def load_tokenizer_for_bench(model_id: str) -> Any:
|
||||
|
||||
return hf_tokenizer
|
||||
|
||||
# TODO: Change back to using only transformers
|
||||
try:
|
||||
return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
||||
except (AttributeError, ValueError):
|
||||
from huggingface_hub import snapshot_download
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
model_path = Path(
|
||||
snapshot_download(
|
||||
model_id,
|
||||
allow_patterns=[
|
||||
"*.json",
|
||||
"*.py",
|
||||
"tokenizer.model",
|
||||
"*.tiktoken",
|
||||
"tiktoken.model",
|
||||
"*.txt",
|
||||
"*.jsonl",
|
||||
"*.jinja",
|
||||
],
|
||||
)
|
||||
)
|
||||
stub_kwargs: dict[str, Any] = {}
|
||||
config_file = model_path / "config.json"
|
||||
if config_file.exists():
|
||||
with open(config_file) as f:
|
||||
raw = json.load(f)
|
||||
for key in (
|
||||
"model_type",
|
||||
"max_position_embeddings",
|
||||
"vocab_size",
|
||||
"bos_token_id",
|
||||
"eos_token_id",
|
||||
"pad_token_id",
|
||||
):
|
||||
if key in raw:
|
||||
stub_kwargs[key] = raw[key]
|
||||
return AutoTokenizer.from_pretrained(
|
||||
str(model_path),
|
||||
config=PretrainedConfig(**stub_kwargs),
|
||||
trust_remote_code=True,
|
||||
)
|
||||
# Default: use AutoTokenizer
|
||||
return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
|
||||
def format_peak_memory(b: float) -> str:
|
||||
@@ -175,91 +130,6 @@ def format_peak_memory(b: float) -> str:
|
||||
raise ValueError("You're using petabytes of memory. Something went wrong...")
|
||||
|
||||
|
||||
_SAMPLER_METRICS = ("gpuUsage", "temp", "sysPower", "pcpuUsage", "ecpuUsage")
|
||||
|
||||
|
||||
class SystemMetricsSampler:
|
||||
def __init__(self, client: ExoClient, node_ids: list[str], interval_s: float = 1.0):
|
||||
self._client = client
|
||||
self._node_ids = node_ids
|
||||
self._interval_s = interval_s
|
||||
self._samples: dict[str, list[tuple[float, dict[str, float]]]] = {
|
||||
nid: [] for nid in node_ids
|
||||
}
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
self._stop.clear()
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
t = time.monotonic()
|
||||
for nid in self._node_ids:
|
||||
try:
|
||||
data = self._client.get_node_system(nid)
|
||||
if data:
|
||||
self._samples[nid].append(
|
||||
(t, {k: data.get(k, 0.0) for k in _SAMPLER_METRICS})
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self._stop.wait(self._interval_s)
|
||||
|
||||
def energy_between(self, t0: float, t1: float) -> float:
|
||||
total_joules = 0.0
|
||||
for _nid, samples in self._samples.items():
|
||||
window = [(t, s["sysPower"]) for t, s in samples if t0 <= t <= t1]
|
||||
if len(window) >= 2:
|
||||
for i in range(1, len(window)):
|
||||
dt = window[i][0] - window[i - 1][0]
|
||||
avg_power = (window[i][1] + window[i - 1][1]) / 2
|
||||
total_joules += avg_power * dt
|
||||
elif len(window) == 1:
|
||||
total_joules += window[0][1] * (t1 - t0)
|
||||
return total_joules
|
||||
|
||||
def summarize(self) -> dict[str, dict[str, dict[str, float]]]:
|
||||
result: dict[str, dict[str, dict[str, float]]] = {}
|
||||
for nid, samples in self._samples.items():
|
||||
if not samples:
|
||||
continue
|
||||
metrics: dict[str, dict[str, float]] = {}
|
||||
for key in _SAMPLER_METRICS:
|
||||
values = [s[key] for t, s in samples]
|
||||
metrics[key] = {
|
||||
"min": round(min(values), 2),
|
||||
"max": round(max(values), 2),
|
||||
"mean": round(mean(values), 2),
|
||||
"samples": len(values),
|
||||
}
|
||||
result[nid] = metrics
|
||||
return result
|
||||
|
||||
def print_summary(self, placement_label: str) -> None:
|
||||
summary = self.summarize()
|
||||
if not summary:
|
||||
return
|
||||
logger.info(f"--- System Metrics ({placement_label}) ---")
|
||||
for nid, metrics in summary.items():
|
||||
gpu = metrics.get("gpuUsage", {})
|
||||
temp = metrics.get("temp", {})
|
||||
power = metrics.get("sysPower", {})
|
||||
logger.info(
|
||||
f" {nid}: "
|
||||
f"GPU {gpu.get('mean', 0) * 100:.0f}% avg ({gpu.get('min', 0) * 100:.0f}–{gpu.get('max', 0) * 100:.0f}%) | "
|
||||
f"{temp.get('mean', 0):.1f}°C avg | "
|
||||
f"{power.get('mean', 0):.1f}W avg"
|
||||
)
|
||||
|
||||
|
||||
def parse_int_list(values: list[str]) -> list[int]:
|
||||
items: list[int] = []
|
||||
for v in values:
|
||||
@@ -271,79 +141,27 @@ def parse_int_list(values: list[str]) -> list[int]:
|
||||
|
||||
|
||||
def run_one_completion(
|
||||
client: ExoClient,
|
||||
model_id: str,
|
||||
pp_hint: int,
|
||||
tg: int,
|
||||
prompt_sizer: PromptSizer,
|
||||
*,
|
||||
use_prefix_cache: bool = False,
|
||||
stream: bool = False,
|
||||
client: ExoClient, model_id: str, pp_hint: int, tg: int, prompt_sizer: PromptSizer
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
content, pp_tokens = prompt_sizer.build(pp_hint)
|
||||
payload: dict[str, Any] = {
|
||||
"model": model_id,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"stream": False,
|
||||
"max_tokens": tg,
|
||||
"logprobs": False,
|
||||
"use_prefix_cache": use_prefix_cache,
|
||||
}
|
||||
|
||||
if not stream:
|
||||
payload["stream"] = False
|
||||
t0 = time.perf_counter()
|
||||
out = client.post_bench_chat_completions(payload)
|
||||
elapsed = time.perf_counter() - t0
|
||||
t0 = time.perf_counter()
|
||||
out = client.post_bench_chat_completions(payload)
|
||||
elapsed = time.perf_counter() - t0
|
||||
|
||||
stats = out.get("generation_stats")
|
||||
choices = out.get("choices") or [{}]
|
||||
message = choices[0].get("message", {}) if choices else {}
|
||||
content = message.get("content") or ""
|
||||
preview = content[:200] if content else ""
|
||||
else:
|
||||
tokens = 0
|
||||
first_token_time = None
|
||||
t0 = time.perf_counter()
|
||||
text_parts: list[str] = []
|
||||
stats = None
|
||||
stats = out.get("generation_stats")
|
||||
|
||||
for raw_line in client.stream_bench_chat_completions(payload):
|
||||
line = raw_line.strip()
|
||||
if line.startswith(": generation_stats "):
|
||||
with contextlib.suppress(json.JSONDecodeError):
|
||||
stats = json.loads(line[len(": generation_stats ") :])
|
||||
continue
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
data = line[6:]
|
||||
if data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(data)
|
||||
delta = chunk.get("choices", [{}])[0].get("delta", {})
|
||||
if delta.get("content"):
|
||||
if first_token_time is None:
|
||||
first_token_time = time.perf_counter()
|
||||
tokens += 1
|
||||
text_parts.append(delta["content"])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
elapsed = time.perf_counter() - t0
|
||||
preview = "".join(text_parts)[:200]
|
||||
|
||||
if not stats:
|
||||
ttft = (first_token_time - t0) if first_token_time else elapsed
|
||||
gen_time = elapsed - ttft if tokens > 1 else elapsed
|
||||
gen_tps = (tokens - 1) / gen_time if tokens > 1 and gen_time > 0 else 0.0
|
||||
prompt_tps = pp_tokens / ttft if ttft > 0 else 0.0
|
||||
stats = {
|
||||
"prompt_tokens": pp_tokens,
|
||||
"generation_tokens": tokens,
|
||||
"prompt_tps": round(prompt_tps, 2),
|
||||
"generation_tps": round(gen_tps, 2),
|
||||
"peak_memory_usage": {"inBytes": 0},
|
||||
}
|
||||
# Extract preview, handling None content (common for thinking models)
|
||||
choices = out.get("choices") or [{}]
|
||||
message = choices[0].get("message", {}) if choices else {}
|
||||
content = message.get("content") or ""
|
||||
preview = content[:200] if content else ""
|
||||
|
||||
return {
|
||||
"elapsed_s": elapsed,
|
||||
@@ -363,19 +181,9 @@ class PromptSizer:
|
||||
def _make_counter(tokenizer: Any) -> Callable[[str], int]:
|
||||
def count_fn(user_content: str) -> int:
|
||||
messages = [{"role": "user", "content": user_content}]
|
||||
try:
|
||||
ids = tokenizer.apply_chat_template(
|
||||
messages, tokenize=True, add_generation_prompt=True
|
||||
)
|
||||
except ValueError:
|
||||
# Models without a Jinja chat template (e.g. DeepSeek V4 which
|
||||
# ships its own Python encoder). Use the exo-side V4 encoder.
|
||||
from exo.worker.engines.mlx.deepseek_v4_encoding import (
|
||||
encode_messages as encode_v4,
|
||||
)
|
||||
|
||||
prompt = encode_v4(messages, thinking_mode="thinking")
|
||||
ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
ids = tokenizer.apply_chat_template(
|
||||
messages, tokenize=True, add_generation_prompt=True
|
||||
)
|
||||
# Fix for transformers 5.x
|
||||
if hasattr(ids, "input_ids"):
|
||||
ids = ids.input_ids
|
||||
@@ -444,12 +252,6 @@ def main() -> int:
|
||||
ap.add_argument(
|
||||
"--repeat", type=int, default=1, help="Repetitions per (pp,tg) pair."
|
||||
)
|
||||
ap.add_argument(
|
||||
"--concurrency",
|
||||
nargs="+",
|
||||
default=["1"],
|
||||
help="Concurrency levels (ints). Accepts commas. E.g. --concurrency 1,2,4,8. Default 1.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--warmup",
|
||||
type=int,
|
||||
@@ -470,27 +272,6 @@ def main() -> int:
|
||||
action="store_true",
|
||||
help="Force all pp×tg combinations (cartesian product) even when lists have equal length.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
help="Use /bench/chat/completions with streaming SSE response (bench=True still applies: no EOS detection, no KV cache).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--no-system-metrics",
|
||||
action="store_true",
|
||||
help="Disable GPU utilization, temperature, and power collection during inference.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--metrics-interval",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="System metrics polling interval in seconds (default: 1.0).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--use-prefix-cache",
|
||||
action="store_true",
|
||||
help="Enable KV prefix cache during bench (default: disabled for cold-cache measurements).",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
pp_list = parse_int_list(args.pp)
|
||||
@@ -501,19 +282,6 @@ def main() -> int:
|
||||
if args.repeat <= 0:
|
||||
logger.error("--repeat must be >= 1")
|
||||
return 2
|
||||
concurrency_list = parse_int_list(args.concurrency)
|
||||
if not concurrency_list or any(c <= 0 for c in concurrency_list):
|
||||
logger.error("--concurrency values must be >= 1")
|
||||
return 2
|
||||
|
||||
if args.use_prefix_cache:
|
||||
logger.warning(
|
||||
"--use-prefix-cache: prompt TPS will be approximate. See METHODOLOGY.md for details."
|
||||
)
|
||||
if pp_list != sorted(pp_list):
|
||||
logger.warning(
|
||||
"--pp values are not in ascending order: prompt TPS will be less accurate. Use ascending --pp for best results."
|
||||
)
|
||||
|
||||
# Log pairing mode
|
||||
use_combinations = args.all_combinations or len(pp_list) != len(tg_list)
|
||||
@@ -525,9 +293,7 @@ def main() -> int:
|
||||
logger.info(f"pp/tg mode: tandem (zip) - {len(pp_list)} pairs")
|
||||
|
||||
client = ExoClient(args.host, args.port, timeout_s=args.timeout)
|
||||
short_id, full_model_id = resolve_model_short_id(
|
||||
client, args.model, force_download=args.force_download
|
||||
)
|
||||
short_id, full_model_id = resolve_model_short_id(client, args.model)
|
||||
|
||||
tokenizer = load_tokenizer_for_bench(full_model_id)
|
||||
if tokenizer is None:
|
||||
@@ -540,146 +306,78 @@ def main() -> int:
|
||||
logger.error("[exo-bench] tokenizer usable but prompt sizing failed")
|
||||
raise
|
||||
|
||||
# Optionally reuse a running instance for this model
|
||||
reused_instance_id: str | None = None
|
||||
if args.reuse_instance:
|
||||
existing = find_existing_instance(client, full_model_id)
|
||||
if existing:
|
||||
reused_instance_id = existing
|
||||
logger.info(f"Reusing existing instance {reused_instance_id}")
|
||||
else:
|
||||
logger.warning(
|
||||
"--reuse-instance: no existing instance found, creating a new one"
|
||||
)
|
||||
selected = settle_and_fetch_placements(
|
||||
client, full_model_id, args, settle_timeout=args.settle_timeout
|
||||
)
|
||||
|
||||
if reused_instance_id is not None:
|
||||
# Use the existing instance directly — skip placement iteration
|
||||
selected = []
|
||||
download_duration_s = None
|
||||
else:
|
||||
selected = settle_and_fetch_placements(
|
||||
client, full_model_id, args, settle_timeout=args.settle_timeout
|
||||
if not selected:
|
||||
logger.error("No valid placements matched your filters.")
|
||||
return 1
|
||||
|
||||
selected.sort(
|
||||
key=lambda p: (
|
||||
str(p.get("instance_meta", "")),
|
||||
str(p.get("sharding", "")),
|
||||
-nodes_used_in_instance(p["instance"]),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
logger.debug(f"exo-bench model: short_id={short_id} full_id={full_model_id}")
|
||||
logger.info(f"placements: {len(selected)}")
|
||||
for p in selected:
|
||||
logger.info(
|
||||
f" - {p['sharding']} / {p['instance_meta']} / nodes={nodes_used_in_instance(p['instance'])}"
|
||||
)
|
||||
|
||||
if not selected:
|
||||
logger.error("No valid placements matched your filters.")
|
||||
return 1
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
selected.sort(
|
||||
key=lambda p: (
|
||||
str(p.get("instance_meta", "")),
|
||||
str(p.get("sharding", "")),
|
||||
nodes_used_in_instance(p["instance"]),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
settle_deadline = (
|
||||
time.monotonic() + args.settle_timeout if args.settle_timeout > 0 else None
|
||||
)
|
||||
|
||||
logger.debug(f"exo-bench model: short_id={short_id} full_id={full_model_id}")
|
||||
logger.info(f"placements: {len(selected)}")
|
||||
for p in selected:
|
||||
logger.info(
|
||||
f" - {p['sharding']} / {p['instance_meta']} / nodes={nodes_used_in_instance(p['instance'])}"
|
||||
)
|
||||
logger.info("Planning phase: checking downloads...")
|
||||
run_planning_phase(
|
||||
client,
|
||||
full_model_id,
|
||||
selected[0],
|
||||
args.danger_delete_downloads,
|
||||
args.timeout,
|
||||
settle_deadline,
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
settle_deadline = (
|
||||
time.monotonic() + args.settle_timeout if args.settle_timeout > 0 else None
|
||||
)
|
||||
|
||||
logger.info("Planning phase: checking downloads...")
|
||||
download_duration_s = run_planning_phase(
|
||||
client,
|
||||
full_model_id,
|
||||
selected[0],
|
||||
args.danger_delete_downloads,
|
||||
args.timeout,
|
||||
settle_deadline,
|
||||
)
|
||||
if download_duration_s is not None:
|
||||
logger.info(f"Download: {download_duration_s:.1f}s (freshly downloaded)")
|
||||
else:
|
||||
logger.info("Download: model already cached")
|
||||
|
||||
cluster_snapshot = capture_cluster_snapshot(client)
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
all_system_metrics: dict[str, dict[str, dict[str, float]]] = {}
|
||||
|
||||
# If reusing an existing instance, run a single benchmark pass against it
|
||||
if reused_instance_id is not None:
|
||||
selected = [None]
|
||||
|
||||
for preview in selected:
|
||||
created_instance = False
|
||||
if preview is not None:
|
||||
instance = preview["instance"]
|
||||
instance_id = instance_id_from_instance(instance)
|
||||
instance = preview["instance"]
|
||||
instance_id = instance_id_from_instance(instance)
|
||||
|
||||
sharding = str(preview["sharding"])
|
||||
instance_meta = str(preview["instance_meta"])
|
||||
n_nodes = nodes_used_in_instance(instance)
|
||||
sharding = str(preview["sharding"])
|
||||
instance_meta = str(preview["instance_meta"])
|
||||
n_nodes = nodes_used_in_instance(instance)
|
||||
|
||||
logger.info("=" * 80)
|
||||
logger.info(
|
||||
f"PLACEMENT: {sharding} / {instance_meta} / nodes={n_nodes} / instance_id={instance_id}"
|
||||
)
|
||||
logger.info("=" * 80)
|
||||
logger.info(
|
||||
f"PLACEMENT: {sharding} / {instance_meta} / nodes={n_nodes} / instance_id={instance_id}"
|
||||
)
|
||||
|
||||
# Delete any existing instances to free resources before placing
|
||||
try:
|
||||
state = client.request_json("GET", "/state")
|
||||
for old_id in list(state.get("instances", {}).keys()):
|
||||
logger.info(f"Deleting stale instance {old_id}")
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{old_id}")
|
||||
if state.get("instances"):
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up stale instances: {e}")
|
||||
client.request_json("POST", "/instance", body={"instance": instance})
|
||||
try:
|
||||
wait_for_instance_ready(client, instance_id)
|
||||
except (RuntimeError, TimeoutError) as e:
|
||||
logger.error(f"Failed to initialize placement: {e}")
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{instance_id}")
|
||||
continue
|
||||
|
||||
client.request_json("POST", "/instance", body={"instance": instance})
|
||||
try:
|
||||
wait_for_instance_ready(client, instance_id)
|
||||
except (RuntimeError, TimeoutError) as e:
|
||||
logger.error(f"Failed to initialize placement: {e}")
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{instance_id}")
|
||||
continue
|
||||
|
||||
time.sleep(1)
|
||||
created_instance = True
|
||||
else:
|
||||
instance_id = reused_instance_id
|
||||
sharding = "reused"
|
||||
instance_meta = "reused"
|
||||
n_nodes = 0
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"Using existing instance {instance_id}")
|
||||
|
||||
sampler: SystemMetricsSampler | None = None
|
||||
if not args.no_system_metrics and preview is not None:
|
||||
nids = node_ids_from_instance(instance)
|
||||
sampler = SystemMetricsSampler(
|
||||
ExoClient(args.host, args.port, timeout_s=30),
|
||||
nids,
|
||||
interval_s=args.metrics_interval,
|
||||
)
|
||||
sampler.start()
|
||||
|
||||
def _do_one(c: ExoClient, pp: int, tg: int) -> tuple[dict[str, Any], int]:
|
||||
return run_one_completion(
|
||||
c,
|
||||
full_model_id,
|
||||
pp,
|
||||
tg,
|
||||
prompt_sizer,
|
||||
use_prefix_cache=args.use_prefix_cache,
|
||||
stream=args.stream,
|
||||
)
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
for i in range(args.warmup):
|
||||
_do_one(client, pp_list[0], tg_list[0])
|
||||
run_one_completion(
|
||||
client, full_model_id, pp_list[0], tg_list[0], prompt_sizer
|
||||
)
|
||||
logger.debug(f" warmup {i + 1}/{args.warmup} done")
|
||||
|
||||
# If pp and tg lists have same length, run in tandem (zip)
|
||||
@@ -690,218 +388,63 @@ def main() -> int:
|
||||
pp_tg_pairs = list(zip(pp_list, tg_list, strict=True))
|
||||
|
||||
for pp, tg in pp_tg_pairs:
|
||||
for concurrency in concurrency_list:
|
||||
logger.info(f"--- pp={pp} tg={tg} concurrency={concurrency} ---")
|
||||
runs: list[dict[str, Any]] = []
|
||||
inference_windows: list[tuple[float, float]] = []
|
||||
for r in range(args.repeat):
|
||||
time.sleep(3)
|
||||
|
||||
if concurrency <= 1:
|
||||
# Sequential: single request
|
||||
try:
|
||||
inf_t0 = time.monotonic()
|
||||
row, actual_pp_tokens = _do_one(client, pp, tg)
|
||||
inference_windows.append((inf_t0, time.monotonic()))
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
continue
|
||||
row.update(
|
||||
{
|
||||
"model_short_id": short_id,
|
||||
"model_id": full_model_id,
|
||||
"placement_sharding": sharding,
|
||||
"placement_instance_meta": instance_meta,
|
||||
"placement_nodes": n_nodes,
|
||||
"instance_id": instance_id,
|
||||
"pp_tokens": actual_pp_tokens,
|
||||
"tg": tg,
|
||||
"repeat_index": r,
|
||||
"concurrency": 1,
|
||||
**(
|
||||
{"download_duration_s": download_duration_s}
|
||||
if download_duration_s is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
runs.append(row)
|
||||
all_rows.append(row)
|
||||
else:
|
||||
# Concurrent: fire N requests in parallel
|
||||
# Pre-build prompt once, barrier ensures simultaneous dispatch
|
||||
content, actual_pp = prompt_sizer.build(pp)
|
||||
pre_built_payload: dict[str, Any] = {
|
||||
"model": full_model_id,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"stream": False,
|
||||
"max_tokens": tg,
|
||||
"logprobs": False,
|
||||
"use_prefix_cache": args.use_prefix_cache,
|
||||
}
|
||||
barrier = threading.Barrier(concurrency)
|
||||
batch_start = threading.Event()
|
||||
batch_t0: float = 0.0
|
||||
batch_results: list[tuple[dict[str, Any], int]] = []
|
||||
batch_errors = 0
|
||||
|
||||
def _run_concurrent(
|
||||
idx: int,
|
||||
_barrier: threading.Barrier = barrier,
|
||||
_batch_start: threading.Event = batch_start,
|
||||
_payload: dict[str, Any] = pre_built_payload,
|
||||
_actual_pp: int = actual_pp,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
nonlocal batch_t0
|
||||
c = ExoClient(
|
||||
args.host, args.port, timeout_s=args.timeout
|
||||
)
|
||||
if _barrier.wait() == 0:
|
||||
batch_t0 = time.perf_counter()
|
||||
_batch_start.set()
|
||||
else:
|
||||
_batch_start.wait()
|
||||
t0 = batch_t0
|
||||
out = c.post_bench_chat_completions(_payload)
|
||||
elapsed = time.perf_counter() - t0
|
||||
stats = out.get("generation_stats")
|
||||
choices = out.get("choices") or [{}]
|
||||
message = (
|
||||
choices[0].get("message", {}) if choices else {}
|
||||
)
|
||||
text = message.get("content") or ""
|
||||
return {
|
||||
"elapsed_s": elapsed,
|
||||
"output_text_preview": text[:200],
|
||||
"stats": stats,
|
||||
}, _actual_pp
|
||||
|
||||
inf_t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = {
|
||||
pool.submit(_run_concurrent, i): i
|
||||
for i in range(concurrency)
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
try:
|
||||
batch_results.append(fut.result())
|
||||
except Exception as e:
|
||||
logger.error(f"Concurrent request failed: {e}")
|
||||
batch_errors += 1
|
||||
batch_wall_s = (
|
||||
max(x["elapsed_s"] for x, _ in batch_results)
|
||||
if batch_results
|
||||
else time.perf_counter() - batch_t0
|
||||
)
|
||||
inference_windows.append((inf_t0, time.monotonic()))
|
||||
|
||||
for idx, (row, actual_pp_tokens) in enumerate(
|
||||
batch_results
|
||||
):
|
||||
row.update(
|
||||
{
|
||||
"model_short_id": short_id,
|
||||
"model_id": full_model_id,
|
||||
"placement_sharding": sharding,
|
||||
"placement_instance_meta": instance_meta,
|
||||
"placement_nodes": n_nodes,
|
||||
"instance_id": instance_id,
|
||||
"pp_tokens": actual_pp_tokens,
|
||||
"tg": tg,
|
||||
"repeat_index": r,
|
||||
"concurrency": concurrency,
|
||||
"concurrent_index": idx,
|
||||
**(
|
||||
{"download_duration_s": download_duration_s}
|
||||
if download_duration_s is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
runs.append(row)
|
||||
all_rows.append(row)
|
||||
|
||||
if batch_results:
|
||||
valid_gen_tps = [
|
||||
x["stats"]["generation_tps"]
|
||||
for x, _ in batch_results
|
||||
if x["stats"]["generation_tps"] > 0
|
||||
]
|
||||
per_req_tps = (
|
||||
max(valid_gen_tps) if valid_gen_tps else 0.0
|
||||
)
|
||||
agg_gen_tps = per_req_tps * concurrency
|
||||
logger.info(
|
||||
f"[concurrent {concurrency}x] "
|
||||
f"agg_gen_tps={agg_gen_tps:.2f} "
|
||||
f"per_req_tps={per_req_tps:.2f} "
|
||||
f"wall_s={batch_wall_s:.2f} "
|
||||
f"errors={batch_errors}"
|
||||
)
|
||||
|
||||
if runs:
|
||||
prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
|
||||
valid_gen = [
|
||||
x["stats"]["generation_tps"]
|
||||
for x in runs
|
||||
if x["stats"]["generation_tps"] > 0
|
||||
]
|
||||
per_req_tps = max(valid_gen) if valid_gen else 0.0
|
||||
gen_tps = per_req_tps * concurrency
|
||||
ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
|
||||
gtok = mean(x["stats"]["generation_tokens"] for x in runs)
|
||||
|
||||
def _peak_bytes(s: dict[str, Any]) -> float:
|
||||
pm = s["peak_memory_usage"]
|
||||
return pm.get("inBytes") or pm.get("in_bytes", 0)
|
||||
|
||||
peak = mean(_peak_bytes(x["stats"]) for x in runs)
|
||||
summary = (
|
||||
f"prompt_tps={prompt_tps:.2f} gen_tps={gen_tps:.2f} "
|
||||
f"prompt_tokens={ptok} gen_tokens={gtok} "
|
||||
f"peak_memory={format_peak_memory(peak)}"
|
||||
runs: list[dict[str, Any]] = []
|
||||
for r in range(args.repeat):
|
||||
time.sleep(3)
|
||||
try:
|
||||
row, actual_pp_tokens = run_one_completion(
|
||||
client, full_model_id, pp, tg, prompt_sizer
|
||||
)
|
||||
if sampler and inference_windows:
|
||||
joules = sum(
|
||||
sampler.energy_between(t0, t1)
|
||||
for t0, t1 in inference_windows
|
||||
)
|
||||
inf_seconds = sum(t1 - t0 for t0, t1 in inference_windows)
|
||||
avg_watts = joules / inf_seconds if inf_seconds > 0 else 0
|
||||
summary += f" energy={joules:.1f}J ({avg_watts:.1f}W avg over {inf_seconds:.1f}s inference)"
|
||||
logger.info(f"{summary}\n")
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
continue
|
||||
row.update(
|
||||
{
|
||||
"model_short_id": short_id,
|
||||
"model_id": full_model_id,
|
||||
"placement_sharding": sharding,
|
||||
"placement_instance_meta": instance_meta,
|
||||
"placement_nodes": n_nodes,
|
||||
"instance_id": instance_id,
|
||||
"pp_tokens": actual_pp_tokens,
|
||||
"tg": tg,
|
||||
"repeat_index": r,
|
||||
}
|
||||
)
|
||||
runs.append(row)
|
||||
all_rows.append(row)
|
||||
|
||||
if runs:
|
||||
prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
|
||||
gen_tps = mean(x["stats"]["generation_tps"] for x in runs)
|
||||
ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
|
||||
gtok = mean(x["stats"]["generation_tokens"] for x in runs)
|
||||
peak = mean(
|
||||
x["stats"]["peak_memory_usage"]["inBytes"] for x in runs
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"prompt_tps={prompt_tps:.2f} gen_tps={gen_tps:.2f} "
|
||||
f"prompt_tokens={ptok} gen_tokens={gtok} "
|
||||
f"peak_memory={format_peak_memory(peak)}\n"
|
||||
)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if sampler:
|
||||
sampler.stop()
|
||||
placement_label = f"{sharding}/{instance_meta}/{n_nodes} nodes"
|
||||
sampler.print_summary(placement_label)
|
||||
placement_metrics = sampler.summarize()
|
||||
if placement_metrics:
|
||||
all_system_metrics.update(placement_metrics)
|
||||
try:
|
||||
client.request_json("DELETE", f"/instance/{instance_id}")
|
||||
except ExoHttpError as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
wait_for_instance_gone(client, instance_id)
|
||||
logger.debug(f"Deleted instance {instance_id}")
|
||||
|
||||
if created_instance and instance_id is not None:
|
||||
try:
|
||||
client.request_json("DELETE", f"/instance/{instance_id}")
|
||||
except ExoHttpError as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
wait_for_instance_gone(client, instance_id)
|
||||
logger.debug(f"Deleted instance {instance_id}")
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
output: dict[str, Any] = {"runs": all_rows}
|
||||
if cluster_snapshot:
|
||||
output["cluster"] = cluster_snapshot
|
||||
if all_system_metrics:
|
||||
output["system_metrics"] = all_system_metrics
|
||||
time.sleep(5)
|
||||
|
||||
if args.stdout:
|
||||
json.dump(output, sys.stdout, indent=2, ensure_ascii=False)
|
||||
json.dump(all_rows, sys.stdout, indent=2, ensure_ascii=False)
|
||||
elif args.json_out:
|
||||
with open(args.json_out, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, indent=2, ensure_ascii=False)
|
||||
json.dump(all_rows, f, indent=2, ensure_ascii=False)
|
||||
logger.debug(f"\nWrote results JSON: {args.json_out}")
|
||||
|
||||
return 0
|
||||
|
||||
-1753
File diff suppressed because it is too large.
Load diff
+45
-195
@@ -6,7 +6,6 @@ import http.client
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -70,59 +69,6 @@ class ExoClient:
|
||||
def post_bench_chat_completions(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self.request_json("POST", "/bench/chat/completions", body=payload)
|
||||
|
||||
def stream_bench_chat_completions(self, payload: dict[str, Any]) -> Iterator[str]:
|
||||
"""POST /bench/chat/completions with stream=True, yielding raw SSE lines."""
|
||||
payload = {**payload, "stream": True}
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
conn = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout_s)
|
||||
try:
|
||||
conn.request(
|
||||
"POST",
|
||||
"/bench/chat/completions",
|
||||
body=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
},
|
||||
)
|
||||
resp = conn.getresponse()
|
||||
if resp.status >= 400:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
raise ExoHttpError(resp.status, resp.reason, raw[:300])
|
||||
for line in resp:
|
||||
yield line.decode("utf-8", errors="replace")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_state_path(self, path: str) -> Any:
|
||||
try:
|
||||
return self.request_json("GET", f"/state/{path}")
|
||||
except ExoHttpError as e:
|
||||
if e.status == 404:
|
||||
return None
|
||||
raise
|
||||
|
||||
def get_instance(self, instance_id: str) -> dict[str, Any] | None:
|
||||
return self.get_state_path(f"instances/{instance_id}")
|
||||
|
||||
def get_runner(self, runner_id: str) -> dict[str, Any] | None:
|
||||
return self.get_state_path(f"runners/{runner_id}")
|
||||
|
||||
def get_node_downloads(self, node_id: str) -> list[dict[str, Any]] | None:
|
||||
return self.get_state_path(f"downloads/{node_id}")
|
||||
|
||||
def get_node_disk(self, node_id: str) -> dict[str, Any] | None:
|
||||
return self.get_state_path(f"nodeDisk/{node_id}")
|
||||
|
||||
def get_node_system(self, node_id: str) -> dict[str, Any] | None:
|
||||
return self.get_state_path(f"nodeSystem/{node_id}")
|
||||
|
||||
def get_node_identities(self) -> dict[str, Any] | None:
|
||||
return self.get_state_path("nodeIdentities")
|
||||
|
||||
def get_topology(self) -> dict[str, Any] | None:
|
||||
return self.get_state_path("topology")
|
||||
|
||||
|
||||
def unwrap_instance(instance: dict[str, Any]) -> dict[str, Any]:
|
||||
if len(instance) != 1:
|
||||
@@ -151,11 +97,6 @@ def runner_ids_from_instance(instance: dict[str, Any]) -> list[str]:
|
||||
return list(runner_to_shard.keys())
|
||||
|
||||
|
||||
def node_ids_from_instance(instance: dict[str, Any]) -> list[str]:
|
||||
inner = unwrap_instance(instance)
|
||||
return list(inner["shardAssignments"]["nodeToRunner"].keys())
|
||||
|
||||
|
||||
def runner_ready(runner: dict[str, Any]) -> bool:
|
||||
return "RunnerReady" in runner
|
||||
|
||||
@@ -175,12 +116,13 @@ def wait_for_instance_ready(
|
||||
) -> None:
|
||||
start_time = time.time()
|
||||
instance_existed = False
|
||||
last_loaded: dict[str, int] = {}
|
||||
while time.time() - start_time < timeout:
|
||||
instance = client.get_instance(instance_id)
|
||||
state = client.request_json("GET", "/state")
|
||||
instances = state.get("instances", {})
|
||||
|
||||
if instance is None:
|
||||
if instance_id not in instances:
|
||||
if instance_existed:
|
||||
# Instance was deleted after being created - likely due to runner failure
|
||||
raise RuntimeError(
|
||||
f"Instance {instance_id} was deleted (runner may have failed)"
|
||||
)
|
||||
@@ -188,25 +130,18 @@ def wait_for_instance_ready(
|
||||
continue
|
||||
|
||||
instance_existed = True
|
||||
rids = runner_ids_from_instance(instance)
|
||||
instance = instances[instance_id]
|
||||
runner_ids = runner_ids_from_instance(instance)
|
||||
runners = state.get("runners", {})
|
||||
|
||||
all_ready = True
|
||||
for rid in rids:
|
||||
runner = client.get_runner(rid) or {}
|
||||
# Check for failed runners first
|
||||
for rid in runner_ids:
|
||||
runner = runners.get(rid, {})
|
||||
if runner_failed(runner):
|
||||
error_msg = get_runner_failed_message(runner) or "Unknown error"
|
||||
raise RuntimeError(f"Runner {rid} failed: {error_msg}")
|
||||
if "RunnerLoading" in runner:
|
||||
loading = runner["RunnerLoading"]
|
||||
loaded = loading.get("layersLoaded", 0)
|
||||
total = loading.get("totalLayers", 0)
|
||||
if total > 0 and last_loaded.get(rid) != loaded:
|
||||
last_loaded[rid] = loaded
|
||||
logger.debug(f"Runner {rid}: loading layers {loaded}/{total}")
|
||||
if not runner_ready(runner):
|
||||
all_ready = False
|
||||
|
||||
if all_ready:
|
||||
if all(runner_ready(runners.get(rid, {})) for rid in runner_ids):
|
||||
return
|
||||
|
||||
time.sleep(0.1)
|
||||
@@ -230,26 +165,7 @@ def wait_for_instance_gone(
|
||||
raise TimeoutError(f"Instance {instance_id} did not get deleted within {timeout=}")
|
||||
|
||||
|
||||
def capture_cluster_snapshot(client: ExoClient) -> dict[str, Any]:
|
||||
snapshot: dict[str, Any] = {}
|
||||
identities = client.get_node_identities()
|
||||
if identities:
|
||||
snapshot["nodeIdentities"] = identities
|
||||
topology = client.get_topology()
|
||||
if topology:
|
||||
snapshot["topology"] = topology
|
||||
node_memory = client.get_state_path("nodeMemory")
|
||||
if node_memory:
|
||||
snapshot["nodeMemory"] = node_memory
|
||||
node_system = client.get_state_path("nodeSystem")
|
||||
if node_system:
|
||||
snapshot["nodeSystem"] = node_system
|
||||
return snapshot
|
||||
|
||||
|
||||
def resolve_model_short_id(
|
||||
client: ExoClient, model_arg: str, *, force_download: bool = False
|
||||
) -> tuple[str, str]:
|
||||
def resolve_model_short_id(client: ExoClient, model_arg: str) -> tuple[str, str]:
|
||||
models = client.request_json("GET", "/models") or {}
|
||||
data = models.get("data") or []
|
||||
|
||||
@@ -265,16 +181,6 @@ def resolve_model_short_id(
|
||||
full_id = str(m["hugging_face_id"])
|
||||
return short_id, full_id
|
||||
|
||||
if force_download and "/" in model_arg:
|
||||
logger.info(f"Model not in /models, adding from HuggingFace: {model_arg}")
|
||||
result = client.request_json(
|
||||
"POST", "/models/add", body={"model_id": model_arg}
|
||||
)
|
||||
if result:
|
||||
short_id = str(result.get("name") or model_arg.rsplit("/", 1)[-1])
|
||||
full_id = str(result.get("hugging_face_id") or model_arg)
|
||||
return short_id, full_id
|
||||
|
||||
raise ValueError(f"Model not found in /models: {model_arg}")
|
||||
|
||||
|
||||
@@ -293,15 +199,11 @@ def sharding_filter(sharding: str, wanted: str) -> bool:
|
||||
|
||||
|
||||
def fetch_and_filter_placements(
|
||||
client: ExoClient,
|
||||
full_model_id: str,
|
||||
args: argparse.Namespace,
|
||||
node_id: str | None = None,
|
||||
client: ExoClient, full_model_id: str, args: argparse.Namespace
|
||||
) -> list[dict[str, Any]]:
|
||||
params: dict[str, str] = {"model_id": full_model_id}
|
||||
if node_id is not None:
|
||||
params["node_ids"] = node_id
|
||||
previews_resp = client.request_json("GET", "/instance/previews", params=params)
|
||||
previews_resp = client.request_json(
|
||||
"GET", "/instance/previews", params={"model_id": full_model_id}
|
||||
)
|
||||
previews = previews_resp.get("previews") or []
|
||||
|
||||
selected: list[dict[str, Any]] = []
|
||||
@@ -361,9 +263,8 @@ def settle_and_fetch_placements(
|
||||
full_model_id: str,
|
||||
args: argparse.Namespace,
|
||||
settle_timeout: float = 0,
|
||||
node_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
selected = fetch_and_filter_placements(client, full_model_id, args, node_id=node_id)
|
||||
selected = fetch_and_filter_placements(client, full_model_id, args)
|
||||
|
||||
if not selected and settle_timeout > 0:
|
||||
backoff = _SETTLE_INITIAL_BACKOFF_S
|
||||
@@ -376,9 +277,7 @@ def settle_and_fetch_placements(
|
||||
)
|
||||
time.sleep(min(backoff, remaining))
|
||||
backoff = min(backoff * _SETTLE_BACKOFF_MULTIPLIER, _SETTLE_MAX_BACKOFF_S)
|
||||
selected = fetch_and_filter_placements(
|
||||
client, full_model_id, args, node_id=node_id
|
||||
)
|
||||
selected = fetch_and_filter_placements(client, full_model_id, args)
|
||||
|
||||
return selected
|
||||
|
||||
@@ -390,12 +289,8 @@ def run_planning_phase(
|
||||
danger_delete: bool,
|
||||
timeout: float,
|
||||
settle_deadline: float | None,
|
||||
) -> float | None:
|
||||
"""Check disk space and ensure model is downloaded before benchmarking.
|
||||
|
||||
Returns the wall-clock download duration in seconds if a fresh download
|
||||
was needed, or None if the model was already cached on all nodes.
|
||||
"""
|
||||
) -> None:
|
||||
"""Check disk space and ensure model is downloaded before benchmarking."""
|
||||
# Get model size from /models
|
||||
models = client.request_json("GET", "/models") or {}
|
||||
model_bytes = 0
|
||||
@@ -408,18 +303,21 @@ def run_planning_phase(
|
||||
logger.warning(
|
||||
f"Could not determine size for {full_model_id}, skipping disk check"
|
||||
)
|
||||
return None
|
||||
return
|
||||
|
||||
# Get nodes from preview
|
||||
inner = unwrap_instance(preview["instance"])
|
||||
node_ids = list(inner["shardAssignments"]["nodeToRunner"].keys())
|
||||
runner_to_shard = inner["shardAssignments"]["runnerToShard"]
|
||||
|
||||
needs_download = False
|
||||
state = client.request_json("GET", "/state")
|
||||
downloads = state.get("downloads", {})
|
||||
node_disk = state.get("nodeDisk", {})
|
||||
|
||||
for node_id in node_ids:
|
||||
node_downloads = client.get_node_downloads(node_id) or []
|
||||
node_downloads = downloads.get(node_id, [])
|
||||
|
||||
# Check if model already downloaded on this node
|
||||
already_downloaded = any(
|
||||
"DownloadCompleted" in p
|
||||
and unwrap_instance(p["DownloadCompleted"]["shardMetadata"])["modelCard"][
|
||||
@@ -431,9 +329,8 @@ def run_planning_phase(
|
||||
if already_downloaded:
|
||||
continue
|
||||
|
||||
needs_download = True
|
||||
|
||||
disk_info = client.get_node_disk(node_id) or {}
|
||||
# Wait for disk info if settle_deadline is set
|
||||
disk_info = node_disk.get(node_id, {})
|
||||
backoff = _SETTLE_INITIAL_BACKOFF_S
|
||||
while not disk_info and settle_deadline and time.monotonic() < settle_deadline:
|
||||
remaining = settle_deadline - time.monotonic()
|
||||
@@ -442,7 +339,9 @@ def run_planning_phase(
|
||||
)
|
||||
time.sleep(min(backoff, remaining))
|
||||
backoff = min(backoff * _SETTLE_BACKOFF_MULTIPLIER, _SETTLE_MAX_BACKOFF_S)
|
||||
disk_info = client.get_node_disk(node_id) or {}
|
||||
state = client.request_json("GET", "/state")
|
||||
node_disk = state.get("nodeDisk", {})
|
||||
disk_info = node_disk.get(node_id, {})
|
||||
|
||||
if not disk_info:
|
||||
logger.warning(f"No disk info for {node_id}, skipping space check")
|
||||
@@ -458,16 +357,16 @@ def run_planning_phase(
|
||||
f"have {avail // (1024**3)}GB. Use --danger-delete-downloads to free space."
|
||||
)
|
||||
|
||||
# Delete from smallest to largest
|
||||
completed = [
|
||||
(
|
||||
unwrap_instance(p["DownloadCompleted"]["shardMetadata"])["modelCard"][
|
||||
"modelId"
|
||||
],
|
||||
p["DownloadCompleted"]["total"]["inBytes"],
|
||||
p["DownloadCompleted"]["totalBytes"]["inBytes"],
|
||||
)
|
||||
for p in node_downloads
|
||||
if "DownloadCompleted" in p
|
||||
and not p["DownloadCompleted"].get("readOnly", False)
|
||||
]
|
||||
for del_model, size in sorted(completed, key=lambda x: x[1]):
|
||||
logger.info(f"Deleting {del_model} from {node_id} ({size // (1024**2)}MB)")
|
||||
@@ -480,7 +379,6 @@ def run_planning_phase(
|
||||
raise RuntimeError(f"Could not free enough space on {node_id}")
|
||||
|
||||
# Start downloads (idempotent)
|
||||
download_t0 = time.perf_counter() if needs_download else None
|
||||
for node_id in node_ids:
|
||||
runner_id = inner["shardAssignments"]["nodeToRunner"][node_id]
|
||||
shard = runner_to_shard[runner_id]
|
||||
@@ -494,22 +392,24 @@ def run_planning_phase(
|
||||
)
|
||||
logger.info(f"Started download on {node_id}")
|
||||
|
||||
# Wait for downloads (no timeout — poll until complete or failed)
|
||||
while True:
|
||||
# Wait for downloads
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
state = client.request_json("GET", "/state")
|
||||
downloads = state.get("downloads", {})
|
||||
all_done = True
|
||||
for node_id in node_ids:
|
||||
node_downloads = client.get_node_downloads(node_id) or []
|
||||
done = any(
|
||||
"DownloadCompleted" in p
|
||||
and unwrap_instance(p["DownloadCompleted"]["shardMetadata"])[
|
||||
"modelCard"
|
||||
]["modelId"]
|
||||
== full_model_id
|
||||
for p in node_downloads
|
||||
for p in downloads.get(node_id, [])
|
||||
)
|
||||
failed = [
|
||||
p["DownloadFailed"]["errorMessage"]
|
||||
for p in node_downloads
|
||||
for p in downloads.get(node_id, [])
|
||||
if "DownloadFailed" in p
|
||||
and unwrap_instance(p["DownloadFailed"]["shardMetadata"])["modelCard"][
|
||||
"modelId"
|
||||
@@ -520,49 +420,11 @@ def run_planning_phase(
|
||||
raise RuntimeError(f"Download failed on {node_id}: {failed[0]}")
|
||||
if not done:
|
||||
all_done = False
|
||||
ongoing = [
|
||||
p
|
||||
for p in node_downloads
|
||||
if "DownloadOngoing" in p
|
||||
and unwrap_instance(p["DownloadOngoing"]["shardMetadata"])[
|
||||
"modelCard"
|
||||
]["modelId"]
|
||||
== full_model_id
|
||||
]
|
||||
if ongoing:
|
||||
prog = ongoing[0]["DownloadOngoing"]["downloadProgress"]
|
||||
speed_mb = prog.get("speed", 0) / (1024 * 1024)
|
||||
eta_s = prog.get("etaMs", 0) / 1000
|
||||
dl_bytes = prog.get("downloaded", {}).get("inBytes", 0)
|
||||
total_bytes = prog.get("total", {}).get("inBytes", 0)
|
||||
pct = (dl_bytes / total_bytes * 100) if total_bytes else 0
|
||||
logger.info(
|
||||
f"Downloading on {node_id}: {pct:.1f}% @ {speed_mb:.1f} MB/s, "
|
||||
f"ETA {eta_s:.0f}s "
|
||||
f"({prog.get('completedFiles', 0)}/{prog.get('totalFiles', 0)} files)"
|
||||
)
|
||||
if all_done:
|
||||
if download_t0 is not None:
|
||||
return time.perf_counter() - download_t0
|
||||
return None
|
||||
time.sleep(10)
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def find_existing_instance(client: ExoClient, model_id: str) -> str | None:
|
||||
"""Find an existing running instance for the given model."""
|
||||
try:
|
||||
state = client.request_json("GET", "/state")
|
||||
except Exception:
|
||||
return None
|
||||
for inst_id, inst in state.get("instances", {}).items():
|
||||
# Instance structure is nested: {"MlxJacclInstance": {"shardAssignments": {"modelId": ...}}}
|
||||
for _inst_type, inner in inst.items():
|
||||
if not isinstance(inner, dict):
|
||||
continue
|
||||
sa = inner.get("shardAssignments", {})
|
||||
if sa.get("modelId") == model_id:
|
||||
return inst_id
|
||||
return None
|
||||
raise TimeoutError("Downloads did not complete in time")
|
||||
|
||||
|
||||
def add_common_instance_args(ap: argparse.ArgumentParser) -> None:
|
||||
@@ -571,11 +433,6 @@ def add_common_instance_args(ap: argparse.ArgumentParser) -> None:
|
||||
"--port", type=int, default=int(os.environ.get("EXO_PORT", "52415"))
|
||||
)
|
||||
ap.add_argument("--model", required=True, help="Model short id or huggingface id")
|
||||
ap.add_argument(
|
||||
"--force-download",
|
||||
action="store_true",
|
||||
help="If model not in /models, add it from HuggingFace via exo and download.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--max-nodes",
|
||||
type=int,
|
||||
@@ -589,9 +446,7 @@ def add_common_instance_args(ap: argparse.ArgumentParser) -> None:
|
||||
help="Only consider placements using >= this many nodes.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--instance-meta",
|
||||
choices=["ring", "jaccl", "vllm", "both"],
|
||||
default="both",
|
||||
"--instance-meta", choices=["ring", "jaccl", "both"], default="both"
|
||||
)
|
||||
ap.add_argument(
|
||||
"--sharding", choices=["pipeline", "tensor", "both"], default="both"
|
||||
@@ -612,7 +467,7 @@ def add_common_instance_args(ap: argparse.ArgumentParser) -> None:
|
||||
ap.add_argument(
|
||||
"--settle-timeout",
|
||||
type=float,
|
||||
default=60.0,
|
||||
default=0,
|
||||
help="Max seconds to wait for the cluster to produce valid placements (0 = try once).",
|
||||
)
|
||||
ap.add_argument(
|
||||
@@ -620,8 +475,3 @@ def add_common_instance_args(ap: argparse.ArgumentParser) -> None:
|
||||
action="store_true",
|
||||
help="Delete existing models from smallest to largest to make room for benchmark model.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--reuse-instance",
|
||||
action="store_true",
|
||||
help="Reuse an existing running instance for this model instead of creating a new one.",
|
||||
)
|
||||
@@ -1,255 +0,0 @@
|
||||
# type: ignore
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
import tty
|
||||
|
||||
import aiohttp
|
||||
|
||||
NUM_REQUESTS = 10
|
||||
BASE_URL = ""
|
||||
|
||||
QUESTIONS = [
|
||||
"What is the capital of Australia?",
|
||||
"How many bones are in the human body?",
|
||||
"What year did World War II end?",
|
||||
"What is the speed of light in meters per second?",
|
||||
"Who wrote Romeo and Juliet?",
|
||||
"What is the chemical formula for water?",
|
||||
"How many planets are in our solar system?",
|
||||
"What is the largest ocean on Earth?",
|
||||
"Who painted the Mona Lisa?",
|
||||
"What is the boiling point of water in Celsius?",
|
||||
]
|
||||
|
||||
|
||||
def write(s: str) -> None:
|
||||
sys.stdout.write(s)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model picker (same style as exo_eval)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def fetch_models() -> list[str]:
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
with urllib.request.urlopen(f"{BASE_URL}/state") as resp:
|
||||
data = json.loads(resp.read())
|
||||
model_ids: set[str] = set()
|
||||
for instance in data.get("instances", {}).values():
|
||||
for variant in instance.values():
|
||||
sa = variant.get("shardAssignments", {})
|
||||
model_id = sa.get("modelId")
|
||||
if model_id:
|
||||
model_ids.add(model_id)
|
||||
return sorted(model_ids)
|
||||
|
||||
|
||||
def pick_model() -> str | None:
|
||||
models = fetch_models()
|
||||
if not models:
|
||||
print("No models found.")
|
||||
return None
|
||||
|
||||
cursor = 0
|
||||
total_lines = len(models) + 4
|
||||
|
||||
def render(first: bool = False) -> None:
|
||||
if not first:
|
||||
write(f"\033[{total_lines}A")
|
||||
write("\033[J")
|
||||
write("\033[1mSelect model\033[0m (up/down, enter confirm, q quit)\r\n\r\n")
|
||||
for i, model in enumerate(models):
|
||||
line = f" {'>' if i == cursor else ' '} {model}"
|
||||
write(f"\033[7m{line}\033[0m\r\n" if i == cursor else f"{line}\r\n")
|
||||
write("\r\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
fd = sys.stdin.fileno()
|
||||
old = termios.tcgetattr(fd)
|
||||
try:
|
||||
tty.setraw(fd)
|
||||
write("\033[?25l")
|
||||
render(first=True)
|
||||
while True:
|
||||
ch = sys.stdin.read(1)
|
||||
if ch in ("q", "\x03"):
|
||||
write("\033[?25h\033[0m\r\n")
|
||||
return None
|
||||
elif ch in ("\r", "\n"):
|
||||
break
|
||||
elif ch == "\x1b":
|
||||
seq = sys.stdin.read(2)
|
||||
if seq == "[A":
|
||||
cursor = (cursor - 1) % len(models)
|
||||
elif seq == "[B":
|
||||
cursor = (cursor + 1) % len(models)
|
||||
render()
|
||||
finally:
|
||||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
||||
write(f"\033[{total_lines}A\033[J") # clear picker UI
|
||||
write("\033[?25h\033[0m")
|
||||
sys.stdout.flush()
|
||||
|
||||
return models[cursor]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parallel requests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
statuses: list[str] = []
|
||||
times: list[str] = []
|
||||
previews: list[str] = []
|
||||
tokens: list[str] = []
|
||||
full_responses: list[dict | None] = []
|
||||
total_lines = 0
|
||||
start_time: float = 0
|
||||
selected_model: str = ""
|
||||
|
||||
|
||||
def render_progress(first: bool = False) -> None:
|
||||
if not first:
|
||||
write(f"\033[{total_lines}A")
|
||||
write("\033[J")
|
||||
elapsed = time.monotonic() - start_time if start_time else 0
|
||||
done = sum(1 for s in statuses if s == "done")
|
||||
write(
|
||||
f"\033[1m{selected_model}\033[0m [{done}/{NUM_REQUESTS}] {elapsed:.1f}s\r\n\r\n"
|
||||
)
|
||||
|
||||
for i in range(NUM_REQUESTS):
|
||||
q = QUESTIONS[i % len(QUESTIONS)]
|
||||
status = statuses[i]
|
||||
if status == "pending":
|
||||
color = "\033[33m" # yellow
|
||||
elif status == "running":
|
||||
color = "\033[36m" # cyan
|
||||
elif status == "done":
|
||||
color = "\033[32m" # green
|
||||
else:
|
||||
color = "\033[31m" # red
|
||||
write(
|
||||
f" {i:>2} {color}{status:<8}\033[0m {times[i]:>6} {tokens[i]:>5}tok {q[:40]:<40} {previews[i][:50]}\r\n"
|
||||
)
|
||||
|
||||
write("\r\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
async def send_request(
|
||||
session: aiohttp.ClientSession, i: int, lock: asyncio.Lock
|
||||
) -> None:
|
||||
payload = {
|
||||
"model": selected_model,
|
||||
"messages": [{"role": "user", "content": QUESTIONS[i % len(QUESTIONS)]}],
|
||||
"max_tokens": 1024,
|
||||
}
|
||||
statuses[i] = "running"
|
||||
async with lock:
|
||||
render_progress()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
async with session.post(
|
||||
f"{BASE_URL}/v1/chat/completions", json=payload
|
||||
) as resp:
|
||||
data = await resp.json()
|
||||
elapsed = time.monotonic() - t0
|
||||
full_responses[i] = data
|
||||
times[i] = f"{elapsed:.1f}s"
|
||||
if resp.status == 200:
|
||||
choice = data["choices"][0]
|
||||
msg = choice["message"]
|
||||
content = msg.get("content", "")
|
||||
previews[i] = content[:50].replace("\n", " ") or "(empty)"
|
||||
if "usage" in data:
|
||||
tokens[i] = str(data["usage"].get("total_tokens", ""))
|
||||
statuses[i] = "done"
|
||||
else:
|
||||
statuses[i] = f"err:{resp.status}"
|
||||
previews[i] = str(data.get("error", {}).get("message", ""))[:50]
|
||||
except Exception as e:
|
||||
elapsed = time.monotonic() - t0
|
||||
times[i] = f"{elapsed:.1f}s"
|
||||
statuses[i] = "error"
|
||||
previews[i] = str(e)[:50]
|
||||
async with lock:
|
||||
render_progress()
|
||||
|
||||
|
||||
async def run_requests(print_stdout: bool = False) -> None:
|
||||
global start_time, total_lines, statuses, times, previews, tokens, full_responses
|
||||
|
||||
statuses = ["pending"] * NUM_REQUESTS
|
||||
times = ["-"] * NUM_REQUESTS
|
||||
previews = ["-"] * NUM_REQUESTS
|
||||
tokens = ["-"] * NUM_REQUESTS
|
||||
full_responses = [None] * NUM_REQUESTS
|
||||
total_lines = NUM_REQUESTS + 4
|
||||
|
||||
write("\033[?25l") # hide cursor
|
||||
start_time = time.monotonic()
|
||||
render_progress(first=True)
|
||||
lock = asyncio.Lock()
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
tasks = [send_request(session, i, lock) for i in range(NUM_REQUESTS)]
|
||||
await asyncio.gather(*tasks)
|
||||
total = time.monotonic() - start_time
|
||||
write(
|
||||
f"\033[1m=== All {NUM_REQUESTS} requests done in {total:.1f}s ===\033[0m\r\n\r\n"
|
||||
)
|
||||
|
||||
if print_stdout:
|
||||
for i in range(NUM_REQUESTS):
|
||||
data = full_responses[i]
|
||||
if not data or "choices" not in data:
|
||||
continue
|
||||
choice = data["choices"][0]
|
||||
msg = choice["message"]
|
||||
q = QUESTIONS[i % len(QUESTIONS)]
|
||||
write(f"\033[1m--- #{i}: {q} ---\033[0m\r\n")
|
||||
if msg.get("reasoning_content"):
|
||||
write(f"\033[2m[Thinking]: {msg['reasoning_content']}\033[0m\r\n")
|
||||
write(f"{msg.get('content', '')}\r\n")
|
||||
if "usage" in data:
|
||||
u = data["usage"]
|
||||
write(
|
||||
f"\033[2m[Usage: prompt={u.get('prompt_tokens')}, "
|
||||
f"completion={u.get('completion_tokens')}, "
|
||||
f"total={u.get('total_tokens')}]\033[0m\r\n"
|
||||
)
|
||||
write("\r\n")
|
||||
finally:
|
||||
write("\033[?25h") # show cursor
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
global selected_model, BASE_URL
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Send parallel requests to an exo cluster"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host", required=True, help="Hostname of the exo node (e.g. s1)"
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=52415, help="Port (default: 52415)")
|
||||
parser.add_argument(
|
||||
"--stdout", action="store_true", help="Print full responses after completion"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
BASE_URL = f"http://{args.host}:{args.port}"
|
||||
model = pick_model()
|
||||
if not model:
|
||||
return
|
||||
selected_model = model
|
||||
asyncio.run(run_requests(print_stdout=args.stdout))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,36 +0,0 @@
|
||||
# Prefill/Decode disaggregation benchmark config.
|
||||
#
|
||||
# Top-level keys are bench-wide. [prefill] and [decode] sections set per-side
|
||||
# placement filters and (optionally) per-side model.
|
||||
#
|
||||
# Example:
|
||||
# uv run python bench/prefill_decode_bench.py --config bench/prefill-decode.toml
|
||||
|
||||
host = "james"
|
||||
port = 52415
|
||||
timeout = 7200.0
|
||||
settle_timeout = 60.0
|
||||
|
||||
# Workload
|
||||
pp = [4096, 8192]
|
||||
tg = [128]
|
||||
repeat = 1
|
||||
warmup = 0
|
||||
|
||||
json_out = "bench/prefill_decode_results.json"
|
||||
|
||||
[prefill]
|
||||
model = "sakamakismile/Qwen3.6-27B-NVFP4"
|
||||
node = "gx10-de89"
|
||||
instance_meta = "vllm"
|
||||
sharding = "pipeline"
|
||||
min_nodes = 1
|
||||
max_nodes = 1
|
||||
|
||||
[decode]
|
||||
model = "mlx-community/Qwen3.6-27B-4bit"
|
||||
node = "Ryuichi’s MacBook Pro"
|
||||
instance_meta = "ring"
|
||||
sharding = "pipeline"
|
||||
min_nodes = 1
|
||||
max_nodes = 1
|
||||
@@ -1,869 +0,0 @@
|
||||
# type: ignore
|
||||
#!/usr/bin/env python3
|
||||
"""Disaggregated prefill-decode benchmark for exo (MLX → MLX).
|
||||
|
||||
Spins up two MLX instances on the cluster, marks one as Prefill source and
|
||||
the other as Decode target via /v1/instance-links, then sends chat
|
||||
completions to the API. The master routes the request to the decode
|
||||
instance and stamps `prefill_endpoint` pointing at the prefill instance —
|
||||
the worker decides per-request whether to ship prefill remotely
|
||||
(uncached_count > REMOTE_PREFILL_MIN_TOKENS).
|
||||
|
||||
Usage:
|
||||
uv run python bench/prefill_decode_bench.py --model <id> --pp 2048,8192 --tg 128
|
||||
uv run python bench/prefill_decode_bench.py --model <id> --pp 4096 --tg 128 --repeat 3
|
||||
uv run python bench/prefill_decode_bench.py --model <id> --pp 2048 --tg 128 --dry-run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import copy
|
||||
import itertools
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Any
|
||||
|
||||
from exo_bench import (
|
||||
PromptSizer,
|
||||
SystemMetricsSampler,
|
||||
format_peak_memory,
|
||||
load_tokenizer_for_bench,
|
||||
parse_int_list,
|
||||
)
|
||||
from harness import (
|
||||
ExoClient,
|
||||
ExoHttpError,
|
||||
add_common_instance_args,
|
||||
instance_id_from_instance,
|
||||
node_ids_from_instance,
|
||||
nodes_used_in_instance,
|
||||
resolve_model_short_id,
|
||||
run_planning_phase,
|
||||
settle_and_fetch_placements,
|
||||
unwrap_instance,
|
||||
wait_for_instance_gone,
|
||||
wait_for_instance_ready,
|
||||
)
|
||||
from loguru import logger
|
||||
|
||||
|
||||
def _node_id_to_friendly(client: ExoClient) -> dict[str, str]:
|
||||
identities = client.get_node_identities() or {}
|
||||
out: dict[str, str] = {}
|
||||
for node_id, identity in identities.items():
|
||||
if isinstance(identity, dict):
|
||||
name = identity.get("friendlyName") or identity.get("friendly_name")
|
||||
if isinstance(name, str):
|
||||
out[str(node_id)] = name
|
||||
return out
|
||||
|
||||
|
||||
def _placement_node_friendly_names(
|
||||
placement: dict[str, Any], id_to_friendly: dict[str, str]
|
||||
) -> list[str]:
|
||||
instance = placement["instance"]
|
||||
return [id_to_friendly.get(nid, nid) for nid in node_ids_from_instance(instance)]
|
||||
|
||||
|
||||
def _filter_by_node(
|
||||
placements: list[dict[str, Any]],
|
||||
friendly_name: str,
|
||||
id_to_friendly: dict[str, str],
|
||||
) -> list[dict[str, Any]]:
|
||||
target = friendly_name.lower()
|
||||
matched: list[dict[str, Any]] = []
|
||||
for p in placements:
|
||||
names = [n.lower() for n in _placement_node_friendly_names(p, id_to_friendly)]
|
||||
if any(target == n or target in n for n in names):
|
||||
matched.append(p)
|
||||
return matched
|
||||
|
||||
|
||||
def _node_id_by_friendly(id_to_friendly: dict[str, str], target: str) -> str | None:
|
||||
target_lc = target.lower()
|
||||
for nid, name in id_to_friendly.items():
|
||||
if target_lc == name.lower() or target_lc in name.lower():
|
||||
return nid
|
||||
return None
|
||||
|
||||
|
||||
def _load_toml(path: str) -> dict[str, Any]:
|
||||
with Path(path).open("rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
_TOP_LEVEL_TOML_KEYS = {
|
||||
"host",
|
||||
"port",
|
||||
"timeout",
|
||||
"settle_timeout",
|
||||
"model",
|
||||
"pp",
|
||||
"tg",
|
||||
"repeat",
|
||||
"warmup",
|
||||
"json_out",
|
||||
"instance_meta",
|
||||
"sharding",
|
||||
"min_nodes",
|
||||
"max_nodes",
|
||||
"force_download",
|
||||
"danger_delete_downloads",
|
||||
"all_combinations",
|
||||
}
|
||||
|
||||
|
||||
def _inject_toml_into_argv() -> None:
|
||||
"""If --config X is in sys.argv, pre-load it and inject required CLI args
|
||||
(--model, --pp, --tg) so argparse's required=True checks pass."""
|
||||
argv = sys.argv
|
||||
if "--config" not in argv:
|
||||
return
|
||||
idx = argv.index("--config")
|
||||
if idx + 1 >= len(argv):
|
||||
return
|
||||
cfg_path = argv[idx + 1]
|
||||
cfg = _load_toml(cfg_path)
|
||||
decode = cfg.get("decode", {})
|
||||
|
||||
def _has(flag: str) -> bool:
|
||||
return any(a == flag or a.startswith(flag + "=") for a in argv)
|
||||
|
||||
# --model: prefer top-level, then [decode].model
|
||||
if not _has("--model"):
|
||||
model = cfg.get("model") or decode.get("model")
|
||||
if model:
|
||||
argv += ["--model", str(model)]
|
||||
if not _has("--pp"):
|
||||
pp = cfg.get("pp")
|
||||
if pp:
|
||||
argv += (
|
||||
["--pp", *(str(x) for x in pp)]
|
||||
if isinstance(pp, list)
|
||||
else [
|
||||
"--pp",
|
||||
str(pp),
|
||||
]
|
||||
)
|
||||
if not _has("--tg"):
|
||||
tg = cfg.get("tg")
|
||||
if tg:
|
||||
argv += (
|
||||
["--tg", *(str(x) for x in tg)]
|
||||
if isinstance(tg, list)
|
||||
else [
|
||||
"--tg",
|
||||
str(tg),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _merge_toml_into_args(args: argparse.Namespace, cfg: dict[str, Any]) -> None:
|
||||
"""Apply top-level toml keys onto args namespace where args has a default."""
|
||||
for key, value in cfg.items():
|
||||
if key in {"prefill", "decode"}:
|
||||
continue
|
||||
if key not in _TOP_LEVEL_TOML_KEYS:
|
||||
continue
|
||||
attr = key
|
||||
current = getattr(args, attr, None)
|
||||
if current in (None, [], False):
|
||||
setattr(args, attr, value)
|
||||
|
||||
|
||||
def _side_args(
|
||||
base: argparse.Namespace, overrides: dict[str, Any]
|
||||
) -> argparse.Namespace:
|
||||
out = copy.copy(base)
|
||||
for k in (
|
||||
"instance_meta",
|
||||
"sharding",
|
||||
"min_nodes",
|
||||
"max_nodes",
|
||||
"skip_pipeline_jaccl",
|
||||
"skip_tensor_ring",
|
||||
):
|
||||
if k in overrides:
|
||||
setattr(out, k, overrides[k])
|
||||
return out
|
||||
|
||||
|
||||
def _pick_two_distinct_placements(
|
||||
placements: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
if len(placements) < 2:
|
||||
return None
|
||||
seen_nodes: set[tuple[str, ...]] = set()
|
||||
chosen: list[dict[str, Any]] = []
|
||||
for p in placements:
|
||||
nodes = tuple(sorted(str(n) for n in p.get("nodes", [])))
|
||||
if nodes in seen_nodes:
|
||||
continue
|
||||
seen_nodes.add(nodes)
|
||||
chosen.append(p)
|
||||
if len(chosen) == 2:
|
||||
return chosen[0], chosen[1]
|
||||
return None
|
||||
|
||||
|
||||
def _create_instance_link(
|
||||
client: ExoClient,
|
||||
prefill_instance_id: str,
|
||||
decode_instance_id: str,
|
||||
) -> str:
|
||||
out = client.request_json(
|
||||
"POST",
|
||||
"/v1/instance-links",
|
||||
body={
|
||||
"prefill_instances": [prefill_instance_id],
|
||||
"decode_instances": [decode_instance_id],
|
||||
},
|
||||
)
|
||||
return str(out.get("commandId", ""))
|
||||
|
||||
|
||||
def _list_instance_links(client: ExoClient) -> list[dict[str, Any]]:
|
||||
out = client.request_json("GET", "/v1/instance-links")
|
||||
return out if isinstance(out, list) else []
|
||||
|
||||
|
||||
def _delete_instance_link(client: ExoClient, link_id: str) -> None:
|
||||
client.request_json("DELETE", f"/v1/instance-links/{link_id}")
|
||||
|
||||
|
||||
def run_one(
|
||||
client: ExoClient,
|
||||
model_id: str,
|
||||
pp_hint: int,
|
||||
tg: int,
|
||||
prompt_sizer: PromptSizer,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
content, pp_tokens = prompt_sizer.build(pp_hint)
|
||||
payload: dict[str, Any] = {
|
||||
"model": model_id,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"stream": False,
|
||||
"max_tokens": tg,
|
||||
}
|
||||
|
||||
t0 = time.perf_counter()
|
||||
out = client.post_bench_chat_completions(payload)
|
||||
elapsed = time.perf_counter() - t0
|
||||
|
||||
stats = out.get("generation_stats")
|
||||
choices = out.get("choices") or [{}]
|
||||
message = choices[0].get("message", {}) if choices else {}
|
||||
text = message.get("content") or ""
|
||||
preview = text[:200] if text else ""
|
||||
|
||||
return {
|
||||
"elapsed_s": elapsed,
|
||||
"output_text_preview": preview,
|
||||
"stats": stats,
|
||||
}, pp_tokens
|
||||
|
||||
|
||||
def _run_phase(
|
||||
*,
|
||||
client: ExoClient,
|
||||
label: str,
|
||||
pp_tg_pairs: list[tuple[int, int]],
|
||||
model_id: str,
|
||||
prompt_sizer: PromptSizer,
|
||||
warmup: int,
|
||||
repeat: int,
|
||||
common_meta: dict[str, Any],
|
||||
sampler: SystemMetricsSampler | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
logger.info(f"=== phase: {label} (model={model_id}) ===")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for i in range(warmup):
|
||||
run_one(client, model_id, pp_tg_pairs[0][0], pp_tg_pairs[0][1], prompt_sizer)
|
||||
logger.debug(f" warmup {i + 1}/{warmup} done")
|
||||
|
||||
for pp, tg in pp_tg_pairs:
|
||||
logger.info(f"--- {label}: pp={pp} tg={tg} ---")
|
||||
runs: list[dict[str, Any]] = []
|
||||
inference_windows: list[tuple[float, float]] = []
|
||||
for r in range(repeat):
|
||||
time.sleep(2)
|
||||
try:
|
||||
inf_t0 = time.monotonic()
|
||||
row, actual_pp_tokens = run_one(client, model_id, pp, tg, prompt_sizer)
|
||||
inference_windows.append((inf_t0, time.monotonic()))
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
continue
|
||||
row.update(common_meta)
|
||||
row.update(
|
||||
{
|
||||
"phase": label,
|
||||
"phase_model_id": model_id,
|
||||
"pp_tokens": actual_pp_tokens,
|
||||
"tg": tg,
|
||||
"repeat_index": r,
|
||||
}
|
||||
)
|
||||
runs.append(row)
|
||||
rows.append(row)
|
||||
|
||||
if runs:
|
||||
prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
|
||||
gen_tps = mean(x["stats"]["generation_tps"] for x in runs)
|
||||
ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
|
||||
gtok = mean(x["stats"]["generation_tokens"] for x in runs)
|
||||
peak = mean(x["stats"]["peak_memory_usage"]["inBytes"] for x in runs)
|
||||
avg_elapsed = mean(x["elapsed_s"] for x in runs)
|
||||
energy_str = ""
|
||||
if sampler is not None and inference_windows:
|
||||
joules = sum(
|
||||
sampler.energy_between(t0, t1) for t0, t1 in inference_windows
|
||||
)
|
||||
inf_seconds = sum(t1 - t0 for t0, t1 in inference_windows)
|
||||
avg_watts = joules / inf_seconds if inf_seconds > 0 else 0.0
|
||||
energy_per_run = joules / len(runs) if runs else 0.0
|
||||
energy_str = (
|
||||
f" energy={joules:.1f}J ({avg_watts:.1f}W avg over "
|
||||
f"{inf_seconds:.1f}s inference, {energy_per_run:.1f}J/run)"
|
||||
)
|
||||
for run_row, (t0, t1) in zip(runs, inference_windows, strict=False):
|
||||
run_row["energy_joules"] = sampler.energy_between(t0, t1)
|
||||
run_row["inference_window_s"] = t1 - t0
|
||||
logger.info(
|
||||
f"[{label}] prompt_tps={prompt_tps:.2f} gen_tps={gen_tps:.2f} "
|
||||
f"prompt_tokens={ptok} gen_tokens={gtok} "
|
||||
f"peak_memory={format_peak_memory(peak)} "
|
||||
f"avg_elapsed={avg_elapsed:.2f}s{energy_str}"
|
||||
)
|
||||
time.sleep(2)
|
||||
return rows
|
||||
|
||||
|
||||
def _summarise(rows: list[dict[str, Any]]) -> dict[tuple[int, int], dict[str, float]]:
|
||||
grouped: dict[tuple[int, int], list[dict[str, Any]]] = {}
|
||||
for r in rows:
|
||||
key = (int(r["pp_tokens"]), int(r["tg"]))
|
||||
grouped.setdefault(key, []).append(r)
|
||||
out: dict[tuple[int, int], dict[str, float]] = {}
|
||||
for key, runs in grouped.items():
|
||||
energy_runs = [x.get("energy_joules") for x in runs if "energy_joules" in x]
|
||||
window_runs = [
|
||||
x.get("inference_window_s") for x in runs if "inference_window_s" in x
|
||||
]
|
||||
out[key] = {
|
||||
"prompt_tps": mean(x["stats"]["prompt_tps"] for x in runs),
|
||||
"gen_tps": mean(x["stats"]["generation_tps"] for x in runs),
|
||||
"elapsed_s": mean(x["elapsed_s"] for x in runs),
|
||||
"prompt_tokens": mean(x["stats"]["prompt_tokens"] for x in runs),
|
||||
"gen_tokens": mean(x["stats"]["generation_tokens"] for x in runs),
|
||||
"energy_j": mean(energy_runs) if energy_runs else 0.0,
|
||||
"inference_window_s": mean(window_runs) if window_runs else 0.0,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def _normalised_seconds(summary: dict[str, float], pp: int, tg: int) -> float | None:
|
||||
"""Wall-clock time implied by reported tps for the *configured* pp/tg.
|
||||
|
||||
elapsed_s is not comparable across phases when models EOS at different
|
||||
lengths. This formula reconstructs "what would this phase take to do
|
||||
pp prompt tokens + tg generation tokens" using its own reported rates.
|
||||
"""
|
||||
p_tps = summary.get("prompt_tps", 0.0)
|
||||
g_tps = summary.get("gen_tps", 0.0)
|
||||
if p_tps <= 0 or g_tps <= 0:
|
||||
return None
|
||||
return pp / p_tps + tg / g_tps
|
||||
|
||||
|
||||
def _print_diff(
|
||||
disagg_rows: list[dict[str, Any]],
|
||||
decode_alone_rows: list[dict[str, Any]],
|
||||
prefill_alone_rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
disagg = _summarise(disagg_rows)
|
||||
decode_alone = _summarise(decode_alone_rows)
|
||||
prefill_alone = _summarise(prefill_alone_rows)
|
||||
keys = set(disagg.keys()) | set(decode_alone.keys()) | set(prefill_alone.keys())
|
||||
|
||||
width = 110
|
||||
for key in sorted(keys):
|
||||
pp, tg = key
|
||||
logger.info("─" * width)
|
||||
logger.info(f" pp={pp} tg={tg}")
|
||||
logger.info("─" * width)
|
||||
logger.info(
|
||||
f" {'phase':<16} {'elapsed':>9} {'norm':>9} "
|
||||
f"{'prompt_tps':>11} {'gen_tps':>8} "
|
||||
f"{'p_tok':>6} {'g_tok':>6} "
|
||||
f"{'energy':>9} {'avg_W':>7}"
|
||||
)
|
||||
for label, summary in (
|
||||
("disaggregated", disagg.get(key)),
|
||||
("decode_alone", decode_alone.get(key)),
|
||||
("prefill_alone", prefill_alone.get(key)),
|
||||
):
|
||||
if summary is None:
|
||||
logger.info(
|
||||
f" {label:<16} {'—':>9} {'—':>9} "
|
||||
f"{'—':>11} {'—':>8} {'—':>6} {'—':>6} "
|
||||
f"{'—':>9} {'—':>7}"
|
||||
)
|
||||
continue
|
||||
norm = _normalised_seconds(summary, pp, tg)
|
||||
norm_str = f"{norm:>8.2f}s" if norm is not None else f"{'—':>9}"
|
||||
energy = summary.get("energy_j", 0.0)
|
||||
window = summary.get("inference_window_s", 0.0)
|
||||
energy_str = f"{energy:>8.1f}J" if energy > 0 else f"{'—':>9}"
|
||||
avg_w = energy / window if window > 0 else 0.0
|
||||
avg_w_str = f"{avg_w:>6.1f}W" if avg_w > 0 else f"{'—':>7}"
|
||||
logger.info(
|
||||
f" {label:<16} "
|
||||
f"{summary['elapsed_s']:>8.2f}s "
|
||||
f"{norm_str} "
|
||||
f"{summary['prompt_tps']:>11.1f} "
|
||||
f"{summary['gen_tps']:>8.2f} "
|
||||
f"{summary['prompt_tokens']:>6.0f} "
|
||||
f"{summary['gen_tokens']:>6.0f} "
|
||||
f"{energy_str} "
|
||||
f"{avg_w_str}"
|
||||
)
|
||||
|
||||
d = disagg.get(key)
|
||||
da = decode_alone.get(key)
|
||||
pa = prefill_alone.get(key)
|
||||
d_norm = _normalised_seconds(d, pp, tg) if d else None
|
||||
if d_norm and da:
|
||||
da_norm = _normalised_seconds(da, pp, tg)
|
||||
if da_norm:
|
||||
logger.info(
|
||||
f" norm speedup vs decode_alone: {da_norm / d_norm:.2f}x "
|
||||
f"(prefill {d['prompt_tps'] / da['prompt_tps']:.2f}x, "
|
||||
f"decode {d['gen_tps'] / da['gen_tps']:.2f}x)"
|
||||
)
|
||||
if d_norm and pa:
|
||||
pa_norm = _normalised_seconds(pa, pp, tg)
|
||||
if pa_norm:
|
||||
logger.info(
|
||||
f" norm speedup vs prefill_alone: {pa_norm / d_norm:.2f}x "
|
||||
f"(prefill {d['prompt_tps'] / pa['prompt_tps']:.2f}x, "
|
||||
f"decode {d['gen_tps'] / pa['gen_tps']:.2f}x)"
|
||||
)
|
||||
logger.info("─" * width)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_inject_toml_into_argv()
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="prefill-decode-bench",
|
||||
description="Benchmark MLX-MLX disaggregated prefill/decode via instance links.",
|
||||
)
|
||||
add_common_instance_args(ap)
|
||||
ap.add_argument(
|
||||
"--pp",
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Prompt-size hints (ints, must be >1000). Accepts commas.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--tg",
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Generation lengths (ints). Accepts commas.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--repeat", type=int, default=1, help="Repetitions per (pp,tg) pair."
|
||||
)
|
||||
ap.add_argument(
|
||||
"--warmup",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Warmup runs (uses first pp/tg).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--json-out",
|
||||
default="bench/prefill_decode_results.json",
|
||||
help="Write raw per-run results JSON to this path.",
|
||||
)
|
||||
ap.add_argument("--stdout", action="store_true", help="Write results to stdout")
|
||||
ap.add_argument(
|
||||
"--dry-run", action="store_true", help="List selected placements and exit."
|
||||
)
|
||||
ap.add_argument(
|
||||
"--all-combinations",
|
||||
action="store_true",
|
||||
help="Force all pp×tg combinations even when lists have equal length.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--prefill-model",
|
||||
default=None,
|
||||
help="Model id for the prefill instance. Defaults to --model.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--prefill-node",
|
||||
default=None,
|
||||
help="friendly_name of the node hosting the prefill instance.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--decode-node",
|
||||
default=None,
|
||||
help="friendly_name of the node hosting the decode instance.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--config",
|
||||
default=None,
|
||||
help="TOML config file. CLI flags override toml values.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--compare-baseline",
|
||||
action="store_true",
|
||||
help="Also run each (pp,tg) pair without the prefill/decode link "
|
||||
"(decode instance does its own prefill) and report the diff.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
cfg = _load_toml(args.config) if args.config else {}
|
||||
_merge_toml_into_args(args, cfg)
|
||||
prefill_overrides = cfg.get("prefill", {}) if cfg else {}
|
||||
decode_overrides = cfg.get("decode", {}) if cfg else {}
|
||||
if args.prefill_model is None and "model" in prefill_overrides:
|
||||
args.prefill_model = prefill_overrides["model"]
|
||||
if args.prefill_node is None and "node" in prefill_overrides:
|
||||
args.prefill_node = prefill_overrides["node"]
|
||||
if args.decode_node is None and "node" in decode_overrides:
|
||||
args.decode_node = decode_overrides["node"]
|
||||
if "model" in decode_overrides and not args.model:
|
||||
args.model = decode_overrides["model"]
|
||||
|
||||
pp_list = parse_int_list(args.pp)
|
||||
tg_list = parse_int_list(args.tg)
|
||||
if not pp_list or not tg_list:
|
||||
logger.error("pp and tg lists must be non-empty")
|
||||
return 2
|
||||
for pp in pp_list:
|
||||
if pp <= 1000:
|
||||
logger.error(
|
||||
f"pp={pp} must be >1000 (remote prefill triggers when uncached >1000)"
|
||||
)
|
||||
return 2
|
||||
if args.repeat <= 0:
|
||||
logger.error("--repeat must be >= 1")
|
||||
return 2
|
||||
|
||||
use_combinations = args.all_combinations or len(pp_list) != len(tg_list)
|
||||
if use_combinations:
|
||||
logger.info(
|
||||
f"pp/tg mode: combinations (product) — {len(pp_list) * len(tg_list)} pairs"
|
||||
)
|
||||
else:
|
||||
logger.info(f"pp/tg mode: tandem (zip) — {len(pp_list)} pairs")
|
||||
|
||||
client = ExoClient(args.host, args.port, timeout_s=args.timeout)
|
||||
|
||||
decode_short_id, decode_full_id = resolve_model_short_id(
|
||||
client, args.model, force_download=args.force_download
|
||||
)
|
||||
if args.prefill_model:
|
||||
prefill_short_id, prefill_full_id = resolve_model_short_id(
|
||||
client, args.prefill_model, force_download=args.force_download
|
||||
)
|
||||
else:
|
||||
prefill_short_id, prefill_full_id = decode_short_id, decode_full_id
|
||||
|
||||
tokenizer = load_tokenizer_for_bench(decode_full_id)
|
||||
if tokenizer is None:
|
||||
raise RuntimeError("[prefill-decode-bench] decode tokenizer load failed")
|
||||
try:
|
||||
decode_prompt_sizer = PromptSizer(tokenizer)
|
||||
except Exception:
|
||||
logger.error("[prefill-decode-bench] decode prompt sizing failed")
|
||||
raise
|
||||
|
||||
if prefill_full_id == decode_full_id:
|
||||
prefill_prompt_sizer = decode_prompt_sizer
|
||||
else:
|
||||
prefill_tokenizer = load_tokenizer_for_bench(prefill_full_id)
|
||||
if prefill_tokenizer is None:
|
||||
raise RuntimeError("[prefill-decode-bench] prefill tokenizer load failed")
|
||||
prefill_prompt_sizer = PromptSizer(prefill_tokenizer)
|
||||
|
||||
id_to_friendly = _node_id_to_friendly(client)
|
||||
|
||||
prefill_args = _side_args(args, prefill_overrides)
|
||||
decode_args = _side_args(args, decode_overrides)
|
||||
|
||||
if prefill_full_id == decode_full_id and prefill_overrides == decode_overrides:
|
||||
placements = settle_and_fetch_placements(
|
||||
client, decode_full_id, args, settle_timeout=args.settle_timeout
|
||||
)
|
||||
prefill_candidates = (
|
||||
_filter_by_node(placements, args.prefill_node, id_to_friendly)
|
||||
if args.prefill_node
|
||||
else placements
|
||||
)
|
||||
decode_candidates = (
|
||||
_filter_by_node(placements, args.decode_node, id_to_friendly)
|
||||
if args.decode_node
|
||||
else placements
|
||||
)
|
||||
if args.prefill_node and not prefill_candidates:
|
||||
logger.error(f"No placement on prefill node {args.prefill_node!r}.")
|
||||
return 1
|
||||
if args.decode_node and not decode_candidates:
|
||||
logger.error(f"No placement on decode node {args.decode_node!r}.")
|
||||
return 1
|
||||
if args.prefill_node and args.decode_node:
|
||||
prefill_p = prefill_candidates[0]
|
||||
decode_p = decode_candidates[0]
|
||||
else:
|
||||
pair = _pick_two_distinct_placements(placements)
|
||||
if pair is None:
|
||||
logger.error(
|
||||
"Need at least two distinct-node MLX placements for the same model."
|
||||
)
|
||||
return 1
|
||||
prefill_p, decode_p = pair
|
||||
if args.prefill_node:
|
||||
prefill_p = prefill_candidates[0]
|
||||
if args.decode_node:
|
||||
decode_p = decode_candidates[0]
|
||||
else:
|
||||
prefill_node_id = (
|
||||
_node_id_by_friendly(id_to_friendly, args.prefill_node)
|
||||
if args.prefill_node
|
||||
else None
|
||||
)
|
||||
decode_node_id = (
|
||||
_node_id_by_friendly(id_to_friendly, args.decode_node)
|
||||
if args.decode_node
|
||||
else None
|
||||
)
|
||||
if args.prefill_node and prefill_node_id is None:
|
||||
logger.error(f"Unknown node {args.prefill_node!r}.")
|
||||
return 1
|
||||
if args.decode_node and decode_node_id is None:
|
||||
logger.error(f"Unknown node {args.decode_node!r}.")
|
||||
return 1
|
||||
prefill_placements = settle_and_fetch_placements(
|
||||
client,
|
||||
prefill_full_id,
|
||||
prefill_args,
|
||||
settle_timeout=args.settle_timeout,
|
||||
node_id=prefill_node_id,
|
||||
)
|
||||
decode_placements = settle_and_fetch_placements(
|
||||
client,
|
||||
decode_full_id,
|
||||
decode_args,
|
||||
settle_timeout=args.settle_timeout,
|
||||
node_id=decode_node_id,
|
||||
)
|
||||
if not prefill_placements:
|
||||
logger.error(
|
||||
f"No placement found for prefill model {prefill_full_id}"
|
||||
f"{f' on node {args.prefill_node!r}' if args.prefill_node else ''}."
|
||||
)
|
||||
return 1
|
||||
if not decode_placements:
|
||||
logger.error(
|
||||
f"No placement found for decode model {decode_full_id}"
|
||||
f"{f' on node {args.decode_node!r}' if args.decode_node else ''}."
|
||||
)
|
||||
return 1
|
||||
prefill_p = prefill_placements[0]
|
||||
decode_p = decode_placements[0]
|
||||
|
||||
prefill_node_names = _placement_node_friendly_names(prefill_p, id_to_friendly)
|
||||
decode_node_names = _placement_node_friendly_names(decode_p, id_to_friendly)
|
||||
_ = unwrap_instance
|
||||
|
||||
prefill_instance = prefill_p["instance"]
|
||||
decode_instance = decode_p["instance"]
|
||||
prefill_id = instance_id_from_instance(prefill_instance)
|
||||
decode_id = instance_id_from_instance(decode_instance)
|
||||
prefill_meta = str(prefill_p.get("instance_meta", ""))
|
||||
decode_meta = str(decode_p.get("instance_meta", ""))
|
||||
prefill_nodes = nodes_used_in_instance(prefill_instance)
|
||||
decode_nodes = nodes_used_in_instance(decode_instance)
|
||||
|
||||
logger.info("=" * 80)
|
||||
logger.info(
|
||||
f"PREFILL: {prefill_meta} / nodes={prefill_nodes} ({','.join(prefill_node_names)}) "
|
||||
f"/ {prefill_short_id} ({prefill_full_id}) / instance_id={prefill_id}"
|
||||
)
|
||||
logger.info(
|
||||
f"DECODE: {decode_meta} / nodes={decode_nodes} ({','.join(decode_node_names)}) "
|
||||
f"/ {decode_short_id} ({decode_full_id}) / instance_id={decode_id}"
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
settle_deadline = (
|
||||
time.monotonic() + args.settle_timeout if args.settle_timeout > 0 else None
|
||||
)
|
||||
|
||||
logger.info("Planning phase: prefill...")
|
||||
run_planning_phase(
|
||||
client,
|
||||
prefill_full_id,
|
||||
prefill_p,
|
||||
args.danger_delete_downloads,
|
||||
args.timeout,
|
||||
settle_deadline,
|
||||
)
|
||||
logger.info("Planning phase: decode...")
|
||||
run_planning_phase(
|
||||
client,
|
||||
decode_full_id,
|
||||
decode_p,
|
||||
args.danger_delete_downloads,
|
||||
args.timeout,
|
||||
settle_deadline,
|
||||
)
|
||||
|
||||
if use_combinations:
|
||||
pp_tg_pairs = list(itertools.product(pp_list, tg_list))
|
||||
else:
|
||||
pp_tg_pairs = list(zip(pp_list, tg_list, strict=True))
|
||||
|
||||
common_meta = {
|
||||
"decode_model_short_id": decode_short_id,
|
||||
"decode_model_id": decode_full_id,
|
||||
"prefill_model_short_id": prefill_short_id,
|
||||
"prefill_model_id": prefill_full_id,
|
||||
"prefill_instance_id": prefill_id,
|
||||
"prefill_instance_meta": prefill_meta,
|
||||
"prefill_nodes": prefill_nodes,
|
||||
"decode_instance_id": decode_id,
|
||||
"decode_instance_meta": decode_meta,
|
||||
"decode_nodes": decode_nodes,
|
||||
}
|
||||
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
disagg_rows: list[dict[str, Any]] = []
|
||||
decode_alone_rows: list[dict[str, Any]] = []
|
||||
prefill_alone_rows: list[dict[str, Any]] = []
|
||||
link_id = ""
|
||||
prefill_alive = False
|
||||
decode_alive = False
|
||||
sampler_nodes = sorted(
|
||||
{
|
||||
*node_ids_from_instance(prefill_instance),
|
||||
*node_ids_from_instance(decode_instance),
|
||||
}
|
||||
)
|
||||
sampler = SystemMetricsSampler(
|
||||
ExoClient(args.host, args.port, timeout_s=30), sampler_nodes
|
||||
)
|
||||
sampler.start()
|
||||
try:
|
||||
logger.info("Creating prefill instance...")
|
||||
client.request_json("POST", "/instance", body={"instance": prefill_instance})
|
||||
wait_for_instance_ready(client, prefill_id)
|
||||
prefill_alive = True
|
||||
logger.info("Prefill instance ready")
|
||||
|
||||
if args.compare_baseline:
|
||||
time.sleep(2)
|
||||
prefill_alone_rows = _run_phase(
|
||||
client=client,
|
||||
label="prefill_alone",
|
||||
pp_tg_pairs=pp_tg_pairs,
|
||||
model_id=prefill_full_id,
|
||||
prompt_sizer=prefill_prompt_sizer,
|
||||
warmup=args.warmup,
|
||||
repeat=args.repeat,
|
||||
common_meta=common_meta,
|
||||
sampler=sampler,
|
||||
)
|
||||
all_rows.extend(prefill_alone_rows)
|
||||
|
||||
logger.info("Creating decode instance...")
|
||||
client.request_json("POST", "/instance", body={"instance": decode_instance})
|
||||
wait_for_instance_ready(client, decode_id)
|
||||
decode_alive = True
|
||||
logger.info("Decode instance ready")
|
||||
|
||||
logger.info("Linking instances (prefill → decode)...")
|
||||
_create_instance_link(client, prefill_id, decode_id)
|
||||
time.sleep(1)
|
||||
links = _list_instance_links(client)
|
||||
if not links:
|
||||
logger.error("Link did not appear in state.")
|
||||
return 1
|
||||
link_id = str(links[-1].get("linkId") or links[-1].get("link_id") or "")
|
||||
logger.info(f"Link created: {link_id}")
|
||||
time.sleep(2)
|
||||
|
||||
disagg_rows = _run_phase(
|
||||
client=client,
|
||||
label="disaggregated",
|
||||
pp_tg_pairs=pp_tg_pairs,
|
||||
model_id=decode_full_id,
|
||||
prompt_sizer=decode_prompt_sizer,
|
||||
warmup=args.warmup,
|
||||
repeat=args.repeat,
|
||||
common_meta=common_meta,
|
||||
sampler=sampler,
|
||||
)
|
||||
all_rows.extend(disagg_rows)
|
||||
|
||||
if args.compare_baseline:
|
||||
logger.info("Removing link and prefill instance to isolate decode_alone.")
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
if link_id:
|
||||
_delete_instance_link(client, link_id)
|
||||
link_id = ""
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{prefill_id}")
|
||||
wait_for_instance_gone(client, prefill_id)
|
||||
prefill_alive = False
|
||||
time.sleep(2)
|
||||
|
||||
decode_alone_rows = _run_phase(
|
||||
client=client,
|
||||
label="decode_alone",
|
||||
pp_tg_pairs=pp_tg_pairs,
|
||||
model_id=decode_full_id,
|
||||
prompt_sizer=decode_prompt_sizer,
|
||||
warmup=args.warmup,
|
||||
repeat=args.repeat,
|
||||
common_meta=common_meta,
|
||||
sampler=sampler,
|
||||
)
|
||||
all_rows.extend(decode_alone_rows)
|
||||
|
||||
_print_diff(disagg_rows, decode_alone_rows, prefill_alone_rows)
|
||||
finally:
|
||||
sampler.stop()
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
if link_id:
|
||||
_delete_instance_link(client, link_id)
|
||||
if decode_alive:
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{decode_id}")
|
||||
wait_for_instance_gone(client, decode_id)
|
||||
if prefill_alive:
|
||||
with contextlib.suppress(ExoHttpError):
|
||||
client.request_json("DELETE", f"/instance/{prefill_id}")
|
||||
wait_for_instance_gone(client, prefill_id)
|
||||
logger.debug("Deleted both instances")
|
||||
|
||||
if args.stdout:
|
||||
json.dump(all_rows, sys.stdout, indent=2, ensure_ascii=False)
|
||||
elif args.json_out:
|
||||
with open(args.json_out, "w", encoding="utf-8") as f:
|
||||
json.dump(all_rows, f, indent=2, ensure_ascii=False)
|
||||
logger.debug(f"\nWrote results JSON: {args.json_out}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+6
-12
@@ -4,18 +4,12 @@ version = "0.1.0"
|
||||
description = "Benchmarking tool for exo distributed inference"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"httpx>=0.27.0",
|
||||
"loguru>=0.7.3",
|
||||
"transformers>=5.0.0",
|
||||
"huggingface-hub>=0.33.4",
|
||||
"tiktoken>=0.12.0",
|
||||
"jinja2>=3.1.0",
|
||||
"protobuf>=5.29.0",
|
||||
"datasets>=2.0.0",
|
||||
"math-verify>=0.7.0",
|
||||
"lm-eval[api,math]>=0.4.0",
|
||||
"human-eval>=1.0.3",
|
||||
"numpy>=1.24.0",
|
||||
"httpx>=0.27.0",
|
||||
"loguru>=0.7.3",
|
||||
"transformers>=5.0.0",
|
||||
"huggingface-hub>=0.33.4",
|
||||
"tiktoken>=0.12.0",
|
||||
"jinja2>=3.1.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
#
|
||||
# Shared constraints applied to ALL benchmarks in this file.
|
||||
constraints = [
|
||||
"All(MacOsBuild(=25D125))",
|
||||
"Hosts(=1)",
|
||||
"All(Chip(m3_ultra))",
|
||||
"All(GpuCores(=80))",
|
||||
"All(MacOsBuild(=25D125))",
|
||||
"Hosts(=1)",
|
||||
"All(Chip(m3_ultra))",
|
||||
"All(GpuCores(=80))",
|
||||
]
|
||||
|
||||
[topology]
|
||||
|
||||
Vendored
Whitespace-only changes.
Vendored
-593
@@ -1,593 +0,0 @@
|
||||
# type: ignore
|
||||
# Vendored from LiveCodeBench (https://github.com/LiveCodeBench/LiveCodeBench)
|
||||
# File: lcb_runner/evaluation/testing_util.py
|
||||
# License: MIT
|
||||
# Vendored 2026-03-07 — do not modify without updating from upstream.
|
||||
|
||||
import ast
|
||||
import faulthandler
|
||||
import json
|
||||
import platform
|
||||
|
||||
# to run the solution files we're using a timing based approach
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
# used for debugging to time steps
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from io import StringIO
|
||||
|
||||
# from pyext import RuntimeModule
|
||||
from types import ModuleType
|
||||
|
||||
# used for testing the code that reads from input
|
||||
from unittest.mock import mock_open, patch
|
||||
|
||||
import_string = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\n"
|
||||
|
||||
|
||||
def truncatefn(s, length=300):
|
||||
if isinstance(s, str):
|
||||
pass
|
||||
else:
|
||||
s = str(s)
|
||||
if len(s) <= length:
|
||||
return s
|
||||
|
||||
return s[: length // 2] + "...(truncated) ..." + s[-length // 2 :]
|
||||
|
||||
|
||||
class CODE_TYPE(Enum):
|
||||
call_based = 0
|
||||
standard_input = 1
|
||||
|
||||
|
||||
# stuff for setting up signal timer
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
print("timeout occured: alarm went off")
|
||||
raise TimeoutException
|
||||
|
||||
|
||||
# used to capture stdout as a list
|
||||
# from https://stackoverflow.com/a/16571630/6416660
|
||||
# alternative use redirect_stdout() from contextlib
|
||||
class Capturing(list):
|
||||
def __enter__(self):
|
||||
self._stdout = sys.stdout
|
||||
sys.stdout = self._stringio = StringIO()
|
||||
# Make closing the StringIO a no-op
|
||||
self._stringio.close = lambda x: 1
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.append(self._stringio.getvalue())
|
||||
del self._stringio # free up some memory
|
||||
sys.stdout = self._stdout
|
||||
|
||||
|
||||
# Custom mock for sys.stdin that supports buffer attribute
|
||||
class MockStdinWithBuffer:
|
||||
def __init__(self, inputs: str):
|
||||
self.inputs = inputs
|
||||
self._stringio = StringIO(inputs)
|
||||
self.buffer = MockBuffer(inputs)
|
||||
|
||||
def read(self, *args):
|
||||
return self.inputs
|
||||
|
||||
def readline(self, *args):
|
||||
return self._stringio.readline(*args)
|
||||
|
||||
def readlines(self, *args):
|
||||
return self.inputs.split("\n")
|
||||
|
||||
def __getattr__(self, name):
|
||||
# Delegate other attributes to StringIO
|
||||
return getattr(self._stringio, name)
|
||||
|
||||
|
||||
class MockBuffer:
|
||||
def __init__(self, inputs: str):
|
||||
self.inputs = inputs.encode("utf-8") # Convert to bytes
|
||||
|
||||
def read(self, *args):
|
||||
# Return as byte strings that can be split
|
||||
return self.inputs
|
||||
|
||||
def readline(self, *args):
|
||||
return self.inputs.split(b"\n")[0] + b"\n"
|
||||
|
||||
|
||||
def clean_if_name(code: str) -> str:
|
||||
try:
|
||||
astree = ast.parse(code)
|
||||
last_block = astree.body[-1]
|
||||
if isinstance(last_block, ast.If):
|
||||
condition = last_block.test
|
||||
if ast.unparse(condition).strip() == "__name__ == '__main__'":
|
||||
code = (
|
||||
ast.unparse(astree.body[:-1]) + "\n" + ast.unparse(last_block.body) # type: ignore
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
return code
|
||||
|
||||
|
||||
def make_function(code: str) -> str:
|
||||
try:
|
||||
import_stmts = []
|
||||
all_other_stmts = []
|
||||
astree = ast.parse(code)
|
||||
for stmt in astree.body:
|
||||
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
|
||||
import_stmts.append(stmt)
|
||||
else:
|
||||
all_other_stmts.append(stmt)
|
||||
|
||||
function_ast = ast.FunctionDef(
|
||||
name="wrapped_function",
|
||||
args=ast.arguments(
|
||||
posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]
|
||||
),
|
||||
body=all_other_stmts,
|
||||
decorator_list=[],
|
||||
lineno=-1,
|
||||
)
|
||||
main_code = (
|
||||
import_string
|
||||
+ "\n"
|
||||
+ ast.unparse(import_stmts)
|
||||
+ "\n"
|
||||
+ ast.unparse(function_ast)
|
||||
)
|
||||
return main_code
|
||||
except Exception:
|
||||
return code
|
||||
|
||||
|
||||
def call_method(method, inputs):
|
||||
if isinstance(inputs, list):
|
||||
inputs = "\n".join(inputs)
|
||||
|
||||
inputs_line_iterator = iter(inputs.split("\n"))
|
||||
|
||||
# Create custom stdin mock with buffer support
|
||||
mock_stdin = MockStdinWithBuffer(inputs)
|
||||
|
||||
# sys.setrecursionlimit(10000)
|
||||
|
||||
# @patch('builtins.input', side_effect=inputs.split("\n"))
|
||||
@patch("builtins.open", mock_open(read_data=inputs))
|
||||
@patch("sys.stdin", mock_stdin) # Use our custom mock instead of StringIO
|
||||
@patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator))
|
||||
@patch("sys.stdin.readlines", lambda *args: inputs.split("\n"))
|
||||
@patch("sys.stdin.read", lambda *args: inputs)
|
||||
# @patch('sys.stdout.write', print)
|
||||
def _inner_call_method(_method):
|
||||
try:
|
||||
return _method()
|
||||
except SystemExit:
|
||||
pass
|
||||
finally:
|
||||
pass
|
||||
|
||||
return _inner_call_method(method)
|
||||
|
||||
|
||||
def get_function(compiled_sol, fn_name: str): # type: ignore
|
||||
try:
|
||||
assert hasattr(compiled_sol, fn_name)
|
||||
return getattr(compiled_sol, fn_name)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def compile_code(code: str, timeout: int):
|
||||
signal.alarm(timeout)
|
||||
try:
|
||||
tmp_sol = ModuleType("tmp_sol", "")
|
||||
exec(code, tmp_sol.__dict__)
|
||||
if "class Solution" in code:
|
||||
# leetcode wraps solutions in `Solution`
|
||||
# this is a hack to check if it is leetcode solution or not
|
||||
# currently livecodebench only supports LeetCode but
|
||||
# else condition allows future extensibility to other platforms
|
||||
compiled_sol = tmp_sol.Solution()
|
||||
else:
|
||||
# do nothing in the other case since function is accesible
|
||||
compiled_sol = tmp_sol
|
||||
|
||||
assert compiled_sol is not None
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
|
||||
return compiled_sol
|
||||
|
||||
|
||||
def convert_line_to_decimals(line: str) -> tuple[bool, list[Decimal]]:
|
||||
try:
|
||||
decimal_line = [Decimal(elem) for elem in line.split()]
|
||||
except:
|
||||
return False, []
|
||||
return True, decimal_line
|
||||
|
||||
|
||||
def get_stripped_lines(val: str):
|
||||
## you don't want empty lines to add empty list after splitlines!
|
||||
val = val.strip()
|
||||
|
||||
return [val_line.strip() for val_line in val.split("\n")]
|
||||
|
||||
|
||||
def grade_call_based(
|
||||
code: str, all_inputs: list, all_outputs: list, fn_name: str, timeout: int
|
||||
):
|
||||
# call-based clean up logic
|
||||
# need to wrap in try-catch logic after to catch the correct errors, but for now this is fine.
|
||||
code = import_string + "\n\n" + code
|
||||
compiled_sol = compile_code(code, timeout)
|
||||
|
||||
if compiled_sol is None:
|
||||
return
|
||||
|
||||
method = get_function(compiled_sol, fn_name)
|
||||
|
||||
if method is None:
|
||||
return
|
||||
|
||||
all_inputs = [
|
||||
[json.loads(line) for line in inputs.split("\n")] for inputs in all_inputs
|
||||
]
|
||||
|
||||
all_outputs = [json.loads(output) for output in all_outputs]
|
||||
|
||||
total_execution = 0
|
||||
all_results = []
|
||||
for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
|
||||
signal.alarm(timeout)
|
||||
faulthandler.enable()
|
||||
try:
|
||||
# can lock here so time is useful
|
||||
start = time.time()
|
||||
prediction = method(*gt_inp)
|
||||
total_execution += time.time() - start
|
||||
signal.alarm(0)
|
||||
|
||||
# don't penalize model if it produces tuples instead of lists
|
||||
# ground truth sequences are not tuples
|
||||
if isinstance(prediction, tuple):
|
||||
prediction = list(prediction)
|
||||
|
||||
tmp_result = prediction == gt_out
|
||||
|
||||
# handle floating point comparisons
|
||||
|
||||
all_results.append(tmp_result)
|
||||
|
||||
if not tmp_result:
|
||||
return all_results, {
|
||||
"output": truncatefn(prediction),
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
"error_code": -2,
|
||||
"error_message": "Wrong Answer",
|
||||
}
|
||||
except Exception as e:
|
||||
signal.alarm(0)
|
||||
if "timeoutexception" in repr(e).lower():
|
||||
all_results.append(-3)
|
||||
return all_results, {
|
||||
"error": repr(e),
|
||||
"error_code": -3,
|
||||
"error_message": "Time Limit Exceeded",
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
}
|
||||
else:
|
||||
all_results.append(-4)
|
||||
return all_results, {
|
||||
"error": repr(e),
|
||||
"error_code": -4,
|
||||
"error_message": "Runtime Error",
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
}
|
||||
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
faulthandler.disable()
|
||||
|
||||
return all_results, {"execution time": total_execution}
|
||||
|
||||
|
||||
def grade_stdio(
|
||||
code: str,
|
||||
all_inputs: list,
|
||||
all_outputs: list,
|
||||
timeout: int,
|
||||
):
|
||||
## runtime doesn't interact well with __name__ == '__main__'
|
||||
code = clean_if_name(code)
|
||||
|
||||
## we wrap the given code inside another function
|
||||
code = make_function(code)
|
||||
|
||||
compiled_sol = compile_code(code, timeout)
|
||||
if compiled_sol is None:
|
||||
return
|
||||
|
||||
method = get_function(compiled_sol, "wrapped_function")
|
||||
|
||||
if method is None:
|
||||
return
|
||||
|
||||
all_results = []
|
||||
total_execution_time = 0
|
||||
for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
|
||||
signal.alarm(timeout)
|
||||
faulthandler.enable()
|
||||
|
||||
signal.alarm(timeout)
|
||||
with Capturing() as captured_output:
|
||||
try:
|
||||
start = time.time()
|
||||
call_method(method, gt_inp)
|
||||
total_execution_time += time.time() - start
|
||||
# reset the alarm
|
||||
signal.alarm(0)
|
||||
except Exception as e:
|
||||
signal.alarm(0)
|
||||
if "timeoutexception" in repr(e).lower():
|
||||
all_results.append(-3)
|
||||
return all_results, {
|
||||
"error": repr(e),
|
||||
"error_code": -3,
|
||||
"error_message": "Time Limit Exceeded",
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
}
|
||||
else:
|
||||
all_results.append(-4)
|
||||
return all_results, {
|
||||
"error": repr(e),
|
||||
"error_code": -4,
|
||||
"error_message": "Runtime Error",
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
}
|
||||
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
faulthandler.disable()
|
||||
|
||||
prediction = captured_output[0]
|
||||
|
||||
stripped_prediction_lines = get_stripped_lines(prediction)
|
||||
stripped_gt_out_lines = get_stripped_lines(gt_out)
|
||||
|
||||
## WA happens in multiple circumstances
|
||||
## so cache the return to make it clean!
|
||||
WA_send_args = {
|
||||
"output": truncatefn(prediction),
|
||||
"inputs": truncatefn(gt_inp),
|
||||
"expected": truncatefn(gt_out),
|
||||
"error_code": -2,
|
||||
}
|
||||
|
||||
if len(stripped_prediction_lines) != len(stripped_gt_out_lines):
|
||||
all_results.append(-2)
|
||||
WA_send_args["error_message"] = "Wrong answer: mismatched output length"
|
||||
return all_results, WA_send_args
|
||||
|
||||
for output_line_idx, (
|
||||
stripped_prediction_line,
|
||||
stripped_gt_out_line,
|
||||
) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)):
|
||||
WA_send_args["error_message"] = (
|
||||
f"Wrong answer at {output_line_idx=}: {truncatefn(stripped_prediction_line)} != {truncatefn(stripped_gt_out_line)}"
|
||||
)
|
||||
|
||||
## CASE 1: exact match
|
||||
if stripped_prediction_line == stripped_gt_out_line:
|
||||
continue
|
||||
|
||||
## CASE 2: element-wise comparision
|
||||
## if there are floating elements
|
||||
## use `decimal` library for good floating point comparision
|
||||
## otherwise gotcha: np.isclose(50000000000000000, 50000000000000001) = True
|
||||
## note that we should always be able to convert to decimals
|
||||
|
||||
success, decimal_prediction_line = convert_line_to_decimals(
|
||||
stripped_prediction_line
|
||||
)
|
||||
if not success:
|
||||
all_results.append(-2)
|
||||
return all_results, WA_send_args
|
||||
success, decimal_gtout_line = convert_line_to_decimals(stripped_gt_out_line)
|
||||
if not success:
|
||||
all_results.append(-2)
|
||||
return all_results, WA_send_args
|
||||
|
||||
if decimal_prediction_line == decimal_gtout_line:
|
||||
continue
|
||||
|
||||
all_results.append(-2)
|
||||
return all_results, WA_send_args
|
||||
all_results.append(True)
|
||||
|
||||
return all_results, {"execution time": total_execution_time}
|
||||
|
||||
|
||||
def run_test(sample, test=None, debug=False, timeout=6):
|
||||
"""
|
||||
if test(generated_code) is not None it'll try to run the code.
|
||||
otherwise it'll just return an input and output pair.
|
||||
"""
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
# Disable functionalities that can make destructive changes to the test.
|
||||
# max memory is set to 4GB
|
||||
reliability_guard()
|
||||
|
||||
if debug:
|
||||
print(f"start = {datetime.now().time()}")
|
||||
|
||||
try:
|
||||
in_outs = json.loads(sample["input_output"])
|
||||
except ValueError as e:
|
||||
raise e
|
||||
in_outs = None
|
||||
|
||||
if in_outs:
|
||||
if in_outs.get("fn_name") is None:
|
||||
which_type = CODE_TYPE.standard_input # Standard input
|
||||
method_name = None
|
||||
|
||||
else:
|
||||
which_type = CODE_TYPE.call_based # Call-based
|
||||
method_name = in_outs["fn_name"]
|
||||
|
||||
if debug:
|
||||
print(f"loaded input_output = {datetime.now().time()}")
|
||||
|
||||
if test is None:
|
||||
assert False, "should not happen: test code is none"
|
||||
return in_outs, {"error": "no test code provided"}
|
||||
elif test is not None:
|
||||
results = []
|
||||
sol = import_string
|
||||
if debug:
|
||||
print(f"loading test code = {datetime.now().time()}")
|
||||
|
||||
if which_type == CODE_TYPE.call_based:
|
||||
signal.alarm(timeout)
|
||||
try:
|
||||
results, metadata = grade_call_based(
|
||||
code=test,
|
||||
all_inputs=in_outs["inputs"],
|
||||
all_outputs=in_outs["outputs"],
|
||||
fn_name=method_name,
|
||||
timeout=timeout,
|
||||
)
|
||||
return results, metadata
|
||||
except Exception as e:
|
||||
return [-4], {
|
||||
"error_code": -4,
|
||||
"error_message": f"Error during testing: {e}",
|
||||
}
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
elif which_type == CODE_TYPE.standard_input:
|
||||
# sol
|
||||
# if code has if __name__ == "__main__": then remove it
|
||||
|
||||
signal.alarm(timeout)
|
||||
try:
|
||||
results, metadata = grade_stdio(
|
||||
code=test,
|
||||
all_inputs=in_outs["inputs"],
|
||||
all_outputs=in_outs["outputs"],
|
||||
timeout=timeout,
|
||||
)
|
||||
return results, metadata
|
||||
except Exception as e:
|
||||
return [-4], {
|
||||
"error_code": -4,
|
||||
"error_message": f"Error during testing: {e}",
|
||||
}
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
|
||||
|
||||
def reliability_guard(maximum_memory_bytes=None):
|
||||
"""
|
||||
This disables various destructive functions and prevents the generated code
|
||||
from interfering with the test (e.g. fork bomb, killing other processes,
|
||||
removing filesystem files, etc.)
|
||||
WARNING
|
||||
This function is NOT a security sandbox. Untrusted code, including, model-
|
||||
generated code, should not be blindly executed outside of one. See the
|
||||
Codex paper for more information about OpenAI's code sandbox, and proceed
|
||||
with caution.
|
||||
"""
|
||||
|
||||
if maximum_memory_bytes is not None:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
|
||||
)
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
|
||||
)
|
||||
if not platform.uname().system == "Darwin":
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
|
||||
)
|
||||
|
||||
faulthandler.disable()
|
||||
|
||||
import builtins
|
||||
|
||||
# builtins.exit = None
|
||||
builtins.quit = None
|
||||
|
||||
import os
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
|
||||
os.kill = None
|
||||
os.system = None
|
||||
os.putenv = None
|
||||
os.remove = None
|
||||
os.removedirs = None
|
||||
os.rmdir = None
|
||||
os.fchdir = None
|
||||
os.setuid = None
|
||||
os.fork = None
|
||||
os.forkpty = None
|
||||
os.killpg = None
|
||||
os.rename = None
|
||||
os.renames = None
|
||||
os.truncate = None
|
||||
os.replace = None
|
||||
os.unlink = None
|
||||
os.fchmod = None
|
||||
os.fchown = None
|
||||
os.chmod = None
|
||||
os.chown = None
|
||||
os.chroot = None
|
||||
os.fchdir = None
|
||||
os.lchflags = None
|
||||
os.lchmod = None
|
||||
os.lchown = None
|
||||
os.getcwd = None
|
||||
os.chdir = None
|
||||
|
||||
import shutil
|
||||
|
||||
shutil.rmtree = None
|
||||
shutil.move = None
|
||||
shutil.chown = None
|
||||
|
||||
import subprocess
|
||||
|
||||
subprocess.Popen = None
|
||||
|
||||
__builtins__["help"] = None
|
||||
|
||||
import sys
|
||||
|
||||
sys.modules["ipdb"] = None
|
||||
sys.modules["joblib"] = None
|
||||
sys.modules["resource"] = None
|
||||
sys.modules["psutil"] = None
|
||||
sys.modules["tkinter"] = None
|
||||
Generated
+1
-272
@@ -11,8 +11,7 @@
|
||||
"highlight.js": "^11.11.1",
|
||||
"katex": "^0.16.27",
|
||||
"marked": "^17.0.1",
|
||||
"mode-watcher": "^1.1.0",
|
||||
"pdfjs-dist": "^5.6.205"
|
||||
"mode-watcher": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@sveltejs/adapter-static": "^3.0.10",
|
||||
@@ -519,256 +518,6 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.97.tgz",
|
||||
"integrity": "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.97",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.97",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.97",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.97",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.97",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.97",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.97",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.97",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.97",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.97"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.97.tgz",
|
||||
"integrity": "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.97.tgz",
|
||||
"integrity": "sha512-ok+SCEF4YejcxuJ9Rm+WWunHHpf2HmiPxfz6z1a/NFQECGXtsY7A4B8XocK1LmT1D7P174MzwPF9Wy3AUAwEPw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.97.tgz",
|
||||
"integrity": "sha512-PUP6e6/UGlclUvAQNnuXCcnkpdUou6VYZfQOQxExLp86epOylmiwLkqXIvpFmjoTEDmPmXrI+coL/9EFU1gKPA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.97.tgz",
|
||||
"integrity": "sha512-XyXH2L/cic8eTNtbrXCcvqHtMX/nEOxN18+7rMrAM2XtLYC/EB5s0wnO1FsLMWmK+04ZSLN9FBGipo7kpIkcOw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.97.tgz",
|
||||
"integrity": "sha512-Kuq/M3djq0K8ktgz6nPlK7Ne5d4uWeDxPpyKWOjWDK2RIOhHVtLtyLiJw2fuldw7Vn4mhw05EZXCEr4Q76rs9w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.97.tgz",
|
||||
"integrity": "sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.97.tgz",
|
||||
"integrity": "sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.97.tgz",
|
||||
"integrity": "sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.97.tgz",
|
||||
"integrity": "sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.97.tgz",
|
||||
"integrity": "sha512-u883Yr6A6fO7Vpsy9YE4FVCIxzzo5sO+7pIUjjoDLjS3vQaNMkVzx5bdIpEL+ob+gU88WDK4VcxYMZ6nmnoX9A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.97",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.97.tgz",
|
||||
"integrity": "sha512-sWtD2EE3fV0IzN+iiQUqr/Q1SwqWhs2O1FKItFlxtdDkikpEj5g7DKQpY3x55H/MAOnL8iomnlk3mcEeGiUMoQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@polka/url": {
|
||||
"version": "1.0.0-next.29",
|
||||
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
|
||||
@@ -2886,26 +2635,6 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/node-readable-to-web-readable-stream": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz",
|
||||
"integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.6.205",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz",
|
||||
"integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20.19.0 || >=22.13.0 || >=24"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.96",
|
||||
"node-readable-to-web-readable-stream": "^0.4.2"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
@@ -31,7 +31,6 @@
|
||||
"highlight.js": "^11.11.1",
|
||||
"katex": "^0.16.27",
|
||||
"marked": "^17.0.1",
|
||||
"mode-watcher": "^1.1.0",
|
||||
"pdfjs-dist": "^5.6.205"
|
||||
"mode-watcher": "^1.1.0"
|
||||
}
|
||||
}
|
||||
@@ -202,23 +202,6 @@
|
||||
filter: drop-shadow(0 0 3px oklch(0.85 0.18 85 / 0.5));
|
||||
}
|
||||
|
||||
/* Onboarding step 2: connection line between devices */
|
||||
.onboarding-connection-line {
|
||||
stroke: oklch(0.85 0.18 85 / 0.5);
|
||||
stroke-width: 1.5px;
|
||||
stroke-dasharray: 6, 6;
|
||||
animation: flowAnimation 1s linear infinite;
|
||||
filter: drop-shadow(0 0 4px oklch(0.85 0.18 85 / 0.4));
|
||||
}
|
||||
|
||||
/* Onboarding step 4: red connection line for disconnect */
|
||||
.onboarding-connection-line-red {
|
||||
stroke: rgba(220, 38, 38, 0.7);
|
||||
stroke-width: 1.5px;
|
||||
stroke-dasharray: 6, 6;
|
||||
filter: drop-shadow(0 0 2px rgba(220, 38, 38, 0.3));
|
||||
}
|
||||
|
||||
.graph-link-active {
|
||||
stroke: oklch(0.85 0.18 85 / 0.8);
|
||||
stroke-width: 2px;
|
||||
@@ -337,51 +320,3 @@ input:focus, textarea:focus {
|
||||
transform: translate(400px, 400px);
|
||||
}
|
||||
}
|
||||
|
||||
/* Onboarding smooth fade-in */
|
||||
@keyframes onb-fade-in {
|
||||
from { opacity: 0; transform: translateY(8px); }
|
||||
to { opacity: 1; transform: none; }
|
||||
}
|
||||
|
||||
/* Opacity-only fade-in — no transform, safe for containers with fixed-position children */
|
||||
@keyframes onb-fade-opacity {
|
||||
from { opacity: 0; }
|
||||
to { opacity: 1; }
|
||||
}
|
||||
|
||||
/* Respect reduced motion preference */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.shooting-star,
|
||||
.shooting-star::before {
|
||||
animation: none !important;
|
||||
opacity: 0 !important;
|
||||
}
|
||||
.graph-link {
|
||||
animation: none;
|
||||
}
|
||||
.status-pulse {
|
||||
animation: none;
|
||||
}
|
||||
.cursor-blink {
|
||||
animation: none;
|
||||
}
|
||||
.onboarding-connection-line {
|
||||
animation: none;
|
||||
}
|
||||
.onboarding-connection-line-red {
|
||||
animation: none;
|
||||
}
|
||||
[style*="onb-fade-in"],
|
||||
[style*="onb-fade-opacity"] {
|
||||
animation: none !important;
|
||||
opacity: 1 !important;
|
||||
}
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
transition-duration: 0.01ms !important;
|
||||
animation-duration: 0.01ms !important;
|
||||
animation-iteration-count: 1 !important;
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,14 @@
|
||||
<script lang="ts">
|
||||
import {
|
||||
isLoading,
|
||||
sendMessage,
|
||||
generateImage,
|
||||
editImage,
|
||||
editingImage,
|
||||
clearEditingImage,
|
||||
selectedChatModel,
|
||||
setSelectedChatModel,
|
||||
instances,
|
||||
ttftMs,
|
||||
tps,
|
||||
totalTokens,
|
||||
@@ -24,19 +29,6 @@
|
||||
showModelSelector?: boolean;
|
||||
modelTasks?: Record<string, string[]>;
|
||||
modelCapabilities?: Record<string, string[]>;
|
||||
onSend?: () => void;
|
||||
onAutoSend: (
|
||||
content: string,
|
||||
files?: {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
textContent?: string;
|
||||
preview?: string;
|
||||
}[],
|
||||
) => void;
|
||||
onOpenModelPicker?: () => void;
|
||||
modelDisplayOverride?: string;
|
||||
}
|
||||
|
||||
let {
|
||||
@@ -47,10 +39,6 @@
|
||||
showModelSelector = false,
|
||||
modelTasks = {},
|
||||
modelCapabilities = {},
|
||||
onSend,
|
||||
onAutoSend,
|
||||
onOpenModelPicker,
|
||||
modelDisplayOverride,
|
||||
}: Props = $props();
|
||||
|
||||
let message = $state("");
|
||||
@@ -61,12 +49,27 @@
|
||||
const thinkingEnabled = $derived(thinkingEnabledStore());
|
||||
let loading = $derived(isLoading());
|
||||
const currentModel = $derived(selectedChatModel());
|
||||
const instanceData = $derived(instances());
|
||||
const currentTtft = $derived(ttftMs());
|
||||
const currentTps = $derived(tps());
|
||||
const currentTokens = $derived(totalTokens());
|
||||
const currentEditingImage = $derived(editingImage());
|
||||
const isEditMode = $derived(currentEditingImage !== null);
|
||||
|
||||
// Custom dropdown state
|
||||
let isModelDropdownOpen = $state(false);
|
||||
let dropdownButtonRef: HTMLButtonElement | undefined = $state();
|
||||
let dropdownPosition = $derived(() => {
|
||||
if (!dropdownButtonRef || !isModelDropdownOpen)
|
||||
return { top: 0, left: 0, width: 0 };
|
||||
const rect = dropdownButtonRef.getBoundingClientRect();
|
||||
return {
|
||||
top: rect.top,
|
||||
left: rect.left,
|
||||
width: rect.width,
|
||||
};
|
||||
});
|
||||
|
||||
// Accept all supported file types
|
||||
const acceptString = getAcceptString(["image", "text", "pdf"]);
|
||||
|
||||
@@ -119,14 +122,73 @@
|
||||
uploadedFiles.length > 0),
|
||||
);
|
||||
|
||||
// Short label for the currently selected model
|
||||
const currentModelLabel = $derived(
|
||||
currentModel
|
||||
? currentModel.split("/").pop() || currentModel
|
||||
: modelDisplayOverride
|
||||
? modelDisplayOverride.split("/").pop() || modelDisplayOverride
|
||||
: "",
|
||||
);
|
||||
// Extract available models from running instances
|
||||
const availableModels = $derived(() => {
|
||||
const models: Array<{ id: string; label: string; isImageModel: boolean }> =
|
||||
[];
|
||||
for (const [, instance] of Object.entries(instanceData)) {
|
||||
const modelId = getInstanceModelId(instance);
|
||||
if (
|
||||
modelId &&
|
||||
modelId !== "Unknown" &&
|
||||
!models.some((m) => m.id === modelId)
|
||||
) {
|
||||
models.push({
|
||||
id: modelId,
|
||||
label: modelId.split("/").pop() || modelId,
|
||||
isImageModel: modelSupportsImageGeneration(modelId),
|
||||
});
|
||||
}
|
||||
}
|
||||
return models;
|
||||
});
|
||||
|
||||
// Track previous model IDs to detect newly added models (plain variable to avoid reactive loop)
|
||||
let previousModelIds: Set<string> = new Set();
|
||||
|
||||
// Auto-select the first available model if none is selected, if current selection is stale, or if a new model is added
|
||||
$effect(() => {
|
||||
const models = availableModels();
|
||||
const currentModelIds = new Set(models.map((m) => m.id));
|
||||
|
||||
if (models.length > 0) {
|
||||
// Find newly added models (in current but not in previous)
|
||||
const newModels = models.filter((m) => !previousModelIds.has(m.id));
|
||||
|
||||
// If no model selected, select the first available
|
||||
if (!currentModel) {
|
||||
setSelectedChatModel(models[0].id);
|
||||
}
|
||||
// If current model is stale (no longer has a running instance), reset to first available
|
||||
else if (!models.some((m) => m.id === currentModel)) {
|
||||
setSelectedChatModel(models[0].id);
|
||||
}
|
||||
// If a new model was just added, select it
|
||||
else if (newModels.length > 0 && previousModelIds.size > 0) {
|
||||
setSelectedChatModel(newModels[0].id);
|
||||
}
|
||||
} else {
|
||||
// No instances running - clear the selected model
|
||||
if (currentModel) {
|
||||
setSelectedChatModel("");
|
||||
}
|
||||
}
|
||||
|
||||
// Update previous model IDs for next comparison
|
||||
previousModelIds = currentModelIds;
|
||||
});
|
||||
|
||||
function getInstanceModelId(instanceWrapped: unknown): string {
|
||||
if (!instanceWrapped || typeof instanceWrapped !== "object") return "";
|
||||
const keys = Object.keys(instanceWrapped as Record<string, unknown>);
|
||||
if (keys.length === 1) {
|
||||
const instance = (instanceWrapped as Record<string, unknown>)[
|
||||
keys[0]
|
||||
] as { shardAssignments?: { modelId?: string } };
|
||||
return instance?.shardAssignments?.modelId || "";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
async function handleFiles(files: File[]) {
|
||||
if (files.length === 0) return;
|
||||
@@ -213,10 +275,38 @@
|
||||
uploadedFiles = [];
|
||||
resetTextareaHeight();
|
||||
|
||||
// Parent controls all send logic (including image routing,
|
||||
// launching non-running models before sending, etc.)
|
||||
onAutoSend(content, files);
|
||||
onSend?.();
|
||||
// Use image editing if in edit mode
|
||||
if (isEditMode && currentEditingImage && content) {
|
||||
editImage(content, currentEditingImage.imageDataUrl);
|
||||
}
|
||||
// If user attached an image with an ImageToImage model, use edit endpoint
|
||||
else if (
|
||||
currentModel &&
|
||||
modelSupportsImageEditing(currentModel) &&
|
||||
files.length > 0 &&
|
||||
content
|
||||
) {
|
||||
// Use the first attached image for editing
|
||||
const imageFile = files[0];
|
||||
if (imageFile.preview) {
|
||||
editImage(content, imageFile.preview);
|
||||
}
|
||||
} else if (
|
||||
currentModel &&
|
||||
modelSupportsTextToImage(currentModel) &&
|
||||
content
|
||||
) {
|
||||
// Use image generation for text-to-image models
|
||||
generateImage(content);
|
||||
} else {
|
||||
sendMessage(
|
||||
content,
|
||||
files,
|
||||
modelSupportsThinking() ? thinkingEnabled : null,
|
||||
);
|
||||
}
|
||||
|
||||
// Refocus the textarea after sending
|
||||
setTimeout(() => textareaRef?.focus(), 10);
|
||||
}
|
||||
|
||||
@@ -326,7 +416,7 @@
|
||||
{/if}
|
||||
|
||||
<!-- Model selector (when enabled) -->
|
||||
{#if showModelSelector}
|
||||
{#if showModelSelector && availableModels().length > 0}
|
||||
<div
|
||||
class="flex items-center justify-between gap-2 px-3 py-2 border-b border-exo-medium-gray/30"
|
||||
>
|
||||
@@ -335,22 +425,33 @@
|
||||
class="text-xs text-exo-light-gray uppercase tracking-wider flex-shrink-0"
|
||||
>MODEL:</span
|
||||
>
|
||||
<!-- Model button — opens the full model picker -->
|
||||
<!-- Custom dropdown -->
|
||||
<div class="relative flex-1 max-w-xs">
|
||||
<button
|
||||
bind:this={dropdownButtonRef}
|
||||
type="button"
|
||||
onclick={() => onOpenModelPicker?.()}
|
||||
class="w-full bg-exo-medium-gray/50 border border-exo-yellow/30 rounded pl-3 pr-8 py-1.5 text-xs font-mono text-left tracking-wide cursor-pointer transition-all duration-200 hover:border-exo-yellow/50 focus:outline-none focus:border-exo-yellow/70"
|
||||
onclick={() => (isModelDropdownOpen = !isModelDropdownOpen)}
|
||||
class="w-full bg-exo-medium-gray/50 border border-exo-yellow/30 rounded pl-3 pr-8 py-1.5 text-xs font-mono text-left tracking-wide cursor-pointer transition-all duration-200 hover:border-exo-yellow/50 focus:outline-none focus:border-exo-yellow/70 {isModelDropdownOpen
|
||||
? 'border-exo-yellow/70'
|
||||
: ''}"
|
||||
>
|
||||
{#if currentModelLabel}
|
||||
<span class="text-exo-yellow truncate">{currentModelLabel}</span
|
||||
{#if availableModels().find((m) => m.id === currentModel)}
|
||||
<span class="text-exo-yellow truncate"
|
||||
>{availableModels().find((m) => m.id === currentModel)
|
||||
?.label}</span
|
||||
>
|
||||
{:else if availableModels().length > 0}
|
||||
<span class="text-exo-yellow truncate"
|
||||
>{availableModels()[0].label}</span
|
||||
>
|
||||
{:else}
|
||||
<span class="text-exo-light-gray/50">— SELECT MODEL —</span>
|
||||
{/if}
|
||||
</button>
|
||||
<div
|
||||
class="absolute right-2 top-1/2 -translate-y-1/2 pointer-events-none"
|
||||
class="absolute right-2 top-1/2 -translate-y-1/2 pointer-events-none transition-transform duration-200 {isModelDropdownOpen
|
||||
? 'rotate-180'
|
||||
: ''}"
|
||||
>
|
||||
<svg
|
||||
class="w-3 h-3 text-exo-yellow/60"
|
||||
@@ -367,6 +468,78 @@
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{#if isModelDropdownOpen}
|
||||
<!-- Backdrop to close dropdown -->
|
||||
<button
|
||||
type="button"
|
||||
class="fixed inset-0 z-[9998] cursor-default"
|
||||
onclick={() => (isModelDropdownOpen = false)}
|
||||
aria-label="Close dropdown"
|
||||
></button>
|
||||
|
||||
<!-- Dropdown Panel - fixed positioning to escape overflow:hidden -->
|
||||
<div
|
||||
class="fixed bg-exo-dark-gray border border-exo-yellow/30 rounded shadow-lg shadow-black/50 z-[9999] max-h-48 overflow-y-auto"
|
||||
style="bottom: calc(100vh - {dropdownPosition()
|
||||
.top}px + 4px); left: {dropdownPosition()
|
||||
.left}px; width: {dropdownPosition().width}px;"
|
||||
>
|
||||
<div class="py-1">
|
||||
{#each availableModels() as model}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => {
|
||||
setSelectedChatModel(model.id);
|
||||
isModelDropdownOpen = false;
|
||||
}}
|
||||
class="w-full px-3 py-2 text-left text-xs font-mono tracking-wide transition-colors duration-100 flex items-center gap-2 {currentModel ===
|
||||
model.id
|
||||
? 'bg-transparent text-exo-yellow'
|
||||
: 'text-exo-light-gray hover:text-exo-yellow'}"
|
||||
>
|
||||
{#if currentModel === model.id}
|
||||
<svg
|
||||
class="w-3 h-3 flex-shrink-0"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 20 20"
|
||||
>
|
||||
<path
|
||||
fill-rule="evenodd"
|
||||
d="M16.707 5.293a1 1 0 010 1.414l-8 8a1 1 0 01-1.414 0l-4-4a1 1 0 011.414-1.414L8 12.586l7.293-7.293a1 1 0 011.414 0z"
|
||||
clip-rule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
{:else}
|
||||
<span class="w-3"></span>
|
||||
{/if}
|
||||
{#if model.isImageModel}
|
||||
<svg
|
||||
class="w-3.5 h-3.5 flex-shrink-0 text-exo-yellow"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
aria-label="Image generation model"
|
||||
>
|
||||
<rect
|
||||
x="3"
|
||||
y="3"
|
||||
width="18"
|
||||
height="18"
|
||||
rx="2"
|
||||
ry="2"
|
||||
/>
|
||||
<circle cx="8.5" cy="8.5" r="1.5" />
|
||||
<polyline points="21 15 16 10 5 21" />
|
||||
</svg>
|
||||
{/if}
|
||||
<span class="truncate flex-1">{model.label}</span>
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<!-- Thinking toggle -->
|
||||
{#if modelSupportsThinking()}
|
||||
|
||||
@@ -139,8 +139,6 @@
|
||||
return "🖼";
|
||||
case "text":
|
||||
return "📄";
|
||||
case "pdf":
|
||||
return "📑";
|
||||
default:
|
||||
return "📎";
|
||||
}
|
||||
@@ -809,8 +807,8 @@
|
||||
>
|
||||
AWAITING INPUT
|
||||
</p>
|
||||
<p class="text-xs text-white/30 tracking-wider mt-1.5 font-mono">
|
||||
Type a message below · Shift+Enter for newline
|
||||
<p class="text-sm sm:text-xs text-exo-light-gray tracking-wider mt-1">
|
||||
ENTER A QUERY TO BEGIN
|
||||
</p>
|
||||
</div>
|
||||
{/if}
|
||||
@@ -825,7 +823,6 @@
|
||||
onclick={scrollToBottom}
|
||||
class="sticky bottom-4 left-1/2 -translate-x-1/2 w-10 h-10 rounded-full bg-exo-dark-gray/90 border border-exo-medium-gray/50 flex items-center justify-center text-exo-light-gray hover:text-exo-yellow hover:border-exo-yellow/50 transition-all shadow-lg cursor-pointer z-10"
|
||||
title="Scroll to bottom"
|
||||
aria-label="Scroll to bottom of messages"
|
||||
>
|
||||
<svg
|
||||
class="w-5 h-5"
|
||||
|
||||
@@ -1,401 +0,0 @@
|
||||
<script lang="ts" module>
|
||||
export interface ChatModelInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
base_model: string;
|
||||
storage_size_megabytes: number;
|
||||
capabilities: string[];
|
||||
family: string;
|
||||
quantization: string;
|
||||
}
|
||||
|
||||
// Auto mode tier list (for when user just starts typing)
|
||||
export const AUTO_TIERS: string[][] = [
|
||||
// Tier 1 (frontier)
|
||||
["DeepSeek V3.1", "GLM-5", "Kimi K2.5", "Qwen3 Coder Next"],
|
||||
// Tier 2 (excellent)
|
||||
[
|
||||
"Kimi K2",
|
||||
"Qwen3 235B",
|
||||
"MiniMax M2.5",
|
||||
"Step 3.5 Flash",
|
||||
"Qwen3 Next 80B",
|
||||
],
|
||||
// Tier 3 (great)
|
||||
[
|
||||
"GLM 4.7",
|
||||
"MiniMax M2.1",
|
||||
"Qwen3 Coder 480B",
|
||||
"GLM 4.5 Air",
|
||||
"Llama 3.3 70B",
|
||||
],
|
||||
// Tier 4 (good)
|
||||
["GPT-OSS 120B", "Qwen3 30B", "Llama 3.1 70B", "GLM 4.7 Flash"],
|
||||
// Tier 5 (small/fast)
|
||||
[
|
||||
"Llama 3.1 8B",
|
||||
"GPT-OSS 20B",
|
||||
"Llama 3.2 3B",
|
||||
"Qwen3 0.6B",
|
||||
"Llama 3.2 1B",
|
||||
],
|
||||
];
|
||||
|
||||
/** Return the tier index (0 = best) for a base_model name. */
|
||||
export function getAutoTierIndex(baseModel: string): number {
|
||||
for (let i = 0; i < AUTO_TIERS.length; i++) {
|
||||
if (AUTO_TIERS[i].includes(baseModel)) return i;
|
||||
}
|
||||
return AUTO_TIERS.length; // not in any tier → lowest priority
|
||||
}
|
||||
|
||||
/** Auto mode: walk tiers top-down, pick biggest fitting variant from highest tier. */
|
||||
export function pickAutoModel(
|
||||
modelList: ChatModelInfo[],
|
||||
memoryGB: number,
|
||||
): ChatModelInfo | null {
|
||||
for (const tier of AUTO_TIERS) {
|
||||
const candidates: ChatModelInfo[] = [];
|
||||
for (const baseModel of tier) {
|
||||
const variants = modelList
|
||||
.filter(
|
||||
(m) =>
|
||||
m.base_model === baseModel &&
|
||||
(m.storage_size_megabytes || 0) / 1024 <= memoryGB &&
|
||||
(m.storage_size_megabytes || 0) > 0,
|
||||
)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(b.storage_size_megabytes || 0) - (a.storage_size_megabytes || 0),
|
||||
);
|
||||
if (variants[0]) candidates.push(variants[0]);
|
||||
}
|
||||
if (candidates.length > 0) {
|
||||
candidates.sort(
|
||||
(a, b) =>
|
||||
(b.storage_size_megabytes || 0) - (a.storage_size_megabytes || 0),
|
||||
);
|
||||
return candidates[0];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
interface CategoryRecommendation {
|
||||
category: "coding" | "writing" | "agentic" | "biggest";
|
||||
label: string;
|
||||
model: ChatModelInfo | null;
|
||||
tooltip: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
models: ChatModelInfo[];
|
||||
clusterLabel: string;
|
||||
totalMemoryGB: number;
|
||||
onSelect: (modelId: string, category: string) => void;
|
||||
onAddModel: () => void;
|
||||
class?: string;
|
||||
}
|
||||
|
||||
let {
|
||||
models,
|
||||
clusterLabel,
|
||||
totalMemoryGB,
|
||||
onSelect,
|
||||
onAddModel,
|
||||
class: className = "",
|
||||
}: Props = $props();
|
||||
|
||||
// --- Hardcoded Rankings ---
|
||||
const CODING_RANKING = [
|
||||
"Qwen3 Coder Next",
|
||||
"Qwen3 Coder 480B",
|
||||
"Qwen3 30B",
|
||||
"GPT-OSS 20B",
|
||||
"Llama 3.1 8B",
|
||||
"Llama 3.2 3B",
|
||||
"Qwen3 0.6B",
|
||||
];
|
||||
|
||||
const WRITING_RANKING = [
|
||||
"Kimi K2.5",
|
||||
"Kimi K2",
|
||||
"Qwen3 Next 80B",
|
||||
"Llama 3.3 70B",
|
||||
"MiniMax M2.5",
|
||||
"GLM 4.5 Air",
|
||||
"GLM 4.7 Flash",
|
||||
"GPT-OSS 20B",
|
||||
"Llama 3.1 8B",
|
||||
"Llama 3.2 3B",
|
||||
"Qwen3 0.6B",
|
||||
];
|
||||
|
||||
const AGENTIC_RANKING = [
|
||||
"DeepSeek V3.1",
|
||||
"GLM-5",
|
||||
"Qwen3 235B",
|
||||
"Step 3.5 Flash",
|
||||
"GLM 4.7",
|
||||
"MiniMax M2.1",
|
||||
"GPT-OSS 120B",
|
||||
"Llama 3.3 70B",
|
||||
"Llama 3.1 70B",
|
||||
"GLM 4.7 Flash",
|
||||
"GPT-OSS 20B",
|
||||
"Qwen3 30B",
|
||||
"Llama 3.1 8B",
|
||||
"Llama 3.2 3B",
|
||||
"Qwen3 0.6B",
|
||||
];
|
||||
|
||||
function getModelSizeGB(m: ChatModelInfo): number {
|
||||
return (m.storage_size_megabytes || 0) / 1024;
|
||||
}
|
||||
|
||||
function fitsInMemory(m: ChatModelInfo): boolean {
|
||||
return getModelSizeGB(m) <= totalMemoryGB && getModelSizeGB(m) > 0;
|
||||
}
|
||||
|
||||
/** For a given base_model name, find the biggest quant variant that fits in memory. */
|
||||
function pickBestVariant(baseModel: string): ChatModelInfo | null {
|
||||
const variants = models
|
||||
.filter((m) => m.base_model === baseModel && fitsInMemory(m))
|
||||
.sort((a, b) => getModelSizeGB(b) - getModelSizeGB(a));
|
||||
return variants[0] ?? null;
|
||||
}
|
||||
|
||||
/** Walk a ranked list of base_model names, return the first that has a fitting variant. */
|
||||
function pickFromRanking(ranking: string[]): ChatModelInfo | null {
|
||||
for (const baseModel of ranking) {
|
||||
const pick = pickBestVariant(baseModel);
|
||||
if (pick) return pick;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Pick the single biggest model that fits. */
|
||||
function pickBiggest(): ChatModelInfo | null {
|
||||
const fitting = models
|
||||
.filter((m) => fitsInMemory(m))
|
||||
.sort((a, b) => getModelSizeGB(b) - getModelSizeGB(a));
|
||||
return fitting[0] ?? null;
|
||||
}
|
||||
|
||||
const recommendations = $derived.by((): CategoryRecommendation[] => {
|
||||
return [
|
||||
{
|
||||
category: "coding",
|
||||
label: "Best for Coding",
|
||||
model: pickFromRanking(CODING_RANKING),
|
||||
tooltip:
|
||||
"Ranked by coding benchmark performance (LiveCodeBench, SWE-bench)",
|
||||
},
|
||||
{
|
||||
category: "writing",
|
||||
label: "Best for Writing",
|
||||
model: pickFromRanking(WRITING_RANKING),
|
||||
tooltip: "Ranked by creative writing quality and instruction following",
|
||||
},
|
||||
{
|
||||
category: "agentic",
|
||||
label: "Best Agentic",
|
||||
model: pickFromRanking(AGENTIC_RANKING),
|
||||
tooltip: "Ranked by reasoning, planning, and tool-use capability",
|
||||
},
|
||||
{
|
||||
category: "biggest",
|
||||
label: "Biggest",
|
||||
model: pickBiggest(),
|
||||
tooltip: "Largest model that fits in your available memory",
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
function formatSize(mb: number): string {
|
||||
const gb = mb / 1024;
|
||||
if (gb >= 100) return `${Math.round(gb)} GB`;
|
||||
return `${gb.toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
// Category icons (SVG paths)
|
||||
const categoryIcons: Record<string, string> = {
|
||||
coding:
|
||||
"M8 9l3 3-3 3m5 0h3M5 20h14a2 2 0 002-2V6a2 2 0 00-2-2H5a2 2 0 00-2 2v12a2 2 0 002 2z",
|
||||
writing:
|
||||
"M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z",
|
||||
agentic:
|
||||
"M9.663 17h4.673M12 3v1m6.364 1.636l-.707.707M21 12h-1M4 12H3m3.343-5.657l-.707-.707m2.828 9.9a5 5 0 117.072 0l-.548.547A3.374 3.374 0 0014 18.469V19a2 2 0 11-4 0v-.531c0-.895-.356-1.754-.988-2.386l-.548-.547z",
|
||||
biggest:
|
||||
"M19 11H5m14 0a2 2 0 012 2v6a2 2 0 01-2 2H5a2 2 0 01-2-2v-6a2 2 0 012-2m14 0V9a2 2 0 00-2-2M5 11V9a2 2 0 012-2m0 0V5a2 2 0 012-2h6a2 2 0 012 2v2M7 7h10",
|
||||
};
|
||||
|
||||
let hoveredTooltip = $state<string | null>(null);
|
||||
let tooltipAnchor = $state<{ x: number; y: number } | null>(null);
|
||||
|
||||
function showTooltip(category: string, e: MouseEvent | FocusEvent) {
|
||||
hoveredTooltip = category;
|
||||
const target = e.currentTarget as HTMLElement;
|
||||
const rect = target.getBoundingClientRect();
|
||||
tooltipAnchor = { x: rect.left + rect.width / 2, y: rect.top };
|
||||
}
|
||||
|
||||
function hideTooltip() {
|
||||
hoveredTooltip = null;
|
||||
tooltipAnchor = null;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col items-center justify-center gap-6 {className}">
|
||||
<!-- Header -->
|
||||
<div class="text-center">
|
||||
<p class="text-xs text-exo-light-gray uppercase tracking-[0.2em] mb-1">
|
||||
Recommended for your
|
||||
</p>
|
||||
<p class="text-sm text-white font-mono tracking-wide">{clusterLabel}</p>
|
||||
</div>
|
||||
|
||||
<!-- Category Cards Grid -->
|
||||
<div class="grid grid-cols-2 gap-3 w-full max-w-md">
|
||||
{#each recommendations as rec}
|
||||
{#if rec.model}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => rec.model && onSelect(rec.model.id, rec.category)}
|
||||
class="group relative flex flex-col items-start gap-2 p-4 rounded-lg border border-exo-medium-gray/50 bg-exo-dark-gray/50 hover:border-exo-yellow/40 hover:bg-exo-dark-gray transition-all duration-200 cursor-pointer text-left"
|
||||
>
|
||||
<!-- Category icon + label -->
|
||||
<div class="flex items-center gap-2 w-full">
|
||||
<svg
|
||||
class="w-4 h-4 text-exo-yellow/70 group-hover:text-exo-yellow transition-colors flex-shrink-0"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="1.5"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d={categoryIcons[rec.category]}
|
||||
/>
|
||||
</svg>
|
||||
<span
|
||||
class="text-xs font-mono uppercase tracking-wider text-exo-light-gray group-hover:text-white transition-colors"
|
||||
>
|
||||
{rec.label}
|
||||
</span>
|
||||
<!-- Info tooltip -->
|
||||
<div class="ml-auto flex-shrink-0">
|
||||
<span
|
||||
role="button"
|
||||
tabindex="-1"
|
||||
class="text-exo-light-gray/40 hover:text-exo-light-gray transition-colors cursor-help inline-flex"
|
||||
onmouseenter={(e: MouseEvent) => showTooltip(rec.category, e)}
|
||||
onmouseleave={() => hideTooltip()}
|
||||
onclick={(e: MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
if (hoveredTooltip === rec.category) {
|
||||
hideTooltip();
|
||||
} else {
|
||||
showTooltip(rec.category, e);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10" />
|
||||
<path d="M12 16v-4m0-4h.01" />
|
||||
</svg>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Model name + size -->
|
||||
<div class="w-full">
|
||||
<p class="text-sm text-white font-mono truncate">
|
||||
{rec.model.base_model}
|
||||
</p>
|
||||
<p class="text-xs text-exo-light-gray/60 font-mono mt-0.5">
|
||||
{formatSize(rec.model.storage_size_megabytes)}
|
||||
{#if rec.model.quantization}
|
||||
<span class="text-exo-light-gray/40"
|
||||
>· {rec.model.quantization}</span
|
||||
>
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
</button>
|
||||
{:else}
|
||||
<!-- No model fits for this category -->
|
||||
<div
|
||||
class="flex flex-col items-start gap-2 p-4 rounded-lg border border-exo-medium-gray/30 bg-exo-dark-gray/30 opacity-50"
|
||||
>
|
||||
<div class="flex items-center gap-2">
|
||||
<svg
|
||||
class="w-4 h-4 text-exo-light-gray/40 flex-shrink-0"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="1.5"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d={categoryIcons[rec.category]}
|
||||
/>
|
||||
</svg>
|
||||
<span
|
||||
class="text-xs font-mono uppercase tracking-wider text-exo-light-gray/50"
|
||||
>{rec.label}</span
|
||||
>
|
||||
</div>
|
||||
<p class="text-xs text-exo-light-gray/40 font-mono">No model fits</p>
|
||||
</div>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
<!-- Add Model Button -->
|
||||
<button
|
||||
type="button"
|
||||
onclick={onAddModel}
|
||||
class="flex items-center gap-2 px-4 py-2 text-xs font-mono uppercase tracking-wider text-exo-light-gray hover:text-exo-yellow border border-exo-medium-gray/30 hover:border-exo-yellow/30 rounded-lg transition-all duration-200 cursor-pointer"
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M12 4v16m8-8H4" />
|
||||
</svg>
|
||||
Add Model
|
||||
</button>
|
||||
|
||||
<!-- Auto hint -->
|
||||
<p class="text-xs text-exo-light-gray/40 font-mono tracking-wide text-center">
|
||||
Or just start typing — we'll pick the best model automatically
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Fixed-position tooltip (escapes overflow-hidden ancestors) -->
|
||||
{#if hoveredTooltip && tooltipAnchor}
|
||||
{@const rec = recommendations.find((r) => r.category === hoveredTooltip)}
|
||||
{#if rec}
|
||||
<div
|
||||
class="fixed z-[9999] px-3 py-2 bg-exo-black border border-exo-medium-gray/50 rounded text-xs text-exo-light-gray whitespace-nowrap shadow-lg pointer-events-none"
|
||||
style="left: {tooltipAnchor.x}px; top: {tooltipAnchor.y -
|
||||
8}px; transform: translate(-50%, -100%);"
|
||||
>
|
||||
{rec.tooltip}
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
@@ -2,6 +2,7 @@
|
||||
import {
|
||||
conversations,
|
||||
activeConversationId,
|
||||
createConversation,
|
||||
loadConversation,
|
||||
deleteConversation,
|
||||
deleteAllConversations,
|
||||
@@ -16,21 +17,9 @@
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
onNewChat?: () => void;
|
||||
onSelectConversation?: () => void;
|
||||
isMobileDrawer?: boolean;
|
||||
isOpen?: boolean;
|
||||
onClose?: () => void;
|
||||
}
|
||||
|
||||
let {
|
||||
class: className = "",
|
||||
onNewChat,
|
||||
onSelectConversation,
|
||||
isMobileDrawer = false,
|
||||
isOpen = false,
|
||||
onClose,
|
||||
}: Props = $props();
|
||||
let { class: className = "" }: Props = $props();
|
||||
|
||||
const conversationList = $derived(conversations());
|
||||
const activeId = $derived(activeConversationId());
|
||||
@@ -53,16 +42,11 @@
|
||||
);
|
||||
|
||||
function handleNewChat() {
|
||||
onNewChat?.();
|
||||
createConversation();
|
||||
}
|
||||
|
||||
function handleSelectConversation(id: string) {
|
||||
onSelectConversation?.();
|
||||
loadConversation(id);
|
||||
// Close mobile drawer when selecting a conversation
|
||||
if (isMobileDrawer && isOpen) {
|
||||
onClose?.();
|
||||
}
|
||||
}
|
||||
|
||||
function handleStartEdit(id: string, name: string, event: MouseEvent) {
|
||||
@@ -201,8 +185,11 @@
|
||||
|
||||
let instanceType: string | null = null;
|
||||
if (instanceTag === "MlxRingInstance") instanceType = "MLX Ring";
|
||||
else if (instanceTag === "MlxJacclInstance") instanceType = "MLX RDMA";
|
||||
else if (instanceTag === "VllmInstance") instanceType = "vLLM";
|
||||
else if (
|
||||
instanceTag === "MlxIbvInstance" ||
|
||||
instanceTag === "MlxJacclInstance"
|
||||
)
|
||||
instanceType = "MLX RDMA";
|
||||
|
||||
let sharding: string | null = null;
|
||||
const inst = instance as {
|
||||
@@ -263,7 +250,9 @@
|
||||
}
|
||||
</script>
|
||||
|
||||
{#snippet sidebarContent()}
|
||||
<aside
|
||||
class="flex flex-col h-full bg-exo-dark-gray border-r border-exo-yellow/10 {className}"
|
||||
>
|
||||
<!-- Header -->
|
||||
<div class="p-4">
|
||||
<button
|
||||
@@ -318,7 +307,7 @@
|
||||
<div class="py-2">
|
||||
<div class="px-4 py-2">
|
||||
<span
|
||||
class="text-xs text-exo-light-gray font-mono tracking-wider uppercase"
|
||||
class="text-sm text-white/70 font-mono tracking-wider uppercase"
|
||||
>
|
||||
{searchQuery ? "SEARCH RESULTS" : "CONVERSATIONS"}
|
||||
</span>
|
||||
@@ -387,37 +376,39 @@
|
||||
onkeydown={(e) =>
|
||||
e.key === "Enter" &&
|
||||
handleSelectConversation(conversation.id)}
|
||||
class="group w-full flex items-center justify-between p-2.5 rounded-lg mb-1 transition-all text-left cursor-pointer
|
||||
class="group w-full flex items-center justify-between p-2 rounded mb-1 transition-all text-left cursor-pointer
|
||||
{activeId === conversation.id
|
||||
? 'bg-exo-yellow/5 border border-exo-yellow/30'
|
||||
: 'hover:bg-white/[0.03] hover:border-white/10 border border-transparent'}"
|
||||
? 'bg-transparent border border-exo-yellow/30'
|
||||
: 'hover:border-exo-yellow/20 border border-transparent'}"
|
||||
>
|
||||
<div class="flex-1 min-w-0 pr-2">
|
||||
<div
|
||||
class="text-sm font-medium truncate {activeId ===
|
||||
conversation.id
|
||||
class="text-sm truncate {activeId === conversation.id
|
||||
? 'text-exo-yellow'
|
||||
: 'text-white'}"
|
||||
: 'text-white/90'}"
|
||||
>
|
||||
{conversation.name}
|
||||
</div>
|
||||
<div class="text-xs text-white/60 mt-0.5">
|
||||
<div class="text-sm text-white/50 mt-0.5">
|
||||
{formatDate(conversation.updatedAt)}
|
||||
</div>
|
||||
<div class="text-xs text-exo-light-gray truncate">
|
||||
<div class="text-sm text-white/70 truncate">
|
||||
{info.modelLabel}
|
||||
</div>
|
||||
<div class="text-xs text-white/60 font-mono">
|
||||
Strategy: <span class="text-white/80"
|
||||
>{info.strategyLabel}</span
|
||||
>
|
||||
</div>
|
||||
{#if stats}
|
||||
<div class="text-xs text-white/70 font-mono mt-1">
|
||||
{#if stats.ttftMs}<span class="text-white/50">TTFT</span>
|
||||
<span class="text-exo-yellow/80"
|
||||
>{stats.ttftMs.toFixed(0)}ms</span
|
||||
>{/if}{#if stats.ttftMs && stats.tps}<span
|
||||
class="text-white/30 mx-1.5">·</span
|
||||
>{/if}{#if stats.tps}<span class="text-exo-yellow/80"
|
||||
>{stats.tps.toFixed(1)}</span
|
||||
>
|
||||
<span class="text-white/50">tok/s</span>{/if}
|
||||
<div class="text-xs text-white/60 font-mono mt-1">
|
||||
{#if stats.ttftMs}<span class="text-white/40">TTFT</span>
|
||||
{stats.ttftMs.toFixed(
|
||||
0,
|
||||
)}ms{/if}{#if stats.ttftMs && stats.tps}<span
|
||||
class="text-white/30 mx-1.5">•</span
|
||||
>{/if}{#if stats.tps}{stats.tps.toFixed(1)}
|
||||
<span class="text-white/40">tok/s</span>{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -600,30 +591,4 @@
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
{#if isMobileDrawer}
|
||||
<!-- Mobile drawer with overlay -->
|
||||
{#if isOpen}
|
||||
<!-- Overlay backdrop -->
|
||||
<button
|
||||
type="button"
|
||||
class="fixed inset-0 bg-black/60 backdrop-blur-sm z-40 md:hidden"
|
||||
onclick={() => onClose?.()}
|
||||
aria-label="Close sidebar"
|
||||
></button>
|
||||
<!-- Drawer panel -->
|
||||
<aside
|
||||
class="fixed left-0 top-0 bottom-0 w-72 bg-exo-dark-gray border-r border-exo-yellow/10 z-50 flex flex-col md:hidden"
|
||||
>
|
||||
{@render sidebarContent()}
|
||||
</aside>
|
||||
{/if}
|
||||
{:else}
|
||||
<!-- Desktop sidebar -->
|
||||
<aside
|
||||
class="flex flex-col h-full bg-exo-dark-gray border-r border-exo-yellow/10 {className}"
|
||||
>
|
||||
{@render sidebarContent()}
|
||||
</aside>
|
||||
{/if}
|
||||
</aside>
|
||||
@@ -1,20 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { isConnected } from "$lib/stores/app.svelte";
|
||||
import { slide } from "svelte/transition";
|
||||
|
||||
const connected = $derived(isConnected());
|
||||
</script>
|
||||
|
||||
{#if !connected}
|
||||
<div
|
||||
transition:slide={{ duration: 200 }}
|
||||
class="relative z-50 flex items-center justify-center gap-2 px-4 py-2 bg-red-950/80 border-b border-red-500/30"
|
||||
role="alert"
|
||||
aria-live="assertive"
|
||||
>
|
||||
<div class="w-2 h-2 bg-red-500 rounded-full animate-pulse"></div>
|
||||
<span class="text-xs font-mono text-red-300 tracking-wider uppercase">
|
||||
Connection lost — Reconnecting to backend…
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
@@ -1,567 +0,0 @@
|
||||
<script lang="ts">
|
||||
/**
|
||||
* DeviceIcon — renders a device icon as an SVG <g> element.
|
||||
* Uses the exact same proportional math as TopologyGraph.svelte
|
||||
* so that devices look identical in both the topology view and
|
||||
* the onboarding animation.
|
||||
*
|
||||
* Must be placed inside an <svg> element.
|
||||
*/
|
||||
|
||||
interface Props {
|
||||
/** "macbook pro" | "mac studio" | "mac mini" | "dgx spark" | "linux" etc. */
|
||||
deviceType: string;
|
||||
/** Center X coordinate in SVG space */
|
||||
cx: number;
|
||||
/** Center Y coordinate in SVG space */
|
||||
cy: number;
|
||||
/** Base sizing factor (equivalent to TopologyGraph's nodeRadius) */
|
||||
size?: number;
|
||||
/** RAM usage 0–100 */
|
||||
ramPercent?: number;
|
||||
/** Unique id suffix for clip-path ids */
|
||||
uid?: string;
|
||||
}
|
||||
|
||||
let {
|
||||
deviceType,
|
||||
cx,
|
||||
cy,
|
||||
size = 60,
|
||||
ramPercent = 60,
|
||||
uid = "dev",
|
||||
}: Props = $props();
|
||||
|
||||
// Apple logo path — same constant used by TopologyGraph
|
||||
const APPLE_LOGO_PATH =
|
||||
"M788.1 340.9c-5.8 4.5-108.2 62.2-108.2 190.5 0 148.4 130.3 200.9 134.2 202.2-.6 3.2-20.7 71.9-68.7 141.9-42.8 61.6-87.5 123.1-155.5 123.1s-85.5-39.5-164-39.5c-76.5 0-103.7 40.8-165.9 40.8s-105.6-57-155.5-127C46.7 790.7 0 663 0 541.8c0-194.4 126.4-297.5 250.8-297.5 66.1 0 121.2 43.4 162.7 43.4 39.5 0 101.1-46 176.3-46 28.5 0 130.9 2.6 198.3 99.2zm-234-181.5c31.1-36.9 53.1-88.1 53.1-139.3 0-7.1-.6-14.3-1.9-20.1-50.6 1.9-110.8 33.7-147.1 75.8-28.5 32.4-55.1 83.6-55.1 135.5 0 7.8 1.3 15.6 1.9 18.1 3.2.6 8.4 1.3 13.6 1.3 45.4 0 102.5-30.4 135.5-71.3z";
|
||||
const LOGO_NATIVE_WIDTH = 814;
|
||||
const LOGO_NATIVE_HEIGHT = 1000;
|
||||
|
||||
// NVIDIA logo SVG path
|
||||
const NVIDIA_LOGO_PATH =
|
||||
"M0.81 0.429V0.299c0.013 -0.001 0.026 -0.002 0.038 -0.002 0.355 -0.011 0.588 0.306 0.588 0.306S1.186 0.952 0.916 0.952c-0.036 0 -0.071 -0.006 -0.105 -0.017V0.542c0.138 0.017 0.166 0.078 0.249 0.216l0.185 -0.155s-0.135 -0.177 -0.362 -0.177c-0.024 -0.001 -0.048 0.001 -0.072 0.003m0 -0.429v0.194l0.038 -0.002c0.494 -0.017 0.816 0.405 0.816 0.405s-0.37 0.45 -0.754 0.45c-0.034 0 -0.066 -0.003 -0.099 -0.009v0.12c0.027 0.003 0.055 0.006 0.082 0.006 0.358 0 0.618 -0.183 0.869 -0.399 0.042 0.034 0.212 0.114 0.247 0.15 -0.238 0.2 -0.794 0.361 -1.11 0.361 -0.03 0 -0.059 -0.002 -0.088 -0.005v0.169h1.362V0zm0 0.935v0.102c-0.331 -0.059 -0.423 -0.404 -0.423 -0.404s0.159 -0.176 0.423 -0.205v0.112h-0.001C0.671 0.524 0.562 0.654 0.562 0.654s0.062 0.218 0.248 0.282m-0.588 -0.316s0.196 -0.29 0.589 -0.32V0.194C0.376 0.229 0 0.597 0 0.597s0.213 0.616 0.81 0.672v-0.112c-0.438 -0.054 -0.588 -0.538 -0.588 -0.538";
|
||||
|
||||
const wireColor = "rgba(179,179,179,0.8)";
|
||||
const strokeWidth = 1.5;
|
||||
|
||||
const modelLower = $derived(deviceType.toLowerCase());
|
||||
const isSpark = $derived(
|
||||
modelLower.includes("dgx") || modelLower.includes("gx10"),
|
||||
);
|
||||
const isLinux = $derived(!isSpark && modelLower.startsWith("linux"));
|
||||
const isLinuxLaptop = $derived(isLinux && modelLower.includes("laptop"));
|
||||
|
||||
// ── DGX Spark dimensions ──
|
||||
const dgxW = $derived(size * 1.55);
|
||||
const dgxH = $derived(size * 0.58);
|
||||
const dgxX = $derived(cx - dgxW / 2);
|
||||
const dgxY = $derived(cy - dgxH / 2);
|
||||
const dgxChassisX = $derived(dgxX - dgxW * 0.03);
|
||||
const dgxChassisW = $derived(dgxW * 1.05);
|
||||
const dgxHandleW = $derived(dgxW * 0.27);
|
||||
const dgxHandleGap = $derived(dgxH * 0.05);
|
||||
const dgxHandleH = $derived(dgxH - dgxHandleGap * 2);
|
||||
const dgxHandleY = $derived(dgxY + dgxHandleGap);
|
||||
const dgxInnerHandleW = $derived(dgxW * 0.12);
|
||||
const dgxInnerHandleH = $derived(dgxHandleH - dgxH * 0.06);
|
||||
const dgxLeftHandleX = $derived(dgxX + 4);
|
||||
const dgxRightHandleX = $derived(dgxX + dgxW - dgxHandleW - 4);
|
||||
const dgxClipId = $derived(`di-dgx-${uid}`);
|
||||
const dgxTextureId = $derived(`di-dgx-tex-${uid}`);
|
||||
|
||||
// ── Linux Desktop dimensions (reuses Mac Studio proportions) ──
|
||||
const linuxDesktopClipId = $derived(`di-linux-desktop-${uid}`);
|
||||
|
||||
// ── Linux Laptop dimensions (reuses MacBook proportions) ──
|
||||
const linuxScreenClipId = $derived(`di-linux-screen-${uid}`);
|
||||
|
||||
// ── Mac Studio dimensions (same ratios as TopologyGraph) ──
|
||||
const studioW = $derived(size * 1.25);
|
||||
const studioH = $derived(size * 0.85);
|
||||
const studioX = $derived(cx - studioW / 2);
|
||||
const studioY = $derived(cy - studioH / 2);
|
||||
const studioCorner = 4;
|
||||
const studioTopH = $derived(studioH * 0.15);
|
||||
|
||||
// Studio front panel details
|
||||
const studioSlotH = $derived(studioH * 0.14);
|
||||
const studioVSlotW = $derived(studioW * 0.05);
|
||||
const studioVSlotY = $derived(
|
||||
studioY + studioTopH + (studioH - studioTopH) * 0.6,
|
||||
);
|
||||
const studioVSlot1X = $derived(studioX + studioW * 0.18);
|
||||
const studioVSlot2X = $derived(studioX + studioW * 0.28);
|
||||
const studioHSlotW = $derived(studioW * 0.2);
|
||||
const studioHSlotX = $derived(studioX + studioW * 0.5 - studioHSlotW / 2);
|
||||
|
||||
// Studio memory fill
|
||||
const studioMemTotalH = $derived(studioH - studioTopH);
|
||||
const studioMemH = $derived((ramPercent / 100) * studioMemTotalH);
|
||||
|
||||
// ── MacBook dimensions (same ratios as TopologyGraph) ──
|
||||
const mbW = $derived((size * 1.6 * 0.85) / 1.15);
|
||||
const mbH = $derived(size * 0.85);
|
||||
const mbX = $derived(cx - mbW / 2);
|
||||
const mbY = $derived(cy - mbH / 2);
|
||||
|
||||
const mbScreenH = $derived(mbH * 0.7);
|
||||
const mbBaseH = $derived(mbH * 0.3);
|
||||
const mbScreenW = $derived(mbW * 0.85);
|
||||
const mbScreenX = $derived(cx - mbScreenW / 2);
|
||||
const mbBezel = 3;
|
||||
|
||||
// MacBook memory fill
|
||||
const mbMemTotalH = $derived(mbScreenH - mbBezel * 2);
|
||||
const mbMemH = $derived((ramPercent / 100) * mbMemTotalH);
|
||||
|
||||
// Apple logo sizing
|
||||
const mbLogoTargetH = $derived(mbScreenH * 0.22);
|
||||
const mbLogoScale = $derived(mbLogoTargetH / LOGO_NATIVE_HEIGHT);
|
||||
const mbLogoX = $derived(cx - (LOGO_NATIVE_WIDTH * mbLogoScale) / 2);
|
||||
const mbLogoY = $derived(
|
||||
mbY + mbScreenH / 2 - (LOGO_NATIVE_HEIGHT * mbLogoScale) / 2,
|
||||
);
|
||||
|
||||
// MacBook base (trapezoidal)
|
||||
const mbBaseY = $derived(mbY + mbScreenH);
|
||||
const mbBaseTopW = $derived(mbScreenW);
|
||||
const mbBaseBottomW = $derived(mbW);
|
||||
const mbBaseTopX = $derived(cx - mbBaseTopW / 2);
|
||||
const mbBaseBottomX = $derived(cx - mbBaseBottomW / 2);
|
||||
|
||||
// Keyboard
|
||||
const mbKbX = $derived(mbBaseTopX + 6);
|
||||
const mbKbY = $derived(mbBaseY + 3);
|
||||
const mbKbW = $derived(mbBaseTopW - 12);
|
||||
const mbKbH = $derived(mbBaseH * 0.55);
|
||||
|
||||
// Trackpad
|
||||
const mbTpW = $derived(mbBaseTopW * 0.4);
|
||||
const mbTpX = $derived(cx - mbTpW / 2);
|
||||
const mbTpY = $derived(mbBaseY + mbKbH + 5);
|
||||
const mbTpH = $derived(mbBaseH * 0.3);
|
||||
|
||||
// Clip IDs
|
||||
const screenClipId = $derived(`di-screen-${uid}`);
|
||||
const studioClipId = $derived(`di-studio-${uid}`);
|
||||
</script>
|
||||
|
||||
{#if isSpark}
|
||||
<!-- DGX Spark -->
|
||||
<defs>
|
||||
<clipPath id={dgxClipId}>
|
||||
<rect x={dgxX} y={dgxY} width={dgxW} height={dgxH} rx="3" />
|
||||
</clipPath>
|
||||
<pattern
|
||||
id={dgxTextureId}
|
||||
patternUnits="userSpaceOnUse"
|
||||
width="8"
|
||||
height="8"
|
||||
>
|
||||
<rect width="8" height="8" fill="#6f6248" />
|
||||
<circle cx="2" cy="2" r="1" fill="#5a4f3b" opacity="0.5" />
|
||||
<circle cx="6" cy="6" r="1" fill="#4a4232" opacity="0.45" />
|
||||
</pattern>
|
||||
</defs>
|
||||
|
||||
<!-- Main body -->
|
||||
<rect
|
||||
x={dgxChassisX}
|
||||
y={dgxY}
|
||||
width={dgxChassisW}
|
||||
height={dgxH}
|
||||
rx="3"
|
||||
fill="url(#{dgxTextureId})"
|
||||
stroke={wireColor}
|
||||
stroke-width={strokeWidth}
|
||||
/>
|
||||
|
||||
<!-- Side border accents -->
|
||||
<rect
|
||||
x={dgxChassisX}
|
||||
y={dgxY}
|
||||
width={dgxW * 0.02}
|
||||
height={dgxH}
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
<rect
|
||||
x={dgxChassisX + dgxChassisW - dgxW * 0.02}
|
||||
y={dgxY}
|
||||
width={dgxW * 0.02}
|
||||
height={dgxH}
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
|
||||
<!-- Memory fill -->
|
||||
{#if ramPercent > 0}
|
||||
<rect
|
||||
x={dgxX}
|
||||
y={dgxY + dgxH - (ramPercent / 100) * dgxH}
|
||||
width={dgxW}
|
||||
height={(ramPercent / 100) * dgxH}
|
||||
fill="rgba(255,215,0,0.45)"
|
||||
clip-path="url(#{dgxClipId})"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Left handle -->
|
||||
<rect
|
||||
x={dgxLeftHandleX}
|
||||
y={dgxHandleY}
|
||||
width={dgxHandleW}
|
||||
height={dgxHandleH}
|
||||
rx="2.4"
|
||||
fill="#b3a170"
|
||||
stroke="#403723"
|
||||
stroke-width="0.7"
|
||||
/>
|
||||
<rect
|
||||
x={dgxLeftHandleX + dgxHandleW * 0.06}
|
||||
y={dgxHandleY + dgxH * 0.03}
|
||||
width={dgxInnerHandleW}
|
||||
height={dgxInnerHandleH}
|
||||
rx="1.6"
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
|
||||
<!-- Right handle -->
|
||||
<rect
|
||||
x={dgxRightHandleX}
|
||||
y={dgxHandleY}
|
||||
width={dgxHandleW}
|
||||
height={dgxHandleH}
|
||||
rx="2.4"
|
||||
fill="#b3a170"
|
||||
stroke="#403723"
|
||||
stroke-width="0.7"
|
||||
/>
|
||||
<rect
|
||||
x={dgxRightHandleX + dgxHandleW - dgxInnerHandleW - dgxHandleW * 0.08}
|
||||
y={dgxHandleY + dgxH * 0.03}
|
||||
width={dgxInnerHandleW}
|
||||
height={dgxInnerHandleH}
|
||||
rx="1.6"
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
|
||||
<!-- NVIDIA logo (rotated 90deg on left handle) -->
|
||||
{@const badgeW = dgxW * 0.09}
|
||||
{@const badgeH = dgxHandleH * 0.5}
|
||||
{@const badgeX = dgxLeftHandleX + dgxHandleW - badgeW - dgxHandleW * 0.06}
|
||||
{@const badgeYPos = dgxHandleY + (dgxHandleH - badgeH) / 2}
|
||||
{@const textSz = badgeW * 0.58}
|
||||
{@const logoW = textSz * 1.2}
|
||||
{@const logoH = logoW * (1.438 / 2.174)}
|
||||
{@const ctrX = badgeX + badgeW / 2 - badgeW * 0.03}
|
||||
{@const ctrY = badgeYPos + badgeH / 2}
|
||||
{@const labelGap = badgeW * 0.15}
|
||||
{@const totalW = logoW + labelGap + textSz * 3.6}
|
||||
<g transform="rotate(90 {ctrX} {ctrY})">
|
||||
<svg
|
||||
x={ctrX - totalW / 2}
|
||||
y={ctrY - logoH / 2}
|
||||
width={logoW}
|
||||
height={logoH}
|
||||
viewBox="0 0 2.174 1.438"
|
||||
>
|
||||
<path d={NVIDIA_LOGO_PATH} fill="#76b900" />
|
||||
</svg>
|
||||
<text
|
||||
x={ctrX - totalW / 2 + logoW + labelGap}
|
||||
y={ctrY}
|
||||
text-anchor="start"
|
||||
dominant-baseline="middle"
|
||||
fill="#8a7a56"
|
||||
font-size={textSz}
|
||||
font-family="monospace"
|
||||
font-weight="700">NVIDIA</text
|
||||
>
|
||||
</g>
|
||||
{:else if isLinuxLaptop}
|
||||
<!-- Linux Laptop — MacBook shape with Tux logo -->
|
||||
<defs>
|
||||
<clipPath id={linuxScreenClipId}>
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbScreenH - mbBezel * 2}
|
||||
rx="2"
|
||||
/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
|
||||
<rect
|
||||
x={mbScreenX}
|
||||
y={mbY}
|
||||
width={mbScreenW}
|
||||
height={mbScreenH}
|
||||
rx="3"
|
||||
fill="#1a1a1a"
|
||||
stroke={wireColor}
|
||||
stroke-width={strokeWidth}
|
||||
/>
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbScreenH - mbBezel * 2}
|
||||
rx="2"
|
||||
fill="#0a0a12"
|
||||
/>
|
||||
{#if ramPercent > 0}
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel + (mbMemTotalH - mbMemH)}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbMemH}
|
||||
fill="rgba(255,215,0,0.85)"
|
||||
clip-path="url(#{linuxScreenClipId})"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Terminal prompt on screen -->
|
||||
<text
|
||||
x={cx}
|
||||
y={mbY + mbScreenH / 2}
|
||||
text-anchor="middle"
|
||||
dominant-baseline="middle"
|
||||
fill="#FFFFFF"
|
||||
opacity="0.9"
|
||||
font-size={mbScreenH * 0.25}
|
||||
font-family="SF Mono, Monaco, monospace"
|
||||
font-weight="700">{">_"}</text
|
||||
>
|
||||
|
||||
<path
|
||||
d="M {mbBaseTopX} {mbBaseY} L {mbBaseTopX +
|
||||
mbBaseTopW} {mbBaseY} L {mbBaseBottomX + mbBaseBottomW} {mbBaseY +
|
||||
mbBaseH} L {mbBaseBottomX} {mbBaseY + mbBaseH} Z"
|
||||
fill="#2c2c2c"
|
||||
stroke={wireColor}
|
||||
stroke-width="1"
|
||||
/>
|
||||
<rect
|
||||
x={mbKbX}
|
||||
y={mbKbY}
|
||||
width={mbKbW}
|
||||
height={mbKbH}
|
||||
fill="rgba(0,0,0,0.2)"
|
||||
rx="2"
|
||||
/>
|
||||
<rect
|
||||
x={mbTpX}
|
||||
y={mbTpY}
|
||||
width={mbTpW}
|
||||
height={mbTpH}
|
||||
fill="rgba(255,255,255,0.08)"
|
||||
rx="2"
|
||||
/>
|
||||
{:else if isLinux}
|
||||
<!-- Linux Desktop — Mac Studio shape with Tux logo -->
|
||||
<defs>
|
||||
<clipPath id={linuxDesktopClipId}>
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY + studioTopH}
|
||||
width={studioW}
|
||||
height={studioH - studioTopH}
|
||||
rx={studioCorner - 1}
|
||||
/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY}
|
||||
width={studioW}
|
||||
height={studioH}
|
||||
rx={studioCorner}
|
||||
fill="#1a1a1a"
|
||||
stroke={wireColor}
|
||||
stroke-width={strokeWidth}
|
||||
/>
|
||||
{#if ramPercent > 0}
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY + studioTopH + (studioMemTotalH - studioMemH)}
|
||||
width={studioW}
|
||||
height={studioMemH}
|
||||
fill="rgba(255,215,0,0.75)"
|
||||
clip-path="url(#{linuxDesktopClipId})"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Terminal prompt on front face -->
|
||||
<text
|
||||
x={cx}
|
||||
y={studioY + studioTopH + (studioH - studioTopH) / 2}
|
||||
text-anchor="middle"
|
||||
dominant-baseline="middle"
|
||||
fill="rgba(255,255,255,0.5)"
|
||||
font-size={(studioH - studioTopH) * 0.4}
|
||||
font-family="SF Mono, Monaco, monospace"
|
||||
font-weight="700">{">_"}</text
|
||||
>
|
||||
{:else if modelLower === "mac studio" || modelLower === "mac mini"}
|
||||
<!-- Mac Studio / Mac Mini -->
|
||||
<defs>
|
||||
<clipPath id={studioClipId}>
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY + studioTopH}
|
||||
width={studioW}
|
||||
height={studioH - studioTopH}
|
||||
rx={studioCorner - 1}
|
||||
/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
|
||||
<!-- Main body -->
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY}
|
||||
width={studioW}
|
||||
height={studioH}
|
||||
rx={studioCorner}
|
||||
fill="#1a1a1a"
|
||||
stroke={wireColor}
|
||||
stroke-width={strokeWidth}
|
||||
/>
|
||||
|
||||
<!-- Memory fill -->
|
||||
{#if ramPercent > 0}
|
||||
<rect
|
||||
x={studioX}
|
||||
y={studioY + studioTopH + (studioMemTotalH - studioMemH)}
|
||||
width={studioW}
|
||||
height={studioMemH}
|
||||
fill="rgba(255,215,0,0.75)"
|
||||
clip-path="url(#{studioClipId})"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Top surface divider -->
|
||||
<line
|
||||
x1={studioX}
|
||||
y1={studioY + studioTopH}
|
||||
x2={studioX + studioW}
|
||||
y2={studioY + studioTopH}
|
||||
stroke="rgba(179,179,179,0.3)"
|
||||
stroke-width="0.5"
|
||||
/>
|
||||
|
||||
<!-- Front panel: vertical slots -->
|
||||
<rect
|
||||
x={studioVSlot1X - studioVSlotW / 2}
|
||||
y={studioVSlotY}
|
||||
width={studioVSlotW}
|
||||
height={studioSlotH}
|
||||
fill="rgba(0,0,0,0.35)"
|
||||
rx="1.5"
|
||||
/>
|
||||
<rect
|
||||
x={studioVSlot2X - studioVSlotW / 2}
|
||||
y={studioVSlotY}
|
||||
width={studioVSlotW}
|
||||
height={studioSlotH}
|
||||
fill="rgba(0,0,0,0.35)"
|
||||
rx="1.5"
|
||||
/>
|
||||
|
||||
<!-- Horizontal slot (SD card) -->
|
||||
<rect
|
||||
x={studioHSlotX}
|
||||
y={studioVSlotY}
|
||||
width={studioHSlotW}
|
||||
height={studioSlotH * 0.6}
|
||||
fill="rgba(0,0,0,0.35)"
|
||||
rx="1"
|
||||
/>
|
||||
{:else}
|
||||
<!-- MacBook Pro -->
|
||||
<defs>
|
||||
<clipPath id={screenClipId}>
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbScreenH - mbBezel * 2}
|
||||
rx="2"
|
||||
/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
|
||||
<!-- Screen outer frame -->
|
||||
<rect
|
||||
x={mbScreenX}
|
||||
y={mbY}
|
||||
width={mbScreenW}
|
||||
height={mbScreenH}
|
||||
rx="3"
|
||||
fill="#1a1a1a"
|
||||
stroke={wireColor}
|
||||
stroke-width={strokeWidth}
|
||||
/>
|
||||
|
||||
<!-- Screen inner (dark) -->
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbScreenH - mbBezel * 2}
|
||||
rx="2"
|
||||
fill="#0a0a12"
|
||||
/>
|
||||
|
||||
<!-- Memory fill on screen -->
|
||||
{#if ramPercent > 0}
|
||||
<rect
|
||||
x={mbScreenX + mbBezel}
|
||||
y={mbY + mbBezel + (mbMemTotalH - mbMemH)}
|
||||
width={mbScreenW - mbBezel * 2}
|
||||
height={mbMemH}
|
||||
fill="rgba(255,215,0,0.85)"
|
||||
clip-path="url(#{screenClipId})"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Apple logo -->
|
||||
<path
|
||||
d={APPLE_LOGO_PATH}
|
||||
transform="translate({mbLogoX}, {mbLogoY}) scale({mbLogoScale})"
|
||||
fill="#FFFFFF"
|
||||
opacity="0.9"
|
||||
/>
|
||||
|
||||
<!-- Keyboard base (trapezoidal) -->
|
||||
<path
|
||||
d="M {mbBaseTopX} {mbBaseY} L {mbBaseTopX +
|
||||
mbBaseTopW} {mbBaseY} L {mbBaseBottomX + mbBaseBottomW} {mbBaseY +
|
||||
mbBaseH} L {mbBaseBottomX} {mbBaseY + mbBaseH} Z"
|
||||
fill="#2c2c2c"
|
||||
stroke={wireColor}
|
||||
stroke-width="1"
|
||||
/>
|
||||
|
||||
<!-- Keyboard area -->
|
||||
<rect
|
||||
x={mbKbX}
|
||||
y={mbKbY}
|
||||
width={mbKbW}
|
||||
height={mbKbH}
|
||||
fill="rgba(0,0,0,0.2)"
|
||||
rx="2"
|
||||
/>
|
||||
|
||||
<!-- Trackpad -->
|
||||
<rect
|
||||
x={mbTpX}
|
||||
y={mbTpY}
|
||||
width={mbTpW}
|
||||
height={mbTpH}
|
||||
fill="rgba(255,255,255,0.08)"
|
||||
rx="2"
|
||||
/>
|
||||
{/if}
|
||||
@@ -82,24 +82,6 @@
|
||||
d="M12.025 1.13c-5.77 0-10.449 4.647-10.449 10.378 0 1.112.178 2.181.503 3.185.064-.222.203-.444.416-.577a.96.96 0 0 1 .524-.15c.293 0 .584.124.84.284.278.173.48.408.71.694.226.282.458.611.684.951v-.014c.017-.324.106-.622.264-.874s.403-.487.762-.543c.3-.047.596.06.787.203s.31.313.4.467c.15.257.212.468.233.542.01.026.653 1.552 1.657 2.54.616.605 1.01 1.223 1.082 1.912.055.537-.096 1.059-.38 1.572.637.121 1.294.187 1.967.187.657 0 1.298-.063 1.921-.178-.287-.517-.44-1.041-.384-1.581.07-.69.465-1.307 1.081-1.913 1.004-.987 1.647-2.513 1.657-2.539.021-.074.083-.285.233-.542.09-.154.208-.323.4-.467a1.08 1.08 0 0 1 .787-.203c.359.056.604.29.762.543s.247.55.265.874v.015c.225-.34.457-.67.683-.952.23-.286.432-.52.71-.694.257-.16.547-.284.84-.285a.97.97 0 0 1 .524.151c.228.143.373.388.43.625l.006.04a10.3 10.3 0 0 0 .534-3.273c0-5.731-4.678-10.378-10.449-10.378M8.327 6.583a1.5 1.5 0 0 1 .713.174 1.487 1.487 0 0 1 .617 2.013c-.183.343-.762-.214-1.102-.094-.38.134-.532.914-.917.71a1.487 1.487 0 0 1 .69-2.803m7.486 0a1.487 1.487 0 0 1 .689 2.803c-.385.204-.536-.576-.916-.71-.34-.12-.92.437-1.103.094a1.487 1.487 0 0 1 .617-2.013 1.5 1.5 0 0 1 .713-.174m-10.68 1.55a.96.96 0 1 1 0 1.921.96.96 0 0 1 0-1.92m13.838 0a.96.96 0 1 1 0 1.92.96.96 0 0 1 0-1.92M8.489 11.458c.588.01 1.965 1.157 3.572 1.164 1.607-.007 2.984-1.155 3.572-1.164.196-.003.305.12.305.454 0 .886-.424 2.328-1.563 3.202-.22-.756-1.396-1.366-1.63-1.32q-.011.001-.02.006l-.044.026-.01.008-.03.024q-.018.017-.035.036l-.032.04a1 1 0 0 0-.058.09l-.014.025q-.049.088-.11.19a1 1 0 0 1-.083.116 1.2 1.2 0 0 1-.173.18q-.035.029-.075.058a1.3 1.3 0 0 1-.251-.243 1 1 0 0 1-.076-.107c-.124-.193-.177-.363-.337-.444-.034-.016-.104-.008-.2.022q-.094.03-.216.087-.06.028-.125.063l-.13.074q-.067.04-.136.086a3 3 0 0 0-.135.096 3 3 0 0 0-.26.219 2 2 0 0 0-.12.121 2 2 0 0 0-.106.128l-.002.002a2 2 0 0 0-.09.132l-.001.001a1.2 1.2 0 0 0-.105.212q-.013.036-.024.073c-1.139-.875-1.563-2.317-1.563-3.203 0-.334.109-.457.305-.454m.836 10.354c.824-1.19.766-2.082-.365-3.194-1.13-1.112-1.789-2.738-1.789-2.738s-.246-.945-.806-.858-.97 1.499.202 2.362c1.173.864-.233 1.45-.685.64-.45-.812-1.683-2.896-2.322-3.295s-1.089-.175-.938.647 2.822 2.813 2.562 3.244-1.176-.506-1.176-.506-2.866-2.567-3.49-1.898.473 1.23 2.037 2.16c1.564.932 1.686 1.178 1.464 1.53s-3.675-2.511-4-1.297c-.323 1.214 3.524 1.567 3.287 2.405-.238.839-2.71-1.587-3.216-.642-.506.946 3.49 2.056 3.522 2.064 1.29.33 4.568 1.028 5.713-.624m5.349 0c-.824-1.19-.766-2.082.365-3.194 1.13-1.112 1.789-2.738 1.789-2.738s.246-.945.806-.858.97 1.499-.202 2.362c-1.173.864.233 1.45.685.64.451-.812 1.683-2.896 2.322-3.295s1.089-.175.938.647-2.822 2.813-2.562 3.244 1.176-.506 1.176-.506 2.866-2.567 3.49-1.898-.473 1.23-2.037 2.16c-1.564.932-1.686 1.178-1.464 1.53s3.675-2.511 4-1.297c.323 1.214-3.524 1.567-3.287 2.405.238.839 2.71-1.587 3.216-.642.506.946-3.49 2.056-3.522 2.064-1.29.33-4.568 1.028-5.713-.624"
|
||||
/>
|
||||
</svg>
|
||||
{:else if family === "step"}
|
||||
<svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path
|
||||
d="M22.012 0h1.032v.927H24v.968h-.956V3.78h-1.032V1.896h-1.878v-.97h1.878V0zM2.6 12.371V1.87h.969v10.502h-.97zm10.423.66h10.95v.918h-6.208v9.579h-4.742V13.03zM5.629 3.333v12.356H0v4.51h10.386V8L20.859 8l-.003-4.668-15.227.001z"
|
||||
/>
|
||||
</svg>
|
||||
{:else if family === "nemotron"}
|
||||
<svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path
|
||||
d="M8.948 8.798v-1.43a6.7 6.7 0 0 1 .424-.018c3.922-.124 6.493 3.374 6.493 3.374s-2.774 3.851-5.75 3.851c-.398 0-.787-.062-1.158-.185v-4.346c1.528.185 1.837.857 2.747 2.385l2.04-1.714s-1.492-1.952-4-1.952a6.016 6.016 0 0 0-.796.035m0-4.735v2.138l.424-.027c5.45-.185 9.01 4.47 9.01 4.47s-4.08 4.964-8.33 4.964c-.37 0-.733-.035-1.095-.097v1.325c.3.035.61.062.91.062 3.957 0 6.82-2.023 9.593-4.408.459.371 2.34 1.263 2.73 1.652-2.633 2.208-8.772 3.984-12.253 3.984-.335 0-.653-.018-.971-.053v1.864H24V4.063zm0 10.326v1.131c-3.657-.654-4.673-4.46-4.673-4.46s1.758-1.944 4.673-2.262v1.237H8.94c-1.528-.186-2.73 1.245-2.73 1.245s.68 2.412 2.739 3.11M2.456 10.9s2.164-3.197 6.5-3.533V6.201C4.153 6.59 0 10.653 0 10.653s2.35 6.802 8.948 7.42v-1.237c-4.84-.6-6.492-5.936-6.492-5.936z"
|
||||
/>
|
||||
</svg>
|
||||
{:else if family === "gemma"}
|
||||
<svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path
|
||||
d="M12.48 10.92v3.28h7.84c-.24 1.84-.853 3.187-1.787 4.133-1.147 1.147-2.933 2.4-6.053 2.4-4.827 0-8.6-3.893-8.6-8.72s3.773-8.72 8.6-8.72c2.6 0 4.507 1.027 5.907 2.347l2.307-2.307C18.747 1.44 16.133 0 12.48 0 5.867 0 .307 5.387.307 12s5.56 12 12.173 12c3.573 0 6.267-1.173 8.373-3.36 2.16-2.16 2.84-5.213 2.84-7.667 0-.76-.053-1.467-.173-2.053H12.48z"
|
||||
/>
|
||||
</svg>
|
||||
{:else}
|
||||
<svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path
|
||||
|
||||
@@ -31,8 +31,6 @@
|
||||
kimi: "Kimi",
|
||||
flux: "FLUX",
|
||||
"qwen-image": "Qwen Img",
|
||||
nemotron: "NVIDIA",
|
||||
gemma: "Google",
|
||||
};
|
||||
|
||||
function getFamilyName(family: string): string {
|
||||
@@ -43,20 +41,31 @@
|
||||
</script>
|
||||
|
||||
<div
|
||||
class="flex flex-col gap-1 py-2 px-1 border-r border-exo-yellow/10 bg-exo-medium-gray/30 min-w-[80px] sm:min-w-[72px] overflow-y-auto scrollbar-hide"
|
||||
class="flex flex-col gap-1 py-2 px-1 border-r border-exo-yellow/10 bg-exo-medium-gray/30 min-w-[64px] overflow-y-auto scrollbar-hide"
|
||||
>
|
||||
<!-- All models (no filter) -->
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => onSelect(null)}
|
||||
class="group flex items-center justify-center px-3 py-2.5 rounded transition-all duration-200 cursor-pointer min-h-[44px] sm:min-h-0 {selectedFamily ===
|
||||
class="group flex flex-col items-center justify-center p-2 rounded transition-all duration-200 cursor-pointer {selectedFamily ===
|
||||
null
|
||||
? 'bg-exo-yellow/20 border-l-2 border-exo-yellow'
|
||||
: 'hover:bg-white/5 border-l-2 border-transparent'}"
|
||||
title="All models"
|
||||
>
|
||||
<svg
|
||||
class="w-5 h-5 {selectedFamily === null
|
||||
? 'text-exo-yellow'
|
||||
: 'text-white/50 group-hover:text-white/70'}"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path
|
||||
d="M4 8h4V4H4v4zm6 12h4v-4h-4v4zm-6 0h4v-4H4v4zm0-6h4v-4H4v4zm6 0h4v-4h-4v4zm6-10v4h4V4h-4zm-6 4h4V4h-4v4zm6 6h4v-4h-4v4zm0 6h4v-4h-4v4z"
|
||||
/>
|
||||
</svg>
|
||||
<span
|
||||
class="text-[12px] font-mono font-medium {selectedFamily === null
|
||||
class="text-[9px] font-mono mt-0.5 {selectedFamily === null
|
||||
? 'text-exo-yellow'
|
||||
: 'text-white/40 group-hover:text-white/60'}">All</span
|
||||
>
|
||||
@@ -80,7 +89,7 @@
|
||||
: "text-white/50 group-hover:text-amber-400/70"}
|
||||
/>
|
||||
<span
|
||||
class="text-[11px] font-mono mt-0.5 {selectedFamily === 'favorites'
|
||||
class="text-[9px] font-mono mt-0.5 {selectedFamily === 'favorites'
|
||||
? 'text-amber-400'
|
||||
: 'text-white/40 group-hover:text-white/60'}">Faves</span
|
||||
>
|
||||
@@ -105,7 +114,7 @@
|
||||
: "text-white/50 group-hover:text-white/70"}
|
||||
/>
|
||||
<span
|
||||
class="text-[11px] font-mono mt-0.5 {selectedFamily === 'recents'
|
||||
class="text-[9px] font-mono mt-0.5 {selectedFamily === 'recents'
|
||||
? 'text-exo-yellow'
|
||||
: 'text-white/40 group-hover:text-white/60'}">Recent</span
|
||||
>
|
||||
@@ -129,7 +138,7 @@
|
||||
: "text-white/50 group-hover:text-orange-400/70"}
|
||||
/>
|
||||
<span
|
||||
class="text-[11px] font-mono mt-0.5 {selectedFamily === 'huggingface'
|
||||
class="text-[9px] font-mono mt-0.5 {selectedFamily === 'huggingface'
|
||||
? 'text-orange-400'
|
||||
: 'text-white/40 group-hover:text-white/60'}">Hub</span
|
||||
>
|
||||
@@ -155,7 +164,7 @@
|
||||
: "text-white/50 group-hover:text-white/70"}
|
||||
/>
|
||||
<span
|
||||
class="text-[11px] font-mono mt-0.5 truncate max-w-full {selectedFamily ===
|
||||
class="text-[9px] font-mono mt-0.5 truncate max-w-full {selectedFamily ===
|
||||
family
|
||||
? 'text-exo-yellow'
|
||||
: 'text-white/40 group-hover:text-white/60'}"
|
||||
|
||||
@@ -1,41 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { browser } from "$app/environment";
|
||||
import { featureFlags } from "$lib/stores/app.svelte";
|
||||
|
||||
const showAdvanced = $derived(featureFlags()["disaggregation"] === true);
|
||||
|
||||
interface Props {
|
||||
showHome?: boolean;
|
||||
onHome?: (() => void) | null;
|
||||
showSidebarToggle?: boolean;
|
||||
sidebarVisible?: boolean;
|
||||
onToggleSidebar?: (() => void) | null;
|
||||
showMobileMenuToggle?: boolean;
|
||||
mobileMenuOpen?: boolean;
|
||||
onToggleMobileMenu?: (() => void) | null;
|
||||
showMobileRightToggle?: boolean;
|
||||
mobileRightOpen?: boolean;
|
||||
onToggleMobileRight?: (() => void) | null;
|
||||
downloadProgress?: {
|
||||
count: number;
|
||||
percentage: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
let {
|
||||
showHome = true,
|
||||
onHome = null,
|
||||
showSidebarToggle = false,
|
||||
sidebarVisible = true,
|
||||
onToggleSidebar = null,
|
||||
showMobileMenuToggle = false,
|
||||
mobileMenuOpen = false,
|
||||
onToggleMobileMenu = null,
|
||||
showMobileRightToggle = false,
|
||||
mobileRightOpen = false,
|
||||
onToggleMobileRight = null,
|
||||
downloadProgress = null,
|
||||
}: Props = $props();
|
||||
export let showHome = true;
|
||||
export let onHome: (() => void) | null = null;
|
||||
export let showSidebarToggle = false;
|
||||
export let sidebarVisible = true;
|
||||
export let onToggleSidebar: (() => void) | null = null;
|
||||
|
||||
function handleHome(): void {
|
||||
if (onHome) {
|
||||
@@ -53,96 +23,45 @@
|
||||
onToggleSidebar();
|
||||
}
|
||||
}
|
||||
|
||||
function handleToggleMobileMenu(): void {
|
||||
if (onToggleMobileMenu) {
|
||||
onToggleMobileMenu();
|
||||
}
|
||||
}
|
||||
|
||||
function handleToggleMobileRight(): void {
|
||||
if (onToggleMobileRight) {
|
||||
onToggleMobileRight();
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<header
|
||||
class="relative z-20 flex items-center justify-center px-4 md:px-6 pt-4 md:pt-8 pb-3 md:pb-4 bg-exo-dark-gray"
|
||||
class="relative z-20 flex items-center justify-center px-6 pt-8 pb-4 bg-exo-dark-gray"
|
||||
>
|
||||
<!-- Left: Sidebar Toggle (desktop) or Mobile Sidebar Toggle (mobile) -->
|
||||
<div
|
||||
class="absolute left-4 md:left-6 top-1/2 -translate-y-1/2 flex items-center gap-2"
|
||||
>
|
||||
<!-- Mobile sidebar toggle -->
|
||||
<button
|
||||
onclick={handleToggleMobileMenu}
|
||||
class="p-2 rounded border border-exo-light-gray/30 hover:border-exo-yellow/50 hover:bg-exo-medium-gray/30 transition-colors cursor-pointer md:hidden"
|
||||
title={mobileMenuOpen ? "Hide sidebar" : "Show sidebar"}
|
||||
aria-label={mobileMenuOpen
|
||||
? "Hide conversation sidebar"
|
||||
: "Show conversation sidebar"}
|
||||
aria-pressed={mobileMenuOpen}
|
||||
>
|
||||
<svg
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
class="w-5 h-5 {mobileMenuOpen
|
||||
? 'text-exo-yellow'
|
||||
: 'text-exo-light-gray'}"
|
||||
<!-- Left: Sidebar Toggle -->
|
||||
{#if showSidebarToggle}
|
||||
<div class="absolute left-6 top-1/2 -translate-y-1/2">
|
||||
<button
|
||||
onclick={handleToggleSidebar}
|
||||
class="p-2 rounded border border-exo-medium-gray/40 hover:border-exo-yellow/50 transition-colors cursor-pointer"
|
||||
title={sidebarVisible ? "Hide sidebar" : "Show sidebar"}
|
||||
>
|
||||
{#if mobileMenuOpen}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M11 19l-7-7 7-7m8 14l-7-7 7-7"
|
||||
></path>
|
||||
{:else}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M13 5l7 7-7 7M5 5l7 7-7 7"
|
||||
></path>
|
||||
{/if}
|
||||
</svg>
|
||||
</button>
|
||||
<!-- Desktop sidebar toggle -->
|
||||
<button
|
||||
onclick={handleToggleSidebar}
|
||||
class="p-2 rounded border border-exo-light-gray/30 hover:border-exo-yellow/50 hover:bg-exo-medium-gray/30 transition-colors cursor-pointer hidden md:block"
|
||||
title={sidebarVisible ? "Hide sidebar" : "Show sidebar"}
|
||||
aria-label={sidebarVisible
|
||||
? "Hide conversation sidebar"
|
||||
: "Show conversation sidebar"}
|
||||
aria-pressed={sidebarVisible}
|
||||
>
|
||||
<svg
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
class="w-5 h-5 {sidebarVisible
|
||||
? 'text-exo-yellow'
|
||||
: 'text-exo-light-gray'}"
|
||||
>
|
||||
{#if sidebarVisible}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M11 19l-7-7 7-7m8 14l-7-7 7-7"
|
||||
></path>
|
||||
{:else}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M13 5l7 7-7 7M5 5l7 7-7 7"
|
||||
></path>
|
||||
{/if}
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
<svg
|
||||
class="w-5 h-5 {sidebarVisible
|
||||
? 'text-exo-yellow'
|
||||
: 'text-exo-medium-gray'}"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
{#if sidebarVisible}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M11 19l-7-7 7-7m8 14l-7-7 7-7"
|
||||
/>
|
||||
{:else}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M13 5l7 7-7 7M5 5l7 7-7 7"
|
||||
/>
|
||||
{/if}
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Center: Logo (clickable to go home) -->
|
||||
<button
|
||||
@@ -156,55 +75,18 @@
|
||||
<img
|
||||
src="/exo-logo.png"
|
||||
alt="EXO"
|
||||
class="h-12 md:h-18 drop-shadow-[0_0_4px_rgba(255,215,0,0.3)]"
|
||||
class="h-18 drop-shadow-[0_0_20px_rgba(255,215,0,0.5)]"
|
||||
/>
|
||||
</button>
|
||||
|
||||
<!-- Right: Home + Downloads + Mobile Right Toggle -->
|
||||
<nav
|
||||
class="absolute right-4 md:right-6 top-1/2 -translate-y-1/2 flex items-center gap-2 md:gap-4"
|
||||
aria-label="Main navigation"
|
||||
<!-- Right: Home + Downloads -->
|
||||
<div
|
||||
class="absolute right-6 top-1/2 -translate-y-1/2 flex items-center gap-4"
|
||||
>
|
||||
<!-- Mobile right sidebar toggle (instances/models) - only show when not in chat mode -->
|
||||
{#if showMobileRightToggle}
|
||||
<button
|
||||
onclick={handleToggleMobileRight}
|
||||
class="p-2 rounded border border-exo-light-gray/30 hover:border-exo-yellow/50 hover:bg-exo-medium-gray/30 transition-colors cursor-pointer md:hidden"
|
||||
title={mobileRightOpen ? "Hide instances" : "Show instances"}
|
||||
aria-label={mobileRightOpen
|
||||
? "Hide instances panel"
|
||||
: "Show instances panel"}
|
||||
aria-pressed={mobileRightOpen}
|
||||
>
|
||||
<svg
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
class="w-5 h-5 {mobileRightOpen
|
||||
? 'text-exo-yellow'
|
||||
: 'text-exo-light-gray'}"
|
||||
>
|
||||
{#if mobileRightOpen}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M13 5l7 7-7 7M5 5l7 7-7 7"
|
||||
></path>
|
||||
{:else}
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M11 19l-7-7 7-7m8 14l-7-7 7-7"
|
||||
></path>
|
||||
{/if}
|
||||
</svg>
|
||||
</button>
|
||||
{/if}
|
||||
{#if showHome}
|
||||
<button
|
||||
onclick={handleHome}
|
||||
class="flex text-sm text-white/70 hover:text-exo-yellow transition-colors tracking-wider uppercase items-center gap-2 cursor-pointer"
|
||||
class="text-sm text-exo-light-gray hover:text-exo-yellow transition-colors tracking-wider uppercase flex items-center gap-2 cursor-pointer"
|
||||
title="Back to topology view"
|
||||
>
|
||||
<svg
|
||||
@@ -220,69 +102,13 @@
|
||||
d="M3 12l2-2m0 0l7-7 7 7M5 10v10a1 1 0 001 1h3m10-11l2 2m-2-2v10a1 1 0 01-1 1h-3m-6 0a1 1 0 001-1v-4a1 1 0 011-1h2a1 1 0 011 1v4a1 1 0 001 1m-6 0h6"
|
||||
/>
|
||||
</svg>
|
||||
<span class="hidden sm:inline">Home</span>
|
||||
Home
|
||||
</button>
|
||||
{/if}
|
||||
<a
|
||||
href="/#/downloads"
|
||||
class="text-xs md:text-sm text-white/70 hover:text-exo-yellow transition-colors tracking-wider uppercase flex items-center gap-1.5 md:gap-2 cursor-pointer"
|
||||
class="text-sm text-exo-light-gray hover:text-exo-yellow transition-colors tracking-wider uppercase flex items-center gap-2 cursor-pointer"
|
||||
title="View downloads overview"
|
||||
>
|
||||
{#if downloadProgress}
|
||||
<!-- Compact download progress indicator -->
|
||||
<div class="relative w-4 h-4 flex-shrink-0">
|
||||
<svg class="w-4 h-4 -rotate-90" viewBox="0 0 20 20">
|
||||
<circle
|
||||
cx="10"
|
||||
cy="10"
|
||||
r="8"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
opacity="0.2"
|
||||
/>
|
||||
<circle
|
||||
cx="10"
|
||||
cy="10"
|
||||
r="8"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-dasharray={2 * Math.PI * 8}
|
||||
stroke-dashoffset={2 *
|
||||
Math.PI *
|
||||
8 *
|
||||
(1 - downloadProgress.percentage / 100)}
|
||||
class="text-blue-400 transition-all duration-300"
|
||||
/>
|
||||
</svg>
|
||||
<div
|
||||
class="absolute inset-0 flex items-center justify-center text-[6px] font-mono text-blue-400"
|
||||
>
|
||||
{downloadProgress.count}
|
||||
</div>
|
||||
</div>
|
||||
{:else}
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path d="M12 3v12" />
|
||||
<path d="M7 12l5 5 5-5" />
|
||||
<path d="M5 21h14" />
|
||||
</svg>
|
||||
{/if}
|
||||
<span class="hidden sm:inline">Downloads</span>
|
||||
</a>
|
||||
<a
|
||||
href="/#/integrations"
|
||||
class="text-xs md:text-sm text-white/70 hover:text-exo-yellow transition-colors tracking-wider uppercase flex items-center gap-1.5 md:gap-2 cursor-pointer"
|
||||
title="Integration configs for external tools"
|
||||
>
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
@@ -293,35 +119,11 @@
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71" />
|
||||
<path
|
||||
d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"
|
||||
/>
|
||||
<path d="M12 3v12" />
|
||||
<path d="M7 12l5 5 5-5" />
|
||||
<path d="M5 21h14" />
|
||||
</svg>
|
||||
<span class="hidden sm:inline">Integrations</span>
|
||||
Downloads
|
||||
</a>
|
||||
{#if showAdvanced}
|
||||
<a
|
||||
href="/#/advanced"
|
||||
class="text-xs md:text-sm text-white/70 hover:text-exo-yellow transition-colors tracking-wider uppercase flex items-center gap-1.5 md:gap-2 cursor-pointer"
|
||||
title="Advanced cluster settings"
|
||||
>
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
<path
|
||||
d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-4 0v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1 0-4h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"
|
||||
/>
|
||||
</svg>
|
||||
<span class="hidden sm:inline">Advanced</span>
|
||||
</a>
|
||||
{/if}
|
||||
</nav>
|
||||
</div>
|
||||
</header>
|
||||
@@ -36,12 +36,8 @@
|
||||
return num.toString();
|
||||
}
|
||||
|
||||
// Show short name for mlx-community models, full ID for everything else
|
||||
const modelName = $derived(
|
||||
model.author === "mlx-community"
|
||||
? model.id.split("/").pop() || model.id
|
||||
: model.id,
|
||||
);
|
||||
// Extract model name from full ID (e.g., "mlx-community/Llama-3.2-1B" -> "Llama-3.2-1B")
|
||||
const modelName = $derived(model.id.split("/").pop() || model.id);
|
||||
</script>
|
||||
|
||||
<div
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { copyText } from "$lib/utils/clipboard";
|
||||
|
||||
interface Props {
|
||||
title: string;
|
||||
subtitle: string;
|
||||
config: string;
|
||||
description?: string;
|
||||
language?: "json" | "bash";
|
||||
}
|
||||
|
||||
let {
|
||||
title,
|
||||
subtitle,
|
||||
config,
|
||||
description = "",
|
||||
language = "json",
|
||||
}: Props = $props();
|
||||
|
||||
let copied = $state(false);
|
||||
let failed = $state(false);
|
||||
|
||||
async function copyToClipboard() {
|
||||
const ok = await copyText(config);
|
||||
if (ok) {
|
||||
copied = true;
|
||||
setTimeout(() => (copied = false), 2000);
|
||||
} else {
|
||||
failed = true;
|
||||
setTimeout(() => (failed = false), 2000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div
|
||||
class="border border-exo-light-gray/20 rounded-lg bg-exo-medium-gray/20 overflow-hidden"
|
||||
>
|
||||
<div class="flex items-center justify-between px-5 py-4">
|
||||
<div>
|
||||
<h3 class="text-white text-sm font-semibold tracking-wide">{title}</h3>
|
||||
<p class="text-exo-light-gray/60 text-xs mt-0.5 font-mono">{subtitle}</p>
|
||||
</div>
|
||||
<button
|
||||
onclick={copyToClipboard}
|
||||
class="px-3 py-1.5 text-xs rounded border transition-all duration-200 cursor-pointer
|
||||
{copied
|
||||
? 'border-green-500/50 text-green-400 bg-green-500/10'
|
||||
: failed
|
||||
? 'border-red-500/50 text-red-400 bg-red-500/10'
|
||||
: 'border-exo-light-gray/30 text-exo-light-gray hover:border-exo-yellow/50 hover:text-exo-yellow'}"
|
||||
>
|
||||
{copied ? "Copied!" : failed ? "Copy failed" : "Copy"}
|
||||
</button>
|
||||
</div>
|
||||
{#if description}
|
||||
<p class="text-exo-light-gray/70 text-xs px-5 pb-3">{description}</p>
|
||||
{/if}
|
||||
<div class="bg-black/30 border-t border-exo-light-gray/10">
|
||||
<pre
|
||||
class="text-xs text-exo-light-gray/90 font-mono p-4 overflow-x-auto whitespace-pre">{config}</pre>
|
||||
</div>
|
||||
</div>
|
||||
@@ -507,29 +507,9 @@
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
if (!containerRef || !browser) return;
|
||||
|
||||
function handleDelegatedClick(event: MouseEvent) {
|
||||
const codeBtn = (event.target as HTMLElement).closest(
|
||||
".copy-code-btn",
|
||||
) as HTMLButtonElement | null;
|
||||
if (codeBtn) {
|
||||
handleCopyClick({ currentTarget: codeBtn } as unknown as Event);
|
||||
return;
|
||||
}
|
||||
const mathBtn = (event.target as HTMLElement).closest(
|
||||
".copy-math-btn",
|
||||
) as HTMLButtonElement | null;
|
||||
if (mathBtn) {
|
||||
handleMathCopyClick({ currentTarget: mathBtn } as unknown as Event);
|
||||
return;
|
||||
}
|
||||
if (containerRef && processedHtml) {
|
||||
setupCopyButtons();
|
||||
}
|
||||
|
||||
containerRef.addEventListener("click", handleDelegatedClick);
|
||||
return () => {
|
||||
containerRef?.removeEventListener("click", handleDelegatedClick);
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
<script lang="ts">
|
||||
import type {
|
||||
MetaInstance,
|
||||
MetaInstanceStatus,
|
||||
NodeInfo,
|
||||
} from "$lib/stores/app.svelte";
|
||||
import {
|
||||
getMetaInstanceStatus,
|
||||
getMetaInstanceBackingNodes,
|
||||
topologyData,
|
||||
} from "$lib/stores/app.svelte";
|
||||
|
||||
interface Props {
|
||||
metaInstance: MetaInstance;
|
||||
onDelete?: (metaInstanceId: string) => void;
|
||||
}
|
||||
|
||||
let { metaInstance, onDelete }: Props = $props();
|
||||
|
||||
const status: MetaInstanceStatus = $derived(
|
||||
getMetaInstanceStatus(metaInstance),
|
||||
);
|
||||
const backingNodeIds: string[] = $derived(
|
||||
getMetaInstanceBackingNodes(metaInstance),
|
||||
);
|
||||
|
||||
const statusConfig = $derived.by(() => {
|
||||
switch (status) {
|
||||
case "active":
|
||||
return {
|
||||
label: "ACTIVE",
|
||||
dotClass: "bg-green-400",
|
||||
borderClass:
|
||||
"border-green-500/30 border-l-green-400",
|
||||
cornerClass: "border-green-500/50",
|
||||
glowClass: "shadow-[0_0_6px_rgba(74,222,128,0.4)]",
|
||||
animate: false,
|
||||
};
|
||||
case "provisioning":
|
||||
return {
|
||||
label: "PROVISIONING",
|
||||
dotClass: "bg-yellow-400",
|
||||
borderClass:
|
||||
"border-exo-yellow/30 border-l-yellow-400",
|
||||
cornerClass: "border-yellow-500/50",
|
||||
glowClass: "shadow-[0_0_6px_rgba(250,204,21,0.4)]",
|
||||
animate: true,
|
||||
};
|
||||
case "error":
|
||||
return {
|
||||
label: "ERROR",
|
||||
dotClass: "bg-red-400",
|
||||
borderClass: "border-red-500/30 border-l-red-400",
|
||||
cornerClass: "border-red-500/50",
|
||||
glowClass: "shadow-[0_0_6px_rgba(248,113,113,0.4)]",
|
||||
animate: false,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
function getNodeName(nodeId: string): string {
|
||||
const topo = topologyData();
|
||||
if (!topo?.nodes) return nodeId.slice(0, 8);
|
||||
const node = topo.nodes[nodeId];
|
||||
return node?.friendly_name || node?.system_info?.model_id || nodeId.slice(0, 8);
|
||||
}
|
||||
|
||||
function formatModelId(modelId: string): string {
|
||||
// Show just the model name part after the org prefix
|
||||
const parts = modelId.split("/");
|
||||
return parts.length > 1 ? parts[parts.length - 1] : modelId;
|
||||
}
|
||||
|
||||
function handleDelete() {
|
||||
if (
|
||||
onDelete &&
|
||||
confirm(
|
||||
`Delete meta-instance for ${formatModelId(metaInstance.modelId)}?`,
|
||||
)
|
||||
) {
|
||||
onDelete(metaInstance.metaInstanceId);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="relative group">
|
||||
<!-- Corner accents -->
|
||||
<div
|
||||
class="absolute -top-px -left-px w-2 h-2 border-l border-t {statusConfig.cornerClass}"
|
||||
></div>
|
||||
<div
|
||||
class="absolute -top-px -right-px w-2 h-2 border-r border-t {statusConfig.cornerClass}"
|
||||
></div>
|
||||
<div
|
||||
class="absolute -bottom-px -left-px w-2 h-2 border-l border-b {statusConfig.cornerClass}"
|
||||
></div>
|
||||
<div
|
||||
class="absolute -bottom-px -right-px w-2 h-2 border-r border-b {statusConfig.cornerClass}"
|
||||
></div>
|
||||
|
||||
<div
|
||||
class="bg-exo-dark-gray/60 border border-l-2 {statusConfig.borderClass} p-3"
|
||||
>
|
||||
<!-- Header: Status + Delete -->
|
||||
<div class="flex justify-between items-start mb-2 pl-2">
|
||||
<div class="flex items-center gap-2">
|
||||
<div
|
||||
class="w-1.5 h-1.5 {statusConfig.dotClass} rounded-full {statusConfig.glowClass} {statusConfig.animate
|
||||
? 'animate-pulse'
|
||||
: ''}"
|
||||
></div>
|
||||
<span
|
||||
class="text-xs font-mono tracking-[0.15em] uppercase {status === 'active'
|
||||
? 'text-green-400'
|
||||
: status === 'error'
|
||||
? 'text-red-400'
|
||||
: 'text-yellow-400'}"
|
||||
>
|
||||
{statusConfig.label}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
onclick={handleDelete}
|
||||
class="text-xs px-2 py-1 font-mono tracking-wider uppercase border border-red-500/30 text-red-400 hover:bg-red-500/20 hover:text-red-400 hover:border-red-500/50 transition-all duration-200 cursor-pointer"
|
||||
>
|
||||
DELETE
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Model Info -->
|
||||
<div class="pl-2 space-y-1">
|
||||
<div class="text-exo-yellow text-xs font-mono tracking-wide truncate">
|
||||
{metaInstance.modelId}
|
||||
</div>
|
||||
|
||||
<!-- Sharding + Runtime badges -->
|
||||
<div class="flex items-center gap-2">
|
||||
<span
|
||||
class="inline-flex items-center px-1.5 py-0.5 text-[10px] font-mono tracking-wider uppercase border border-white/10 text-white/50"
|
||||
>
|
||||
{metaInstance.sharding}
|
||||
</span>
|
||||
<span
|
||||
class="inline-flex items-center px-1.5 py-0.5 text-[10px] font-mono tracking-wider uppercase border border-white/10 text-white/50"
|
||||
>
|
||||
{metaInstance.instanceMeta}
|
||||
</span>
|
||||
{#if metaInstance.minNodes > 1}
|
||||
<span
|
||||
class="inline-flex items-center px-1.5 py-0.5 text-[10px] font-mono tracking-wider uppercase border border-white/10 text-white/50"
|
||||
>
|
||||
{metaInstance.minNodes}+ nodes
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Node Assignments (when active) -->
|
||||
{#if backingNodeIds.length > 0}
|
||||
<div class="flex items-center gap-1.5 mt-1">
|
||||
<svg
|
||||
class="w-3 h-3 text-green-400/70 flex-shrink-0"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path
|
||||
d="M22 12h-4l-3 9L9 3l-3 9H2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
<span class="text-white/60 text-xs font-mono truncate">
|
||||
{backingNodeIds.map((id) => getNodeName(id)).join(", ")}
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Pinned nodes constraint -->
|
||||
{#if metaInstance.nodeIds && metaInstance.nodeIds.length > 0}
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg
|
||||
class="w-3 h-3 text-white/40 flex-shrink-0"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<rect x="3" y="11" width="18" height="11" rx="2" ry="2" />
|
||||
<path d="M7 11V7a5 5 0 0 1 10 0v4" />
|
||||
</svg>
|
||||
<span class="text-white/40 text-[11px] font-mono">
|
||||
Pinned: {metaInstance.nodeIds
|
||||
.map((id) => getNodeName(id))
|
||||
.join(", ")}
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Error details -->
|
||||
{#if metaInstance.placementError}
|
||||
<div
|
||||
class="mt-1.5 p-2 bg-red-500/5 border border-red-500/15 rounded-sm"
|
||||
>
|
||||
<div class="text-red-400 text-[11px] font-mono leading-relaxed">
|
||||
{metaInstance.placementError}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Retry counter -->
|
||||
{#if metaInstance.consecutiveFailures > 0}
|
||||
<div class="flex items-center gap-1.5 mt-1">
|
||||
<svg
|
||||
class="w-3 h-3 text-yellow-500/60 flex-shrink-0"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<polyline points="23 4 23 10 17 10" />
|
||||
<path d="M20.49 15a9 9 0 1 1-2.12-9.36L23 10" />
|
||||
</svg>
|
||||
<span class="text-yellow-500/60 text-[11px] font-mono">
|
||||
{metaInstance.consecutiveFailures} consecutive
|
||||
failure{metaInstance.consecutiveFailures !== 1 ? "s" : ""}
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -16,14 +16,12 @@
|
||||
perNode?: Array<{
|
||||
nodeId: string;
|
||||
nodeName: string;
|
||||
status: "completed" | "partial" | "pending" | "downloading";
|
||||
percentage: number;
|
||||
progress: DownloadProgress | null;
|
||||
progress: DownloadProgress;
|
||||
}>;
|
||||
} | null;
|
||||
nodes?: Record<string, NodeInfo>;
|
||||
sharding?: "Pipeline" | "Tensor";
|
||||
runtime?: "MlxRing" | "MlxJaccl" | "Vllm";
|
||||
runtime?: "MlxRing" | "MlxIbv" | "MlxJaccl";
|
||||
onLaunch?: () => void;
|
||||
tags?: string[];
|
||||
apiPreview?: PlacementPreview | null;
|
||||
@@ -147,7 +145,10 @@
|
||||
return `${s}s`;
|
||||
}
|
||||
|
||||
const perNode = $derived(downloadStatus?.perNode ?? []);
|
||||
const isDownloading = $derived(downloadStatus?.isDownloading ?? false);
|
||||
const progress = $derived(downloadStatus?.progress);
|
||||
const percentage = $derived(progress?.percentage ?? 0);
|
||||
let expandedNodes = $state<Set<string>>(new Set());
|
||||
|
||||
function toggleNodeDetails(nodeId: string): void {
|
||||
const next = new Set(expandedNodes);
|
||||
@@ -168,10 +169,8 @@
|
||||
|
||||
function getDeviceType(
|
||||
name: string,
|
||||
): "macbook" | "studio" | "mini" | "dgx" | "linux" | "unknown" {
|
||||
): "macbook" | "studio" | "mini" | "unknown" {
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.includes("dgx") || lower.includes("gx10")) return "dgx";
|
||||
if (lower.includes("linux")) return "linux";
|
||||
if (lower.includes("macbook")) return "macbook";
|
||||
if (lower.includes("studio")) return "studio";
|
||||
if (lower.includes("mini")) return "mini";
|
||||
@@ -349,7 +348,7 @@
|
||||
// Debug mode state
|
||||
const isDebugMode = $derived(debugMode());
|
||||
const topology = $derived(topologyData());
|
||||
const isRdma = $derived(runtime === "MlxJaccl");
|
||||
const isRdma = $derived(runtime === "MlxIbv" || runtime === "MlxJaccl");
|
||||
|
||||
// Get interface name for an IP from node data
|
||||
function getInterfaceForIp(nodeId: string, ip?: string): string | null {
|
||||
@@ -568,76 +567,20 @@
|
||||
<div class="flex items-center gap-1.5 mb-2">
|
||||
<span
|
||||
class="px-1.5 py-0.5 text-xs font-mono tracking-wider uppercase bg-exo-medium-gray/30 text-exo-light-gray border border-exo-medium-gray/40"
|
||||
title={sharding === "Pipeline"
|
||||
? "Pipeline: splits model into sequential stages across devices. Lower network overhead."
|
||||
: "Tensor: splits each layer across devices. Best with high-bandwidth connections (Thunderbolt)."}
|
||||
>
|
||||
{sharding}
|
||||
</span>
|
||||
<span
|
||||
class="px-1.5 py-0.5 text-xs font-mono tracking-wider uppercase bg-exo-medium-gray/30 text-exo-light-gray border border-exo-medium-gray/40"
|
||||
title={runtime === "MlxRing"
|
||||
? "Ring: standard networking. Works over any connection (Wi-Fi, Ethernet, Thunderbolt)."
|
||||
: runtime === "MlxJaccl"
|
||||
? "RDMA: direct memory access over Thunderbolt. Significantly faster for multi-device inference."
|
||||
: "vLLM: NVIDIA CUDA inference engine."}
|
||||
>
|
||||
{runtime === "MlxRing"
|
||||
? "MLX Ring"
|
||||
: runtime === "MlxJaccl"
|
||||
: runtime === "MlxIbv" || runtime === "MlxJaccl"
|
||||
? "MLX RDMA"
|
||||
: runtime === "Vllm"
|
||||
? "vLLM"
|
||||
: runtime}
|
||||
: runtime}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Download Status (per-node) -->
|
||||
{#if perNode.length > 0}
|
||||
<div class="mb-2 space-y-1">
|
||||
<div
|
||||
class="text-[10px] font-mono text-white/20 tracking-widest uppercase"
|
||||
>
|
||||
Download progress
|
||||
</div>
|
||||
{#each perNode as node}
|
||||
<div class="flex items-center gap-2 text-xs font-mono">
|
||||
<span class="text-white/40 w-20 truncate" title={node.nodeId}
|
||||
>{node.nodeName}</span
|
||||
>
|
||||
<div
|
||||
class="flex-1 h-1 bg-exo-medium-gray/30 rounded overflow-hidden"
|
||||
>
|
||||
<div
|
||||
class="h-full transition-all duration-300 {node.status ===
|
||||
'downloading'
|
||||
? 'bg-blue-500/70'
|
||||
: node.status === 'completed'
|
||||
? 'bg-exo-yellow/40'
|
||||
: 'bg-white/20'}"
|
||||
style="width: {node.percentage}%"
|
||||
></div>
|
||||
</div>
|
||||
<span
|
||||
class="text-right {node.status === 'completed'
|
||||
? 'text-exo-yellow/60'
|
||||
: node.status === 'downloading'
|
||||
? 'text-blue-400/60'
|
||||
: 'text-white/30'}"
|
||||
>
|
||||
{#if node.status === "downloading" && node.progress}
|
||||
{Math.round(node.percentage)}% {formatSpeed(
|
||||
node.progress.speed,
|
||||
)}
|
||||
{:else}
|
||||
{node.percentage > 0 ? `${Math.round(node.percentage)}%` : "0%"}
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Mini Topology Preview -->
|
||||
{#if placementPreview().nodes.length > 0}
|
||||
{@const preview = placementPreview()}
|
||||
@@ -693,7 +636,15 @@
|
||||
{@const allConnections =
|
||||
isDebugMode && usedNodes.length > 1
|
||||
? (() => {
|
||||
const conns: Array = [];
|
||||
const conns: Array<{
|
||||
ip: string;
|
||||
iface: string | null;
|
||||
from: string;
|
||||
to: string;
|
||||
midX: number;
|
||||
midY: number;
|
||||
arrow: string;
|
||||
}> = [];
|
||||
for (let i = 0; i < usedNodes.length; i++) {
|
||||
for (let j = i + 1; j < usedNodes.length; j++) {
|
||||
const n1 = usedNodes[i];
|
||||
@@ -705,12 +656,7 @@
|
||||
const toPos = nodePositions[c.to];
|
||||
const arrow =
|
||||
fromPos && toPos ? getArrow(fromPos, toPos) : "→";
|
||||
conns.push({
|
||||
...c,
|
||||
midX,
|
||||
midY,
|
||||
arrow,
|
||||
});
|
||||
conns.push({ ...c, midX, midY, arrow });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -996,81 +942,6 @@
|
||||
/>
|
||||
{/if}
|
||||
</g>
|
||||
{:else if node.deviceType === "dgx"}
|
||||
<!-- DGX Spark icon -->
|
||||
{@const s = node.iconSize}
|
||||
{@const dgxW = s * 1.4}
|
||||
{@const dgxH = s * 0.52}
|
||||
<g transform="translate({-dgxW / 2}, {-dgxH / 2})">
|
||||
<!-- Chassis -->
|
||||
<rect
|
||||
x="0"
|
||||
y="0"
|
||||
width={dgxW}
|
||||
height={dgxH}
|
||||
rx="2"
|
||||
fill="#6f6248"
|
||||
stroke={node.isUsed ? "#FFD700" : "#4B5563"}
|
||||
stroke-width="1.5"
|
||||
/>
|
||||
<!-- Side accents -->
|
||||
<rect
|
||||
x="0"
|
||||
y="0"
|
||||
width={dgxW * 0.02}
|
||||
height={dgxH}
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
<rect
|
||||
x={dgxW - dgxW * 0.02}
|
||||
y="0"
|
||||
width={dgxW * 0.02}
|
||||
height={dgxH}
|
||||
fill="#8a7a56"
|
||||
/>
|
||||
<!-- Left handle -->
|
||||
<rect
|
||||
x={dgxW * 0.04}
|
||||
y={dgxH * 0.08}
|
||||
width={dgxW * 0.22}
|
||||
height={dgxH * 0.84}
|
||||
rx="2"
|
||||
fill="#b3a170"
|
||||
stroke="#403723"
|
||||
stroke-width="0.5"
|
||||
/>
|
||||
<!-- Right handle -->
|
||||
<rect
|
||||
x={dgxW - dgxW * 0.04 - dgxW * 0.22}
|
||||
y={dgxH * 0.08}
|
||||
width={dgxW * 0.22}
|
||||
height={dgxH * 0.84}
|
||||
rx="2"
|
||||
fill="#b3a170"
|
||||
stroke="#403723"
|
||||
stroke-width="0.5"
|
||||
/>
|
||||
<!-- Memory fill -->
|
||||
<rect
|
||||
x="2"
|
||||
y={dgxH - dgxH * (node.currentPercent / 100)}
|
||||
width={dgxW - 4}
|
||||
height={dgxH * (node.currentPercent / 100)}
|
||||
fill="rgba(255,215,0,0.35)"
|
||||
/>
|
||||
{#if node.modelUsageGB > 0 && node.isUsed}
|
||||
<rect
|
||||
x="2"
|
||||
y={dgxH - dgxH * (node.newPercent / 100)}
|
||||
width={dgxW - 4}
|
||||
height={dgxH *
|
||||
((node.newPercent - node.currentPercent) / 100)}
|
||||
fill="#FFD700"
|
||||
filter="url(#memGlow-{filterId})"
|
||||
class="animate-pulse-slow"
|
||||
/>
|
||||
{/if}
|
||||
</g>
|
||||
{:else}
|
||||
<!-- Unknown device - hexagon -->
|
||||
<g
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
capabilities: string[];
|
||||
sizeRange: { min: number; max: number } | null;
|
||||
downloadedOnly: boolean;
|
||||
readyOnly: boolean;
|
||||
}
|
||||
|
||||
type ModelFilterPopoverProps = {
|
||||
@@ -73,7 +72,7 @@
|
||||
|
||||
<!-- svelte-ignore a11y_no_static_element_interactions -->
|
||||
<div
|
||||
class="filter-popover absolute right-0 top-full mt-2 w-64 bg-exo-dark-gray border border-exo-yellow/10 rounded-lg shadow-xl z-20"
|
||||
class="filter-popover absolute right-0 top-full mt-2 w-64 bg-exo-dark-gray border border-exo-yellow/10 rounded-lg shadow-xl z-10"
|
||||
transition:fly={{ y: -10, duration: 200, easing: cubicOut }}
|
||||
onclick={(e) => e.stopPropagation()}
|
||||
role="dialog"
|
||||
@@ -191,58 +190,34 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Availability filters -->
|
||||
<!-- Downloaded only -->
|
||||
<div>
|
||||
<h4 class="text-xs font-mono text-white/50 mb-2">Availability</h4>
|
||||
<div class="flex flex-wrap gap-1.5">
|
||||
<button
|
||||
type="button"
|
||||
class="px-2 py-1 text-xs font-mono rounded transition-colors {filters.downloadedOnly
|
||||
? 'bg-green-500/20 text-green-400 border border-green-500/30'
|
||||
: 'bg-white/5 text-white/60 hover:bg-white/10 border border-transparent'}"
|
||||
onclick={() =>
|
||||
onChange({ ...filters, downloadedOnly: !filters.downloadedOnly })}
|
||||
<button
|
||||
type="button"
|
||||
class="px-2 py-1 text-xs font-mono rounded transition-colors {filters.downloadedOnly
|
||||
? 'bg-green-500/20 text-green-400 border border-green-500/30'
|
||||
: 'bg-white/5 text-white/60 hover:bg-white/10 border border-transparent'}"
|
||||
onclick={() =>
|
||||
onChange({ ...filters, downloadedOnly: !filters.downloadedOnly })}
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5 inline-block"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5 inline-block"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path
|
||||
class="text-white/40"
|
||||
d="M20 20a2 2 0 0 0 2-2V8a2 2 0 0 0-2-2h-7.9a2 2 0 0 1-1.69-.9L9.6 3.9A2 2 0 0 0 7.93 3H4a2 2 0 0 0-2 2v13a2 2 0 0 0 2 2Z"
|
||||
/>
|
||||
<path class="text-green-400" d="m9 13 2 2 4-4" />
|
||||
</svg>
|
||||
<span class="ml-1">Downloaded</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="px-2 py-1 text-xs font-mono rounded transition-colors {filters.readyOnly
|
||||
? 'bg-green-500/20 text-green-400 border border-green-500/30'
|
||||
: 'bg-white/5 text-white/60 hover:bg-white/10 border border-transparent'}"
|
||||
onclick={() =>
|
||||
onChange({ ...filters, readyOnly: !filters.readyOnly })}
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5 inline-block"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10" />
|
||||
<path d="m9 12 2 2 4-4" />
|
||||
</svg>
|
||||
<span class="ml-1">Ready</span>
|
||||
</button>
|
||||
</div>
|
||||
<path
|
||||
class="text-white/40"
|
||||
d="M20 20a2 2 0 0 0 2-2V8a2 2 0 0 0-2-2h-7.9a2 2 0 0 1-1.69-.9L9.6 3.9A2 2 0 0 0 7.93 3H4a2 2 0 0 0-2 2v13a2 2 0 0 0 2 2Z"
|
||||
/>
|
||||
<path class="text-green-400" d="m9 13 2 2 4-4" />
|
||||
</svg>
|
||||
<span class="ml-1">Downloaded</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Size range -->
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
capabilities?: string[];
|
||||
family?: string;
|
||||
is_custom?: boolean;
|
||||
requires_vllm?: boolean;
|
||||
}
|
||||
|
||||
interface ModelGroup {
|
||||
@@ -20,7 +19,6 @@
|
||||
variants: ModelInfo[];
|
||||
smallestVariant: ModelInfo;
|
||||
hasMultipleVariants: boolean;
|
||||
requiresVllm: boolean;
|
||||
}
|
||||
|
||||
type DownloadAvailability = {
|
||||
@@ -34,7 +32,6 @@
|
||||
group: ModelGroup;
|
||||
isExpanded: boolean;
|
||||
isFavorite: boolean;
|
||||
isHighlighted?: boolean;
|
||||
selectedModelId: string | null;
|
||||
canModelFit: (id: string) => boolean;
|
||||
getModelFitStatus: (id: string) => ModelFitStatus;
|
||||
@@ -44,14 +41,12 @@
|
||||
onShowInfo: (group: ModelGroup) => void;
|
||||
downloadStatusMap?: Map<string, DownloadAvailability>;
|
||||
launchedAt?: number;
|
||||
instanceStatuses?: Record<string, { status: string; statusClass: string }>;
|
||||
};
|
||||
|
||||
let {
|
||||
group,
|
||||
isExpanded,
|
||||
isFavorite,
|
||||
isHighlighted = false,
|
||||
selectedModelId,
|
||||
canModelFit,
|
||||
getModelFitStatus,
|
||||
@@ -61,7 +56,6 @@
|
||||
onShowInfo,
|
||||
downloadStatusMap,
|
||||
launchedAt,
|
||||
instanceStatuses = {},
|
||||
}: ModelPickerGroupProps = $props();
|
||||
|
||||
// Group-level download status: show if any variant is downloaded
|
||||
@@ -98,12 +92,6 @@
|
||||
const anyVariantFits = $derived(
|
||||
group.variants.some((v) => canModelFit(v.id)),
|
||||
);
|
||||
// Check if any variant has an active instance (ready, loading, downloading)
|
||||
const anyVariantHasInstance = $derived(
|
||||
instanceStatuses
|
||||
? group.variants.some((v) => instanceStatuses[v.id] != null)
|
||||
: false,
|
||||
);
|
||||
const groupFitStatus = $derived.by((): ModelFitStatus => {
|
||||
let hasClusterCapacityOnly = false;
|
||||
for (const variant of group.variants) {
|
||||
@@ -134,36 +122,16 @@
|
||||
!group.hasMultipleVariants &&
|
||||
group.variants.some((v) => v.id === selectedModelId),
|
||||
);
|
||||
|
||||
// Group-level instance status: show the "best" status across all variants
|
||||
const groupInstanceStatus = $derived.by(() => {
|
||||
if (!instanceStatuses) return null;
|
||||
const readyStatuses = ["READY", "LOADED", "RUNNING"];
|
||||
const loadingStatuses = ["LOADING", "WARMING UP"];
|
||||
let bestStatus: { status: string; statusClass: string } | null = null;
|
||||
for (const variant of group.variants) {
|
||||
const s = instanceStatuses[variant.id];
|
||||
if (!s) continue;
|
||||
if (readyStatuses.includes(s.status)) return s; // Ready is best
|
||||
if (loadingStatuses.includes(s.status) || s.status === "DOWNLOADING") {
|
||||
bestStatus = s;
|
||||
}
|
||||
}
|
||||
return bestStatus;
|
||||
});
|
||||
</script>
|
||||
|
||||
<div
|
||||
data-model-ids={group.variants.map((v) => v.id).join(" ")}
|
||||
class="border-b border-white/5 last:border-b-0 {!anyVariantFits &&
|
||||
!anyVariantHasInstance
|
||||
class="border-b border-white/5 last:border-b-0 {!anyVariantFits
|
||||
? 'opacity-50'
|
||||
: ''} {isHighlighted ? 'model-just-added' : ''}"
|
||||
: ''}"
|
||||
>
|
||||
<!-- Main row -->
|
||||
<div
|
||||
class="flex items-center gap-2 px-3 py-2.5 transition-colors {anyVariantFits ||
|
||||
anyVariantHasInstance
|
||||
class="flex items-center gap-2 px-3 py-2.5 transition-colors {anyVariantFits
|
||||
? 'hover:bg-white/5 cursor-pointer'
|
||||
: 'cursor-not-allowed'} {isMainSelected
|
||||
? 'bg-exo-yellow/10 border-l-2 border-exo-yellow'
|
||||
@@ -173,7 +141,7 @@
|
||||
onToggleExpand();
|
||||
} else {
|
||||
const modelId = group.variants[0]?.id;
|
||||
if (modelId && (canModelFit(modelId) || instanceStatuses[modelId])) {
|
||||
if (modelId && canModelFit(modelId)) {
|
||||
onSelectModel(modelId);
|
||||
}
|
||||
}
|
||||
@@ -187,7 +155,7 @@
|
||||
onToggleExpand();
|
||||
} else {
|
||||
const modelId = group.variants[0]?.id;
|
||||
if (modelId && (canModelFit(modelId) || instanceStatuses[modelId])) {
|
||||
if (modelId && canModelFit(modelId)) {
|
||||
onSelectModel(modelId);
|
||||
}
|
||||
}
|
||||
@@ -215,14 +183,6 @@
|
||||
<span class="font-mono text-sm text-white truncate">
|
||||
{group.name}
|
||||
</span>
|
||||
{#if group.requiresVllm}
|
||||
<span
|
||||
class="text-[10px] font-mono px-1.5 py-0.5 rounded bg-orange-500/15 text-orange-300 border border-orange-400/30 flex-shrink-0 tracking-wider uppercase"
|
||||
title="Requires vLLM runtime"
|
||||
>
|
||||
vLLM
|
||||
</span>
|
||||
{/if}
|
||||
<!-- Capability icons -->
|
||||
{#each group.capabilities.filter((c) => c !== "text") as cap}
|
||||
{#if cap === "thinking"}
|
||||
@@ -386,47 +346,6 @@
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
<!-- Instance status badge -->
|
||||
{#if groupInstanceStatus}
|
||||
{#if groupInstanceStatus.status === "READY" || groupInstanceStatus.status === "LOADED" || groupInstanceStatus.status === "RUNNING"}
|
||||
<span class="flex-shrink-0" title="Running">
|
||||
<svg
|
||||
class="w-3 h-3 text-green-400"
|
||||
viewBox="0 0 12 12"
|
||||
fill="currentColor"
|
||||
>
|
||||
<circle cx="6" cy="6" r="5" />
|
||||
</svg>
|
||||
</span>
|
||||
{:else if groupInstanceStatus.status === "DOWNLOADING"}
|
||||
<span class="flex-shrink-0 animate-pulse" title="Downloading">
|
||||
<svg
|
||||
class="w-3.5 h-3.5 text-blue-400"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path d="M21 15v4a2 2 0 01-2 2H5a2 2 0 01-2-2v-4" />
|
||||
<polyline points="7 10 12 15 17 10" />
|
||||
<line x1="12" y1="15" x2="12" y2="3" />
|
||||
</svg>
|
||||
</span>
|
||||
{:else if groupInstanceStatus.status === "LOADING" || groupInstanceStatus.status === "WARMING UP"}
|
||||
<span class="flex-shrink-0 animate-pulse" title="Loading">
|
||||
<svg
|
||||
class="w-3 h-3 text-yellow-400"
|
||||
viewBox="0 0 12 12"
|
||||
fill="currentColor"
|
||||
>
|
||||
<circle cx="6" cy="6" r="5" />
|
||||
</svg>
|
||||
</span>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<!-- Check mark if selected (single-variant) -->
|
||||
{#if isMainSelected}
|
||||
<svg
|
||||
@@ -501,30 +420,20 @@
|
||||
{#each group.variants as variant}
|
||||
{@const fitStatus = getModelFitStatus(variant.id)}
|
||||
{@const modelCanFit = canModelFit(variant.id)}
|
||||
{@const variantHasInstance = instanceStatuses[variant.id] != null}
|
||||
{@const isSelected = selectedModelId === variant.id}
|
||||
<div
|
||||
class="w-full flex items-center gap-3 px-3 py-2 pl-10 hover:bg-white/5 transition-colors text-left {!modelCanFit &&
|
||||
!variantHasInstance
|
||||
<button
|
||||
type="button"
|
||||
class="w-full flex items-center gap-3 px-3 py-2 pl-10 hover:bg-white/5 transition-colors text-left {!modelCanFit
|
||||
? 'opacity-50 cursor-not-allowed'
|
||||
: 'cursor-pointer'} {isSelected
|
||||
? 'bg-exo-yellow/10 border-l-2 border-exo-yellow'
|
||||
: 'border-l-2 border-transparent'}"
|
||||
role="button"
|
||||
tabindex="0"
|
||||
disabled={!modelCanFit}
|
||||
onclick={() => {
|
||||
if (modelCanFit || variantHasInstance) {
|
||||
if (modelCanFit) {
|
||||
onSelectModel(variant.id);
|
||||
}
|
||||
}}
|
||||
onkeydown={(e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
if (modelCanFit) {
|
||||
onSelectModel(variant.id);
|
||||
}
|
||||
}
|
||||
}}
|
||||
>
|
||||
<!-- Quantization badge -->
|
||||
<span
|
||||
@@ -533,15 +442,6 @@
|
||||
{variant.quantization || "default"}
|
||||
</span>
|
||||
|
||||
{#if variant.requires_vllm}
|
||||
<span
|
||||
class="text-[10px] font-mono px-1.5 py-0.5 rounded bg-orange-500/15 text-orange-300 border border-orange-400/30 flex-shrink-0 tracking-wider uppercase"
|
||||
title="Requires vLLM runtime"
|
||||
>
|
||||
vLLM
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
<!-- Size -->
|
||||
<span
|
||||
class="text-xs font-mono flex-1 {getSizeClassForFitStatus(
|
||||
@@ -578,48 +478,6 @@
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<!-- Instance status badge -->
|
||||
{#if instanceStatuses[variant.id]}
|
||||
{@const instStatus = instanceStatuses[variant.id]}
|
||||
{#if instStatus.status === "READY" || instStatus.status === "LOADED" || instStatus.status === "RUNNING"}
|
||||
<span class="flex-shrink-0" title="Running">
|
||||
<svg
|
||||
class="w-3 h-3 text-green-400"
|
||||
viewBox="0 0 12 12"
|
||||
fill="currentColor"
|
||||
>
|
||||
<circle cx="6" cy="6" r="5" />
|
||||
</svg>
|
||||
</span>
|
||||
{:else if instStatus.status === "DOWNLOADING"}
|
||||
<span class="flex-shrink-0 animate-pulse" title="Downloading">
|
||||
<svg
|
||||
class="w-3.5 h-3.5 text-blue-400"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
>
|
||||
<path d="M21 15v4a2 2 0 01-2 2H5a2 2 0 01-2-2v-4" />
|
||||
<polyline points="7 10 12 15 17 10" />
|
||||
<line x1="12" y1="15" x2="12" y2="3" />
|
||||
</svg>
|
||||
</span>
|
||||
{:else if instStatus.status === "LOADING" || instStatus.status === "WARMING UP"}
|
||||
<span class="flex-shrink-0 animate-pulse" title="Loading">
|
||||
<svg
|
||||
class="w-3 h-3 text-yellow-400"
|
||||
viewBox="0 0 12 12"
|
||||
fill="currentColor"
|
||||
>
|
||||
<circle cx="6" cy="6" r="5" />
|
||||
</svg>
|
||||
</span>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<!-- Check mark if selected -->
|
||||
{#if isSelected}
|
||||
<svg
|
||||
@@ -632,56 +490,8 @@
|
||||
/>
|
||||
</svg>
|
||||
{/if}
|
||||
|
||||
<!-- Info button -->
|
||||
<button
|
||||
type="button"
|
||||
class="p-1 rounded hover:bg-white/10 transition-colors flex-shrink-0"
|
||||
onclick={(e) => {
|
||||
e.stopPropagation();
|
||||
onShowInfo({
|
||||
id: variant.id,
|
||||
name: variant.name || variant.id,
|
||||
capabilities: group.capabilities,
|
||||
family: group.family,
|
||||
variants: [variant],
|
||||
smallestVariant: variant,
|
||||
hasMultipleVariants: false,
|
||||
requiresVllm: variant.requires_vllm === true,
|
||||
});
|
||||
}}
|
||||
title="View variant details"
|
||||
>
|
||||
<svg
|
||||
class="w-4 h-4 text-white/30 hover:text-white/50"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path
|
||||
d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm1 15h-2v-6h2v6zm0-8h-2V7h2v2z"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.model-just-added {
|
||||
animation: highlightFade 4s ease-out forwards;
|
||||
}
|
||||
|
||||
@keyframes highlightFade {
|
||||
0%,
|
||||
40% {
|
||||
background-color: rgba(20, 83, 45, 0.25);
|
||||
box-shadow: inset 0 0 0 1px rgba(74, 222, 128, 0.4);
|
||||
}
|
||||
100% {
|
||||
background-color: transparent;
|
||||
box-shadow: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -1,5 +1,4 @@
|
||||
<script lang="ts">
|
||||
import { tick } from "svelte";
|
||||
import { fade, fly } from "svelte/transition";
|
||||
import { cubicOut } from "svelte/easing";
|
||||
import FamilySidebar from "./FamilySidebar.svelte";
|
||||
@@ -8,7 +7,6 @@
|
||||
import HuggingFaceResultItem from "./HuggingFaceResultItem.svelte";
|
||||
import { getNodesWithModelDownloaded } from "$lib/utils/downloads";
|
||||
import { getRecentEntries } from "$lib/stores/recents.svelte";
|
||||
import { addToast } from "$lib/stores/toast.svelte";
|
||||
|
||||
interface ModelInfo {
|
||||
id: string;
|
||||
@@ -22,7 +20,6 @@
|
||||
is_custom?: boolean;
|
||||
tasks?: string[];
|
||||
hugging_face_id?: string;
|
||||
requires_vllm?: boolean;
|
||||
}
|
||||
|
||||
interface ModelGroup {
|
||||
@@ -33,14 +30,12 @@
|
||||
variants: ModelInfo[];
|
||||
smallestVariant: ModelInfo;
|
||||
hasMultipleVariants: boolean;
|
||||
requiresVllm: boolean;
|
||||
}
|
||||
|
||||
interface FilterState {
|
||||
capabilities: string[];
|
||||
sizeRange: { min: number; max: number } | null;
|
||||
downloadedOnly: boolean;
|
||||
readyOnly: boolean;
|
||||
}
|
||||
|
||||
interface HuggingFaceModel {
|
||||
@@ -54,11 +49,6 @@
|
||||
|
||||
type ModelFitStatus = "fits_now" | "fits_cluster_capacity" | "too_large";
|
||||
|
||||
export type InstanceStatus = {
|
||||
status: string;
|
||||
statusClass: string;
|
||||
};
|
||||
|
||||
type ModelPickerModalProps = {
|
||||
isOpen: boolean;
|
||||
models: ModelInfo[];
|
||||
@@ -85,7 +75,6 @@
|
||||
macmon_info?: { memory?: { ram_total?: number } };
|
||||
}
|
||||
>;
|
||||
instanceStatuses?: Record<string, InstanceStatus>;
|
||||
};
|
||||
|
||||
let {
|
||||
@@ -107,7 +96,6 @@
|
||||
usedMemoryGB,
|
||||
downloadsData,
|
||||
topologyNodes,
|
||||
instanceStatuses = {},
|
||||
}: ModelPickerModalProps = $props();
|
||||
|
||||
// Local state
|
||||
@@ -119,7 +107,6 @@
|
||||
capabilities: [],
|
||||
sizeRange: null,
|
||||
downloadedOnly: false,
|
||||
readyOnly: false,
|
||||
});
|
||||
let infoGroup = $state<ModelGroup | null>(null);
|
||||
|
||||
@@ -195,13 +182,6 @@
|
||||
let hfSearchDebounceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let manualModelId = $state("");
|
||||
let addModelError = $state<string | null>(null);
|
||||
let justAddedModelId = $state<string | null>(null);
|
||||
let justAddedTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
// Inline HuggingFace search in main search bar
|
||||
let mainSearchHfResults = $state<HuggingFaceModel[]>([]);
|
||||
let mainSearchHfLoading = $state(false);
|
||||
let mainSearchDebounceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
// Reset transient state when modal opens, but preserve tab selection
|
||||
$effect(() => {
|
||||
@@ -211,11 +191,6 @@
|
||||
showFilters = false;
|
||||
manualModelId = "";
|
||||
addModelError = null;
|
||||
justAddedModelId = null;
|
||||
if (justAddedTimer) {
|
||||
clearTimeout(justAddedTimer);
|
||||
justAddedTimer = null;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -230,50 +205,6 @@
|
||||
}
|
||||
});
|
||||
|
||||
// Inline HuggingFace search when local search returns no results
|
||||
$effect(() => {
|
||||
const query = searchQuery.trim();
|
||||
const noLocalResults = filteredGroups.length === 0;
|
||||
|
||||
if (mainSearchDebounceTimer) {
|
||||
clearTimeout(mainSearchDebounceTimer);
|
||||
mainSearchDebounceTimer = null;
|
||||
}
|
||||
|
||||
if (
|
||||
selectedFamily === "huggingface" ||
|
||||
selectedFamily === "recents" ||
|
||||
selectedFamily === "favorites" ||
|
||||
query.length < 2 ||
|
||||
!noLocalResults
|
||||
) {
|
||||
mainSearchHfResults = [];
|
||||
mainSearchHfLoading = false;
|
||||
return;
|
||||
}
|
||||
|
||||
mainSearchHfLoading = true;
|
||||
mainSearchDebounceTimer = setTimeout(async () => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/models/search?query=${encodeURIComponent(query)}&limit=10`,
|
||||
);
|
||||
if (response.ok) {
|
||||
const results: HuggingFaceModel[] = await response.json();
|
||||
mainSearchHfResults = results.filter(
|
||||
(r) => !existingModelIds.has(r.id),
|
||||
);
|
||||
} else {
|
||||
mainSearchHfResults = [];
|
||||
}
|
||||
} catch {
|
||||
mainSearchHfResults = [];
|
||||
} finally {
|
||||
mainSearchHfLoading = false;
|
||||
}
|
||||
}, 500);
|
||||
});
|
||||
|
||||
async function fetchTrendingModels() {
|
||||
hfIsLoadingTrending = true;
|
||||
try {
|
||||
@@ -334,24 +265,6 @@
|
||||
addModelError = null;
|
||||
try {
|
||||
await onAddModel(modelId);
|
||||
// Success: show toast, switch to All Models, highlight the model
|
||||
const shortName = modelId.split("/").pop() || modelId;
|
||||
addToast({ type: "success", message: `Added ${shortName}` });
|
||||
justAddedModelId = modelId;
|
||||
selectedFamily = null;
|
||||
searchQuery = "";
|
||||
// Scroll to the newly added model after DOM update
|
||||
await tick();
|
||||
const el = document.querySelector(
|
||||
`[data-model-ids~="${CSS.escape(modelId)}"]`,
|
||||
);
|
||||
el?.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
// Clear highlight after 4 seconds
|
||||
if (justAddedTimer) clearTimeout(justAddedTimer);
|
||||
justAddedTimer = setTimeout(() => {
|
||||
justAddedModelId = null;
|
||||
justAddedTimer = null;
|
||||
}, 4000);
|
||||
} catch (error) {
|
||||
addModelError =
|
||||
error instanceof Error ? error.message : "Failed to add model";
|
||||
@@ -398,7 +311,6 @@
|
||||
variants: [],
|
||||
smallestVariant: model,
|
||||
hasMultipleVariants: false,
|
||||
requiresVllm: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -433,7 +345,6 @@
|
||||
(a.storage_size_megabytes || 0) - (b.storage_size_megabytes || 0),
|
||||
);
|
||||
group.hasMultipleVariants = group.variants.length > 1;
|
||||
group.requiresVllm = group.variants.every((v) => v.requires_vllm);
|
||||
}
|
||||
|
||||
// Convert to array and sort by smallest variant size (biggest first)
|
||||
@@ -461,10 +372,8 @@
|
||||
"deepseek",
|
||||
"gpt-oss",
|
||||
"llama",
|
||||
"gemma",
|
||||
"flux",
|
||||
"qwen-image",
|
||||
"nemotron",
|
||||
];
|
||||
return Array.from(families).sort((a, b) => {
|
||||
const aIdx = familyOrder.indexOf(a);
|
||||
@@ -531,16 +440,6 @@
|
||||
);
|
||||
}
|
||||
|
||||
// Filter to ready/running models only
|
||||
if (filters.readyOnly) {
|
||||
result = result.filter((g) =>
|
||||
g.variants.some((v) => {
|
||||
const s = instanceStatuses[v.id];
|
||||
return s && s.statusClass === "ready";
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Sort: fits-now first, then fits-cluster-capacity, then too-large
|
||||
result.sort((a, b) => {
|
||||
const getGroupFitRank = (group: ModelGroup): number => {
|
||||
@@ -591,7 +490,6 @@
|
||||
variants: [model],
|
||||
smallestVariant: model,
|
||||
hasMultipleVariants: false,
|
||||
requiresVllm: model.requires_vllm === true,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -614,18 +512,6 @@
|
||||
);
|
||||
});
|
||||
|
||||
// Split filtered groups into recommended (fits_now) and others for visual separation
|
||||
const recommendedGroups = $derived(
|
||||
filteredGroups.filter((g) =>
|
||||
g.variants.some((v) => getModelFitStatus(v.id) === "fits_now"),
|
||||
),
|
||||
);
|
||||
const otherGroups = $derived(
|
||||
filteredGroups.filter(
|
||||
(g) => !g.variants.some((v) => getModelFitStatus(v.id) === "fits_now"),
|
||||
),
|
||||
);
|
||||
|
||||
function toggleGroupExpanded(groupId: string) {
|
||||
const next = new Set(expandedGroups);
|
||||
if (next.has(groupId)) {
|
||||
@@ -652,19 +538,13 @@
|
||||
}
|
||||
|
||||
function clearFilters() {
|
||||
filters = {
|
||||
capabilities: [],
|
||||
sizeRange: null,
|
||||
downloadedOnly: false,
|
||||
readyOnly: false,
|
||||
};
|
||||
filters = { capabilities: [], sizeRange: null, downloadedOnly: false };
|
||||
}
|
||||
|
||||
const hasActiveFilters = $derived(
|
||||
filters.capabilities.length > 0 ||
|
||||
filters.sizeRange !== null ||
|
||||
filters.downloadedOnly ||
|
||||
filters.readyOnly,
|
||||
filters.downloadedOnly,
|
||||
);
|
||||
</script>
|
||||
|
||||
@@ -924,8 +804,6 @@
|
||||
{group}
|
||||
isExpanded={expandedGroups.has(group.id)}
|
||||
isFavorite={favorites.has(group.id)}
|
||||
isHighlighted={justAddedModelId !== null &&
|
||||
group.variants.some((v) => v.id === justAddedModelId)}
|
||||
{selectedModelId}
|
||||
{canModelFit}
|
||||
{getModelFitStatus}
|
||||
@@ -935,7 +813,6 @@
|
||||
onShowInfo={(g) => (infoGroup = g)}
|
||||
downloadStatusMap={getVariantDownloadMap(group)}
|
||||
launchedAt={recentTimestamps.get(group.variants[0]?.id ?? "")}
|
||||
{instanceStatuses}
|
||||
/>
|
||||
{/each}
|
||||
{/if}
|
||||
@@ -963,40 +840,11 @@
|
||||
{/if}
|
||||
</div>
|
||||
{:else}
|
||||
<!-- Recommended for your cluster -->
|
||||
{#if recommendedGroups.length > 0 && otherGroups.length > 0 && !searchQuery.trim()}
|
||||
<div
|
||||
class="sticky top-0 z-10 flex items-center gap-2 px-3 py-2 bg-green-950/60 border-b border-green-500/20 backdrop-blur-sm"
|
||||
>
|
||||
<svg
|
||||
class="w-3.5 h-3.5 text-green-400 flex-shrink-0"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M9 12.75L11.25 15 15 9.75M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
|
||||
/>
|
||||
</svg>
|
||||
<span
|
||||
class="text-xs font-mono text-green-400 tracking-wider uppercase"
|
||||
>Recommended for your cluster</span
|
||||
>
|
||||
<span class="text-xs font-mono text-green-400/50"
|
||||
>— fits in available memory</span
|
||||
>
|
||||
</div>
|
||||
{/if}
|
||||
{#each recommendedGroups as group}
|
||||
{#each filteredGroups as group}
|
||||
<ModelPickerGroup
|
||||
{group}
|
||||
isExpanded={expandedGroups.has(group.id)}
|
||||
isFavorite={favorites.has(group.id)}
|
||||
isHighlighted={justAddedModelId !== null &&
|
||||
group.variants.some((v) => v.id === justAddedModelId)}
|
||||
{selectedModelId}
|
||||
{canModelFit}
|
||||
{getModelFitStatus}
|
||||
@@ -1005,87 +853,8 @@
|
||||
{onToggleFavorite}
|
||||
onShowInfo={(g) => (infoGroup = g)}
|
||||
downloadStatusMap={getVariantDownloadMap(group)}
|
||||
{instanceStatuses}
|
||||
/>
|
||||
{/each}
|
||||
<!-- Other models -->
|
||||
{#if otherGroups.length > 0 && recommendedGroups.length > 0 && !searchQuery.trim()}
|
||||
<div
|
||||
class="sticky top-0 z-10 flex items-center gap-2 px-3 py-2 bg-exo-dark-gray/80 border-y border-exo-medium-gray/20 backdrop-blur-sm"
|
||||
>
|
||||
<span
|
||||
class="text-xs font-mono text-white/40 tracking-wider uppercase"
|
||||
>Other models</span
|
||||
>
|
||||
</div>
|
||||
{/if}
|
||||
{#each otherGroups as group}
|
||||
<ModelPickerGroup
|
||||
{group}
|
||||
isExpanded={expandedGroups.has(group.id)}
|
||||
isFavorite={favorites.has(group.id)}
|
||||
isHighlighted={justAddedModelId !== null &&
|
||||
group.variants.some((v) => v.id === justAddedModelId)}
|
||||
{selectedModelId}
|
||||
{canModelFit}
|
||||
{getModelFitStatus}
|
||||
onToggleExpand={() => toggleGroupExpanded(group.id)}
|
||||
onSelectModel={handleSelect}
|
||||
{onToggleFavorite}
|
||||
onShowInfo={(g) => (infoGroup = g)}
|
||||
downloadStatusMap={getVariantDownloadMap(group)}
|
||||
{instanceStatuses}
|
||||
/>
|
||||
{/each}
|
||||
<!-- Inline HuggingFace search results (shown when no local results match) -->
|
||||
{#if filteredGroups.length === 0 && searchQuery.trim().length >= 2 && selectedFamily !== "huggingface" && selectedFamily !== "recents" && selectedFamily !== "favorites"}
|
||||
{#if mainSearchHfLoading}
|
||||
<div
|
||||
class="flex items-center gap-2 px-3 py-2 border-t border-orange-400/20 bg-orange-950/20"
|
||||
>
|
||||
<span
|
||||
class="w-4 h-4 border-2 border-orange-400 border-t-transparent rounded-full animate-spin"
|
||||
></span>
|
||||
<span class="text-xs font-mono text-orange-400/60"
|
||||
>Searching HuggingFace...</span
|
||||
>
|
||||
</div>
|
||||
{:else if mainSearchHfResults.length > 0}
|
||||
<div
|
||||
class="sticky top-0 z-10 flex items-center gap-2 px-3 py-2 bg-orange-950/30 border-y border-orange-400/20 backdrop-blur-sm"
|
||||
>
|
||||
<span
|
||||
class="text-xs font-mono text-orange-400 tracking-wider uppercase"
|
||||
>From HuggingFace</span
|
||||
>
|
||||
</div>
|
||||
{#each mainSearchHfResults as model}
|
||||
<HuggingFaceResultItem
|
||||
{model}
|
||||
isAdded={existingModelIds.has(model.id)}
|
||||
isAdding={addingModelId === model.id}
|
||||
onAdd={() => handleAddModel(model.id)}
|
||||
onSelect={() => handleSelectHfModel(model.id)}
|
||||
downloadedOnNodes={downloadsData
|
||||
? getNodesWithModelDownloaded(downloadsData, model.id).map(
|
||||
getNodeName,
|
||||
)
|
||||
: []}
|
||||
/>
|
||||
{/each}
|
||||
<button
|
||||
type="button"
|
||||
class="w-full px-3 py-2 text-xs font-mono text-orange-400/60 hover:text-orange-400 hover:bg-orange-500/10 transition-colors text-center"
|
||||
onclick={() => {
|
||||
hfSearchQuery = searchQuery;
|
||||
searchHuggingFace(searchQuery);
|
||||
selectedFamily = "huggingface";
|
||||
}}
|
||||
>
|
||||
See all results on Hub
|
||||
</button>
|
||||
{/if}
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
@@ -1106,11 +875,6 @@
|
||||
>Downloaded</span
|
||||
>
|
||||
{/if}
|
||||
{#if filters.readyOnly}
|
||||
<span class="px-1.5 py-0.5 bg-green-500/20 text-green-400 rounded"
|
||||
>Ready</span
|
||||
>
|
||||
{/if}
|
||||
{#if filters.sizeRange}
|
||||
<span class="px-1.5 py-0.5 bg-exo-yellow/20 text-exo-yellow rounded">
|
||||
{filters.sizeRange.min}GB - {filters.sizeRange.max}GB
|
||||
@@ -1170,17 +934,6 @@
|
||||
<span class="text-white/40">Variants:</span>
|
||||
<span class="text-white/70">{infoGroup.variants.length}</span>
|
||||
</div>
|
||||
{#if infoGroup.requiresVllm}
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-white/40">Runtime:</span>
|
||||
<span
|
||||
class="text-[10px] font-mono px-1.5 py-0.5 rounded bg-orange-500/15 text-orange-300 border border-orange-400/30 tracking-wider uppercase"
|
||||
>
|
||||
vLLM
|
||||
</span>
|
||||
<span class="text-white/40 text-[11px]">required</span>
|
||||
</div>
|
||||
{/if}
|
||||
{#if infoGroup.variants.length > 0}
|
||||
<div class="mt-3 pt-3 border-t border-exo-yellow/10">
|
||||
<span class="text-white/40">Available quantizations:</span>
|
||||
|
||||
@@ -1,565 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from "svelte";
|
||||
import FamilyLogos from "$lib/components/FamilyLogos.svelte";
|
||||
import {
|
||||
instances,
|
||||
instanceLinks,
|
||||
nodeIdentities,
|
||||
refreshState,
|
||||
createInstanceLink,
|
||||
updateInstanceLink,
|
||||
deleteInstanceLink,
|
||||
type Instance,
|
||||
} from "$lib/stores/app.svelte";
|
||||
import { deriveBaseModel, deriveFamily } from "$lib/utils/model_family";
|
||||
|
||||
type InstanceWrapper = {
|
||||
MlxRingInstance?: Instance;
|
||||
MlxJacclInstance?: Instance;
|
||||
VllmInstance?: Instance;
|
||||
};
|
||||
|
||||
let interval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
onMount(() => {
|
||||
refreshState();
|
||||
interval = setInterval(refreshState, 3000);
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (interval) clearInterval(interval);
|
||||
});
|
||||
|
||||
type InstanceRow = {
|
||||
id: string;
|
||||
modelId: string;
|
||||
family: string;
|
||||
baseModel: string;
|
||||
nodeNames: string[];
|
||||
nodeCount: number;
|
||||
};
|
||||
|
||||
const instanceRows = $derived.by<InstanceRow[]>(() => {
|
||||
const rows: InstanceRow[] = [];
|
||||
const ids = nodeIdentities();
|
||||
for (const [id, raw] of Object.entries(instances())) {
|
||||
const wrapper = raw as InstanceWrapper;
|
||||
const inst =
|
||||
wrapper.MlxRingInstance ??
|
||||
wrapper.MlxJacclInstance ??
|
||||
wrapper.VllmInstance;
|
||||
const modelId = inst?.shardAssignments?.modelId ?? "";
|
||||
const nodeToRunner = inst?.shardAssignments?.nodeToRunner ?? {};
|
||||
const nodeIds = Object.keys(nodeToRunner);
|
||||
const nodeNames = nodeIds
|
||||
.map((nodeId) => ids[nodeId]?.friendlyName ?? nodeId.slice(0, 6))
|
||||
.filter((name) => !!name);
|
||||
rows.push({
|
||||
id,
|
||||
modelId,
|
||||
family: deriveFamily(modelId),
|
||||
baseModel: deriveBaseModel(modelId),
|
||||
nodeNames,
|
||||
nodeCount: nodeIds.length,
|
||||
});
|
||||
}
|
||||
rows.sort((a, b) => a.modelId.localeCompare(b.modelId));
|
||||
return rows;
|
||||
});
|
||||
|
||||
const instanceById = $derived(
|
||||
Object.fromEntries(instanceRows.map((r) => [r.id, r])),
|
||||
);
|
||||
|
||||
type LinkRow = {
|
||||
linkId: string;
|
||||
prefill: string[];
|
||||
decode: string[];
|
||||
families: string[];
|
||||
multiNode: boolean;
|
||||
};
|
||||
|
||||
const linkRows = $derived.by<LinkRow[]>(() => {
|
||||
const rows: LinkRow[] = [];
|
||||
for (const [, link] of Object.entries(instanceLinks())) {
|
||||
const fams = new Set<string>();
|
||||
let multiNode = false;
|
||||
for (const id of [...link.prefillInstances, ...link.decodeInstances]) {
|
||||
const r = instanceById[id];
|
||||
if (r && r.baseModel) fams.add(r.baseModel.toLowerCase());
|
||||
if (r && r.nodeCount > 1) multiNode = true;
|
||||
}
|
||||
rows.push({
|
||||
linkId: link.linkId,
|
||||
prefill: link.prefillInstances,
|
||||
decode: link.decodeInstances,
|
||||
families: Array.from(fams),
|
||||
multiNode,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
|
||||
let editingLinkId = $state<string | null>(null);
|
||||
let editingPrefill = $state<Set<string>>(new Set());
|
||||
let editingDecode = $state<Set<string>>(new Set());
|
||||
let saving = $state(false);
|
||||
let errorMessage = $state<string | null>(null);
|
||||
|
||||
function startCreate() {
|
||||
editingLinkId = "new";
|
||||
editingPrefill = new Set();
|
||||
editingDecode = new Set();
|
||||
errorMessage = null;
|
||||
}
|
||||
|
||||
function startEdit(row: LinkRow) {
|
||||
editingLinkId = row.linkId;
|
||||
editingPrefill = new Set(row.prefill);
|
||||
editingDecode = new Set(row.decode);
|
||||
errorMessage = null;
|
||||
}
|
||||
|
||||
function cancelEdit() {
|
||||
editingLinkId = null;
|
||||
editingPrefill = new Set();
|
||||
editingDecode = new Set();
|
||||
errorMessage = null;
|
||||
}
|
||||
|
||||
type Role = "prefill" | "decode" | "none";
|
||||
|
||||
function roleOf(id: string): Role {
|
||||
if (editingPrefill.has(id)) return "prefill";
|
||||
if (editingDecode.has(id)) return "decode";
|
||||
return "none";
|
||||
}
|
||||
|
||||
function setRole(id: string, role: Role) {
|
||||
const p = new Set(editingPrefill);
|
||||
const d = new Set(editingDecode);
|
||||
p.delete(id);
|
||||
d.delete(id);
|
||||
if (role === "prefill") p.add(id);
|
||||
if (role === "decode") d.add(id);
|
||||
editingPrefill = p;
|
||||
editingDecode = d;
|
||||
}
|
||||
|
||||
const editingFamilies = $derived.by<string[]>(() => {
|
||||
const fams = new Set<string>();
|
||||
for (const id of [...editingPrefill, ...editingDecode]) {
|
||||
const r = instanceById[id];
|
||||
if (r && r.baseModel) fams.add(r.baseModel.toLowerCase());
|
||||
}
|
||||
return Array.from(fams);
|
||||
});
|
||||
|
||||
const editingMultiNode = $derived.by<string[]>(() => {
|
||||
const names: string[] = [];
|
||||
for (const id of [...editingPrefill, ...editingDecode]) {
|
||||
const r = instanceById[id];
|
||||
if (r && r.nodeCount > 1) {
|
||||
names.push(r.baseModel || r.modelId);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
});
|
||||
|
||||
const editingMismatch = $derived(editingFamilies.length > 1);
|
||||
const canSave = $derived(
|
||||
editingLinkId !== null &&
|
||||
editingPrefill.size > 0 &&
|
||||
editingDecode.size > 0 &&
|
||||
!saving,
|
||||
);
|
||||
|
||||
async function save() {
|
||||
if (editingLinkId === null) return;
|
||||
saving = true;
|
||||
errorMessage = null;
|
||||
try {
|
||||
const prefill = Array.from(editingPrefill);
|
||||
const decode = Array.from(editingDecode);
|
||||
if (editingLinkId === "new") {
|
||||
await createInstanceLink(prefill, decode);
|
||||
} else {
|
||||
await updateInstanceLink(editingLinkId, prefill, decode);
|
||||
}
|
||||
cancelEdit();
|
||||
await refreshState();
|
||||
} catch (err) {
|
||||
errorMessage = err instanceof Error ? err.message : String(err);
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function remove(linkId: string) {
|
||||
if (!confirm("Remove this routing?")) return;
|
||||
try {
|
||||
await deleteInstanceLink(linkId);
|
||||
if (editingLinkId === linkId) cancelEdit();
|
||||
await refreshState();
|
||||
} catch (err) {
|
||||
errorMessage = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="font-mono text-foreground">
|
||||
<div class="mb-6 space-y-4">
|
||||
<details open class="group [&_summary::-webkit-details-marker]:hidden">
|
||||
<summary
|
||||
class="cursor-pointer list-none text-exo-yellow text-xs font-mono tracking-widest uppercase flex items-center gap-2 hover:opacity-80 transition-opacity"
|
||||
>
|
||||
<span
|
||||
class="inline-block transition-transform group-open:rotate-90 text-exo-light-gray"
|
||||
>▶</span
|
||||
>
|
||||
Prefill vs Decode
|
||||
</summary>
|
||||
<div class="mt-2 text-white/80 text-sm leading-relaxed">
|
||||
Prefill is the compute-bound pass that consumes the entire prompt and
|
||||
builds a KV cache. Decode is the memory-bandwidth-bound loop that emits
|
||||
tokens sequentially from that cache. The two phases have very different
|
||||
bottlenecks, so running them on different hardware can be substantially
|
||||
faster than doing both on one node.
|
||||
</div>
|
||||
</details>
|
||||
<details class="group [&_summary::-webkit-details-marker]:hidden">
|
||||
<summary
|
||||
class="cursor-pointer list-none text-exo-yellow text-xs font-mono tracking-widest uppercase flex items-center gap-2 hover:opacity-80 transition-opacity"
|
||||
>
|
||||
<span
|
||||
class="inline-block transition-transform group-open:rotate-90 text-exo-light-gray"
|
||||
>▶</span
|
||||
>
|
||||
Linking Instances
|
||||
</summary>
|
||||
<div class="mt-2 text-white/80 text-sm leading-relaxed space-y-2">
|
||||
<p>
|
||||
A linked route here tells the cluster: when a request is sent to a
|
||||
model in that cluster, the decode node (or the least active one if
|
||||
there are multiple) will handle it. If it decides it must do a lot of
|
||||
prefill not already cached in the prefix cache, it routes the request
|
||||
to the prefill node over TCP IP. The prefill node streams the KV cache
|
||||
back to the decode node which picks up from there.
|
||||
</p>
|
||||
<p>
|
||||
Linked instances must be running the same model family — KV layouts
|
||||
differ across architectures. More on the <a
|
||||
class="text-exo-yellow underline underline-offset-2 hover:text-exo-yellow-darker transition-colors"
|
||||
href="https://blog.exolabs.net/nvidia-dgx-spark/"
|
||||
target="_blank"
|
||||
rel="noreferrer noopener">blog</a
|
||||
>.
|
||||
</p>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
{#if errorMessage}
|
||||
<div
|
||||
class="mb-4 px-4 py-3 bg-red-500/10 border border-red-500/40 text-red-300 text-sm"
|
||||
>
|
||||
{errorMessage}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<section class="mt-12">
|
||||
<h2
|
||||
class="text-exo-yellow text-xs font-mono tracking-widest uppercase m-0 mb-3"
|
||||
>
|
||||
Existing routes
|
||||
</h2>
|
||||
|
||||
{#if linkRows.length === 0}
|
||||
{#if editingLinkId === null}
|
||||
<div class="flex items-center justify-between">
|
||||
<p class="text-exo-light-gray italic text-sm m-0">
|
||||
No routes yet. Create one to enable remote prefill.
|
||||
</p>
|
||||
<button
|
||||
class="px-3 py-1.5 text-xs font-mono tracking-wider uppercase bg-exo-yellow/15 border border-exo-yellow/50 text-exo-yellow hover:bg-exo-yellow/25 hover:border-exo-yellow/80 transition-colors"
|
||||
onclick={startCreate}
|
||||
>
|
||||
+ New route
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
{#if editingLinkId === null}
|
||||
<div class="flex justify-end mb-3">
|
||||
<button
|
||||
class="px-3 py-1.5 text-xs font-mono tracking-wider uppercase bg-exo-yellow/15 border border-exo-yellow/50 text-exo-yellow hover:bg-exo-yellow/25 hover:border-exo-yellow/80 transition-colors"
|
||||
onclick={startCreate}
|
||||
>
|
||||
+ New route
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
<div
|
||||
class="bg-exo-dark-gray/60 border border-exo-medium-gray/40 flex flex-col"
|
||||
>
|
||||
{#each linkRows as row (row.linkId)}
|
||||
{#if editingLinkId !== row.linkId}
|
||||
<article
|
||||
class="p-4 border-b border-exo-light-gray/25 last:border-b-0"
|
||||
>
|
||||
{#if row.multiNode}
|
||||
<div
|
||||
class="mb-3 px-3 py-2 bg-red-500/10 border border-red-500/40 text-red-300 text-xs tracking-wide"
|
||||
>
|
||||
⚠ Multi-node instance detected. Remote prefill currently only
|
||||
works on single-node (rank-0) instances. This route will not
|
||||
function until that's supported.
|
||||
</div>
|
||||
{/if}
|
||||
{#if row.families.length > 1}
|
||||
<div
|
||||
class="mb-3 px-3 py-2 bg-amber-500/10 border border-amber-500/40 text-amber-300 text-xs tracking-wide"
|
||||
>
|
||||
⚠ Mixed model families: {row.families.join(", ")}
|
||||
</div>
|
||||
{/if}
|
||||
<div
|
||||
class="grid grid-cols-[1fr_auto_1fr_auto] items-center gap-x-3 gap-y-2"
|
||||
>
|
||||
<span
|
||||
class="inline-block justify-self-start text-[10px] font-mono tracking-widest uppercase px-2 py-0.5 bg-exo-yellow/15 border border-exo-yellow/40 text-exo-yellow"
|
||||
>Prefill</span
|
||||
>
|
||||
<span></span>
|
||||
<span
|
||||
class="inline-block justify-self-start text-[10px] font-mono tracking-widest uppercase px-2 py-0.5 bg-exo-medium-gray/40 border border-exo-medium-gray/60 text-foreground"
|
||||
>Decode</span
|
||||
>
|
||||
<span></span>
|
||||
<div class="min-w-0">
|
||||
<ul class="list-none p-0 m-0 flex flex-col gap-2">
|
||||
{#each row.prefill as id (id)}
|
||||
{@const r = instanceById[id]}
|
||||
{#if r}
|
||||
<li
|
||||
class="flex items-center gap-2 px-2.5 py-2 bg-exo-medium-gray/20 border border-exo-medium-gray/40"
|
||||
>
|
||||
<FamilyLogos family={r.family} />
|
||||
<div class="min-w-0 flex-1">
|
||||
<div
|
||||
class="text-exo-yellow text-xs font-mono truncate"
|
||||
>
|
||||
{r.baseModel || r.modelId}
|
||||
</div>
|
||||
<div
|
||||
class="text-exo-light-gray text-[11px] truncate"
|
||||
>
|
||||
{r.nodeNames.join(", ") || "?"}{r.nodeCount > 1
|
||||
? ` (${r.nodeCount} nodes)`
|
||||
: ""}
|
||||
</div>
|
||||
<div
|
||||
class="text-exo-light-gray/40 text-[10px] font-mono truncate"
|
||||
title={r.id}
|
||||
>
|
||||
{r.id.slice(0, 8)}
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
{/if}
|
||||
{/each}
|
||||
</ul>
|
||||
</div>
|
||||
<div class="text-exo-yellow/60 text-xl px-2" aria-hidden="true">
|
||||
→
|
||||
</div>
|
||||
<div class="min-w-0">
|
||||
<ul class="list-none p-0 m-0 flex flex-col gap-2">
|
||||
{#each row.decode as id (id)}
|
||||
{@const r = instanceById[id]}
|
||||
{#if r}
|
||||
<li
|
||||
class="flex items-center gap-2 px-2.5 py-2 bg-exo-medium-gray/20 border border-exo-medium-gray/40"
|
||||
>
|
||||
<FamilyLogos family={r.family} />
|
||||
<div class="min-w-0 flex-1">
|
||||
<div
|
||||
class="text-exo-yellow text-xs font-mono truncate"
|
||||
>
|
||||
{r.baseModel || r.modelId}
|
||||
</div>
|
||||
<div
|
||||
class="text-exo-light-gray text-[11px] truncate"
|
||||
>
|
||||
{r.nodeNames.join(", ") || "?"}{r.nodeCount > 1
|
||||
? ` (${r.nodeCount} nodes)`
|
||||
: ""}
|
||||
</div>
|
||||
<div
|
||||
class="text-exo-light-gray/40 text-[10px] font-mono truncate"
|
||||
title={r.id}
|
||||
>
|
||||
{r.id.slice(0, 8)}
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
{/if}
|
||||
{/each}
|
||||
</ul>
|
||||
</div>
|
||||
<div class="flex gap-2 pl-3">
|
||||
<button
|
||||
class="px-2 py-0.5 text-[11px] font-mono tracking-wider uppercase bg-exo-medium-gray/30 border border-exo-medium-gray/60 rounded text-foreground hover:border-exo-yellow/60 hover:text-exo-yellow disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
onclick={() => startEdit(row)}
|
||||
disabled={editingLinkId !== null}
|
||||
>
|
||||
Edit
|
||||
</button>
|
||||
<button
|
||||
class="px-2 py-0.5 text-[11px] font-mono tracking-wider uppercase bg-red-500/15 border border-red-500/40 rounded text-red-300 hover:bg-red-500/25 transition-colors"
|
||||
onclick={() => remove(row.linkId)}
|
||||
>
|
||||
Remove
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
{#if editingLinkId !== null && instanceRows.length === 0}
|
||||
<section
|
||||
class="mt-6 bg-exo-dark-gray/60 border border-exo-yellow/30 px-4 py-2.5 flex items-center justify-between gap-3"
|
||||
>
|
||||
<span class="text-exo-light-gray italic text-sm font-mono"
|
||||
>No instances available.</span
|
||||
>
|
||||
<button
|
||||
class="px-3 py-1 text-xs font-mono tracking-wider uppercase bg-exo-medium-gray/30 border border-exo-medium-gray/60 rounded text-foreground hover:border-exo-yellow/60 transition-colors"
|
||||
onclick={cancelEdit}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</section>
|
||||
{:else if editingLinkId !== null}
|
||||
<section class="mt-6 bg-exo-dark-gray/60 border border-exo-yellow/30 p-5">
|
||||
<h2
|
||||
class="text-exo-yellow text-xs font-mono tracking-widest uppercase m-0 mb-3"
|
||||
>
|
||||
{editingLinkId === "new" ? "New route" : "Edit route"}
|
||||
</h2>
|
||||
|
||||
{#if editingMismatch}
|
||||
<div
|
||||
class="mb-3 px-3 py-2 bg-amber-500/10 border border-amber-500/40 text-amber-300 text-xs tracking-wide"
|
||||
>
|
||||
⚠ Selected instances span multiple model families: <strong
|
||||
>{editingFamilies.join(", ")}</strong
|
||||
>. Linking across families produces a corrupt KV cache.
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if editingMultiNode.length > 0}
|
||||
<div
|
||||
class="mb-3 px-3 py-2 bg-red-500/10 border border-red-500/40 text-red-300 text-xs tracking-wide"
|
||||
>
|
||||
⚠ Multi-node instance(s) selected: <strong
|
||||
>{editingMultiNode.join(", ")}</strong
|
||||
>. Remote prefill currently only works on single-node instances. This
|
||||
route will not function until multi-node support lands.
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<p class="text-exo-light-gray text-xs mb-4">
|
||||
Pick a role for each instance:
|
||||
<span class="text-exo-yellow">Prefill</span>
|
||||
serves KV cache,
|
||||
<span class="text-foreground">Decode</span> consumes it.
|
||||
</p>
|
||||
<div
|
||||
class="grid gap-2.5"
|
||||
style="grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));"
|
||||
>
|
||||
{#each instanceRows as row (row.id)}
|
||||
{@const role = roleOf(row.id)}
|
||||
<div
|
||||
class="border p-3 flex flex-col gap-2.5 transition-colors {role ===
|
||||
'prefill'
|
||||
? 'border-exo-yellow/60 bg-exo-dark-gray/60'
|
||||
: role === 'decode'
|
||||
? 'border-exo-light-gray/60 bg-exo-dark-gray/60'
|
||||
: 'border-exo-medium-gray/40 bg-exo-dark-gray/40'}"
|
||||
>
|
||||
<div class="flex items-center gap-2">
|
||||
<FamilyLogos family={row.family} />
|
||||
<div class="min-w-0 flex-1">
|
||||
<div class="text-exo-yellow text-xs font-mono truncate">
|
||||
{row.baseModel || row.modelId}
|
||||
</div>
|
||||
<div class="text-exo-light-gray text-[11px] truncate">
|
||||
{row.nodeNames.join(", ") || "?"}{row.nodeCount > 1
|
||||
? ` (${row.nodeCount} nodes)`
|
||||
: ""}
|
||||
</div>
|
||||
<div
|
||||
class="text-exo-light-gray/40 text-[10px] font-mono truncate"
|
||||
title={row.id}
|
||||
>
|
||||
{row.id.slice(0, 8)}
|
||||
</div>
|
||||
</div>
|
||||
{#if row.nodeCount > 1}
|
||||
<span
|
||||
class="text-[9px] font-mono tracking-widest uppercase px-1.5 py-0.5 bg-red-500/15 border border-red-500/40 text-red-300"
|
||||
title="Multi-node instances are not supported by remote prefill yet."
|
||||
>Unsupported</span
|
||||
>
|
||||
{/if}
|
||||
</div>
|
||||
<div
|
||||
class="flex rounded-md overflow-hidden border border-exo-light-gray/40 divide-x divide-exo-light-gray/40"
|
||||
>
|
||||
<button
|
||||
class="flex-1 px-2 py-1 text-[11px] font-mono tracking-wider uppercase transition-colors {role ===
|
||||
'prefill'
|
||||
? 'bg-exo-yellow/20 text-exo-yellow'
|
||||
: 'bg-transparent text-white/80 hover:text-exo-yellow'}"
|
||||
onclick={() =>
|
||||
setRole(row.id, role === "prefill" ? "none" : "prefill")}
|
||||
>Prefill</button
|
||||
>
|
||||
<button
|
||||
class="flex-1 px-2 py-1 text-[11px] font-mono tracking-wider uppercase transition-colors {role ===
|
||||
'decode'
|
||||
? 'bg-exo-medium-gray/50 text-foreground'
|
||||
: 'bg-transparent text-white/80 hover:text-foreground'}"
|
||||
onclick={() =>
|
||||
setRole(row.id, role === "decode" ? "none" : "decode")}
|
||||
>Decode</button
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
<div class="flex gap-2 mt-5 justify-end">
|
||||
<button
|
||||
class="px-3 py-1.5 text-xs font-mono tracking-wider uppercase bg-exo-yellow/15 border border-exo-yellow/50 text-exo-yellow hover:bg-exo-yellow/25 hover:border-exo-yellow/80 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
onclick={save}
|
||||
disabled={!canSave}
|
||||
>
|
||||
{saving ? "Saving..." : "Save route"}
|
||||
</button>
|
||||
<button
|
||||
class="px-3 py-1.5 text-xs font-mono tracking-wider uppercase bg-exo-medium-gray/30 border border-exo-medium-gray/60 text-foreground hover:border-exo-yellow/60 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
onclick={cancelEdit}
|
||||
disabled={saving}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -14,21 +14,6 @@
|
||||
: 0,
|
||||
);
|
||||
|
||||
const etaText = $derived.by(() => {
|
||||
if (progress.processed <= 0 || progress.total <= 0) return null;
|
||||
const elapsedMs = performance.now() - progress.startedAt;
|
||||
if (elapsedMs < 200) return null; // need a minimum sample window
|
||||
const tokensPerMs = progress.processed / elapsedMs;
|
||||
const remainingTokens = progress.total - progress.processed;
|
||||
const remainingMs = remainingTokens / tokensPerMs;
|
||||
const remainingSec = Math.ceil(remainingMs / 1000);
|
||||
if (remainingSec <= 0) return null;
|
||||
if (remainingSec < 60) return `~${remainingSec}s remaining`;
|
||||
const mins = Math.floor(remainingSec / 60);
|
||||
const secs = remainingSec % 60;
|
||||
return `~${mins}m ${secs}s remaining`;
|
||||
});
|
||||
|
||||
function formatTokenCount(count: number | undefined): string {
|
||||
if (count == null) return "0";
|
||||
if (count >= 1000) {
|
||||
@@ -55,11 +40,8 @@
|
||||
style="width: {percentage}%"
|
||||
></div>
|
||||
</div>
|
||||
<div
|
||||
class="flex items-center justify-between text-xs text-exo-light-gray/70 mt-0.5 font-mono"
|
||||
>
|
||||
<span>{etaText ?? ""}</span>
|
||||
<span>{percentage}%</span>
|
||||
<div class="text-right text-xs text-exo-light-gray/70 mt-0.5 font-mono">
|
||||
{percentage}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { toasts, dismissToast, type Toast } from "$lib/stores/toast.svelte";
|
||||
import { fly, fade } from "svelte/transition";
|
||||
import { flip } from "svelte/animate";
|
||||
|
||||
const items = $derived(toasts());
|
||||
|
||||
const typeStyles: Record<
|
||||
Toast["type"],
|
||||
{ border: string; icon: string; iconColor: string }
|
||||
> = {
|
||||
success: {
|
||||
border: "border-l-green-500",
|
||||
icon: "M9 12.75L11.25 15 15 9.75M21 12a9 9 0 11-18 0 9 9 0 0118 0z",
|
||||
iconColor: "text-green-400",
|
||||
},
|
||||
error: {
|
||||
border: "border-l-red-500",
|
||||
icon: "M12 9v3.75m9-.75a9 9 0 11-18 0 9 9 0 0118 0zm-9 3.75h.008v.008H12v-.008z",
|
||||
iconColor: "text-red-400",
|
||||
},
|
||||
warning: {
|
||||
border: "border-l-yellow-500",
|
||||
icon: "M12 9v3.75m-9.303 3.376c-.866 1.5.217 3.374 1.948 3.374h14.71c1.73 0 2.813-1.874 1.948-3.374L13.949 3.378c-.866-1.5-3.032-1.5-3.898 0L2.697 16.126z",
|
||||
iconColor: "text-yellow-400",
|
||||
},
|
||||
info: {
|
||||
border: "border-l-blue-500",
|
||||
icon: "M11.25 11.25l.041-.02a.75.75 0 011.063.852l-.708 2.836a.75.75 0 001.063.853l.041-.021M21 12a9 9 0 11-18 0 9 9 0 0118 0zm-9-3.75h.008v.008H12V8.25z",
|
||||
iconColor: "text-blue-400",
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
{#if items.length > 0}
|
||||
<div
|
||||
class="fixed bottom-6 right-6 z-[9999] flex flex-col gap-2 pointer-events-none"
|
||||
role="log"
|
||||
aria-live="polite"
|
||||
aria-label="Notifications"
|
||||
>
|
||||
{#each items as toast (toast.id)}
|
||||
{@const style = typeStyles[toast.type]}
|
||||
<div
|
||||
class="pointer-events-auto max-w-sm w-80 bg-exo-dark-gray/95 backdrop-blur-sm border border-exo-medium-gray/60 border-l-[3px] {style.border} rounded shadow-lg shadow-black/40"
|
||||
in:fly={{ x: 80, duration: 250 }}
|
||||
out:fade={{ duration: 150 }}
|
||||
animate:flip={{ duration: 200 }}
|
||||
role="alert"
|
||||
>
|
||||
<div class="flex items-start gap-3 px-4 py-3">
|
||||
<!-- Icon -->
|
||||
<svg
|
||||
class="w-5 h-5 flex-shrink-0 mt-0.5 {style.iconColor}"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke-width="1.5"
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d={style.icon}
|
||||
/>
|
||||
</svg>
|
||||
|
||||
<!-- Message -->
|
||||
<p class="flex-1 text-sm text-white/90 font-mono leading-snug">
|
||||
{toast.message}
|
||||
</p>
|
||||
|
||||
<!-- Dismiss button -->
|
||||
<button
|
||||
onclick={() => dismissToast(toast.id)}
|
||||
class="flex-shrink-0 p-0.5 text-white/40 hover:text-white/80 transition-colors cursor-pointer"
|
||||
aria-label="Dismiss notification"
|
||||
>
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke-width="2"
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M6 18L18 6M6 6l12 12"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Auto-dismiss progress bar -->
|
||||
{#if toast.duration > 0}
|
||||
<div class="h-0.5 bg-white/5 rounded-b overflow-hidden">
|
||||
<div
|
||||
class="h-full {style.border.replace('border-l-', 'bg-')}/60"
|
||||
style="animation: shrink {toast.duration}ms linear forwards"
|
||||
></div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
@keyframes shrink {
|
||||
from {
|
||||
width: 100%;
|
||||
}
|
||||
to {
|
||||
width: 0%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -117,10 +117,6 @@
|
||||
const LOGO_NATIVE_WIDTH = 814;
|
||||
const LOGO_NATIVE_HEIGHT = 1000;
|
||||
|
||||
// NVIDIA logo SVG path (from exo-nvidia)
|
||||
const NVIDIA_LOGO_PATH =
|
||||
"M0.81 0.429V0.299c0.013 -0.001 0.026 -0.002 0.038 -0.002 0.355 -0.011 0.588 0.306 0.588 0.306S1.186 0.952 0.916 0.952c-0.036 0 -0.071 -0.006 -0.105 -0.017V0.542c0.138 0.017 0.166 0.078 0.249 0.216l0.185 -0.155s-0.135 -0.177 -0.362 -0.177c-0.024 -0.001 -0.048 0.001 -0.072 0.003m0 -0.429v0.194l0.038 -0.002c0.494 -0.017 0.816 0.405 0.816 0.405s-0.37 0.45 -0.754 0.45c-0.034 0 -0.066 -0.003 -0.099 -0.009v0.12c0.027 0.003 0.055 0.006 0.082 0.006 0.358 0 0.618 -0.183 0.869 -0.399 0.042 0.034 0.212 0.114 0.247 0.15 -0.238 0.2 -0.794 0.361 -1.11 0.361 -0.03 0 -0.059 -0.002 -0.088 -0.005v0.169h1.362V0zm0 0.935v0.102c-0.331 -0.059 -0.423 -0.404 -0.423 -0.404s0.159 -0.176 0.423 -0.205v0.112h-0.001C0.671 0.524 0.562 0.654 0.562 0.654s0.062 0.218 0.248 0.282m-0.588 -0.316s0.196 -0.29 0.589 -0.32V0.194C0.376 0.229 0 0.597 0 0.597s0.213 0.616 0.81 0.672v-0.112c-0.438 -0.054 -0.588 -0.538 -0.588 -0.538";
|
||||
|
||||
function formatBytes(bytes: number, decimals = 1): string {
|
||||
if (!bytes || bytes === 0) return "0B";
|
||||
const k = 1024;
|
||||
@@ -558,13 +554,6 @@
|
||||
const clipPathId = `clip-${nodeInfo.id.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
|
||||
const modelLower = modelId.toLowerCase();
|
||||
const identity = identitiesData[nodeInfo.id];
|
||||
const nameLower = (friendlyName || "").toLowerCase();
|
||||
const isSpark = modelLower.includes("dgx") || modelLower.includes("gx10");
|
||||
const isLinux =
|
||||
!isSpark &&
|
||||
(modelLower.startsWith("linux") || identity?.osVersion === "Linux");
|
||||
const isLinuxLaptop = isLinux && modelLower.includes("laptop");
|
||||
|
||||
// Check node states for styling
|
||||
const isHighlighted = highlightedNodes.has(nodeInfo.id);
|
||||
@@ -634,382 +623,7 @@
|
||||
`${friendlyName}\nID: ${nodeInfo.id.slice(-8)}\nMemory: ${formatBytes(ramUsed)}/${formatBytes(ramTotal)}`,
|
||||
);
|
||||
|
||||
if (isSpark) {
|
||||
// NVIDIA DGX Spark — gold chassis with textured front, side handles, and NVIDIA badge
|
||||
iconBaseWidth = nodeRadius * 1.55;
|
||||
iconBaseHeight = nodeRadius * 0.58;
|
||||
const x = nodeInfo.x - iconBaseWidth / 2;
|
||||
const y = nodeInfo.y - iconBaseHeight / 2;
|
||||
const chassisX = x - iconBaseWidth * 0.03;
|
||||
const chassisWidth = iconBaseWidth * 1.05;
|
||||
const cornerRadius = 3;
|
||||
|
||||
const dgxClipId = `dgx-clip-${nodeInfo.id.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
defs
|
||||
.append("clipPath")
|
||||
.attr("id", dgxClipId)
|
||||
.append("rect")
|
||||
.attr("x", x)
|
||||
.attr("y", y)
|
||||
.attr("width", iconBaseWidth)
|
||||
.attr("height", iconBaseHeight)
|
||||
.attr("rx", cornerRadius);
|
||||
|
||||
// Chassis texture pattern
|
||||
const textureId = `chassis-texture-${nodeInfo.id.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
defs
|
||||
.append("pattern")
|
||||
.attr("id", textureId)
|
||||
.attr("patternUnits", "userSpaceOnUse")
|
||||
.attr("width", 8)
|
||||
.attr("height", 8);
|
||||
const texturePattern = defs.select(`#${textureId}`);
|
||||
texturePattern
|
||||
.append("rect")
|
||||
.attr("width", 8)
|
||||
.attr("height", 8)
|
||||
.attr("fill", "#6f6248");
|
||||
texturePattern
|
||||
.append("circle")
|
||||
.attr("cx", 2)
|
||||
.attr("cy", 2)
|
||||
.attr("r", 1)
|
||||
.attr("fill", "#5a4f3b")
|
||||
.attr("opacity", 0.5);
|
||||
texturePattern
|
||||
.append("circle")
|
||||
.attr("cx", 6)
|
||||
.attr("cy", 6)
|
||||
.attr("r", 1)
|
||||
.attr("fill", "#4a4232")
|
||||
.attr("opacity", 0.45);
|
||||
|
||||
// Main body
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("class", "node-outline")
|
||||
.attr("x", chassisX)
|
||||
.attr("y", y)
|
||||
.attr("width", chassisWidth)
|
||||
.attr("height", iconBaseHeight)
|
||||
.attr("rx", cornerRadius)
|
||||
.attr("fill", `url(#${textureId})`)
|
||||
.attr("stroke", wireColor)
|
||||
.attr("stroke-width", strokeWidth);
|
||||
|
||||
// Side border accents
|
||||
const sideThickness = iconBaseWidth * 0.02;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", chassisX)
|
||||
.attr("y", y)
|
||||
.attr("width", sideThickness)
|
||||
.attr("height", iconBaseHeight)
|
||||
.attr("fill", "#8a7a56");
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", chassisX + chassisWidth - sideThickness)
|
||||
.attr("y", y)
|
||||
.attr("width", sideThickness)
|
||||
.attr("height", iconBaseHeight)
|
||||
.attr("fill", "#8a7a56");
|
||||
|
||||
// Memory fill (bottom up)
|
||||
if (ramUsagePercent > 0) {
|
||||
const memFillHeight = (ramUsagePercent / 100) * iconBaseHeight;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", x)
|
||||
.attr("y", y + iconBaseHeight - memFillHeight)
|
||||
.attr("width", iconBaseWidth)
|
||||
.attr("height", memFillHeight)
|
||||
.attr("fill", "rgba(255,215,0,0.45)")
|
||||
.attr("clip-path", `url(#${dgxClipId})`);
|
||||
}
|
||||
|
||||
// Side handles with inner recess
|
||||
const handleWidth = iconBaseWidth * 0.27;
|
||||
const handleGap = iconBaseHeight * 0.05;
|
||||
const handleHeight = iconBaseHeight - handleGap * 2;
|
||||
const handleY = y + handleGap;
|
||||
const innerHandleWidth = iconBaseWidth * 0.12;
|
||||
const innerHandleHeight = handleHeight - iconBaseHeight * 0.06;
|
||||
const leftHandleX = x + 4;
|
||||
const rightHandleX = x + iconBaseWidth - handleWidth - 4;
|
||||
|
||||
// Left handle
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", leftHandleX)
|
||||
.attr("y", handleY)
|
||||
.attr("width", handleWidth)
|
||||
.attr("height", handleHeight)
|
||||
.attr("rx", 2.4)
|
||||
.attr("fill", "#b3a170")
|
||||
.attr("stroke", "#403723")
|
||||
.attr("stroke-width", 0.7);
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", leftHandleX + handleWidth * 0.06)
|
||||
.attr("y", handleY + iconBaseHeight * 0.03)
|
||||
.attr("width", innerHandleWidth)
|
||||
.attr("height", innerHandleHeight)
|
||||
.attr("rx", 1.6)
|
||||
.attr("fill", "#8a7a56");
|
||||
|
||||
// Right handle
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", rightHandleX)
|
||||
.attr("y", handleY)
|
||||
.attr("width", handleWidth)
|
||||
.attr("height", handleHeight)
|
||||
.attr("rx", 2.4)
|
||||
.attr("fill", "#b3a170")
|
||||
.attr("stroke", "#403723")
|
||||
.attr("stroke-width", 0.7);
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr(
|
||||
"x",
|
||||
rightHandleX + handleWidth - innerHandleWidth - handleWidth * 0.08,
|
||||
)
|
||||
.attr("y", handleY + iconBaseHeight * 0.03)
|
||||
.attr("width", innerHandleWidth)
|
||||
.attr("height", innerHandleHeight)
|
||||
.attr("rx", 1.6)
|
||||
.attr("fill", "#8a7a56");
|
||||
|
||||
// NVIDIA logo + text label (rotated 90 deg on left handle)
|
||||
const badgeWidth = iconBaseWidth * 0.09;
|
||||
const badgeHeight = handleHeight * 0.5;
|
||||
const badgeX =
|
||||
leftHandleX + handleWidth - badgeWidth - handleWidth * 0.06;
|
||||
const badgeY = handleY + (handleHeight - badgeHeight) / 2;
|
||||
const textSize = badgeWidth * 0.58;
|
||||
const logoWidth = textSize * 1.2;
|
||||
const logoHeight = logoWidth * (1.438 / 2.174);
|
||||
const centerX = badgeX + badgeWidth / 2 - badgeWidth * 0.03;
|
||||
const centerY = badgeY + badgeHeight / 2;
|
||||
const gap = badgeWidth * 0.15;
|
||||
const totalWidth = logoWidth + gap + textSize * 3.6;
|
||||
|
||||
const labelGroup = nodeG
|
||||
.append("g")
|
||||
.attr("transform", `rotate(90 ${centerX} ${centerY})`);
|
||||
|
||||
labelGroup
|
||||
.append("svg")
|
||||
.attr("x", centerX - totalWidth / 2)
|
||||
.attr("y", centerY - logoHeight / 2)
|
||||
.attr("width", logoWidth)
|
||||
.attr("height", logoHeight)
|
||||
.attr("viewBox", "0 0 2.174 1.438")
|
||||
.append("path")
|
||||
.attr("d", NVIDIA_LOGO_PATH)
|
||||
.attr("fill", "#76b900");
|
||||
|
||||
labelGroup
|
||||
.append("text")
|
||||
.attr("x", centerX - totalWidth / 2 + logoWidth + gap)
|
||||
.attr("y", centerY)
|
||||
.attr("text-anchor", "start")
|
||||
.attr("dominant-baseline", "middle")
|
||||
.attr("fill", "#8a7a56")
|
||||
.attr("font-size", textSize)
|
||||
.attr("font-family", "monospace")
|
||||
.attr("font-weight", "700")
|
||||
.text("NVIDIA");
|
||||
} else if (isLinuxLaptop) {
|
||||
// Linux Laptop — same shape as MacBook but with Tux logo
|
||||
iconBaseWidth = nodeRadius * 1.6;
|
||||
iconBaseHeight = nodeRadius * 1.15;
|
||||
const x = nodeInfo.x - iconBaseWidth / 2;
|
||||
const y = nodeInfo.y - iconBaseHeight / 2;
|
||||
|
||||
const screenHeight = iconBaseHeight * 0.7;
|
||||
const baseHeight = iconBaseHeight * 0.3;
|
||||
const screenWidth = iconBaseWidth * 0.85;
|
||||
const screenX = nodeInfo.x - screenWidth / 2;
|
||||
const screenBezel = 3;
|
||||
|
||||
const linuxScreenClipId = `linux-screen-${nodeInfo.id.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
defs
|
||||
.append("clipPath")
|
||||
.attr("id", linuxScreenClipId)
|
||||
.append("rect")
|
||||
.attr("x", screenX + screenBezel)
|
||||
.attr("y", y + screenBezel)
|
||||
.attr("width", screenWidth - screenBezel * 2)
|
||||
.attr("height", screenHeight - screenBezel * 2)
|
||||
.attr("rx", 2);
|
||||
|
||||
// Screen outer frame
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("class", "node-outline")
|
||||
.attr("x", screenX)
|
||||
.attr("y", y)
|
||||
.attr("width", screenWidth)
|
||||
.attr("height", screenHeight)
|
||||
.attr("rx", 3)
|
||||
.attr("fill", "#1a1a1a")
|
||||
.attr("stroke", wireColor)
|
||||
.attr("stroke-width", strokeWidth);
|
||||
|
||||
// Screen inner
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", screenX + screenBezel)
|
||||
.attr("y", y + screenBezel)
|
||||
.attr("width", screenWidth - screenBezel * 2)
|
||||
.attr("height", screenHeight - screenBezel * 2)
|
||||
.attr("rx", 2)
|
||||
.attr("fill", "#0a0a12");
|
||||
|
||||
// Memory fill on screen
|
||||
if (ramUsagePercent > 0) {
|
||||
const memFillTotalHeight = screenHeight - screenBezel * 2;
|
||||
const memFillActualHeight =
|
||||
(ramUsagePercent / 100) * memFillTotalHeight;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", screenX + screenBezel)
|
||||
.attr(
|
||||
"y",
|
||||
y + screenBezel + (memFillTotalHeight - memFillActualHeight),
|
||||
)
|
||||
.attr("width", screenWidth - screenBezel * 2)
|
||||
.attr("height", memFillActualHeight)
|
||||
.attr("fill", "rgba(255,215,0,0.85)")
|
||||
.attr("clip-path", `url(#${linuxScreenClipId})`);
|
||||
}
|
||||
|
||||
// Terminal prompt on screen
|
||||
nodeG
|
||||
.append("text")
|
||||
.attr("x", nodeInfo.x)
|
||||
.attr("y", y + screenHeight / 2)
|
||||
.attr("text-anchor", "middle")
|
||||
.attr("dominant-baseline", "middle")
|
||||
.attr("fill", "#FFFFFF")
|
||||
.attr("opacity", 0.9)
|
||||
.attr("font-size", screenHeight * 0.25)
|
||||
.attr("font-family", "SF Mono, Monaco, monospace")
|
||||
.attr("font-weight", "700")
|
||||
.text(">_");
|
||||
|
||||
// Keyboard base (trapezoidal)
|
||||
const baseY = y + screenHeight;
|
||||
const baseTopWidth = screenWidth;
|
||||
const baseBottomWidth = iconBaseWidth;
|
||||
const baseTopX = nodeInfo.x - baseTopWidth / 2;
|
||||
const baseBottomX = nodeInfo.x - baseBottomWidth / 2;
|
||||
|
||||
nodeG
|
||||
.append("path")
|
||||
.attr(
|
||||
"d",
|
||||
`M ${baseTopX} ${baseY} L ${baseTopX + baseTopWidth} ${baseY} L ${baseBottomX + baseBottomWidth} ${baseY + baseHeight} L ${baseBottomX} ${baseY + baseHeight} Z`,
|
||||
)
|
||||
.attr("fill", "#2c2c2c")
|
||||
.attr("stroke", wireColor)
|
||||
.attr("stroke-width", 1);
|
||||
|
||||
// Keyboard area
|
||||
const keyboardX = baseTopX + 6;
|
||||
const keyboardY = baseY + 3;
|
||||
const keyboardWidth = baseTopWidth - 12;
|
||||
const keyboardHeight = baseHeight * 0.55;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", keyboardX)
|
||||
.attr("y", keyboardY)
|
||||
.attr("width", keyboardWidth)
|
||||
.attr("height", keyboardHeight)
|
||||
.attr("fill", "rgba(0,0,0,0.2)")
|
||||
.attr("rx", 2);
|
||||
|
||||
// Trackpad
|
||||
const trackpadWidth = baseTopWidth * 0.4;
|
||||
const trackpadX = nodeInfo.x - trackpadWidth / 2;
|
||||
const trackpadY = baseY + keyboardHeight + 5;
|
||||
const trackpadHeight = baseHeight * 0.3;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", trackpadX)
|
||||
.attr("y", trackpadY)
|
||||
.attr("width", trackpadWidth)
|
||||
.attr("height", trackpadHeight)
|
||||
.attr("fill", "rgba(255,255,255,0.08)")
|
||||
.attr("rx", 2);
|
||||
} else if (isLinux) {
|
||||
// Linux Desktop — same shape as Mac Studio but with Tux logo
|
||||
iconBaseWidth = nodeRadius * 1.25;
|
||||
iconBaseHeight = nodeRadius * 0.85;
|
||||
const x = nodeInfo.x - iconBaseWidth / 2;
|
||||
const y = nodeInfo.y - iconBaseHeight / 2;
|
||||
const cornerRadius = 4;
|
||||
const topSurfaceHeight = iconBaseHeight * 0.15;
|
||||
|
||||
const linuxDesktopClipId = `linux-desktop-${nodeInfo.id.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
defs
|
||||
.append("clipPath")
|
||||
.attr("id", linuxDesktopClipId)
|
||||
.append("rect")
|
||||
.attr("x", x)
|
||||
.attr("y", y + topSurfaceHeight)
|
||||
.attr("width", iconBaseWidth)
|
||||
.attr("height", iconBaseHeight - topSurfaceHeight)
|
||||
.attr("rx", cornerRadius - 1);
|
||||
|
||||
// Main body
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("class", "node-outline")
|
||||
.attr("x", x)
|
||||
.attr("y", y)
|
||||
.attr("width", iconBaseWidth)
|
||||
.attr("height", iconBaseHeight)
|
||||
.attr("rx", cornerRadius)
|
||||
.attr("fill", "#1a1a1a")
|
||||
.attr("stroke", wireColor)
|
||||
.attr("stroke-width", strokeWidth);
|
||||
|
||||
// Memory fill
|
||||
if (ramUsagePercent > 0) {
|
||||
const memFillTotalHeight = iconBaseHeight - topSurfaceHeight;
|
||||
const memFillActualHeight =
|
||||
(ramUsagePercent / 100) * memFillTotalHeight;
|
||||
nodeG
|
||||
.append("rect")
|
||||
.attr("x", x)
|
||||
.attr(
|
||||
"y",
|
||||
y + topSurfaceHeight + (memFillTotalHeight - memFillActualHeight),
|
||||
)
|
||||
.attr("width", iconBaseWidth)
|
||||
.attr("height", memFillActualHeight)
|
||||
.attr("fill", "rgba(255,215,0,0.75)")
|
||||
.attr("clip-path", `url(#${linuxDesktopClipId})`);
|
||||
}
|
||||
|
||||
// Terminal prompt on front face
|
||||
nodeG
|
||||
.append("text")
|
||||
.attr("x", nodeInfo.x)
|
||||
.attr(
|
||||
"y",
|
||||
y + topSurfaceHeight + (iconBaseHeight - topSurfaceHeight) / 2,
|
||||
)
|
||||
.attr("text-anchor", "middle")
|
||||
.attr("dominant-baseline", "middle")
|
||||
.attr("fill", "rgba(255,255,255,0.5)")
|
||||
.attr("font-size", (iconBaseHeight - topSurfaceHeight) * 0.4)
|
||||
.attr("font-family", "SF Mono, Monaco, monospace")
|
||||
.attr("font-weight", "700")
|
||||
.text(">_");
|
||||
} else if (modelLower === "mac studio") {
|
||||
if (modelLower === "mac studio") {
|
||||
// Mac Studio - classic cube with memory fill
|
||||
iconBaseWidth = nodeRadius * 1.25;
|
||||
iconBaseHeight = nodeRadius * 0.85;
|
||||
@@ -1568,12 +1182,8 @@
|
||||
debugLabelY += debugLineHeight;
|
||||
}
|
||||
|
||||
const dbgIdentity = identitiesData[nodeInfo.id];
|
||||
if (dbgIdentity?.osVersion) {
|
||||
const osLabel =
|
||||
dbgIdentity.osVersion === "Linux"
|
||||
? "Linux"
|
||||
: `macOS ${dbgIdentity.osVersion}${dbgIdentity.osBuildVersion ? ` (${dbgIdentity.osBuildVersion})` : ""}`;
|
||||
const identity = identitiesData[nodeInfo.id];
|
||||
if (identity?.osVersion) {
|
||||
nodeG
|
||||
.append("text")
|
||||
.attr("x", nodeInfo.x)
|
||||
@@ -1582,7 +1192,9 @@
|
||||
.attr("fill", "rgba(179,179,179,0.7)")
|
||||
.attr("font-size", debugFontSize)
|
||||
.attr("font-family", "SF Mono, Monaco, monospace")
|
||||
.text(osLabel);
|
||||
.text(
|
||||
`macOS ${identity.osVersion}${identity.osBuildVersion ? ` (${identity.osBuildVersion})` : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -11,6 +11,5 @@ export { default as FamilySidebar } from "./FamilySidebar.svelte";
|
||||
export { default as HuggingFaceResultItem } from "./HuggingFaceResultItem.svelte";
|
||||
export { default as ModelFilterPopover } from "./ModelFilterPopover.svelte";
|
||||
export { default as ModelPickerGroup } from "./ModelPickerGroup.svelte";
|
||||
export { default as MetaInstanceCard } from "./MetaInstanceCard.svelte";
|
||||
export { default as ModelPickerModal } from "./ModelPickerModal.svelte";
|
||||
export { default as ChatModelSelector } from "./ChatModelSelector.svelte";
|
||||
export { default as IntegrationCard } from "./IntegrationCard.svelte";
|
||||
@@ -72,14 +72,23 @@ export interface Instance {
|
||||
runnerToShard?: Record<string, unknown>;
|
||||
nodeToRunner?: Record<string, string>;
|
||||
};
|
||||
metaInstanceId?: string | null;
|
||||
}
|
||||
|
||||
export interface RawInstanceLink {
|
||||
linkId: string;
|
||||
prefillInstances: string[];
|
||||
decodeInstances: string[];
|
||||
export interface MetaInstance {
|
||||
metaInstanceId: string;
|
||||
modelId: string;
|
||||
sharding: "Pipeline" | "Tensor";
|
||||
instanceMeta: "MlxRing" | "MlxJaccl";
|
||||
minNodes: number;
|
||||
nodeIds: string[] | null;
|
||||
placementError: string | null;
|
||||
consecutiveFailures: number;
|
||||
lastFailureError: string | null;
|
||||
}
|
||||
|
||||
export type MetaInstanceStatus = "active" | "provisioning" | "error";
|
||||
|
||||
// Granular node state types from the new state structure
|
||||
interface RawNodeIdentity {
|
||||
modelId?: string;
|
||||
@@ -174,7 +183,7 @@ export interface ModelDownloadStatus {
|
||||
export interface PlacementPreview {
|
||||
model_id: string;
|
||||
sharding: "Pipeline" | "Tensor";
|
||||
instance_meta: "MlxRing" | "MlxJaccl";
|
||||
instance_meta: "MlxRing" | "MlxIbv" | "MlxJaccl";
|
||||
instance: unknown | null;
|
||||
memory_delta_by_node: Record<string, number> | null;
|
||||
error: string | null;
|
||||
@@ -225,11 +234,12 @@ interface RawStateResponse {
|
||||
string,
|
||||
{
|
||||
MlxRingInstance?: Instance;
|
||||
MlxIbvInstance?: Instance;
|
||||
MlxJacclInstance?: Instance;
|
||||
}
|
||||
>;
|
||||
metaInstances?: Record<string, MetaInstance>;
|
||||
runners?: Record<string, unknown>;
|
||||
instanceLinks?: Record<string, RawInstanceLink>;
|
||||
downloads?: Record<string, unknown[]>;
|
||||
// New granular node state fields
|
||||
nodeIdentities?: Record<string, RawNodeIdentity>;
|
||||
@@ -256,20 +266,14 @@ interface RawStateResponse {
|
||||
>;
|
||||
// Thunderbolt bridge cycles (nodes with bridge enabled forming loops)
|
||||
thunderboltBridgeCycles?: string[][];
|
||||
// Disk usage per node
|
||||
nodeDisk?: Record<
|
||||
string,
|
||||
{ total: { inBytes: number }; available: { inBytes: number } }
|
||||
>;
|
||||
}
|
||||
|
||||
export interface MessageAttachment {
|
||||
type: "image" | "text" | "file" | "generated-image" | "pdf";
|
||||
type: "image" | "text" | "file" | "generated-image";
|
||||
name: string;
|
||||
content?: string;
|
||||
preview?: string;
|
||||
mimeType?: string;
|
||||
pageImages?: string[];
|
||||
}
|
||||
|
||||
export interface TopLogprob {
|
||||
@@ -288,8 +292,6 @@ export interface TokenData {
|
||||
export interface PrefillProgress {
|
||||
processed: number;
|
||||
total: number;
|
||||
/** Timestamp (performance.now()) when prefill started. */
|
||||
startedAt: number;
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
@@ -547,9 +549,8 @@ class AppStore {
|
||||
// Topology state
|
||||
topologyData = $state<TopologyData | null>(null);
|
||||
instances = $state<Record<string, unknown>>({});
|
||||
metaInstances = $state<Record<string, MetaInstance>>({});
|
||||
runners = $state<Record<string, unknown>>({});
|
||||
instanceLinks = $state<Record<string, RawInstanceLink>>({});
|
||||
featureFlags = $state<Record<string, boolean>>({});
|
||||
downloads = $state<Record<string, unknown[]>>({});
|
||||
nodeDisk = $state<
|
||||
Record<
|
||||
@@ -590,8 +591,6 @@ class AppStore {
|
||||
debugMode = $state(false);
|
||||
topologyOnlyMode = $state(false);
|
||||
chatSidebarVisible = $state(true); // Shown by default
|
||||
mobileChatSidebarOpen = $state(false); // Mobile drawer state
|
||||
mobileRightSidebarOpen = $state(false); // Mobile right drawer state
|
||||
|
||||
// Image generation params
|
||||
imageGenerationParams = $state<ImageGenerationParams>({
|
||||
@@ -601,12 +600,6 @@ class AppStore {
|
||||
// Image editing state
|
||||
editingImage = $state<EditingImage | null>(null);
|
||||
|
||||
/** True when the backend is reachable. */
|
||||
isConnected = $state<boolean>(true);
|
||||
/** Number of consecutive fetch failures. */
|
||||
private consecutiveFailures = 0;
|
||||
private static readonly CONNECTION_LOST_THRESHOLD = 3;
|
||||
|
||||
private fetchInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private previewsInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private lastConversationPersistTs = 0;
|
||||
@@ -859,10 +852,6 @@ class AppStore {
|
||||
this.thinkingEnabled = conversation.enableThinking ?? true;
|
||||
this.refreshConversationModelFromInstances();
|
||||
|
||||
// Sync global selection to the loaded conversation's model so reactive
|
||||
// effects in +page.svelte can determine the correct chat launch state.
|
||||
this.selectedChatModel = conversation.modelId || "";
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -933,7 +922,11 @@ class AppStore {
|
||||
|
||||
let instanceType: string | null = null;
|
||||
if (instanceTag === "MlxRingInstance") instanceType = "MLX Ring";
|
||||
else if (instanceTag === "MlxJacclInstance") instanceType = "MLX RDMA";
|
||||
else if (
|
||||
instanceTag === "MlxIbvInstance" ||
|
||||
instanceTag === "MlxJacclInstance"
|
||||
)
|
||||
instanceType = "MLX RDMA";
|
||||
|
||||
let sharding: string | null = null;
|
||||
const inst = instance as {
|
||||
@@ -1257,33 +1250,8 @@ class AppStore {
|
||||
this.saveChatSidebarVisibleToStorage();
|
||||
}
|
||||
|
||||
getMobileChatSidebarOpen(): boolean {
|
||||
return this.mobileChatSidebarOpen;
|
||||
}
|
||||
|
||||
setMobileChatSidebarOpen(open: boolean) {
|
||||
this.mobileChatSidebarOpen = open;
|
||||
}
|
||||
|
||||
toggleMobileChatSidebar() {
|
||||
this.mobileChatSidebarOpen = !this.mobileChatSidebarOpen;
|
||||
}
|
||||
|
||||
getMobileRightSidebarOpen(): boolean {
|
||||
return this.mobileRightSidebarOpen;
|
||||
}
|
||||
|
||||
setMobileRightSidebarOpen(open: boolean) {
|
||||
this.mobileRightSidebarOpen = open;
|
||||
}
|
||||
|
||||
toggleMobileRightSidebar() {
|
||||
this.mobileRightSidebarOpen = !this.mobileRightSidebarOpen;
|
||||
}
|
||||
|
||||
startPolling() {
|
||||
this.fetchState();
|
||||
this.fetchFeatureFlags();
|
||||
this.fetchInterval = setInterval(() => this.fetchState(), 1000);
|
||||
}
|
||||
|
||||
@@ -1295,16 +1263,6 @@ class AppStore {
|
||||
this.stopPreviewsPolling();
|
||||
}
|
||||
|
||||
async fetchFeatureFlags() {
|
||||
try {
|
||||
const response = await fetch("/v1/feature-flags");
|
||||
if (!response.ok) return;
|
||||
this.featureFlags = await response.json();
|
||||
} catch {
|
||||
// Silently ignore — defaults to all-disabled.
|
||||
}
|
||||
}
|
||||
|
||||
async fetchState() {
|
||||
try {
|
||||
const response = await fetch("/state");
|
||||
@@ -1327,14 +1285,12 @@ class AppStore {
|
||||
this.instances = data.instances;
|
||||
this.refreshConversationModelFromInstances();
|
||||
}
|
||||
if (data.metaInstances) {
|
||||
this.metaInstances = data.metaInstances;
|
||||
}
|
||||
if (data.runners) {
|
||||
this.runners = data.runners;
|
||||
}
|
||||
if (data.instanceLinks) {
|
||||
this.instanceLinks = data.instanceLinks;
|
||||
} else {
|
||||
this.instanceLinks = {};
|
||||
}
|
||||
if (data.downloads) {
|
||||
this.downloads = data.downloads;
|
||||
}
|
||||
@@ -1352,23 +1308,84 @@ class AppStore {
|
||||
// Thunderbolt bridge status per node
|
||||
this.nodeThunderboltBridge = data.nodeThunderboltBridge ?? {};
|
||||
this.lastUpdate = Date.now();
|
||||
// Connection recovered
|
||||
if (!this.isConnected) {
|
||||
this.isConnected = true;
|
||||
}
|
||||
this.consecutiveFailures = 0;
|
||||
} catch (error) {
|
||||
this.consecutiveFailures++;
|
||||
if (
|
||||
this.consecutiveFailures >= AppStore.CONNECTION_LOST_THRESHOLD &&
|
||||
this.isConnected
|
||||
) {
|
||||
this.isConnected = false;
|
||||
}
|
||||
console.error("Error fetching state:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async createMetaInstance(
|
||||
modelId: string,
|
||||
sharding: "Pipeline" | "Tensor" = "Pipeline",
|
||||
instanceMeta: "MlxRing" | "MlxJaccl" = "MlxRing",
|
||||
minNodes: number = 1,
|
||||
nodeIds: string[] | null = null,
|
||||
) {
|
||||
try {
|
||||
const response = await fetch("/meta_instance", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model_id: modelId,
|
||||
sharding,
|
||||
instance_meta: instanceMeta,
|
||||
min_nodes: minNodes,
|
||||
node_ids: nodeIds,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
console.error("Failed to create meta-instance:", response.status);
|
||||
}
|
||||
await this.fetchState();
|
||||
} catch (error) {
|
||||
console.error("Error creating meta-instance:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteMetaInstance(metaInstanceId: string) {
|
||||
try {
|
||||
const response = await fetch(`/meta_instance/${metaInstanceId}`, {
|
||||
method: "DELETE",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
console.error("Failed to delete meta-instance:", response.status);
|
||||
}
|
||||
await this.fetchState();
|
||||
} catch (error) {
|
||||
console.error("Error deleting meta-instance:", error);
|
||||
}
|
||||
}
|
||||
|
||||
getMetaInstanceStatus(
|
||||
metaInstance: MetaInstance,
|
||||
): MetaInstanceStatus {
|
||||
// Check if any running instance is bound to this meta-instance
|
||||
for (const instanceWrapper of Object.values(this.instances)) {
|
||||
if (!instanceWrapper || typeof instanceWrapper !== "object") continue;
|
||||
const keys = Object.keys(instanceWrapper as Record<string, unknown>);
|
||||
if (keys.length !== 1) continue;
|
||||
const inner = (instanceWrapper as Record<string, unknown>)[keys[0]];
|
||||
if (inner && typeof inner === "object" && (inner as Instance).metaInstanceId === metaInstance.metaInstanceId) {
|
||||
return "active";
|
||||
}
|
||||
}
|
||||
if (metaInstance.placementError) return "error";
|
||||
return "provisioning";
|
||||
}
|
||||
|
||||
getMetaInstanceBackingNodes(metaInstance: MetaInstance): string[] {
|
||||
for (const instanceWrapper of Object.values(this.instances)) {
|
||||
if (!instanceWrapper || typeof instanceWrapper !== "object") continue;
|
||||
const keys = Object.keys(instanceWrapper as Record<string, unknown>);
|
||||
if (keys.length !== 1) continue;
|
||||
const inner = (instanceWrapper as Record<string, unknown>)[keys[0]] as Instance;
|
||||
if (inner?.metaInstanceId === metaInstance.metaInstanceId && inner?.shardAssignments?.nodeToRunner) {
|
||||
return Object.keys(inner.shardAssignments.nodeToRunner);
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
async fetchPlacementPreviews(modelId: string, showLoading = true) {
|
||||
if (!modelId) return;
|
||||
|
||||
@@ -1603,7 +1620,6 @@ class AppStore {
|
||||
// Remove messages after user message (including the user message for image requests
|
||||
// since generateImage/editImage will re-add it)
|
||||
this.messages = this.messages.slice(0, lastUserIndex);
|
||||
this.updateActiveConversation();
|
||||
|
||||
switch (requestType) {
|
||||
case "image-generation":
|
||||
@@ -1695,15 +1711,7 @@ class AppStore {
|
||||
}
|
||||
}
|
||||
}
|
||||
const out: {
|
||||
role: string;
|
||||
content: string;
|
||||
reasoning_content?: string;
|
||||
} = { role: m.role, content: msgContent };
|
||||
if (m.role === "assistant" && m.thinking) {
|
||||
out.reasoning_content = m.thinking;
|
||||
}
|
||||
return out;
|
||||
return { role: m.role, content: msgContent };
|
||||
}),
|
||||
];
|
||||
|
||||
@@ -1827,14 +1835,6 @@ class AppStore {
|
||||
this.persistConversation(targetConversationId);
|
||||
}
|
||||
},
|
||||
{
|
||||
generation_stats: (data) => {
|
||||
const stats = data as { generation_tps: number };
|
||||
if (stats.generation_tps > 0) {
|
||||
this.tps = stats.generation_tps;
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Final update
|
||||
@@ -1910,15 +1910,7 @@ class AppStore {
|
||||
const apiMessages = [
|
||||
systemPrompt,
|
||||
...targetConversation.messages.slice(0, -1).map((m) => {
|
||||
const out: {
|
||||
role: string;
|
||||
content: string;
|
||||
reasoning_content?: string;
|
||||
} = { role: m.role, content: m.content };
|
||||
if (m.role === "assistant" && m.thinking) {
|
||||
out.reasoning_content = m.thinking;
|
||||
}
|
||||
return out;
|
||||
return { role: m.role, content: m.content };
|
||||
}),
|
||||
];
|
||||
|
||||
@@ -1930,7 +1922,7 @@ class AppStore {
|
||||
assistantMessage.id,
|
||||
(msg) => {
|
||||
msg.content =
|
||||
"No model is loaded yet. Select a model from the sidebar to get started — it will download and load automatically.";
|
||||
"Error: No model available. Please launch an instance first.";
|
||||
},
|
||||
);
|
||||
this.syncActiveMessagesIfNeeded(targetConversationId);
|
||||
@@ -2040,14 +2032,6 @@ class AppStore {
|
||||
this.persistConversation(targetConversationId);
|
||||
}
|
||||
},
|
||||
{
|
||||
generation_stats: (data) => {
|
||||
const stats = data as { generation_tps: number };
|
||||
if (stats.generation_tps > 0) {
|
||||
this.tps = stats.generation_tps;
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Final cleanup of the message (if conversation still exists)
|
||||
@@ -2284,7 +2268,6 @@ class AppStore {
|
||||
type: string;
|
||||
textContent?: string;
|
||||
preview?: string;
|
||||
pageImages?: string[];
|
||||
}[],
|
||||
enableThinking?: boolean | null,
|
||||
): Promise<void> {
|
||||
@@ -2320,20 +2303,6 @@ class AppStore {
|
||||
preview: file.preview,
|
||||
mimeType: file.type,
|
||||
});
|
||||
} else if (
|
||||
file.pageImages ||
|
||||
(file.textContent && file.type === "application/pdf")
|
||||
) {
|
||||
attachments.push({
|
||||
type: "pdf",
|
||||
name: file.name,
|
||||
content: file.textContent,
|
||||
pageImages: file.pageImages,
|
||||
mimeType: file.type,
|
||||
});
|
||||
if (file.textContent) {
|
||||
fileContext += `\n\n[File: ${file.name}]\n\`\`\`\n${file.textContent}\n\`\`\``;
|
||||
}
|
||||
} else if (file.textContent) {
|
||||
attachments.push({
|
||||
type: "text",
|
||||
@@ -2401,89 +2370,22 @@ class AppStore {
|
||||
const apiMessages = [
|
||||
systemPrompt,
|
||||
...targetConversation.messages.slice(0, -1).map((m) => {
|
||||
// Check if this message has image or PDF attachments
|
||||
const visualAttachments = m.attachments?.filter(
|
||||
(a) =>
|
||||
(a.type === "image" && a.preview) ||
|
||||
(a.type === "pdf" && a.pageImages?.length),
|
||||
);
|
||||
|
||||
if (visualAttachments && visualAttachments.length > 0) {
|
||||
// Build multimodal content array (OpenAI vision format)
|
||||
const contentParts: Array<
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } }
|
||||
> = [];
|
||||
|
||||
// Add image parts first
|
||||
for (const att of visualAttachments) {
|
||||
if (att.type === "image" && att.preview) {
|
||||
contentParts.push({
|
||||
type: "image_url",
|
||||
image_url: { url: att.preview },
|
||||
});
|
||||
} else if (att.type === "pdf" && att.pageImages) {
|
||||
for (const pageImg of att.pageImages) {
|
||||
contentParts.push({
|
||||
type: "image_url",
|
||||
image_url: { url: pageImg },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build text content including any text/pdf file attachments
|
||||
let textContent = m.content;
|
||||
if (m.attachments) {
|
||||
for (const attachment of m.attachments) {
|
||||
if (
|
||||
(attachment.type === "text" || attachment.type === "pdf") &&
|
||||
attachment.content
|
||||
) {
|
||||
textContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (textContent) {
|
||||
contentParts.push({ type: "text", text: textContent });
|
||||
}
|
||||
|
||||
const out: {
|
||||
role: string;
|
||||
content: typeof contentParts;
|
||||
reasoning_content?: string;
|
||||
} = { role: m.role, content: contentParts };
|
||||
if (m.role === "assistant" && m.thinking) {
|
||||
out.reasoning_content = m.thinking;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Text-only message (original path)
|
||||
// Build content including any text file attachments
|
||||
let msgContent = m.content;
|
||||
|
||||
// Add text/pdf attachments as context
|
||||
// Add text attachments as context
|
||||
if (m.attachments) {
|
||||
for (const attachment of m.attachments) {
|
||||
if (
|
||||
(attachment.type === "text" || attachment.type === "pdf") &&
|
||||
attachment.content
|
||||
) {
|
||||
if (attachment.type === "text" && attachment.content) {
|
||||
msgContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const out: {
|
||||
role: string;
|
||||
content: string;
|
||||
reasoning_content?: string;
|
||||
} = { role: m.role, content: msgContent };
|
||||
if (m.role === "assistant" && m.thinking) {
|
||||
out.reasoning_content = m.thinking;
|
||||
}
|
||||
return out;
|
||||
return {
|
||||
role: m.role,
|
||||
content: msgContent,
|
||||
};
|
||||
}),
|
||||
];
|
||||
|
||||
@@ -2491,7 +2393,7 @@ class AppStore {
|
||||
const modelToUse = this.getModelForRequest();
|
||||
if (!modelToUse) {
|
||||
throw new Error(
|
||||
"No model is loaded yet. Select a model from the sidebar to get started — it will download and load automatically.",
|
||||
"No model selected and no running instances available. Please launch an instance first.",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2537,7 +2439,7 @@ class AppStore {
|
||||
|
||||
let streamedContent = "";
|
||||
let streamedThinking = "";
|
||||
let serverTpsReceived = false;
|
||||
|
||||
interface ChatCompletionChunk {
|
||||
choices?: Array<{
|
||||
delta?: { content?: string; reasoning_content?: string };
|
||||
@@ -2602,6 +2504,7 @@ class AppStore {
|
||||
tokenCount += 1;
|
||||
this.totalTokens = tokenCount;
|
||||
|
||||
// Update real-time TPS during streaming
|
||||
if (firstTokenTime !== null && tokenCount > 1) {
|
||||
const elapsed = performance.now() - firstTokenTime;
|
||||
this.tps = (tokenCount / elapsed) * 1000;
|
||||
@@ -2649,27 +2552,18 @@ class AppStore {
|
||||
this.prefillProgress = {
|
||||
processed: inner.processed_tokens,
|
||||
total: inner.total_tokens,
|
||||
startedAt: this.prefillProgress?.startedAt ?? performance.now(),
|
||||
};
|
||||
},
|
||||
generation_stats: (data) => {
|
||||
const stats = data as { generation_tps: number };
|
||||
|
||||
if (stats.generation_tps > 0) {
|
||||
this.tps = stats.generation_tps;
|
||||
serverTpsReceived = true;
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Clear prefill progress after stream ends
|
||||
this.prefillProgress = null;
|
||||
|
||||
// Use server-side TPS if available, otherwise fall back to client-side
|
||||
if (!serverTpsReceived && firstTokenTime !== null && tokenCount > 1) {
|
||||
// Calculate final TPS
|
||||
if (firstTokenTime !== null && tokenCount > 1) {
|
||||
const totalGenerationTime = performance.now() - firstTokenTime;
|
||||
this.tps = (tokenCount / totalGenerationTime) * 1000;
|
||||
this.tps = (tokenCount / totalGenerationTime) * 1000; // tokens per second
|
||||
}
|
||||
|
||||
// Final cleanup of the message (if conversation still exists)
|
||||
@@ -2774,9 +2668,6 @@ class AppStore {
|
||||
this.syncActiveMessagesIfNeeded(targetConversationId);
|
||||
this.saveConversationsToStorage();
|
||||
|
||||
const abortController = new AbortController();
|
||||
this.currentAbortController = abortController;
|
||||
|
||||
try {
|
||||
// Determine the model to use
|
||||
const model = this.getModelForRequest(modelId);
|
||||
@@ -2831,7 +2722,6 @@ class AppStore {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -2971,27 +2861,14 @@ class AppStore {
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (abortController.signal.aborted) {
|
||||
this.updateConversationMessage(
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
(msg) => {
|
||||
msg.content = "Cancelled";
|
||||
msg.attachments = [];
|
||||
},
|
||||
);
|
||||
this.syncActiveMessagesIfNeeded(targetConversationId);
|
||||
} else {
|
||||
console.error("Error generating image:", error);
|
||||
this.handleStreamingError(
|
||||
error,
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
"Failed to generate image",
|
||||
);
|
||||
}
|
||||
console.error("Error generating image:", error);
|
||||
this.handleStreamingError(
|
||||
error,
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
"Failed to generate image",
|
||||
);
|
||||
} finally {
|
||||
this.currentAbortController = null;
|
||||
this.isLoading = false;
|
||||
this.saveConversationsToStorage();
|
||||
}
|
||||
@@ -3055,9 +2932,6 @@ class AppStore {
|
||||
// Clear editing state
|
||||
this.editingImage = null;
|
||||
|
||||
const abortController = new AbortController();
|
||||
this.currentAbortController = abortController;
|
||||
|
||||
try {
|
||||
// Determine the model to use
|
||||
const model = this.getModelForRequest(modelId);
|
||||
@@ -3119,7 +2993,6 @@ class AppStore {
|
||||
const apiResponse = await fetch("/v1/images/edits", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
||||
if (!apiResponse.ok) {
|
||||
@@ -3220,27 +3093,14 @@ class AppStore {
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (abortController.signal.aborted) {
|
||||
this.updateConversationMessage(
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
(msg) => {
|
||||
msg.content = "Cancelled";
|
||||
msg.attachments = [];
|
||||
},
|
||||
);
|
||||
this.syncActiveMessagesIfNeeded(targetConversationId);
|
||||
} else {
|
||||
console.error("Error editing image:", error);
|
||||
this.handleStreamingError(
|
||||
error,
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
"Failed to edit image",
|
||||
);
|
||||
}
|
||||
console.error("Error editing image:", error);
|
||||
this.handleStreamingError(
|
||||
error,
|
||||
targetConversationId,
|
||||
assistantMessage.id,
|
||||
"Failed to edit image",
|
||||
);
|
||||
} finally {
|
||||
this.currentAbortController = null;
|
||||
this.isLoading = false;
|
||||
this.saveConversationsToStorage();
|
||||
}
|
||||
@@ -3307,85 +3167,6 @@ class AppStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel/pause an active download on a specific node
|
||||
*/
|
||||
async cancelDownload(nodeId: string, modelId: string): Promise<void> {
|
||||
try {
|
||||
const response = await fetch("/download/cancel", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
targetNodeId: nodeId,
|
||||
modelId: modelId,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(
|
||||
`Failed to cancel download: ${response.status} - ${errorText}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error cancelling download:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async createInstanceLink(
|
||||
prefillInstances: string[],
|
||||
decodeInstances: string[],
|
||||
): Promise<void> {
|
||||
const response = await fetch("/v1/instance-links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
prefill_instances: prefillInstances,
|
||||
decode_instances: decodeInstances,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to create instance link: ${response.status} ${await response.text()}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async updateInstanceLink(
|
||||
linkId: string,
|
||||
prefillInstances: string[],
|
||||
decodeInstances: string[],
|
||||
): Promise<void> {
|
||||
const response = await fetch(
|
||||
`/v1/instance-links/${encodeURIComponent(linkId)}`,
|
||||
{
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
prefill_instances: prefillInstances,
|
||||
decode_instances: decodeInstances,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to update instance link: ${response.status} ${await response.text()}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteInstanceLink(linkId: string): Promise<void> {
|
||||
const response = await fetch(
|
||||
`/v1/instance-links/${encodeURIComponent(linkId)}`,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to delete instance link: ${response.status} ${await response.text()}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a downloaded model from a specific node
|
||||
*/
|
||||
@@ -3445,23 +3226,6 @@ class AppStore {
|
||||
return (await response.json()) as TraceStatsResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete traces by task IDs
|
||||
*/
|
||||
async deleteTraces(
|
||||
taskIds: string[],
|
||||
): Promise<{ deleted: string[]; notFound: string[] }> {
|
||||
const response = await fetch("/v1/traces/delete", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ taskIds }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to delete traces: ${response.status}`);
|
||||
}
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL for the raw trace file (for Perfetto)
|
||||
*/
|
||||
@@ -3483,20 +3247,8 @@ export const totalTokens = () => appStore.totalTokens;
|
||||
export const prefillProgress = () => appStore.prefillProgress;
|
||||
export const topologyData = () => appStore.topologyData;
|
||||
export const instances = () => appStore.instances;
|
||||
export const metaInstances = () => appStore.metaInstances;
|
||||
export const runners = () => appStore.runners;
|
||||
export const instanceLinks = () => appStore.instanceLinks;
|
||||
export const featureFlags = () => appStore.featureFlags;
|
||||
export const createInstanceLink = (
|
||||
prefillInstances: string[],
|
||||
decodeInstances: string[],
|
||||
) => appStore.createInstanceLink(prefillInstances, decodeInstances);
|
||||
export const updateInstanceLink = (
|
||||
linkId: string,
|
||||
prefillInstances: string[],
|
||||
decodeInstances: string[],
|
||||
) => appStore.updateInstanceLink(linkId, prefillInstances, decodeInstances);
|
||||
export const deleteInstanceLink = (linkId: string) =>
|
||||
appStore.deleteInstanceLink(linkId);
|
||||
export const downloads = () => appStore.downloads;
|
||||
export const nodeDisk = () => appStore.nodeDisk;
|
||||
export const placementPreviews = () => appStore.placementPreviews;
|
||||
@@ -3521,7 +3273,6 @@ export const sendMessage = (
|
||||
type: string;
|
||||
textContent?: string;
|
||||
preview?: string;
|
||||
pageImages?: string[];
|
||||
}[],
|
||||
enableThinking?: boolean | null,
|
||||
) => appStore.sendMessage(content, files, enableThinking);
|
||||
@@ -3583,22 +3334,22 @@ export const toggleChatSidebarVisible = () =>
|
||||
appStore.toggleChatSidebarVisible();
|
||||
export const setChatSidebarVisible = (visible: boolean) =>
|
||||
appStore.setChatSidebarVisible(visible);
|
||||
|
||||
// Mobile sidebar state
|
||||
export const mobileChatSidebarOpen = () => appStore.mobileChatSidebarOpen;
|
||||
export const toggleMobileChatSidebar = () => appStore.toggleMobileChatSidebar();
|
||||
export const setMobileChatSidebarOpen = (open: boolean) =>
|
||||
appStore.setMobileChatSidebarOpen(open);
|
||||
export const mobileRightSidebarOpen = () => appStore.mobileRightSidebarOpen;
|
||||
export const toggleMobileRightSidebar = () =>
|
||||
appStore.toggleMobileRightSidebar();
|
||||
export const setMobileRightSidebarOpen = (open: boolean) =>
|
||||
appStore.setMobileRightSidebarOpen(open);
|
||||
|
||||
export const refreshState = () => appStore.fetchState();
|
||||
|
||||
// Connection status
|
||||
export const isConnected = () => appStore.isConnected;
|
||||
// Meta-instance actions
|
||||
export const createMetaInstance = (
|
||||
modelId: string,
|
||||
sharding?: "Pipeline" | "Tensor",
|
||||
instanceMeta?: "MlxRing" | "MlxJaccl",
|
||||
minNodes?: number,
|
||||
nodeIds?: string[] | null,
|
||||
) => appStore.createMetaInstance(modelId, sharding, instanceMeta, minNodes, nodeIds);
|
||||
export const deleteMetaInstance = (metaInstanceId: string) =>
|
||||
appStore.deleteMetaInstance(metaInstanceId);
|
||||
export const getMetaInstanceStatus = (metaInstance: MetaInstance) =>
|
||||
appStore.getMetaInstanceStatus(metaInstance);
|
||||
export const getMetaInstanceBackingNodes = (metaInstance: MetaInstance) =>
|
||||
appStore.getMetaInstanceBackingNodes(metaInstance);
|
||||
|
||||
// Node identities (for OS version mismatch detection)
|
||||
export const nodeIdentities = () => appStore.nodeIdentities;
|
||||
@@ -3620,8 +3371,6 @@ export const resetImageGenerationParams = () =>
|
||||
// Download actions
|
||||
export const startDownload = (nodeId: string, shardMetadata: object) =>
|
||||
appStore.startDownload(nodeId, shardMetadata);
|
||||
export const cancelDownload = (nodeId: string, modelId: string) =>
|
||||
appStore.cancelDownload(nodeId, modelId);
|
||||
export const deleteDownload = (nodeId: string, modelId: string) =>
|
||||
appStore.deleteDownload(nodeId, modelId);
|
||||
|
||||
@@ -3633,5 +3382,3 @@ export const fetchTraceStats = (taskId: string) =>
|
||||
appStore.fetchTraceStats(taskId);
|
||||
export const getTraceRawUrl = (taskId: string) =>
|
||||
appStore.getTraceRawUrl(taskId);
|
||||
export const deleteTraces = (taskIds: string[]) =>
|
||||
appStore.deleteTraces(taskIds);
|
||||
@@ -1,87 +0,0 @@
|
||||
/**
|
||||
* Toast notification store - Global notification system for the EXO dashboard.
|
||||
*
|
||||
* Usage:
|
||||
* import { addToast, dismissToast, toasts } from "$lib/stores/toast.svelte";
|
||||
* addToast({ type: "success", message: "Model launched" });
|
||||
* addToast({ type: "error", message: "Connection lost", persistent: true });
|
||||
*/
|
||||
|
||||
type ToastType = "success" | "error" | "warning" | "info";
|
||||
|
||||
export interface Toast {
|
||||
id: string;
|
||||
type: ToastType;
|
||||
message: string;
|
||||
/** Auto-dismiss after this many ms. 0 = persistent (must be dismissed manually). */
|
||||
duration: number;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
interface ToastInput {
|
||||
type: ToastType;
|
||||
message: string;
|
||||
/** If true, toast stays until manually dismissed. Default: false. */
|
||||
persistent?: boolean;
|
||||
/** Auto-dismiss duration in ms. Default: 4000 for success/info, 6000 for error/warning. */
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_DURATIONS: Record<ToastType, number> = {
|
||||
success: 4000,
|
||||
info: 4000,
|
||||
warning: 6000,
|
||||
error: 6000,
|
||||
};
|
||||
|
||||
let toastList = $state<Toast[]>([]);
|
||||
const timers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
|
||||
function generateId(): string {
|
||||
return `toast-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
export function addToast(input: ToastInput): string {
|
||||
const id = generateId();
|
||||
const duration = input.persistent
|
||||
? 0
|
||||
: (input.duration ?? DEFAULT_DURATIONS[input.type]);
|
||||
|
||||
const toast: Toast = {
|
||||
id,
|
||||
type: input.type,
|
||||
message: input.message,
|
||||
duration,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
|
||||
toastList = [...toastList, toast];
|
||||
|
||||
if (duration > 0) {
|
||||
const timer = setTimeout(() => dismissToast(id), duration);
|
||||
timers.set(id, timer);
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
export function dismissToast(id: string): void {
|
||||
const timer = timers.get(id);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timers.delete(id);
|
||||
}
|
||||
toastList = toastList.filter((t) => t.id !== id);
|
||||
}
|
||||
|
||||
/** Dismiss all toasts matching a message (useful for dedup). */
|
||||
export function dismissByMessage(message: string): void {
|
||||
const matching = toastList.filter((t) => t.message === message);
|
||||
for (const t of matching) {
|
||||
dismissToast(t.id);
|
||||
}
|
||||
}
|
||||
|
||||
export function toasts(): Toast[] {
|
||||
return toastList;
|
||||
}
|
||||
@@ -2,41 +2,6 @@
|
||||
* File attachment types for the chat interface
|
||||
*/
|
||||
|
||||
import { getDocument, GlobalWorkerOptions, version } from "pdfjs-dist";
|
||||
import type { DocumentInitParameters } from "pdfjs-dist/types/src/display/api";
|
||||
|
||||
// Safari (through at least 18/26) does not implement
|
||||
// ReadableStream.prototype[Symbol.asyncIterator], which pdfjs-dist uses
|
||||
// internally in getTextContent(). Without this polyfill, `for await (const n
|
||||
// of stream)` throws "undefined is not a function" and PDF processing fails.
|
||||
if (
|
||||
typeof ReadableStream !== "undefined" &&
|
||||
!(ReadableStream.prototype as unknown as Record<symbol, unknown>)[
|
||||
Symbol.asyncIterator
|
||||
]
|
||||
) {
|
||||
(ReadableStream.prototype as unknown as Record<symbol, unknown>)[
|
||||
Symbol.asyncIterator
|
||||
] = async function* (this: ReadableStream<unknown>) {
|
||||
const reader = this.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) return;
|
||||
yield value;
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
GlobalWorkerOptions.workerSrc = `https://cdn.jsdelivr.net/npm/pdfjs-dist@${version}/build/pdf.worker.mjs`;
|
||||
|
||||
const PDF_PAGE_SCALE = 2.0;
|
||||
const PDF_MAX_PAGES = 20;
|
||||
const PDF_MAX_TEXT_CHARS = 100_000;
|
||||
|
||||
export interface ChatUploadedFile {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -45,7 +10,6 @@ export interface ChatUploadedFile {
|
||||
file: File;
|
||||
preview?: string;
|
||||
textContent?: string;
|
||||
pageImages?: string[];
|
||||
}
|
||||
|
||||
export interface ChatAttachment {
|
||||
@@ -230,58 +194,6 @@ export function readFileAsText(file: File): Promise<string> {
|
||||
});
|
||||
}
|
||||
|
||||
async function extractPdfContent(
|
||||
file: File,
|
||||
): Promise<{ text: string; pageImages: string[] }> {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const pdf = await getDocument({
|
||||
data: new Uint8Array(arrayBuffer),
|
||||
useSystemFonts: true,
|
||||
} as DocumentInitParameters).promise;
|
||||
|
||||
const numPages = Math.min(pdf.numPages, PDF_MAX_PAGES);
|
||||
const pageTexts: string[] = [];
|
||||
const pageImages: string[] = [];
|
||||
|
||||
for (let i = 1; i <= numPages; i++) {
|
||||
const page = await pdf.getPage(i);
|
||||
|
||||
const content = await page.getTextContent();
|
||||
const strings = content.items
|
||||
.filter((item: any) => "str" in item)
|
||||
.map((item: any) => item.str as string);
|
||||
pageTexts.push(strings.join(" "));
|
||||
|
||||
const viewport = page.getViewport({ scale: PDF_PAGE_SCALE });
|
||||
const canvas = new OffscreenCanvas(viewport.width, viewport.height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (ctx) {
|
||||
await page.render({ canvasContext: ctx as any, viewport }).promise;
|
||||
const blob = await canvas.convertToBlob({
|
||||
type: "image/jpeg",
|
||||
quality: 0.8,
|
||||
});
|
||||
const reader = new FileReader();
|
||||
const dataUrl = await new Promise<string>((resolve, reject) => {
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.onerror = () => reject(reader.error);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
pageImages.push(dataUrl);
|
||||
}
|
||||
}
|
||||
|
||||
let text = pageTexts.join("\n\n").trim();
|
||||
if (text.length > PDF_MAX_TEXT_CHARS) {
|
||||
text = text.slice(0, PDF_MAX_TEXT_CHARS) + "\n\n[truncated]";
|
||||
}
|
||||
if (pdf.numPages > PDF_MAX_PAGES) {
|
||||
text += `\n\n[showing ${PDF_MAX_PAGES} of ${pdf.numPages} pages]`;
|
||||
}
|
||||
|
||||
return { text, pageImages };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process uploaded files into ChatUploadedFile format
|
||||
*/
|
||||
@@ -311,12 +223,7 @@ export async function processUploadedFiles(
|
||||
const textContent = await readFileAsText(file);
|
||||
results.push({ ...base, textContent });
|
||||
} else if (category === "pdf") {
|
||||
const { text, pageImages } = await extractPdfContent(file);
|
||||
results.push({
|
||||
...base,
|
||||
textContent: text || undefined,
|
||||
pageImages: pageImages.length > 0 ? pageImages : undefined,
|
||||
});
|
||||
results.push(base);
|
||||
} else if (category === "audio") {
|
||||
const preview = await readFileAsDataURL(file);
|
||||
results.push({ ...base, preview });
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
export async function copyText(text: string): Promise<boolean> {
|
||||
if (
|
||||
typeof window !== "undefined" &&
|
||||
window.isSecureContext &&
|
||||
navigator.clipboard?.writeText
|
||||
) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
return true;
|
||||
} catch {
|
||||
// fall through to execCommand fallback
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof document === "undefined") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const textarea = document.createElement("textarea");
|
||||
textarea.value = text;
|
||||
textarea.setAttribute("readonly", "");
|
||||
textarea.style.position = "fixed";
|
||||
textarea.style.top = "0";
|
||||
textarea.style.left = "0";
|
||||
textarea.style.width = "1px";
|
||||
textarea.style.height = "1px";
|
||||
textarea.style.padding = "0";
|
||||
textarea.style.border = "none";
|
||||
textarea.style.outline = "none";
|
||||
textarea.style.boxShadow = "none";
|
||||
textarea.style.background = "transparent";
|
||||
textarea.style.opacity = "0";
|
||||
document.body.appendChild(textarea);
|
||||
|
||||
const previousSelection = document.getSelection();
|
||||
const previousRange =
|
||||
previousSelection && previousSelection.rangeCount > 0
|
||||
? previousSelection.getRangeAt(0)
|
||||
: null;
|
||||
|
||||
try {
|
||||
textarea.focus();
|
||||
textarea.select();
|
||||
textarea.setSelectionRange(0, text.length);
|
||||
return document.execCommand("copy");
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
document.body.removeChild(textarea);
|
||||
if (previousRange && previousSelection) {
|
||||
previousSelection.removeAllRanges();
|
||||
previousSelection.addRange(previousRange);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loaded 100 of 438 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user